diff --git a/.github/workflows/beta-installer.yml b/.github/workflows/beta-installer.yml index 654477f6..8aa26e2c 100644 --- a/.github/workflows/beta-installer.yml +++ b/.github/workflows/beta-installer.yml @@ -100,12 +100,30 @@ jobs: - name: Download the bundled virtual display driver shell: pwsh run: | + # GitHub's download servers sometimes answer 5xx for a minute or two. + # Up to 5 tries over 2 minutes; another 4xx fails at once. The next + # step checks the driver's pinned SHA-256 either way. + function Get-WithRetry($uri, $out) { + $waits = 10, 20, 30, 60 + for ($try = 1; ; $try++) { + try { + Invoke-WebRequest -Uri $uri -OutFile $out + return + } catch { + $status = [int]$_.Exception.Response.StatusCode + $transient = $status -eq 0 -or $status -eq 429 -or $status -ge 500 + if (-not $transient -or $try -gt $waits.Count) { throw } + Write-Warning "Download of $uri failed (try $try): $($_.Exception.Message). Retrying in $($waits[$try - 1]) s." + Start-Sleep -Seconds $waits[$try - 1] + } + } + } $dest = "installer\vendor\vdd" $url = "https://github.com/VirtualDrivers/Virtual-Display-Driver/releases/download/$env:VDD_RELEASE_TAG/$env:VDD_ASSET" - Invoke-WebRequest -Uri $url -OutFile (Join-Path $dest $env:VDD_ASSET) + Get-WithRetry $url (Join-Path $dest $env:VDD_ASSET) # Its MIT license text, redistributed alongside the binary. - Invoke-WebRequest -Uri "https://raw.githubusercontent.com/VirtualDrivers/Virtual-Display-Driver/$env:VDD_RELEASE_TAG/LICENSE" ` - -OutFile (Join-Path $dest "LICENSE-VirtualDisplayDriver.txt") + Get-WithRetry "https://raw.githubusercontent.com/VirtualDrivers/Virtual-Display-Driver/$env:VDD_RELEASE_TAG/LICENSE" ` + (Join-Path $dest "LICENSE-VirtualDisplayDriver.txt") - name: Verify the driver against the pinned SHA-256 shell: pwsh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 691a0493..633ac21f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,9 @@ jobs: - name: Verify Setup hands its language to the app shell: powershell run: ./installer/tests/Test-InstallerLanguages.ps1 + - name: Verify Setup leaves the virtual display driver off Windows 10 + shell: powershell + run: ./installer/tests/Test-InstallerWindows10.ps1 rust-macos: name: Rust (macOS, full workspace + E2E) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ced5da86..0d5ef1ac 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -7,8 +7,10 @@ name: Beta builds # https://github.com/sidebandstudio/EternalMonitor/releases/download/beta/EternalMonitor-Setup-beta.exe # never changes (docs/download.html?channel=beta links it). No tag here starts # with v, so release.yml and testflight.yml never run for a beta build, and -# docs/latest.json, /releases/latest and the apps' update checks never see -# one. scripts/nightly.py has the version, branch and retention rules. +# docs/latest.json, /releases/latest and stable installs never see one. A +# Beta install reads the beta release's body for the newest Beta +# (host/src/updates.rs). scripts/nightly.py has the version, branch and +# retention rules. # # Only a run on main publishes. A manual run with dry_run, on any other # branch or for a pull request that changes this pipeline, builds and scans @@ -163,6 +165,19 @@ jobs: python3 scripts/nightly.py notes --version "$VERSION" --branch "$BRANCH" \ --sha "$SHA" --stable "$STABLE" --sha256 "$SHA256" > notes.md + # What scripts/sign_beta_manifest.py --sha256/--version must be given: + # the hash the build computed, which the step above checked. + - name: Summarize the build to sign + run: | + { + echo "### Beta build to sign" + echo + echo "| | |" + echo "|---|---|" + echo "| Version | $VERSION |" + echo "| Installer SHA-256 | \`$SHA256\` |" + } >> "$GITHUB_STEP_SUMMARY" + # The dated release first, checked anonymously; then the fixed beta link, # swapped by renames and checked byte for byte before the old asset goes; # the beta tag; and last the marker plan reads to skip a finished commit, diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a66da814..83476248 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -101,12 +101,30 @@ jobs: - name: Download the bundled virtual display driver shell: pwsh run: | + # GitHub's download servers sometimes answer 5xx for a minute or two. + # Up to 5 tries over 2 minutes; another 4xx fails at once. The next + # step checks the driver's pinned SHA-256 either way. + function Get-WithRetry($uri, $out) { + $waits = 10, 20, 30, 60 + for ($try = 1; ; $try++) { + try { + Invoke-WebRequest -Uri $uri -OutFile $out + return + } catch { + $status = [int]$_.Exception.Response.StatusCode + $transient = $status -eq 0 -or $status -eq 429 -or $status -ge 500 + if (-not $transient -or $try -gt $waits.Count) { throw } + Write-Warning "Download of $uri failed (try $try): $($_.Exception.Message). Retrying in $($waits[$try - 1]) s." + Start-Sleep -Seconds $waits[$try - 1] + } + } + } $dest = "installer\vendor\vdd" $url = "https://github.com/VirtualDrivers/Virtual-Display-Driver/releases/download/$env:VDD_RELEASE_TAG/$env:VDD_ASSET" - Invoke-WebRequest -Uri $url -OutFile (Join-Path $dest $env:VDD_ASSET) + Get-WithRetry $url (Join-Path $dest $env:VDD_ASSET) # Its MIT license text, redistributed alongside the binary. - Invoke-WebRequest -Uri "https://raw.githubusercontent.com/VirtualDrivers/Virtual-Display-Driver/$env:VDD_RELEASE_TAG/LICENSE" ` - -OutFile (Join-Path $dest "LICENSE-VirtualDisplayDriver.txt") + Get-WithRetry "https://raw.githubusercontent.com/VirtualDrivers/Virtual-Display-Driver/$env:VDD_RELEASE_TAG/LICENSE" ` + (Join-Path $dest "LICENSE-VirtualDisplayDriver.txt") - name: Verify the driver against the pinned SHA-256 shell: pwsh diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index eb9f289f..9461197c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -75,6 +75,7 @@ Rust and Swift share golden vectors and stable packet-type registry tests. | KEYFRAME_REQUEST | Recovery after repair expiry or decoder error | | PING / PONG | Min-RTT clock estimate for end-to-end latency | | INPUT_EVENT | Normalized mouse/touch events, HID keys and UTF-16 text | +| POINTER_SHAPE / POINTER_STATE | The Mirror's pointer image (chunked premultiplied BGRA and an invert mask) and its position, at most once a frame budget, for an iPad that draws the pointer itself (feature bit 5, host capability bit 6); the video then leaves the pointer out. The Second screen does the same when the running virtual display driver started with a hardware cursor (as the host's settings file sets it), which the host knows from its record of that start (boot and device arrival); a pointer image larger than that cursor's caps stays in the video. When the host can't tell, it does as 0.3.4: no pointer messages, and the pointer in the video. | | Audio | Sequenced Opus packets with discontinuity and capture timing | | BYE | User, background or shutdown teardown; the host ending a silent session | @@ -196,7 +197,15 @@ connected client, writes the requested mode first in VDD XML, and reports actual task failures. Setup and the host log the active display paths around every driver change; when a change leaves the PC without any real screen it had, they re-apply the saved layout once, else switch to "PC screen only" if the built-in -panel was on (`display_guard.rs`, `display-guard.ps1`). It also ships a UDP +panel was on (`display_guard.rs`, `display-guard.ps1`). At runtime the host also +acts while the virtual display is still on: a turn on that leaves it alone +switches Windows to "Extend" (else the saved layout or "PC screen only"), and a +turn off never runs while it is the only screen, until Windows ends the session. +On a hybrid laptop whose panel the virtual display takes away until a restart +(Windows withdraws its target while the lid is open), it tries nothing and +keeps the display on while the host runs; the turn offs at start-up, quit and +in the failure reporter run, so the restart starts without it. Second screen +mirrors there until the user chooses it again. It also ships a UDP firewall rule. Runtime upgrade, task-ACL and uninstall checks remain hardware gates. diff --git a/Cargo.lock b/Cargo.lock index 03c5906c..63450620 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -998,10 +998,9 @@ dependencies = [ [[package]] name = "egui-wgpu" version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cef1f6d1bba40b09a4d0d62cb822b81030922cfbf65c5373398c3a3caafc5bd" dependencies = [ "ahash", + "ash", "bytemuck", "document-features", "egui", diff --git a/Cargo.toml b/Cargo.toml index c25514d0..2b301e23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,13 @@ # docs/ contains the static website (GitHub Pages). Never write build output here. [workspace] members = ["host", "proto", "updater"] +exclude = ["vendor"] resolver = "2" [profile.release] strip = "symbols" opt-level = 3 + +# Patched copies of upstream crates; vendor/README.md says why. +[patch.crates-io] +egui-wgpu = { path = "vendor/egui-wgpu" } diff --git a/SECURITY.md b/SECURITY.md index e058dcc1..825ec52c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,7 +33,16 @@ Knowing these limits helps you judge whether a finding is new. - The installer is not code-signed yet. Check its SHA-256 against the release page. - In-app updates (Windows) trust the network for nothing: the host fetches over HTTPS from an allowlist of GitHub hosts, and the installer must match the - SHA-256 in both eternalmonitor.dev/latest.json and the GitHub release. Against + SHA-256 in both eternalmonitor.dev/latest.json and the GitHub release. A Beta + build may also update to the newer Beta from the fixed `beta` GitHub release. + Beta updates are signed: the release must carry a manifest with a valid + Ed25519 signature by a public key pinned in the app, made with a key the + release workflows don't have, and the manifest's version, SHA-256 and size + must match the release body, GitHub's digest of the installer and the + downloaded file. Without a valid signature no Beta is offered, and a signed + Beta older than one the install has already verified is refused, so an old + signed release can't be replayed to hold testers back. Stable builds never + take a Beta. Against other software already running as your Windows user, the updater adds defense in depth rather than a guarantee. Its helper loads DLLs only from System32 and imports only Windows system libraries. It inherits only its output pipe from diff --git a/SETUP.md b/SETUP.md index 6b0a5ba4..4c60be66 100644 --- a/SETUP.md +++ b/SETUP.md @@ -255,12 +255,13 @@ try fewer words. If your problem isn't here, see [Known issues](https://eternalm | **EternalMonitor can't look for PCs** | The iPad isn't allowed to search your local network. | Tap **Open Settings** and turn on **Local Network**, or go to **Settings > Privacy & Security > Local Network** and turn on **EternalMonitor**. Then tap **Search again**. A USB cable works without this setting. | | **Couldn't search for PCs** | The iPad's search for PCs failed for a reason other than Local Network access. | Tap **Search again**, or scan the QR code or type the **Address** shown in the PC window. If it keeps failing, turn on **Local Network** for EternalMonitor in the iPad's Settings. | | **Couldn't get its address. Type the Address shown on the PC.** under a found PC | The iPad saw the PC's name but not its address. | Type the **Address** shown in the PC window, then tap **Connect to PC**. | -| **Your PC didn't answer** | Nothing came back from the PC. It may be closed, asleep, on another network, or blocked by a firewall. | Check that EternalMonitor is open on the PC and says **Ready for your iPad**, that the PC is awake, and that both are on the same Wi-Fi, not a guest network. If all of that is right, Windows Firewall or security software is probably blocking EternalMonitor: run EternalMonitor-Setup.exe again, or use a USB cable. Then tap **Try again**. | +| **Your PC didn't answer** | Nothing came back from the PC. It may be closed, asleep, on another network, or blocked by a firewall. | Check that EternalMonitor is open on the PC and says **Ready for your iPad**, that the PC is awake, and that both are on the same Wi-Fi, not a guest network. If all of that is right, Windows Firewall or security software is probably blocking EternalMonitor: run EternalMonitor-Setup.exe again, or use a USB cable. Then tap **Try again**. If you open EternalMonitor on the PC within a few minutes of the message appearing, the iPad connects by itself. | | **Your PC didn't answer over the cable** | The cable is connected, but EternalMonitor on the PC didn't reply through it. | Check that EternalMonitor and Apple Devices are both open on the PC, then unplug the cable and plug it in again. | | **Your PC answered, but no picture arrived** | The connection worked, but the PC's second screen didn't start, or the PC couldn't send its screen. | On the PC, choose **Mirror my screen** (**Main display (mirror)** in version 0.3.1), then tap **Try again** on the iPad. If the PC already mirrors, click **Restart** there first. After two attempts in a row end this way, the iPad stops reconnecting by itself until you tap **Try again**. | | **That looks like the pairing code, not the PC's address** | A six-digit number was typed where the address goes. | Type the **Address** from the PC window, for example 192.168.1.20. The iPad asks for the code after it connects. | | **That looks like your router's address, not your PC's** | Nothing answered at an address ending in .1, which is usually the Wi-Fi router. | Type the **Address** shown in the PC window. | | **That address doesn't look right** | The iPad couldn't find anything by that name. | Type the **Address** exactly as the PC window shows it. | +| **Wi-Fi is off** | iPadOS has no network for the connection: Wi-Fi is off on the iPad, or it hasn't joined a network. | Turn on Wi-Fi and join the same network as the PC, or use a USB cable. | | **This iPad isn't connected to a network** | Wi-Fi is off on the iPad, or it hasn't joined a network. | Turn on Wi-Fi and join the same network as the PC, or use a USB cable. | | **Local Network is turned off for EternalMonitor** | Local Network access is off for EternalMonitor on the iPad, so it can't reach the PC over Wi-Fi. | Tap **Open Settings** and turn on **Local Network**, or go to **Settings > Privacy & Security > Local Network** and turn on **EternalMonitor**. Then tap **Try again**. A USB cable works without this setting. | | **Couldn't connect to your PC** | The connection failed for another reason. | Check that both are on the same Wi-Fi and that EternalMonitor is open on the PC, then tap **Try again**. | @@ -274,7 +275,10 @@ try fewer words. If your problem isn't here, see [Known issues](https://eternalm | **The connection over the cable stopped** | The USB connection broke after it had started. | Keep EternalMonitor and Apple Devices open on the PC, unlock the iPad, then unplug the cable and plug it in again. | | **The USB cable was disconnected** | The cable came out, or the PC lost it. | Plug the cable back in. The iPad reconnects by itself. | | **USB connections were turned off** | **Allow USB connections** is off in the iPad app's settings. | Turn on **Allow USB connections** to use the cable again. | -| **Your PC ended the session** | Someone clicked **Stop** on the PC, changed a setting that restarts the stream, or closed EternalMonitor. Or the connection dropped, for example in a Wi-Fi stall, or the PC ran into a problem. | Tap **Try again** when the PC says **Ready for your iPad**. | +| **Your PC ended the session** | Someone clicked **Stop** on the PC, changed a setting that restarts the stream, or closed EternalMonitor. | Tap **Try again** when the PC says **Ready for your iPad**. | +| **The connection to your PC ended** | The connection dropped, for example in a Wi-Fi stall, the PC ran into a problem, or EternalMonitor restarted itself on the PC. | Tap **Try again**. With **Reconnect automatically** on, the iPad tries again by itself first. | +| **Your PC went to sleep** | Windows put the PC to sleep while it streamed. | Wake the PC, then tap **Try again**. | +| **Your PC is shutting down or signing out** | Windows shut down, restarted or signed out while the PC streamed. | Tap **Try again** once the PC is on, someone has signed in, and EternalMonitor is open. | | **Your PC is already streaming to another device** | Only one iPad can connect at a time. | Disconnect the other iPad first, then tap **Try again**. | | **Your PC is updating EternalMonitor** | The PC is installing a new version. | Wait. With **Reconnect automatically** on, the iPad connects again by itself. Otherwise tap **Try again** once EternalMonitor has opened on the PC. | | **Your PC hasn't come back from its update yet** | The update is taking longer than expected. | Check that EternalMonitor opened again on the PC. Windows may be waiting for you to click **Yes** there. Then tap **Try again**. | diff --git a/docs/beta-testing.md b/docs/beta-testing.md index e4e23a7b..95c8cad0 100644 --- a/docs/beta-testing.md +++ b/docs/beta-testing.md @@ -32,6 +32,30 @@ back to Stable by running the Stable installer over it. `docs/latest.json`, else that release's next patch. It sorts below `0.3.4`, so Beta users get the next Stable release from the normal update prompt, and Stable users are never offered a Beta build. +- **Updates on Beta:** a Beta build also reads the `beta` release below and + offers the newer of that Beta and the next Stable release, with the same + banner, popup and helper as a Stable update. Beta updates are signed and + verified against a public key pinned in the app: the release must carry + `beta-manifest.txt` and its Ed25519 signature `beta-manifest.sig`, and the + manifest's version, SHA-256 and size must match the release body + `nightly.py` writes, GitHub's digest of the installer and the downloaded + file. The nightly does not sign; until a build is signed, Beta installs are + offered no Beta. After a nightly publishes, open its run (Actions, Beta + builds) and copy **Installer SHA-256** and **Version** from the Publish + job's summary, "Beta build to sign": the hash the Windows installer job + computed for the build it scanned, which Publish checks against the file + it uploads. Then run + `scripts/sign_beta_manifest.py --sha256 --version ` with + the signing key on stdin. It refuses a release that names another + installer or version, checks the release's parts agree, signs, checks the + signature with the pinned key, uploads both files and verifies them; + `--verify-only` checks what is published. A signed Beta older than one an + install has already seen is refused, so taking a bad Beta back means + signing a newer build, not re-signing an older one. + A new Beta is announced at most once a day, never during a stream. Settings, General, **Get Beta updates** (shown only on a + Beta build) turns this off. Taking a Stable release leaves the Beta channel. + Betas built before this arrived can't update themselves, so their testers + install one newer Beta by hand. - **On GitHub:** each build is a pre-release tagged `beta--`, titled like "EternalMonitor 0.3.4 beta (Oct 30, 2026)". The newest 7 are kept. The pre-release `beta` always holds the newest, so this link never diff --git a/docs/decisions.md b/docs/decisions.md index 8ab265e0..6561517f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -225,6 +225,64 @@ such frames then completed from late or repaired fragments. The iPad now requests a wide gap in several NACKs and asks for a keyframe only when a frame is dropped. Evidence: `EternalMonitor-Handoff/evidence/rc-hardware-20260924/`. +### Repair a keyframe until it completes + +On WiFi with 4-22% loss, iPads held a frozen picture for 1-4 s and then +reconnected. The PC answered every keyframe request at once, but each +replacement keyframe arrived incomplete. A keyframe at 1080p is 80-300 +fragments, so at 10% loss one round of requests and repairs leaves some +missing. The iPad asked again once, and dropped the keyframe at its 25 ms +deadline because the small frames sent after it were already complete. Those +frames need the keyframe, so the decoder held the picture and asked for +another, which failed the same way. + +A keyframe now keeps its place while the frames after it complete. At its +deadline it is held, like a frame with nothing behind it, until it completes, +a newer keyframe completes, a second has passed, or the 32 frames the +assembler holds are full. The hold only moves its deadline. The keyframe's +requests follow one schedule: after its first retry it asks again every RTT + +5 ms, sooner than which no repair can return, and only once something has +arrived since its last request. The repair simulation +(`proto/tests/wifi_repair_sim.rs`, 10% loss each way, 150-fragment keyframes +at 60 fps) completed 148 of 390 keyframes before, with 35 freezes over a +second and the longest 3.4 s. After, it completes 394 of 399, the longest +freeze is 0.61 s, and no freeze lasts a second. The 1% loss rows also lost +fewer frames. + +Delta frames keep the 25 ms rule and their single retry. Holding them too, +for at most 100 ms, with every frame asking again, removed nearly every +remaining drop in the simulation, but raised the p50 latency at 10% loss from +6.6 to 10.7 ms. It would also undo the repair-window rules above, which were +tuned on a physical iPad, so it waits for a device run. + +### Answer a loss after a recovery at once + +With keyframes repaired, the first run on real encoders under loss (NVENC and +AMF, headless hosts with synthetic capture and the fake iPad, 1-10% loss and +300 ms outages) put the longest recoveries somewhere else. The PC's IDR began +to arrive 15-130 ms after a request (medians), AMF's reopen included. But at +10% loss the next frame was often lost within half a second of a recovery +landing. The iPad sends one keyframe request per 500 ms and the PC grants one +per 500 ms, so that loss waited for the decoder's next try 600 ms later. Most +recoveries over half a second began that way. Keyframe size was not the cause: +a P-frame was 20-30 KB, an IDR 30-100 KB, sent within one frame interval. + +A loss after a recovery now gets its own IDR at once. When a keyframe has +arrived since the iPad's last request, its next GapLoss request goes inside +the 500 ms spacing, once per keyframe, without moving that spacing. The PC +grants it inside its own window when the frame the request names is at or +past the first keyframe the PC sent since its last grant, because then the +iPad already had that recovery. That grant doesn't move the window either. A +request that names an older frame, or none, as the decoder's requests do, +waits as before, so one loss still gets one IDR. A PC that does this sets +`HOSTCAP_KEYFRAME_PER_LOSS`, and the iPad asks early only such a PC. An older +PC moves its window for any grant, so an early request it granted could push +the iPad's next one out of its window. + +AMF still reopens for each forced keyframe, at most once per 500 ms, so on +AMF the IDR for such a loss can still wait for that limit. The 300 ms outage +rows, which end with one request after the outage, are unchanged. + ### Use Apple's usbmuxd tunnel for USB Apple's installed device service handles the cable/trust relationship. EMLINK diff --git a/host/Cargo.toml b/host/Cargo.toml index 165e0bca..9b45e2c0 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -51,7 +51,7 @@ socket2 = "0.5" [features] # Lets a release build obey ETERNAL_SIMULATE_STARTUP_FAILURE, for native # failure-path checks. Debug builds always do; shipped builds never do. -fault-injection = [] +fault-injection = ["windows/Win32_Graphics_Direct3D12"] # Lets a release build obey ETERNAL_USB_DIRECT, for the end-to-end scripts. # Debug builds always do; shipped builds never do. usb-direct = [] @@ -86,6 +86,7 @@ features = [ "Win32_System_Kernel", "Win32_System_LibraryLoader", "Win32_System_Com_StructuredStorage", + "Win32_System_Ole", "Win32_System_Threading", "Win32_System_LibraryLoader", "Win32_System_Performance", @@ -93,6 +94,7 @@ features = [ "Win32_System_ProcessStatus", "Win32_Storage_FileSystem", "Win32_Devices_FunctionDiscovery", + "Win32_Devices_Properties", "Win32_UI_Shell_PropertiesSystem", "Win32_Graphics_DXCore", "Win32_Graphics_Dxgi", @@ -102,11 +104,13 @@ features = [ "Win32_Graphics_Gdi", "Win32_Globalization", "Win32_System_Registry", + "Win32_System_RemoteDesktop", "Win32_System_StationsAndDesktops", "Win32_System_SystemInformation", "Win32_System_IO", "Win32_NetworkManagement_IpHelper", "Win32_NetworkManagement_Ndis", + "Win32_NetworkManagement_WindowsFirewall", "Win32_Networking_WinSock", "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_Input_Pointer", diff --git a/host/examples/first_frame_probe.rs b/host/examples/first_frame_probe.rs index f11ed850..c828e80f 100644 --- a/host/examples/first_frame_probe.rs +++ b/host/examples/first_frame_probe.rs @@ -11,7 +11,7 @@ use std::time::{Duration, Instant}; use eternal_wire::reassembly::{AddOutcome, Reassembler}; use eternal_wire::v2::control::{ - encode_control, parse_control, ControlMessage, Hello2, HelloStatus, ReceiverReport, + encode_control, parse_control, ControlMessage, Hello2, HelloStatus, Ping, ReceiverReport, CAP_DECODE_H264, }; use eternal_wire::v2::media::MediaHeader; @@ -102,7 +102,19 @@ fn main() { if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(bytes) { assert_eq!(ack.status, HelloStatus::Ok, "pairing must be off"); session_id = ack.session_id; - acked_at.get_or_insert_with(Instant::now); + if acked_at.is_none() { + acked_at = Some(Instant::now()); + // As the app does when its HELLO_ACK lands: the host + // sends a viewer that has not answered only its + // first picture. + msg_seq += 1; + let ping = ControlMessage::Ping(Ping { + t1_us: started.elapsed().as_micros() as u64, + }); + socket + .send_to(&encode_control(session_id, msg_seq, &ping), &host) + .unwrap(); + } } } Classified::Media { .. } if session_id != 0 => { diff --git a/host/examples/input_soak.rs b/host/examples/input_soak.rs new file mode 100644 index 00000000..6d0141e1 --- /dev/null +++ b/host/examples/input_soak.rs @@ -0,0 +1,326 @@ +//! A fake iPad that streams input at a host while it connects and leaves +//! over and over, so a host set to Second screen turns the virtual display +//! on and off under the input. It logs every gap in what the host sends and +//! how long each HELLO waits for its answer: a stuck transport loop shows up +//! as both. It also logs its own send errors and late wake-ups, so a stall +//! of its own is not mistaken for the host's. +//! +//! The input is harmless on a real desktop: pointer and Pencil hover moves +//! (no clicks, no ink) and taps of Right Ctrl alone. +//! +//! `input_soak ` +//! +//! Keep the log, and stdout if redirected, on a disk nothing else is busy +//! writing: a log write that waits seconds stops the input with it, and the +//! host then rightly ends the session for silence. +use std::io::Write; +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use eternal_wire::v2::control::{ + encode_control, parse_control, ByeReason, ControlMessage, Hello2, HelloStatus, InputEvent, + Ping, ReceiverReport, CAP_DECODE_H264, CAP_DECODE_HEVC, FEATURE_SUPPORTS_NACK, + FEATURE_WANTS_INPUT, +}; +use eternal_wire::v2::{classify, Classified}; + +const KIND_PENCIL: u8 = 1; +const KIND_KEY: u8 = 4; +const KIND_HOVER: u8 = 6; +const PHASE_BEGAN: u8 = 0; +const PHASE_MOVED: u8 = 1; +const PHASE_ENDED: u8 = 2; +const RIGHT_CTRL: u16 = 0xe4; +/// Connected and away times, cycled. The short ones leave while the +/// display is still coming up. +const CYCLE: [(u64, u64); 4] = [(25, 15), (4, 14), (25, 15), (8, 3)]; +const GAP_LOGGED: Duration = Duration::from_millis(1500); +const INPUT_EVERY: Duration = Duration::from_millis(8); +/// A wake-up this late is the soak's own stall. +const LATE_WAKE: Duration = Duration::from_millis(100); + +struct Log { + start: Instant, + file: Mutex, +} + +impl Log { + fn note(&self, line: String) { + let line = format!("{{\"t_ms\":{},{line}}}", self.start.elapsed().as_millis()); + println!("{line}"); + let _ = writeln!(self.file.lock().unwrap(), "{line}"); + } +} + +/// What the receiving thread heard. +#[derive(Default)] +struct Heard { + session: AtomicU32, + frames: AtomicU64, + connected: AtomicBool, + cycle: AtomicU32, + last: Mutex>, + worst_gap_ms: AtomicU64, + recv_errors: AtomicU64, +} + +fn receive(socket: &UdpSocket, heard: &Heard, log: &Log) { + let mut buffer = [0u8; 2048]; + loop { + let (len, _) = match socket.recv_from(&mut buffer) { + Ok(received) => received, + Err(error) => { + if !matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) { + heard.recv_errors.fetch_add(1, Ordering::Relaxed); + } + continue; + } + }; + let bytes = &buffer[..len]; + let now = Instant::now(); + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(bytes) { + if ack.status == HelloStatus::Ok + && ack.client_nonce == heard.cycle.load(Ordering::SeqCst) + && heard.session.load(Ordering::SeqCst) == 0 + { + *heard.last.lock().unwrap() = Some(now); + heard.session.store(ack.session_id, Ordering::SeqCst); + } + continue; + } + if !heard.connected.load(Ordering::SeqCst) { + continue; + } + if let Classified::Media { .. } = classify(bytes) { + heard.frames.fetch_add(1, Ordering::Relaxed); + } + let last = heard.last.lock().unwrap().replace(now); + if let Some(gap) = last.map(|last| now - last) { + heard + .worst_gap_ms + .fetch_max(gap.as_millis() as u64, Ordering::Relaxed); + if gap >= GAP_LOGGED { + log.note(format!( + "\"event\":\"gap\",\"cycle\":{},\"gap_ms\":{}", + heard.cycle.load(Ordering::SeqCst), + gap.as_millis() + )); + } + } + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + let host: SocketAddr = args[1].parse().unwrap(); + let seconds: u64 = args[2].parse().unwrap(); + let start = Instant::now(); + let log = Arc::new(Log { + start, + file: Mutex::new(std::fs::File::create(&args[3]).unwrap()), + }); + let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").unwrap()); + socket + .set_read_timeout(Some(Duration::from_millis(100))) + .unwrap(); + let heard = Arc::new(Heard::default()); + { + let (socket, heard, log) = (socket.clone(), heard.clone(), log.clone()); + std::thread::spawn(move || receive(&socket, &heard, &log)); + } + let mut msg_seq = 0u32; + let send_errors = AtomicU64::new(0); + let send = |msg_seq: &mut u32, session_id: u32, message: &ControlMessage| { + *msg_seq += 1; + if socket + .send_to(&encode_control(session_id, *msg_seq, message), host) + .is_err() + { + send_errors.fetch_add(1, Ordering::Relaxed); + } + }; + let mut event_id = 0u32; + let (mut worst_ack, mut worst_wake, mut cycles) = (Duration::ZERO, Duration::ZERO, 0u32); + 'run: for (on, off) in CYCLE.iter().cycle().copied() { + if start.elapsed() >= Duration::from_secs(seconds) { + break; + } + cycles += 1; + heard.cycle.store(cycles, Ordering::SeqCst); + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: cycles, + listen_port: socket.local_addr().unwrap().port(), + decoder_caps: CAP_DECODE_H264 | CAP_DECODE_HEVC, + feature_caps: FEATURE_WANTS_INPUT | FEATURE_SUPPORTS_NACK, + screen_px_w: 2732, + screen_px_h: 2048, + screen_pt_w: 1366, + screen_pt_h: 1024, + refresh_hz: 60, + device_name: "Input soak".into(), + device_id: 0x50a6, + preferred_fps: 60, + auth_token: [0; 16], + pairing_code: 0, + }); + heard.session.store(0, Ordering::SeqCst); + let asked = Instant::now(); + let mut next_hello = asked; + let session_id = loop { + if start.elapsed() >= Duration::from_secs(seconds) { + break 'run; + } + let session_id = heard.session.load(Ordering::SeqCst); + if session_id != 0 { + break session_id; + } + if Instant::now() >= next_hello { + send(&mut msg_seq, 0, &hello); + next_hello += Duration::from_millis(500); + } + std::thread::sleep(Duration::from_millis(2)); + }; + let waited = asked.elapsed(); + worst_ack = worst_ack.max(waited); + heard.connected.store(true, Ordering::SeqCst); + log.note(format!( + "\"event\":\"ack\",\"cycle\":{cycles},\"waited_ms\":{}", + waited.as_millis() + )); + let connected_until = Instant::now() + Duration::from_secs(on); + let (mut next_report, mut next_ping) = (Instant::now(), Instant::now()); + let mut step = 0u32; + let mut key_down = false; + let mut due = Instant::now(); + while Instant::now() < connected_until { + let late = Instant::now().saturating_duration_since(due); + if late >= LATE_WAKE { + worst_wake = worst_wake.max(late); + log.note(format!( + "\"event\":\"late_wake\",\"cycle\":{cycles},\"late_ms\":{}", + late.as_millis() + )); + } + due = Instant::now() + INPUT_EVERY; + step += 1; + event_id += 1; + let angle = f64::from(step) / 60.0; + let mut event = InputEvent { + input_ver: 1, + kind: KIND_HOVER, + phase: PHASE_MOVED, + buttons: 0, + event_id, + x_norm: (32768.0 + 20000.0 * angle.cos()) as u16, + y_norm: (32768.0 + 20000.0 * angle.sin()) as u16, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: 0, + modifiers: 0, + client_time_us: start.elapsed().as_micros() as u64, + tilt_x: 0, + tilt_y: 0, + }; + match step % 4 { + // Pencil hover: native pen frames without contact. + 1 | 3 => { + event.input_ver = 2; + event.kind = KIND_PENCIL; + event.tilt_x = (step % 90) as i16 - 45; + event.tilt_y = 30; + if step % 400 == 399 { + event.phase = PHASE_ENDED; // leave + } + } + 2 if step % 32 == 2 => { + event.kind = KIND_KEY; + event.keycode = RIGHT_CTRL; + event.phase = if key_down { PHASE_ENDED } else { PHASE_BEGAN }; + key_down = !key_down; + } + _ => {} + } + send(&mut msg_seq, session_id, &ControlMessage::InputEvent(event)); + let now = Instant::now(); + if now >= next_report { + next_report += Duration::from_millis(500); + let complete = heard.frames.load(Ordering::Relaxed) as u32; + send( + &mut msg_seq, + session_id, + &ControlMessage::ReceiverReport(ReceiverReport { + stream_epoch: 1, + highest_seq: complete, + frames_complete: complete, + ..ReceiverReport::default() + }), + ); + } + if now >= next_ping { + next_ping += Duration::from_secs(2); + send( + &mut msg_seq, + session_id, + &ControlMessage::Ping(Ping { + t1_us: start.elapsed().as_micros() as u64, + }), + ); + } + std::thread::sleep(due.saturating_duration_since(Instant::now())); + } + if key_down { + event_id += 1; + send( + &mut msg_seq, + session_id, + &ControlMessage::InputEvent(InputEvent { + input_ver: 1, + kind: KIND_KEY, + phase: PHASE_ENDED, + buttons: 0, + event_id, + x_norm: 0, + y_norm: 0, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: RIGHT_CTRL, + modifiers: 0, + client_time_us: start.elapsed().as_micros() as u64, + tilt_x: 0, + tilt_y: 0, + }), + ); + } + for _ in 0..3 { + send( + &mut msg_seq, + session_id, + &ControlMessage::Bye(ByeReason::UserDisconnect), + ); + } + heard.connected.store(false, Ordering::SeqCst); + *heard.last.lock().unwrap() = None; + log.note(format!( + "\"event\":\"bye\",\"cycle\":{cycles},\"frames\":{},\"send_errors\":{},\"recv_errors\":{}", + heard.frames.swap(0, Ordering::Relaxed), + send_errors.swap(0, Ordering::Relaxed), + heard.recv_errors.swap(0, Ordering::Relaxed) + )); + std::thread::sleep(Duration::from_secs(off)); + } + log.note(format!( + "\"event\":\"summary\",\"cycles\":{cycles},\"worst_gap_ms\":{},\"worst_ack_ms\":{},\"worst_late_wake_ms\":{}", + heard.worst_gap_ms.load(Ordering::Relaxed), + worst_ack.as_millis(), + worst_wake.as_millis() + )); +} diff --git a/host/src/audio/mod.rs b/host/src/audio/mod.rs index a0d30698..5af1b7ae 100644 --- a/host/src/audio/mod.rs +++ b/host/src/audio/mod.rs @@ -50,6 +50,31 @@ pub fn wanted_session(shared: &SharedControl) -> Option { } } +/// Tests that replace the global audio stats run one at a time. +#[cfg(test)] +pub(crate) static STATS_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); + +/// The audio status while another app has the PC's output to itself. +pub const DEVICE_IN_USE_STATUS: &str = "Output in use by another app"; + +/// Another app has the PC's output device to itself (exclusive mode), so +/// loopback capture can't open it. Not the session's failure: the audio +/// stage tries again until the app lets go. +#[derive(Debug)] +pub struct DeviceInUse; + +impl std::fmt::Display for DeviceInUse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("AUDCLNT_E_DEVICE_IN_USE: another app has exclusive use of the output device") + } +} + +impl std::error::Error for DeviceInUse {} + +pub fn is_device_in_use(error: &(dyn std::error::Error + Send + Sync + 'static)) -> bool { + error.downcast_ref::().is_some() +} + pub fn codec_available() -> bool { ffmpeg_next::encoder::find_by_name("libopus").is_some() } diff --git a/host/src/audio/stage.rs b/host/src/audio/stage.rs index 8b3dab70..a0f7f157 100644 --- a/host/src/audio/stage.rs +++ b/host/src/audio/stage.rs @@ -6,7 +6,9 @@ use tokio::sync::mpsc::{error::TrySendError, Sender}; use super::encoder::OpusEncoder; use super::resample::{PcmBlock, PcmFormat, Resampler}; use super::synthetic::SyntheticSource; -use super::{wanted_session, AudioResult, AudioStats, SessionPacket, SAMPLE_RATE}; +use super::{ + wanted_session, AudioResult, AudioStats, SessionPacket, DEVICE_IN_USE_STATUS, SAMPLE_RATE, +}; use crate::capture::timing::FrameTimer; use crate::control::SharedControl; use crate::stats::PIPELINE_STATS; @@ -29,6 +31,8 @@ enum Source { Wasapi(Box), #[cfg(test)] Scripted(std::collections::VecDeque), + #[cfg(test)] + Fake(Box AudioResult>>), } impl Source { @@ -63,6 +67,8 @@ impl Source { Self::Wasapi(source) => source.name(), #[cfg(test)] Self::Scripted(_) => "Scripted test source", + #[cfg(test)] + Self::Fake(_) => "Fake test source", } } @@ -96,6 +102,8 @@ impl Source { Self::Wasapi(source) => source.read(), #[cfg(test)] Self::Scripted(blocks) => Ok(blocks.pop_front()), + #[cfg(test)] + Self::Fake(read) => read(), } } } @@ -121,10 +129,10 @@ struct CaptureSession { } impl CaptureSession { - fn new(id: u32) -> AudioResult { + fn new(id: u32, source: Source) -> AudioResult { Ok(Self { id, - source: Source::open()?, + source, resampler: None, encoder: OpusEncoder::new(MAX_OPUS_PAYLOAD)?, mark_discontinuity: true, @@ -302,48 +310,108 @@ pub(crate) fn restart_stats( } } -/// When to try again to open audio while FFmpeg's encoder lane is stuck: -/// soon at first, then at most every 8 s, so a long stall neither spins nor +/// When to try again to open audio that waits on something outside it: +/// soon at first, then at most every `max`, so a long wait neither spins nor /// floods the log. -#[derive(Default)] -struct StallRetry { +struct Retry { next: Option, wait: Duration, + first: Duration, + max: Duration, } -impl StallRetry { - const FIRST: Duration = Duration::from_millis(500); - const MAX: Duration = Duration::from_secs(8); +impl Retry { + /// FFmpeg's encoder lane is stuck in a driver call. + const STALL: Self = Self::new(Duration::from_millis(500), Duration::from_secs(8)); + /// Another app has the output to itself. A try is a few WASAPI calls and + /// opens no encoder, so audio is back within 2 s of the app letting go. + const DEVICE_IN_USE: Self = Self::new(Duration::from_millis(250), Duration::from_secs(2)); + + const fn new(first: Duration, max: Duration) -> Self { + Self { + next: None, + wait: Duration::ZERO, + first, + max, + } + } fn due(&self, now: Instant) -> bool { self.next.is_none_or(|next| now >= next) } fn failed(&mut self, now: Instant) -> Duration { - self.wait = (self.wait * 2).clamp(Self::FIRST, Self::MAX); + self.wait = (self.wait * 2).clamp(self.first, self.max); self.next = Some(now + self.wait); self.wait } } +/// An output another app has to itself, for one audio session: when to try +/// it again. Its "unavailable" line was logged when it began; its "back" +/// line is logged only if that session's audio starts. +struct OutputWait { + session_id: u32, + retry: Retry, +} + +/// The stage loop's time, so tests can step through retry deadlines. +trait Clock { + fn now(&self) -> Instant; + fn sleep(&self, wait: Duration); +} + +struct SystemClock; + +impl Clock for SystemClock { + fn now(&self) -> Instant { + Instant::now() + } + + fn sleep(&self, wait: Duration) { + std::thread::sleep(wait); + } +} + /// An endpoint is opened only after both peers opt in. A failed capture /// attempt stays disabled for that session; reconnecting or toggling the -/// host audio setting off and on retries without restarting video. +/// host audio setting off and on retries without restarting video. An output +/// another app has to itself is tried again until it lets go. pub fn run_audio_stage( tx: Sender, shared: SharedControl, generation: u64, reporter: HealthReporter, +) { + run_stage(tx, shared, generation, reporter, Source::open, &SystemClock); +} + +fn run_stage( + tx: Sender, + shared: SharedControl, + generation: u64, + reporter: HealthReporter, + mut open: impl FnMut() -> AudioResult, + clock: &impl Clock, ) { let mut active: Option = None; let mut failed_session = None; - let mut stalled = StallRetry::default(); - let mut next_log = Instant::now() + Duration::from_secs(1); + let mut stalled = Retry::STALL; + // Belongs to one session: it ends with it, when audio is turned off, + // or when another session wants audio. + let mut in_use: Option = None; + let mut next_log = clock.now() + Duration::from_secs(1); while shared.running.load(Ordering::SeqCst) && generation_is_current(generation) && !tx.is_closed() { let wanted = wanted_session(&shared); + if in_use + .as_ref() + .is_some_and(|wait| Some(wait.session_id) != wanted) + { + in_use = None; + } if wanted.is_none() { log_final(active.take()); if !shared.audio_enabled.load(Ordering::SeqCst) { @@ -359,12 +427,16 @@ pub fn run_audio_stage( audio.error = None; } }); - std::thread::sleep(Duration::from_millis(20)); + clock.sleep(Duration::from_millis(20)); continue; } let session_id = wanted.unwrap(); - if failed_session == Some(session_id) || !stalled.due(Instant::now()) { - std::thread::sleep(Duration::from_millis(20)); + let now = clock.now(); + if failed_session == Some(session_id) + || !stalled.due(now) + || in_use.as_ref().is_some_and(|wait| !wait.retry.due(now)) + { + clock.sleep(Duration::from_millis(20)); continue; } let result = (|| -> AudioResult { @@ -373,7 +445,7 @@ pub fn run_audio_stage( .is_none_or(|session| session.id != session_id) { active = None; - let mut session = CaptureSession::new(session_id)?; + let mut session = CaptureSession::new(session_id, open()?)?; if !shared.running.load(Ordering::SeqCst) { return Ok(false); } @@ -394,6 +466,12 @@ pub fn run_audio_stage( bitrate = super::BITRATE, "PC audio session started (Opus, 48 kHz stereo, 20 ms)" ); + if in_use.take().is_some() { + tracing::info!( + session_id, + "PC audio is back: the other app let go of the output" + ); + } active = Some(session); } active.as_mut().unwrap().tick(&tx, &shared, generation) @@ -402,14 +480,14 @@ pub fn run_audio_stage( break; } match result { - Ok(true) => stalled = StallRetry::default(), + Ok(true) => stalled = Retry::STALL, Ok(false) => break, // Video's encoder holds FFmpeg's lock in a stuck driver call: // not this session's failure. Video goes on; the open is tried // again later, once the call has returned. Err(error) if super::encoder::is_stall(error.as_ref()) => { log_final(active.take()); - let wait = stalled.failed(Instant::now()); + let wait = stalled.failed(clock.now()); tracing::warn!( session_id, retry_ms = wait.as_millis() as u64, @@ -420,6 +498,28 @@ pub fn run_audio_stage( audio.error = Some(error.to_string()); }); } + // Another app has the output to itself: not this session's + // failure either. Video goes on; the open is tried again until + // the app lets go, and the window says why meanwhile. + Err(error) if super::is_device_in_use(error.as_ref()) => { + log_final(active.take()); + let wait = in_use.get_or_insert_with(|| { + tracing::warn!( + session_id, + reason = %error, + "PC audio unavailable while another app has the output to itself; video continues, and audio starts when it lets go" + ); + OutputWait { + session_id, + retry: Retry::DEVICE_IN_USE, + } + }); + wait.retry.failed(clock.now()); + update_current(generation, |audio| { + audio.stopped(DEVICE_IN_USE_STATUS); + audio.error = None; + }); + } Err(error) => { log_final(active.take()); failed_session = Some(session_id); @@ -431,8 +531,8 @@ pub fn run_audio_stage( reporter.stage_exited(Stage::Audio, StageOutcome::Failed(reason)); } } - if Instant::now() >= next_log { - next_log = Instant::now() + Duration::from_secs(1); + if clock.now() >= next_log { + next_log = clock.now() + Duration::from_secs(1); let stats = PIPELINE_STATS.lock().audio.clone(); // Log only this stage's capture, never a successor's stats. if active @@ -453,7 +553,7 @@ mod tests { #[test] fn audio_retries_a_stalled_open_with_a_bounded_backoff() { - let mut retry = StallRetry::default(); + let mut retry = Retry::STALL; let now = Instant::now(); assert!(retry.due(now)); let waits: Vec = (0..7) @@ -533,6 +633,7 @@ mod tests { #[test] fn loopback_backlog_from_before_the_session_is_not_sent_stale() { + let _stats = crate::audio::STATS_TEST_LOCK.lock(); ffmpeg_next::init().unwrap(); // WASAPI buffers from Start() while the session opens its encoder. A // 0.3.1 host on an i7-5500U took 105 ms, and its first six packets @@ -708,6 +809,7 @@ mod tests { .with_ansi(false) .with_writer(move || Captured(writer.clone())) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, run); let text = String::from_utf8_lossy(&buffer.lock()).into_owned(); text @@ -750,6 +852,321 @@ mod tests { assert_eq!(logged(|| restart_stats(&target, |_| {})), ""); } + /// Virtual time for the stage: a sleep moves it on at once, then runs + /// `on_sleep` with the time since the start, so a test changes the + /// session, the setting or `running` at exact points. + struct StepClock<'a> { + start: Instant, + elapsed: std::cell::Cell, + on_sleep: std::cell::RefCell>, + } + + impl<'a> StepClock<'a> { + fn new(on_sleep: impl FnMut(Duration) + 'a) -> Self { + Self { + start: Instant::now(), + elapsed: Default::default(), + on_sleep: std::cell::RefCell::new(Box::new(on_sleep)), + } + } + + fn ms(&self) -> u64 { + self.elapsed.get().as_millis() as u64 + } + } + + impl Clock for StepClock<'_> { + fn now(&self) -> Instant { + self.start + self.elapsed.get() + } + + fn sleep(&self, wait: Duration) { + self.elapsed.set(self.elapsed.get() + wait); + (self.on_sleep.borrow_mut())(self.elapsed.get()); + } + } + + fn at(elapsed: Duration, ms: u64) -> bool { + elapsed == Duration::from_millis(ms) + } + + /// A running stream with an iPad session that wants audio. + fn audio_session() -> (SharedControl, u32) { + let (shared, id) = + crate::transport::test_session(eternal_wire::v2::control::FEATURE_WANTS_AUDIO); + shared.running.store(true, Ordering::SeqCst); + (shared, id) + } + + fn busy() -> AudioResult { + Err(crate::audio::DeviceInUse.into()) + } + + /// The audio status and error the window shows. + type Status = (String, Option); + + fn audio_status() -> Status { + let audio = PIPELINE_STATS.lock().audio.clone(); + (audio.status, audio.error) + } + + /// Ten 20 ms blocks of tone, then it notes the audio status in `seen` + /// and stops the stream. + /// + /// The tone starts at the first read, as a real capture's audio does. + /// Timed from when the source was made, it predated the session's start + /// whenever the Opus open between the two took over 250 ms, as it can on + /// a loaded PC, and the stage rightly dropped every frame as stale. + fn tone_then_stop( + shared: &SharedControl, + seen: &std::rc::Rc>>, + ) -> Source { + let (shared, seen) = (shared.clone(), seen.clone()); + let mut start = None; + let mut index = 0; + Source::Fake(Box::new(move || { + let start = *start.get_or_insert_with(crate::clock::host_now_us); + if index == 10 { + *seen.borrow_mut() = Some(audio_status()); + shared.running.store(false, Ordering::SeqCst); + return Ok(None); + } + index += 1; + Ok(Some(PcmBlock { + format: PcmFormat::STEREO_48K, + samples: vec![0.25; 2 * super::super::FRAME_SAMPLES], + capture_ts_us: start + (index - 1) * FRAME_US, + discontinuity: index == 1, + })) + })) + } + + fn names_session(line: &str, id: u32) -> bool { + line.split_whitespace() + .any(|field| field == format!("session_id={id}")) + } + + const UNAVAILABLE: &str = "PC audio unavailable while another app has the output to itself"; + const BACK: &str = "PC audio is back"; + + /// Runs the stage on this thread until it stops, opening its source with + /// `open`, on `clock`. Gives its log lines and the packets it queued. + fn run_stepped( + shared: &SharedControl, + open: impl FnMut() -> AudioResult, + clock: &StepClock, + ) -> (String, Vec) { + let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let (tx, mut rx) = tokio::sync::mpsc::channel(super::super::QUEUE_CAPACITY); + let (health_tx, health_rx) = std::sync::mpsc::channel(); + let reporter = HealthReporter::new(health_tx, generation); + let lines = logged(|| run_stage(tx, shared.clone(), generation, reporter, open, clock)); + let mut packets = Vec::new(); + while let Ok(packet) = rx.try_recv() { + packets.push(packet); + } + // Video goes on: the stage reported nothing but its own end, which + // neither stops nor restarts the pipeline. + let health: Vec<_> = health_rx.try_iter().collect(); + let mut machine = crate::supervisor::Machine::new(generation); + for message in &health { + let crate::supervisor::SupMsg::Health { + generation, + stage: Stage::Audio, + outcome: StageOutcome::Completed, + } = message + else { + panic!("{message:?}") + }; + assert_eq!( + machine.on_stage_exit( + *generation, + Stage::Audio, + &StageOutcome::Completed, + clock.now() + ), + crate::supervisor::Effect::None + ); + } + assert_eq!( + machine.state, + crate::supervisor::SupervisorState::Running { generation } + ); + // The lines this change adds name no device. + for line in lines + .lines() + .filter(|line| line.contains(UNAVAILABLE) || line.contains(BACK)) + { + assert!( + !line.contains("device=") && !line.contains("Fake test source"), + "{line}" + ); + } + (lines, packets) + } + + #[test] + fn audio_starts_once_another_app_lets_go_of_the_output() { + let _stats = crate::audio::STATS_TEST_LOCK.lock(); + ffmpeg_next::init().unwrap(); + let (shared, id) = audio_session(); + let clock = StepClock::new(|_| {}); + let seen = Default::default(); + // When the stage opened its source, and the audio status then. + let opens = std::cell::RefCell::new(Vec::new()); + let open = || { + opens.borrow_mut().push((clock.ms(), audio_status().0)); + if opens.borrow().len() <= 3 { + busy() + } else { + Ok(tone_then_stop(&shared, &seen)) + } + }; + let (lines, packets) = run_stepped(&shared, open, &clock); + let opens = opens.into_inner(); + assert_eq!( + opens.iter().map(|(ms, _)| *ms).collect::>(), + [0, 260, 760, 1760], + "retries after 250, 500 and 1000 ms, on the 20 ms poll" + ); + assert!( + opens[1..] + .iter() + .all(|(_, status)| status == DEVICE_IN_USE_STATUS), + "the window said why while it waited: {opens:?}" + ); + assert!(!packets.is_empty(), "audio never started\n{lines}"); + assert!(packets.iter().all(|packet| packet.session_id == id)); + assert_eq!(*seen.borrow(), Some(("Streaming".to_string(), None))); + assert_eq!(lines.matches(UNAVAILABLE).count(), 1, "{lines}"); + assert_eq!(lines.matches(BACK).count(), 1, "{lines}"); + } + + #[test] + fn an_output_that_stays_in_use_is_tried_on_a_bounded_backoff() { + let _stats = crate::audio::STATS_TEST_LOCK.lock(); + let now = Instant::now(); + let mut retry = Retry::DEVICE_IN_USE; + let waits: Vec = (0..6) + .map(|_| retry.failed(now).as_millis() as u64) + .collect(); + assert_eq!(waits, [250, 500, 1000, 2000, 2000, 2000]); + + for (stop_ms, tries) in [ + (4000, &[0, 260, 760, 1760, 3760][..]), + (1000, &[0, 260, 760][..]), + ] { + let (shared, _) = audio_session(); + let clock = StepClock::new(|elapsed| { + if at(elapsed, stop_ms) { + shared.running.store(false, Ordering::SeqCst); + } + }); + let opens = std::cell::RefCell::new(Vec::new()); + let open = || { + opens.borrow_mut().push((clock.ms(), audio_status().0)); + busy() + }; + let (lines, _) = run_stepped(&shared, open, &clock); + let opens = opens.into_inner(); + assert_eq!(opens.iter().map(|(ms, _)| *ms).collect::>(), tries); + assert!(opens[1..] + .iter() + .all(|(_, status)| status == DEVICE_IN_USE_STATUS)); + // Stopped while a retry is pending: the stage ends at its next + // 20 ms poll and tries nothing more. + assert_eq!(clock.ms(), stop_ms); + assert_eq!(lines.matches(UNAVAILABLE).count(), 1, "{lines}"); + assert_eq!(lines.matches(BACK).count(), 0, "{lines}"); + } + } + + #[test] + fn a_new_session_starts_with_no_wait_left_over_from_the_last() { + let _stats = crate::audio::STATS_TEST_LOCK.lock(); + ffmpeg_next::init().unwrap(); + let (shared, first) = audio_session(); + let (next, second) = audio_session(); + assert_ne!(first, second); + let clock = StepClock::new(|elapsed| { + // The iPad goes away while a retry waits until 1760 ms, and + // another session wants audio 100 ms later. + if at(elapsed, 800) { + *shared.session.lock() = crate::transport::session::Session::new(0); + } + if at(elapsed, 900) { + std::mem::swap(&mut *shared.session.lock(), &mut *next.session.lock()); + } + if at(elapsed, 880) { + assert_eq!(audio_status().0, "Waiting for client"); + } + }); + let seen = Default::default(); + let opens = std::cell::RefCell::new(Vec::new()); + let open = || { + opens.borrow_mut().push(clock.ms()); + // The new session finds the output still taken once. + if opens.borrow().len() <= 4 { + busy() + } else { + let source = tone_then_stop(&shared, &seen); + // Real time before the first read, as a loaded PC's Opus + // open takes: the tone must still be sent. + std::thread::sleep(Duration::from_millis(300)); + Ok(source) + } + }; + let (lines, packets) = run_stepped(&shared, open, &clock); + assert_eq!(opens.into_inner(), [0, 260, 760, 900, 1160]); + assert!(packets.iter().all(|packet| packet.session_id == second)); + assert!(!packets.is_empty()); + let unavailable: Vec<_> = lines + .lines() + .filter(|line| line.contains(UNAVAILABLE)) + .collect(); + assert_eq!(unavailable.len(), 2, "{lines}"); + assert!(names_session(unavailable[0], first), "{lines}"); + assert!(names_session(unavailable[1], second), "{lines}"); + let back: Vec<_> = lines.lines().filter(|line| line.contains(BACK)).collect(); + assert_eq!(back.len(), 1, "{lines}"); + assert!(names_session(back[0], second), "{lines}"); + } + + #[test] + fn turning_audio_off_and_on_starts_the_wait_over() { + let _stats = crate::audio::STATS_TEST_LOCK.lock(); + ffmpeg_next::init().unwrap(); + let (shared, _) = audio_session(); + let clock = StepClock::new(|elapsed| { + if at(elapsed, 800) { + shared.audio_enabled.store(false, Ordering::SeqCst); + } + if at(elapsed, 880) { + assert_eq!(audio_status(), ("Off".to_string(), None)); + } + if at(elapsed, 900) { + shared.audio_enabled.store(true, Ordering::SeqCst); + } + }); + let seen = Default::default(); + let opens = std::cell::RefCell::new(Vec::new()); + let open = || { + opens.borrow_mut().push(clock.ms()); + if opens.borrow().len() <= 4 { + busy() + } else { + Ok(tone_then_stop(&shared, &seen)) + } + }; + let (lines, packets) = run_stepped(&shared, open, &clock); + // Back on: tried at once, not when the old wait ran out (1760 ms). + assert_eq!(opens.into_inner(), [0, 260, 760, 900, 1160]); + assert!(!packets.is_empty()); + assert_eq!(*seen.borrow(), Some(("Streaming".to_string(), None))); + assert_eq!(lines.matches(UNAVAILABLE).count(), 2, "{lines}"); + assert_eq!(lines.matches(BACK).count(), 1, "{lines}"); + } + /// Native check of the same startup path with the PC's real loopback. /// The delay stands in for the encoder open that follows WASAPI Start(). /// Inaudible: it only reads the mix. Something must be rendering (even diff --git a/host/src/audio/wasapi.rs b/host/src/audio/wasapi.rs index 320646ab..0f70f057 100644 --- a/host/src/audio/wasapi.rs +++ b/host/src/audio/wasapi.rs @@ -17,7 +17,7 @@ use windows::Win32::System::Threading::{CreateEventW, WaitForSingleObject}; use super::pcm::SampleEncoding; use super::resample::{PcmBlock, PcmFormat}; -use super::AudioResult; +use super::{AudioResult, DeviceInUse}; const PCM_GUID: GUID = GUID::from_u128(0x00000001_0000_0010_8000_00aa00389b71); const FLOAT_GUID: GUID = GUID::from_u128(0x00000003_0000_0010_8000_00aa00389b71); @@ -122,6 +122,21 @@ fn parse_format(mix: &MixFormat) -> AudioResult<(PcmFormat, SampleEncoding)> { Ok((format, encoding)) } +/// Another app took the output in exclusive mode: shared-mode loopback +/// can't open until it lets go, which the audio stage waits out. +fn in_use( + error: Box, +) -> Box { + if error + .downcast_ref::() + .is_some_and(|error| error.code() == AUDCLNT_E_DEVICE_IN_USE) + { + Box::new(DeviceInUse) + } else { + error + } +} + struct Loopback { capture: IAudioCaptureClient, client: IAudioClient, @@ -297,7 +312,7 @@ impl WasapiSource { pub fn new() -> AudioResult { let apartment = ComApartment::new()?; let enumerator = unsafe { CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)? }; - let stream = Loopback::open(&enumerator)?; + let stream = Loopback::open(&enumerator).map_err(in_use)?; Ok(Self { stream: Some(stream), enumerator, @@ -325,11 +340,15 @@ impl WasapiSource { self.stream = Some(stream); } Err(error) => { - if self - .unavailable_since - .get_or_insert_with(Instant::now) - .elapsed() - >= Duration::from_secs(5) + // An app holding the output to itself is waited out by + // the audio stage, which shows why. + let error = in_use(error); + if super::is_device_in_use(error.as_ref()) + || self + .unavailable_since + .get_or_insert_with(Instant::now) + .elapsed() + >= Duration::from_secs(5) { return Err(error); } @@ -373,7 +392,7 @@ impl WasapiSource { self.reopen(); Ok(None) } else { - Err(error) + Err(in_use(error)) } } } @@ -386,3 +405,34 @@ impl WasapiSource { tracing::info!("Default PC audio endpoint changed; reopening loopback"); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_an_output_in_use_waits_for_the_other_app() { + let busy = in_use(windows::core::Error::from(AUDCLNT_E_DEVICE_IN_USE).into()); + assert!(super::super::is_device_in_use(busy.as_ref())); + // Reopened in place, or not something waiting fixes: unchanged. + for code in [ + AUDCLNT_E_DEVICE_INVALIDATED, + AUDCLNT_E_RESOURCES_INVALIDATED, + AUDCLNT_E_SERVICE_NOT_RUNNING, + AUDCLNT_E_ENDPOINT_CREATE_FAILED, + AUDCLNT_E_UNSUPPORTED_FORMAT, + AUDCLNT_E_EXCLUSIVE_MODE_NOT_ALLOWED, + ] { + let other = in_use(windows::core::Error::from(code).into()); + assert!(!super::super::is_device_in_use(other.as_ref()), "{code:?}"); + assert_eq!( + other + .downcast_ref::() + .map(|error| error.code()), + Some(code) + ); + } + let text = in_use("invalid WASAPI mix geometry".into()); + assert_eq!(text.to_string(), "invalid WASAPI mix geometry"); + } +} diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 1139b91f..22d9b70f 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -11,15 +11,17 @@ use std::time::{Duration, Instant}; use tracing::{debug, error, info, warn}; use windows::core::Interface; +use super::listing::Judged; use super::recovery::{ note_capture_resumed, note_gpu_outcome, CaptureBlock, CaptureUnavailable, LossRecovery, - LossStep, OpenStep, LOSS_GIVE_UP, + LossStep, OpenStep, Reopen, LOSS_GIVE_UP, }; use super::{ - client_connected, frame_budget_for, frame_buffer, frame_has_image, heartbeat, idle_heartbeat, - keepalive_allowed, keepalive_due, read_back_image, reconcile_virtual_display, recovery_poll, - resolve_target, FrameSlot, FrameUse, IdleGate, OutputInfo, RawFrame, ACQUIRE_TIMEOUT_MS, - FPS_WINDOW, FRAME_PADDING, + acquired, client_connected, composite_pointer, frame_budget_for, frame_buffer, heartbeat, + idle_heartbeat, keepalive_allowed, keepalive_due, nap_unless, pointer_apart, pointer_place, + read_back_image, reconcile_virtual_display, recovery_poll, redraw_cached, resolve_target, + Acquired, FrameSlot, FrameUse, IdleGate, OutputInfo, Picture, PointerPlace, RawFrame, Redraw, + StartPicture, ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, PICTURE_HOLD, }; use crate::control::{CaptureTarget, SharedControl}; use crate::gpu_preference::Outcome; @@ -32,11 +34,15 @@ use windows::Win32::Graphics::Direct3D11::{ D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_READ, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING, }; -use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_SAMPLE_DESC}; +use windows::Win32::Graphics::Dxgi::Common::{ + DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_MODE_ROTATION, DXGI_MODE_ROTATION_IDENTITY, + DXGI_MODE_ROTATION_UNSPECIFIED, DXGI_SAMPLE_DESC, +}; use windows::Win32::Graphics::Dxgi::{ - CreateDXGIFactory1, IDXGIFactory1, IDXGIOutput, IDXGIOutput1, IDXGIOutputDuplication, - IDXGIResource, DXGI_ADAPTER_DESC1, DXGI_ERROR_ACCESS_LOST, DXGI_ERROR_UNSUPPORTED, - DXGI_ERROR_WAIT_TIMEOUT, DXGI_OUTDUPL_FRAME_INFO, DXGI_OUTDUPL_POINTER_SHAPE_INFO, + CreateDXGIFactory1, IDXGIAdapter1, IDXGIFactory1, IDXGIOutput, IDXGIOutput1, + IDXGIOutputDuplication, IDXGIResource, DXGI_ADAPTER_DESC1, DXGI_ERROR_ACCESS_LOST, + DXGI_ERROR_NOT_FOUND, DXGI_ERROR_UNSUPPORTED, DXGI_ERROR_WAIT_TIMEOUT, DXGI_OUTDUPL_FRAME_INFO, + DXGI_OUTDUPL_POINTER_SHAPE_INFO, }; /// Parse a null-terminated UTF-16 fixed array (DXGI `DeviceName` / `Description`) to a String. @@ -49,10 +55,10 @@ fn utf16_to_string(buf: &[u16]) -> String { ) } -/// Enumerate every desktop-attached display output across all adapters. Read-only and -/// reusable by both the GUI picker and the capture loop. Never panics — on any DXGI error -/// it returns whatever was discovered so far (possibly empty). -pub fn enumerate_outputs() -> Vec { +/// Every desktop-attached display output across all adapters, from a fresh +/// factory. Read-only. Never panics — on any DXGI error it returns whatever +/// was discovered so far (possibly empty). Through [`super::listing`]. +pub(super) fn outputs(judged: Judged) -> Vec { let factory: IDXGIFactory1 = match unsafe { CreateDXGIFactory1() } { Ok(f) => f, Err(error) => { @@ -60,10 +66,25 @@ pub fn enumerate_outputs() -> Vec { return Vec::new(); } }; - enumerate_factory_outputs(&factory) + list_outputs(&factory, judged) + .0 + .into_iter() + .map(|listed| listed.info) + .collect() +} + +/// A desktop-attached output with the adapter and output objects it was +/// listed from. Capture opens these objects rather than looking the +/// output up again by its ordinals. +pub(super) struct Listed { + pub(super) info: OutputInfo, + adapter: IDXGIAdapter1, + output: IDXGIOutput, } -fn enumerate_factory_outputs(factory: &IDXGIFactory1) -> Vec { +/// Every desktop-attached output, and how many adapters were listed. +/// Through [`super::listing`], which judges what it lists. +pub(super) fn list_outputs(factory: &IDXGIFactory1, _: Judged) -> (Vec, u32) { let mut outputs = Vec::new(); let mut adapter_index = 0u32; @@ -72,9 +93,9 @@ fn enumerate_factory_outputs(factory: &IDXGIFactory1) -> Vec { Ok(a) => a, Err(_) => break, }; - let adapter_name = match unsafe { adapter.GetDesc1() } { - Ok(desc) => utf16_to_string(&desc.Description), - Err(_) => String::new(), + let (adapter_name, adapter_luid) = match unsafe { adapter.GetDesc1() } { + Ok(desc) => (utf16_to_string(&desc.Description), adapter_id(desc)), + Err(_) => (String::new(), (0, 0)), }; let mut output_index = 0u32; @@ -86,16 +107,21 @@ fn enumerate_factory_outputs(factory: &IDXGIFactory1) -> Vec { if let Ok(desc) = unsafe { output.GetDesc() } { if desc.AttachedToDesktop.as_bool() { let r = desc.DesktopCoordinates; - outputs.push(OutputInfo { - adapter_index, - output_index, - device_name: utf16_to_string(&desc.DeviceName), - width: (r.right - r.left).max(0) as u32, - height: (r.bottom - r.top).max(0) as u32, - left: r.left, - top: r.top, - is_primary: r.left == 0 && r.top == 0, - adapter_name: adapter_name.clone(), + outputs.push(Listed { + info: OutputInfo { + adapter_index, + output_index, + device_name: utf16_to_string(&desc.DeviceName), + width: (r.right - r.left).max(0) as u32, + height: (r.bottom - r.top).max(0) as u32, + left: r.left, + top: r.top, + is_primary: r.left == 0 && r.top == 0, + adapter_name: adapter_name.clone(), + adapter_id: adapter_luid, + }, + adapter: adapter.clone(), + output, }); } } @@ -103,7 +129,7 @@ fn enumerate_factory_outputs(factory: &IDXGIFactory1) -> Vec { } adapter_index += 1; } - outputs + (outputs, adapter_index) } /// Enumerate adapters, select the output to duplicate, create duplication, @@ -111,13 +137,12 @@ fn enumerate_factory_outputs(factory: &IDXGIFactory1) -> Vec { pub fn run_capture_loop( slot: &FrameSlot, shared: SharedControl, - adapter_index: u32, generation: u64, frame_number: &mut u64, ) -> Result<(), Box> { // The capture adapter FOLLOWS the chosen output (which may differ from the // encoder-detection adapter on a multi-GPU system); the encoder stays vendor-based - // and independent. `adapter_index` is only the last-resort fallback below. + // and independent. let requested_target = shared.capture_target.lock().clone(); // Turn the virtual display on/off to match the request before enumerating, so the // managed virtual output only exists while we're capturing it (and only while an iPad @@ -130,7 +155,8 @@ pub fn run_capture_loop( // DXGI factories snapshot adapters at creation. Enable the driver first, // then enumerate and open the chosen output from the same fresh factory. let factory: IDXGIFactory1 = unsafe { CreateDXGIFactory1()? }; - let outputs = enumerate_factory_outputs(&factory); + let (listed, adapters) = super::listing::listed(&shared, &factory); + let outputs: Vec = listed.iter().map(|listed| listed.info.clone()).collect(); for o in &outputs { info!( device = %o.device_name, @@ -147,67 +173,36 @@ pub fn run_capture_loop( if !outputs.is_empty() && NO_OUTPUTS.swap(false, Ordering::Relaxed) { info!(outputs = outputs.len(), "A display output is back"); } - // A missing adapter or output (DXGI_ERROR_NOT_FOUND) is a display change - // the stage waits out, not a failure. - let (adapter, output, adapter_name, capture_label) = match resolve_target(&outputs, &target) { - Some(idx) => { - let chosen = &outputs[idx]; - let adapter = match unsafe { factory.EnumAdapters1(chosen.adapter_index) } { - Ok(a) => a, - Err(error) => { - warn!( - adapter_index = chosen.adapter_index, - error = %error, - "EnumAdapters1 failed for the chosen output's adapter — falling back to adapter 0" - ); - unsafe { factory.EnumAdapters1(0) }.map_err(opening(OpenStep::Adapter))? - } - }; - let adapter_name = match unsafe { adapter.GetDesc1() } { - Ok(desc) => utf16_to_string(&desc.Description), - Err(_) => chosen.adapter_name.clone(), - }; - let output: IDXGIOutput = unsafe { adapter.EnumOutputs(chosen.output_index) } - .map_err(opening(OpenStep::ListedOutput))?; - let label = format!( - "{} ({}x{})", - chosen.device_name, chosen.width, chosen.height - ); + // No output listed is a display change the stage waits out, not a failure. + let Some(index) = resolve_target(&outputs, &target) else { + if !NO_OUTPUTS.swap(true, Ordering::Relaxed) { info!( - device = %chosen.device_name, - adapter = %adapter_name, - width = chosen.width, - height = chosen.height, - "Capture: selected display output" + adapters, + "No display is attached to the desktop, as while a laptop's lid is closed or the screens are off" ); - (adapter, output, adapter_name, label) - } - None => { - // No outputs enumerated: a closed laptop lid or a screen that is - // off. Try the detection adapter's first output, as before. - if !NO_OUTPUTS.swap(true, Ordering::Relaxed) { - info!("No display outputs enumerated, as while a laptop's lid is closed — trying the detection adapter's first output"); - } - let adapter = match unsafe { factory.EnumAdapters1(adapter_index) } { - Ok(a) => a, - Err(error) => { - warn!( - requested = adapter_index, - error = %error, - "EnumAdapters1 failed for requested adapter index, falling back to adapter 0" - ); - unsafe { factory.EnumAdapters1(0) }.map_err(opening(OpenStep::Adapter))? - } - }; - let adapter_name = match unsafe { adapter.GetDesc1() } { - Ok(desc) => utf16_to_string(&desc.Description), - Err(_) => String::new(), - }; - let output: IDXGIOutput = - unsafe { adapter.EnumOutputs(0) }.map_err(opening(OpenStep::OnlyOutput))?; - (adapter, output, adapter_name, "primary (auto)".to_string()) } + return Err(Box::new(no_output_listed(adapters))); }; + let Listed { + info: chosen, + adapter, + output, + } = listed + .into_iter() + .nth(index) + .expect("resolved from this list"); + let adapter_name = chosen.adapter_name.clone(); + let capture_label = format!( + "{} ({}x{})", + chosen.device_name, chosen.width, chosen.height + ); + info!( + device = %chosen.device_name, + adapter = %adapter_name, + width = chosen.width, + height = chosen.height, + "Capture: selected display output" + ); info!(adapter = %adapter_name, "Capture using adapter"); // The mirror's card, by LUID, for the graphics card checks. @@ -314,8 +309,41 @@ pub fn run_capture_loop( let mut recovery = LossRecovery::default(); let mut resumed = false; let mut idle = IdleGate::new(); - // Set once this duplication has delivered a real desktop image. - let mut seen_image = false; + let mut picture = Picture::default(); + let mut start_picture = StartPicture::new(Instant::now()); + // ACCESS_LOST from a call after the acquire (ReleaseFrame, the staging + // Map). It is handled as if the next acquire had returned it: before, + // it ended the capture loop and restarted the pipeline. + let mut lost_after_acquire: Option = None; + // Whether `last_frame` has the pointer drawn in: it has unless the + // client draws the pointer itself (`SharedControl::client_draws_pointer`). + let mut last_frame_has_pointer = true; + // Where Windows puts the pointer (`pointer_place`): into the picture on + // the second screen without the driver's hardware cursor, or for an + // image too large for it, and then nothing else draws one. Where the + // running driver can't be told, as 0.3.4: no pointer messages. + let driver = if second_screen { + super::running_driver_cursor() + } else { + crate::vdd_settings::DriverCursor::Unknown + }; + shared + .pointer + .set_withheld(second_screen && driver == crate::vdd_settings::DriverCursor::Unknown); + let place_of = |cursor: &CursorState| pointer_place(second_screen, driver, cursor.size()); + let mut place; + { + // The feed outlives this capture: a position from the output + // captured before (the mirror a second screen starts from) would be + // drawn on this one until Desktop Duplication reports its own, + // which it does once the pointer is on this output. + shared.pointer.set_position(super::pointer::Position { + visible: false, + x: 0, + y: 0, + host_time_us: crate::clock::host_now_us(), + }); + } loop { if !shared.running.load(Ordering::SeqCst) @@ -325,6 +353,19 @@ pub fn run_capture_loop( break; } heartbeat(&shared.hb_capture_loop_ms); + if picture.held_too_long(Instant::now(), second_screen) { + info!( + device = %chosen.device_name, + held_s = PICTURE_HOLD.as_secs(), + "The second screen showed no new image since its duplication came back — opening it again" + ); + return Err(Box::new(Reopen { + detail: format!( + "no new image from {} since its duplication came back", + chosen.device_name + ), + })); + } let wait = idle.wait_before_acquire(client_connected(&shared), Instant::now()); if !wait.is_zero() { @@ -346,6 +387,92 @@ pub fn run_capture_loop( continue; } + // An iPad that draws the pointer gets a picture without it; any + // other client, or none, one with it. The staging texture still + // holds the last desktop image, so the cached frame is redrawn from + // it when the connected client's answer differs from the cache's. + place = place_of(&cursor); + let draws_pointer = pointer_apart(shared.client_draws_pointer(), place); + if picture.seen_image && last_frame_has_pointer == draws_pointer { + let mut pixel_buf = frame_buffer(frame_bytes); + // A lost duplication goes to the handler below, which keeps the + // cache; the rebuild is tried again after it. + let copied = redraw_cached( + lost_after_acquire.is_some(), + || { + copy_staging( + &device_context, + &staging_texture, + tex_height, + row_bytes, + &mut pixel_buf, + ) + }, + |error| error.code() == DXGI_ERROR_ACCESS_LOST, + )?; + match copied { + Redraw::Later => {} + Redraw::Lost(error) => lost_after_acquire = Some(error), + Redraw::Copied => { + if composite_pointer(draws_pointer, place, cursor_visible, cursor.has_shape) { + draw_cursor( + &mut pixel_buf, + tex_width, + tex_height, + row_bytes, + &cursor, + cursor_x, + cursor_y, + ); + } + spare = None; + last_frame = Arc::new(pixel_buf); + last_frame_has_pointer = !draws_pointer; + if client_connected(&shared) { + resend_last_frame( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + (tex_width, tex_height), + ); + } + } + } + } + + if start_picture.due(picture.real(), second_screen, Instant::now()) { + let rotation = duplication + .as_ref() + .map(|duplication| unsafe { duplication.GetDesc() }.Rotation); + if let Some(pixels) = read_screen_gdi(&output1, (tex_width, tex_height), rotation) { + info!( + device = %chosen.device_name, + "Desktop duplication gave no image yet — starting from the screen's picture" + ); + spare = None; + last_frame = Arc::new(pixels); + last_frame_has_pointer = false; + picture.cached(); + let now = Instant::now(); + if idle.on_frame(client_connected(&shared), now) == FrameUse::Publish { + *frame_number += 1; + PIPELINE_STATS.lock().record_capture(tex_width, tex_height); + heartbeat(&shared.hb_capture_frame_ms); + slot.publish(RawFrame { + frame_number: *frame_number, + timestamp: now, + data: Arc::clone(&last_frame), + width: tex_width, + height: tex_height, + cached: true, + }); + last_send = now; + } + } + } + let frame_start = Instant::now(); let frame_budget = frame_budget_for(shared.effective_fps()); // Half the frame budget, clamped: at 60 fps this is the classic 8 ms; @@ -361,8 +488,11 @@ pub fn run_capture_loop( let dupl = duplication .as_ref() .expect("the ACCESS_LOST handler re-creates the duplication before continuing"); - let hr = unsafe { - dupl.AcquireNextFrame(acquire_timeout_ms, &mut frame_info, &mut desktop_resource) + let hr = match lost_after_acquire.take() { + Some(lost) => Err(lost), + None => unsafe { + dupl.AcquireNextFrame(acquire_timeout_ms, &mut frame_info, &mut desktop_resource) + }, }; let acquire_us = acquire_start.elapsed().as_micros(); if hr.is_ok() @@ -392,14 +522,37 @@ pub fn run_capture_loop( super::second_screen::note_image(&shared, generation); } - // --- Track cursor position/shape (must read shape before ReleaseFrame) --- - if frame_info.LastMouseUpdateTime != 0 { - cursor_visible = frame_info.PointerPosition.Visible.as_bool(); - cursor_x = frame_info.PointerPosition.Position.x; - cursor_y = frame_info.PointerPosition.Position.y; + // --- Track cursor shape/position (must read shape before ReleaseFrame) --- + // The shape is always read: its size decides whether Windows + // draws it into the picture. + let new_shape = frame_info.PointerShapeBufferSize > 0 + && cursor.update_shape(dupl, frame_info.PointerShapeBufferSize); + place = place_of(&cursor); + if new_shape && place == PointerPlace::Apart { + if let Some(image) = super::pointer::convert_shape( + cursor.shape_type, + cursor.width, + cursor.height, + cursor.pitch, + cursor.hot, + &cursor.shape, + ) { + shared.pointer.set_shape(image); + } } - if frame_info.PointerShapeBufferSize > 0 { - cursor.update_shape(dupl, frame_info.PointerShapeBufferSize); + if frame_info.LastMouseUpdateTime != 0 || new_shape { + if frame_info.LastMouseUpdateTime != 0 { + cursor_visible = frame_info.PointerPosition.Visible.as_bool(); + cursor_x = frame_info.PointerPosition.Position.x; + cursor_y = frame_info.PointerPosition.Position.y; + } + shared.pointer.set_position(super::pointer::Position { + // Only a pointer Windows keeps apart is shown apart. + visible: cursor_visible && place == PointerPlace::Apart, + x: cursor_x, + y: cursor_y, + host_time_us: crate::clock::host_now_us(), + }); } // --- Get dirty rect count --- @@ -438,13 +591,42 @@ pub fn run_capture_loop( // --- Copy desktop texture to staging and read pixels into a // recycled buffer (the readback is the ONLY full-frame copy) --- - if !frame_has_image(frame_info.LastPresentTime, seen_image) { - unsafe { - dupl.ReleaseFrame()?; + // A pointer-only update needs no new frame when the iPad + // draws the pointer: the transport sends the position. + match acquired( + frame_info.LastPresentTime, + picture.seen_image, + draws_pointer, + ) { + Acquired::ReadBack => {} + skip => { + if let Err(error) = unsafe { dupl.ReleaseFrame() } { + lost_after_acquire = Some(access_lost_or(error)?); + continue; + } + if skip == Acquired::PointerOnly + && client_connected(&shared) + && keepalive_due( + last_send.elapsed(), + shared.force_next_idr.load(Ordering::SeqCst), + picture.real(), + ) + && keepalive_allowed(picture.real(), second_screen) + { + resend_last_frame( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + (tex_width, tex_height), + ); + } + continue; } - continue; } - let frame_data = read_back_image(&shared, generation, &mut seen_image, || { + let seen_image = &mut picture.seen_image; + let frame_data = read_back_image(&shared, generation, seen_image, || { windows::core::Result::Ok(if let Some(ref resource) = desktop_resource { match resource.cast::() { Ok(desktop_texture) => { @@ -465,39 +647,21 @@ pub fn run_capture_loop( _ => frame_buffer(frame_bytes), }; - // Map staging texture for CPU read - let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); - unsafe { - device_context.Map( - &staging_texture, - 0, - D3D11_MAP_READ, - 0, - Some(&mut mapped), - )?; - } - - // Copy row-by-row (pitch may differ from width*4) - let src_pitch = mapped.RowPitch as usize; - let src_ptr = mapped.pData as *const u8; - for row in 0..tex_height as usize { - let src_offset = row * src_pitch; - let dst_offset = row * row_bytes; - unsafe { - std::ptr::copy_nonoverlapping( - src_ptr.add(src_offset), - pixel_buf.as_mut_ptr().add(dst_offset), - row_bytes, - ); - } - } - - unsafe { - device_context.Unmap(&staging_texture, 0); - } + copy_staging( + &device_context, + &staging_texture, + tex_height, + row_bytes, + &mut pixel_buf, + )?; // Composite the mouse cursor into the BGRA frame. - if cursor_visible && cursor.has_shape { + if composite_pointer( + draws_pointer, + place, + cursor_visible, + cursor.has_shape, + ) { draw_cursor( &mut pixel_buf, tex_width, @@ -525,9 +689,17 @@ pub fn run_capture_loop( } None }) - })?; + }); + let frame_data = match frame_data { + Ok(frame_data) => frame_data, + Err(error) => { + lost_after_acquire = Some(access_lost_or(error)?); + continue; + } + }; if let Some(data) = frame_data { + picture.cached(); if idle.on_frame(client_connected(&shared), now) == FrameUse::Publish { PIPELINE_STATS.lock().record_capture(tex_width, tex_height); heartbeat(&shared.hb_capture_frame_ms); @@ -538,6 +710,7 @@ pub fn run_capture_loop( data: Arc::clone(&data), width: tex_width, height: tex_height, + cached: false, }); last_send = now; } else { @@ -547,6 +720,7 @@ pub fn run_capture_loop( // two buffers alive; steady state recycles them. spare = Some(Arc::clone(&last_frame)); last_frame = data; + last_frame_has_pointer = !draws_pointer; } } Err(e) if e.code() == DXGI_ERROR_WAIT_TIMEOUT => { @@ -568,9 +742,9 @@ pub fn run_capture_loop( && keepalive_due( last_send.elapsed(), shared.force_next_idr.load(Ordering::SeqCst), - seen_image, + picture.real(), ) - && keepalive_allowed(seen_image, second_screen) + && keepalive_allowed(picture.real(), second_screen) { resend_last_frame( slot, @@ -584,13 +758,21 @@ pub fn run_capture_loop( && last_send.elapsed() >= Duration::from_secs(2) { // Idle throttle: nobody is watching and the desktop is - // static — stop spinning at capture rate. 4 Hz polling - // still notices a client promptly (registration also - // restarts the pipeline for virtual-display targets). - // The 250 ms naps stay far under the supervisor's 3 s - // loop-heartbeat wedge threshold. - idle_heartbeat(&shared, crate::clock::host_now_us() / 1000); - std::thread::sleep(Duration::from_millis(250)); + // static — stop spinning at capture rate. 4 Hz polling, + // whose nap ends within 50 ms of a client connecting + // (registration also restarts the pipeline for + // virtual-display targets). The 250 ms naps stay far + // under the supervisor's 3 s loop-heartbeat wedge + // threshold. + nap_unless( + Duration::from_millis(250), + || { + client_connected(&shared) + || !super::capture_is_current(&shared, generation) + }, + || idle_heartbeat(&shared, crate::clock::host_now_us() / 1000), + std::thread::sleep, + ); } continue; } @@ -598,8 +780,29 @@ pub fn run_capture_loop( // Normal during UAC prompts, the lock screen, mode changes and // full-screen transitions. A duplication re-created too early is // lost again at once, so the retries back off instead of spinning. + // Routine while it is back within a second; see `LOSS_QUIET`. if !recovery.in_episode() { - warn!("Desktop duplication access lost — reinitializing"); + info!("Desktop duplication access lost — reinitializing"); + } + let allowed = keepalive_allowed(picture.real(), second_screen); + let dims = (tex_width, tex_height); + let call = super::second_screen::driver_call(&shared, generation); + let moved = target_moved(&shared, &output1, &chosen, &target, false, || { + if crate::supervisor::generation_is_current(generation) { + keep_picture_alive( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + dims, + allowed, + ); + } + }); + drop(call); + if moved { + return Err(moved_away(&chosen)); } // A second screen that shows no picture before this is // re-created failed for a passing reason. @@ -638,19 +841,16 @@ pub fn run_capture_loop( { return Ok(()); } - let now_ms = crate::clock::host_now_us() / 1000; - if recovery_poll(&shared, now_ms, last_send.elapsed()) - && keepalive_allowed(seen_image, second_screen) - { - resend_last_frame( - slot, - &shared, - frame_number, - &mut last_send, - &last_frame, - (tex_width, tex_height), - ); - } + warn_if_overdue(&mut recovery); + keep_picture_alive( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + (tex_width, tex_height), + keepalive_allowed(picture.real(), second_screen), + ); let now = Instant::now(); if now >= deadline { break; @@ -662,19 +862,16 @@ pub fn run_capture_loop( if !crate::supervisor::generation_is_current(generation) { return; } - let now_ms = crate::clock::host_now_us() / 1000; - if recovery_poll(&shared, now_ms, last_send.elapsed()) - && keepalive_allowed(seen_image, second_screen) - { - resend_last_frame( - slot, - &shared, - frame_number, - &mut last_send, - &last_frame, - (tex_width, tex_height), - ); - } + warn_if_overdue(&mut recovery); + keep_picture_alive( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + (tex_width, tex_height), + keepalive_allowed(picture.real(), second_screen), + ); }; let call = super::second_screen::driver_call(&shared, generation); let retried = duplicate_output(&output1, &device, poll); @@ -695,7 +892,8 @@ pub fn run_capture_loop( match retried { Ok(d) => { duplication = Some(d); - seen_image = false; + picture.recreated(Instant::now()); + start_picture.duplicated(Instant::now()); super::second_screen::note_recovery( &shared, generation, @@ -712,7 +910,29 @@ pub fn run_capture_loop( } // Tell the GUI why while the retries continue. if let Some(block) = CaptureBlock::from_hresult(last_code.0) { - *shared.capture_block.lock() = Some(block); + super::recovery::note_capture_blocked(&shared, block); + } + // The output may be gone: look at the whole layout. + let allowed = keepalive_allowed(picture.real(), second_screen); + let dims = (tex_width, tex_height); + let call = super::second_screen::driver_call(&shared, generation); + let moved = + target_moved(&shared, &output1, &chosen, &target, true, || { + if crate::supervisor::generation_is_current(generation) { + keep_picture_alive( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + dims, + allowed, + ); + } + }); + drop(call); + if moved { + return Err(moved_away(&chosen)); } } } @@ -736,6 +956,8 @@ pub fn run_capture_loop( frame_bytes = row_bytes * tex_height as usize; spare = None; last_frame = Arc::new(frame_buffer(frame_bytes)); + picture.resized(); + start_picture.duplicated(Instant::now()); staging_texture = unsafe { let mut tex = None; device.CreateTexture2D( @@ -836,6 +1058,7 @@ fn resend_last_frame( data: Arc::clone(last_frame), width, height, + cached: true, }); *last_send = now; } @@ -867,11 +1090,129 @@ fn blocked_or_error(error: windows::core::Error) -> Box impl FnOnce(windows::core::Error) -> Box { - move |error| unavailable_or_error(CaptureBlock::opening(step, error.code().0), error) +/// Enumeration listed no display attached to the desktop. The detection +/// adapter's first output, found by its ordinal from startup, was tried +/// next: it never captured anything in the field, and its error text +/// (DXGI_ERROR_NOT_FOUND, "there is no adapter with the specified ordinal") +/// hid what was wrong. +fn no_output_listed(adapters: u32) -> CaptureUnavailable { + let (step, detail) = if adapters == 0 { + ( + OpenStep::Adapter, + "no graphics adapter is listed".to_string(), + ) + } else { + ( + OpenStep::OnlyOutput, + format!("no display is attached to the desktop ({adapters} graphics adapters listed)"), + ) + }; + CaptureUnavailable { + block: CaptureBlock::opening(step, DXGI_ERROR_NOT_FOUND.0) + .expect("DXGI_ERROR_NOT_FOUND is a blocked state"), + detail, + } +} + +/// `error` when it is DXGI_ERROR_ACCESS_LOST, for the loop's handler; +/// anything else ends the capture loop, as before. +fn access_lost_or( + error: windows::core::Error, +) -> Result> { + if error.code() == DXGI_ERROR_ACCESS_LOST { + Ok(error) + } else { + Err(error.into()) + } +} + +/// After ACCESS_LOST: whether the stage should open the target afresh, +/// because it now resolves to another output or adapter than `chosen` (see +/// [`super::target_moved`]). The duplicated output's own description is +/// asked first, and the outputs are listed (a new factory and every adapter) +/// only when it is off the desktop or renamed, while capture runs on the +/// primary for a missing display, or when `list` asks after a failed +/// re-creation. Both run under the stall limits while `progress` keeps the +/// picture alive; one that does not return in time counts as no move, and +/// the re-creation goes on. An output seen off the desktop or renamed, or a +/// listing without it as it was, is a detach for a second screen's proof +/// (see [`super::second_screen::note_detached`]). +fn target_moved( + shared: &SharedControl, + output: &IDXGIOutput1, + chosen: &OutputInfo, + target: &CaptureTarget, + list: bool, + mut progress: impl FnMut(), +) -> bool { + let fallback = matches!(target, CaptureTarget::Output(name) if *name != chosen.device_name); + if !list && !fallback { + let (output, name) = (output.clone(), chosen.device_name.clone()); + let same = crate::stall::run( + Call::ListOutputs, + move || { + unsafe { output.GetDesc() }.is_ok_and(|desc| { + desc.AttachedToDesktop.as_bool() && utf16_to_string(&desc.DeviceName) == name + }) + }, + &mut progress, + ); + if same.unwrap_or(true) { + return false; + } + super::second_screen::note_detached(shared); + } + let (shared, target, chosen) = (shared.clone(), target.clone(), chosen.clone()); + let started = Instant::now(); + let moved = crate::stall::run( + Call::ListOutputs, + move || super::target_moved(&super::listing::outputs(&shared), &target, &chosen), + progress, + ) + .unwrap_or(false); + info!( + took_ms = started.elapsed().as_millis() as u64, + moved, "Listed the display outputs after a lost duplication" + ); + moved +} + +/// [`Reopen`] for a target that moved off `chosen`. +fn moved_away(chosen: &OutputInfo) -> Box { + info!( + device = %chosen.device_name, + "The display layout changed — opening the display to capture again" + ); + Box::new(Reopen { + detail: format!( + "the display layout changed while capturing {}", + chosen.device_name + ), + }) +} + +/// While the duplication is away: resends the last picture when the viewer +/// would otherwise go without one (see [`recovery_poll`]), if `allowed`. +fn keep_picture_alive( + slot: &FrameSlot, + shared: &SharedControl, + frame_number: &mut u64, + last_send: &mut Instant, + last_frame: &Arc>, + dims: (u32, u32), + allowed: bool, +) { + let now_ms = crate::clock::host_now_us() / 1000; + if recovery_poll(shared, now_ms, last_send.elapsed()) && allowed { + resend_last_frame(slot, shared, frame_number, last_send, last_frame, dims); + } +} + +/// Logs, once per loss, that the duplication is not back within a second. +fn warn_if_overdue(recovery: &mut LossRecovery) { + if recovery.overdue(Instant::now()) { + warn!("Desktop duplication not back within a second — still retrying"); + } } fn unavailable_or_error( @@ -903,6 +1244,134 @@ fn publish_capture_geometry(output: &IDXGIOutput1, shared: &SharedControl) { } } +/// The picture `output` shows, read through GDI, as `width`x`height` BGRA +/// rows in a [`frame_buffer`]: for a duplication that gives no image (see +/// [`StartPicture`]). The pointer is not in it. None if the display is +/// rotated (the duplication's `rotation`, or a desktop rect that isn't that +/// size, as for 90 or 270 degrees), if GDI can't read it (no interactive +/// desktop), or if it read only black, as on the secure desktop: the +/// duplication's own image is waited for then. +fn read_screen_gdi( + output: &IDXGIOutput1, + (width, height): (u32, u32), + rotation: Option, +) -> Option> { + use windows::Win32::Graphics::Gdi::{ + BitBlt, CreateCompatibleDC, CreateDIBSection, DeleteDC, DeleteObject, GdiFlush, GetDC, + ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, DIB_RGB_COLORS, SRCCOPY, + }; + if !matches!( + rotation, + Some(DXGI_MODE_ROTATION_IDENTITY | DXGI_MODE_ROTATION_UNSPECIFIED) + ) { + return None; + } + let rect = unsafe { output.GetDesc() }.ok()?.DesktopCoordinates; + if (rect.right - rect.left, rect.bottom - rect.top) != (width as i32, height as i32) { + return None; + } + let row_bytes = width as usize * 4; + let info = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: std::mem::size_of::() as u32, + biWidth: width as i32, + // Negative: top-down rows, as capture's. + biHeight: -(height as i32), + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0, + ..Default::default() + }, + ..Default::default() + }; + unsafe { + let screen = GetDC(None); + if screen.is_invalid() { + return None; + } + let memory = CreateCompatibleDC(screen); + let mut bits = std::ptr::null_mut(); + let pixels = match CreateDIBSection(memory, &info, DIB_RGB_COLORS, &mut bits, None, 0) { + Ok(bitmap) if !bits.is_null() => { + let previous = SelectObject(memory, bitmap); + let copied = BitBlt( + memory, + 0, + 0, + width as i32, + height as i32, + screen, + rect.left, + rect.top, + SRCCOPY, + ) + .is_ok(); + let _ = GdiFlush(); + let pixels = copied + .then(|| { + let mut buf = frame_buffer(row_bytes * height as usize); + std::ptr::copy_nonoverlapping( + bits as *const u8, + buf.as_mut_ptr(), + row_bytes * height as usize, + ); + // GDI leaves the alpha byte 0; Desktop Duplication's is opaque. + let (pixels, _) = buf[..row_bytes * height as usize].as_chunks_mut::<4>(); + let mut lit = false; + for pixel in pixels { + lit |= pixel[..3] != [0, 0, 0]; + pixel[3] = 255; + } + lit.then_some(buf) + }) + .flatten(); + SelectObject(memory, previous); + let _ = DeleteObject(bitmap); + pixels + } + Ok(bitmap) => { + let _ = DeleteObject(bitmap); + None + } + Err(_) => None, + }; + let _ = DeleteDC(memory); + ReleaseDC(None, screen); + pixels + } +} + +/// Reads the staging texture's image into `buf`, row by row (its pitch may +/// differ from `row_bytes`). +fn copy_staging( + device_context: &ID3D11DeviceContext, + staging_texture: &ID3D11Texture2D, + height: u32, + row_bytes: usize, + buf: &mut [u8], +) -> windows::core::Result<()> { + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + unsafe { + device_context.Map(staging_texture, 0, D3D11_MAP_READ, 0, Some(&mut mapped))?; + } + let src_pitch = mapped.RowPitch as usize; + let src_ptr = mapped.pData as *const u8; + debug_assert!(buf.len() >= height as usize * row_bytes); + for row in 0..height as usize { + unsafe { + std::ptr::copy_nonoverlapping( + src_ptr.add(row * src_pitch), + buf.as_mut_ptr().add(row * row_bytes), + row_bytes, + ); + } + } + unsafe { + device_context.Unmap(staging_texture, 0); + } + Ok(()) +} + fn make_staging_desc(width: u32, height: u32) -> D3D11_TEXTURE2D_DESC { D3D11_TEXTURE2D_DESC { Width: width, @@ -948,10 +1417,10 @@ fn get_dirty_rect_count( } } -// DXGI pointer shape types (DXGI_OUTDUPL_POINTER_SHAPE_TYPE_*). -const DXGI_POINTER_SHAPE_MONOCHROME: u32 = 1; -const DXGI_POINTER_SHAPE_COLOR: u32 = 2; -const DXGI_POINTER_SHAPE_MASKED_COLOR: u32 = 4; +use super::pointer::{ + SHAPE_COLOR as DXGI_POINTER_SHAPE_COLOR, SHAPE_MASKED_COLOR as DXGI_POINTER_SHAPE_MASKED_COLOR, + SHAPE_MONOCHROME as DXGI_POINTER_SHAPE_MONOCHROME, +}; /// Caches the latest mouse-cursor shape from DXGI Desktop Duplication. The shape only /// arrives when it changes, so it's kept across frames and composited at the per-frame @@ -963,13 +1432,24 @@ struct CursorState { width: u32, height: u32, pitch: u32, + hot: (u32, u32), has_shape: bool, } impl CursorState { + /// The pointer image's size in pixels, once there is one. A monochrome + /// shape's buffer stacks two masks. + fn size(&self) -> Option<(u32, u32)> { + self.has_shape.then_some(match self.shape_type { + DXGI_POINTER_SHAPE_MONOCHROME => (self.width, self.height / 2), + _ => (self.width, self.height), + }) + } + /// Fetch the current pointer shape. Must be called while a frame is acquired - /// (before `ReleaseFrame`). On failure the previous shape is kept. - fn update_shape(&mut self, duplication: &IDXGIOutputDuplication, buffer_size: u32) { + /// (before `ReleaseFrame`). On failure the previous shape is kept and + /// this returns false. + fn update_shape(&mut self, duplication: &IDXGIOutputDuplication, buffer_size: u32) -> bool { self.shape.resize(buffer_size as usize, 0); let mut required = 0u32; let mut info = DXGI_OUTDUPL_POINTER_SHAPE_INFO::default(); @@ -986,10 +1466,13 @@ impl CursorState { self.width = info.Width; self.height = info.Height; self.pitch = info.Pitch; + self.hot = (info.HotSpot.x.max(0) as u32, info.HotSpot.y.max(0) as u32); self.has_shape = true; + true } Err(e) => { warn!(error = %e, "GetFramePointerShape failed — cursor shape not updated"); + false } } } @@ -1145,6 +1628,59 @@ fn draw_monochrome_cursor( mod tests { use super::*; + /// Native: GDI reads the primary display at its own size, as opaque BGRA + /// in a capture buffer, and nothing at another size or rotation. Needs an + /// interactive desktop that isn't all black; skips with no output listed + /// (a non-interactive SSH session). + /// `cargo test -p eternal-host --lib the_screen_is_read_through_gdi -- --ignored --nocapture` + #[test] + #[ignore = "native: needs an interactive Windows desktop"] + fn the_screen_is_read_through_gdi_only_at_its_own_size_and_rotation() { + use windows::Win32::Graphics::Dxgi::Common::{ + DXGI_MODE_ROTATION_ROTATE180, DXGI_MODE_ROTATION_ROTATE90, + }; + let factory: IDXGIFactory1 = unsafe { CreateDXGIFactory1() }.expect("DXGI factory"); + let shared = SharedControl::new(0, 8_000_000); + let (listed, _) = super::super::listing::listed(&shared, &factory); + let Some(primary) = listed.into_iter().find(|listed| listed.info.is_primary) else { + eprintln!("skipping: no display output is listed in this session"); + return; + }; + let output: IDXGIOutput1 = primary.output.cast().expect("IDXGIOutput1"); + let (width, height) = (primary.info.width, primary.info.height); + let pixels = read_screen_gdi(&output, (width, height), Some(DXGI_MODE_ROTATION_IDENTITY)) + .expect("the desktop read through GDI"); + let picture_bytes = (width * height * 4) as usize; + assert_eq!(pixels.len(), picture_bytes + FRAME_PADDING); + let (read, _) = pixels[..picture_bytes].as_chunks::<4>(); + assert!(read.iter().all(|pixel| pixel[3] == 255), "not opaque"); + assert!(read.iter().any(|pixel| pixel[..3] != [0, 0, 0])); + for (size, rotation, what) in [ + ( + (width + 1, height), + Some(DXGI_MODE_ROTATION_IDENTITY), + "another size", + ), + ( + (width, height), + Some(DXGI_MODE_ROTATION_ROTATE180), + "upside down", + ), + ( + (width, height), + Some(DXGI_MODE_ROTATION_ROTATE90), + "a quarter turn", + ), + ((width, height), None, "no duplication"), + ] { + assert!( + read_screen_gdi(&output, size, rotation).is_none(), + "{what} was read" + ); + } + eprintln!("read {width}x{height} through GDI"); + } + /// Native: a real D3D11CreateDevice held past its limit. The caller gets /// the timeout on time, a second attempt is refused at once, the late /// device is released on its own thread, and then a device is created. diff --git a/host/src/capture/listing.rs b/host/src/capture/listing.rs new file mode 100644 index 00000000..f08b6516 --- /dev/null +++ b/host/src/capture/listing.rs @@ -0,0 +1,236 @@ +//! The one way to list the PC's display outputs. Every listing is judged for +//! a second screen's proof ([`super::second_screen::observe_listing`]): one +//! that does not show the proven output as it was clears the proof. Kept, the +//! same display back before the next reconciliation inherited it, and a dark +//! one cost a watchdog restart. The DXGI enumerator takes a [`Judged`] that +//! only this module makes, so a new listing path cannot skip the judge. + +use super::OutputInfo; +use crate::control::SharedControl; + +/// Made only here: the DXGI enumerator and the debug-build fake display take +/// one, so every listing either makes passes through this module. +#[cfg(any(windows, debug_assertions))] +pub(super) struct Judged(()); + +/// The outputs attached to the desktop, judged for `shared`'s second screen. +/// In debug builds with a fake virtual display installed, the fake's. +pub fn outputs(shared: &SharedControl) -> Vec { + let outputs = unjudged(shared); + super::second_screen::observe_listing(shared, &outputs); + outputs +} + +/// Whether any output is attached to the desktop, judged as [`outputs`]. +pub fn any_attached(shared: &SharedControl) -> bool { + !outputs(shared).is_empty() +} + +/// [`outputs`] with the adapter and output objects DXGI listed them from, +/// for capture to open, and how many adapters were listed. +#[cfg(windows)] +pub(super) fn listed( + shared: &SharedControl, + factory: &windows::Win32::Graphics::Dxgi::IDXGIFactory1, +) -> (Vec, u32) { + let (listed, adapters) = super::dxgi::list_outputs(factory, Judged(())); + let outputs: Vec = listed.iter().map(|listed| listed.info.clone()).collect(); + super::second_screen::observe_listing(shared, &outputs); + (listed, adapters) +} + +/// [`outputs`] before the window's first paint, in the startup graphics +/// lane ([`crate::stall::Call::GpuDetect`]): empty, and nothing judged, if +/// it does not return within the limit or while an earlier startup call is +/// still stuck on the same driver. +pub fn outputs_at_startup(shared: &SharedControl) -> Vec { + let shared = shared.clone(); + crate::stall::run( + crate::stall::Call::GpuDetect, + move || outputs(&shared), + || {}, + ) + .unwrap_or_default() +} + +/// The display the startup banner names: the primary, or the first one. +/// Before any host state exists there is no proof to judge; only this label +/// leaves the module, never the outputs. Listed in the startup graphics +/// lane with GPU detection, which waits on the same driver: "unknown" if it +/// does not return in time. +pub fn startup_display_label() -> String { + let outputs = + crate::stall::run(crate::stall::Call::GpuDetect, from_dxgi, || {}).unwrap_or_default(); + outputs + .iter() + .find(|output| output.is_primary) + .or_else(|| outputs.first()) + .map(|output| { + format!( + "{} ({}x{})", + output.device_name, output.width, output.height + ) + }) + .unwrap_or_else(|| "unknown".to_string()) +} + +fn unjudged(shared: &SharedControl) -> Vec { + #[cfg(debug_assertions)] + if let Some(outputs) = super::second_screen::fake::outputs(shared, Judged(())) { + return outputs; + } + // Only the debug-build fake reads it. + #[cfg(not(debug_assertions))] + let _ = shared; + from_dxgi() +} + +/// Windows-only in substance; other platforms have no selectable outputs +/// (the synthetic source ignores them). +fn from_dxgi() -> Vec { + #[cfg(windows)] + { + super::dxgi::outputs(Judged(())) + } + #[cfg(not(windows))] + { + Vec::new() + } +} + +#[cfg(test)] +mod tests { + /// Each name that sees outputs without the judge, and the files that may + /// name it. The compiler holds the token-taking ones (`list_outputs`, + /// `dxgi::outputs`, `fake::outputs`) to this module; this is the backstop + /// for what it cannot hold: the Windows API itself, and names that must + /// not come back. + const UNJUDGED: [(&str, &[&str]); 6] = [ + // The DXGI enumerator. + ("list_outputs", &["capture/dxgi.rs"]), + // DXGI's own enumeration: capture's, and a per-adapter output count in + // the virtual display's diagnostics snapshot, which names no output. + ("EnumOutputs", &["capture/dxgi.rs", "vdd_snapshot.rs"]), + // The banner's label, which carries no outputs. + ("startup_display_label", &["main.rs"]), + // Removed unjudged entry points. + ("outputs_before_the_host_starts", &[]), + ("enumerate_outputs", &[]), + ("display_attached", &[]), + ]; + + /// Whether `text` uses the identifier `name` (aliases included: a `use … + /// as` names it too). + fn uses(text: &str, name: &str) -> bool { + let word = |c: char| c.is_alphanumeric() || c == '_'; + text.match_indices(name).any(|(at, _)| { + !text[..at].ends_with(word) && !text[at + name.len()..].starts_with(word) + }) + } + + /// `line` without its string literals. + fn code_only(line: &str) -> String { + let mut code = String::new(); + let mut in_string = false; + let mut escaped = false; + for c in line.chars() { + if in_string { + match c { + _ if escaped => escaped = false, + '\\' => escaped = true, + '"' => in_string = false, + _ => {} + } + } else if c == '"' { + in_string = true; + } else { + code.push(c); + } + } + code + } + + /// The unjudged names `file` uses where it may not. Comments and string + /// literals may name them. + fn strays(file: &str, text: &str) -> Vec { + let code: String = text + .lines() + .filter(|line| !line.trim_start().starts_with("//")) + .map(code_only) + .collect::>() + .join("\n"); + UNJUDGED + .iter() + .filter(|(name, places)| uses(&code, name) && !places.contains(&file)) + .map(|(name, _)| format!("{file}: {name}")) + .collect() + } + + /// Review r5 of #205: listings that bypassed the judge let a returning + /// display inherit an old proof, one path at a time. + #[test] + fn every_listing_goes_through_the_judge() { + let src = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let mut files = Vec::new(); + let mut dirs = vec![src.clone()]; + while let Some(dir) = dirs.pop() { + for entry in std::fs::read_dir(&dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + dirs.push(path); + } else if path.extension().is_some_and(|ext| ext == "rs") { + files.push(path); + } + } + } + assert!(files.len() > 50, "{} source files", files.len()); + let mut found = Vec::new(); + for file in &files { + let name = file + .strip_prefix(&src) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + // This module is the judge, and this test names the calls. + if name != "capture/listing.rs" { + found.extend(strays(&name, &std::fs::read_to_string(file).unwrap())); + } + } + assert!( + found.is_empty(), + "list outputs through capture::listing: {found:?}" + ); + } + + /// Review r6 of #205: the scan matched calls, so an alias made through + /// `use … as` went unseen. + #[test] + fn the_listing_scan_catches_aliases_and_ignores_comments() { + let alias = "use crate::capture::listing::startup_display_label as raw;\nfn f() { raw(); }"; + assert_eq!( + strays("gui/mod.rs", alias), + ["gui/mod.rs: startup_display_label"] + ); + assert!(strays("main.rs", alias).is_empty(), "the banner may"); + let grouped = "use windows::Win32::Graphics::Dxgi::{IDXGIAdapter1, EnumOutputs as list};"; + assert_eq!(strays("gpu.rs", grouped), ["gpu.rs: EnumOutputs"]); + let back = "pub fn outputs_before_the_host_starts() {}"; + assert_eq!( + strays("capture/mod.rs", back), + ["capture/mod.rs: outputs_before_the_host_starts"] + ); + // A comment, a string, or a longer name that contains one, is not a + // use; code after a string still counts. + assert!(strays("gui/mod.rs", "// startup_display_label as raw").is_empty()); + assert!(strays( + "capture/recovery.rs", + r#"detail: "EnumOutputs: 0x887a0002".into(),"# + ) + .is_empty()); + assert_eq!( + strays("gpu.rs", r#"let a = "x\"y"; adapter.EnumOutputs(0);"#), + ["gpu.rs: EnumOutputs"] + ); + assert!(strays("gui/mod.rs", "fn startup_display_label_text() {}").is_empty()); + } +} diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 1473f9a9..1b27960a 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -10,9 +10,9 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use parking_lot::{Condvar, Mutex}; -use tracing::error; #[cfg(any(windows, test, debug_assertions))] -use tracing::{info, warn}; +use tracing::warn; +use tracing::{error, info}; use crate::control::SharedControl; #[cfg(any(windows, test, debug_assertions))] @@ -22,6 +22,8 @@ use crate::vdd_device::DeviceState; #[cfg(windows)] pub mod dxgi; +pub mod listing; +pub mod pointer; pub mod recovery; pub mod second_screen; pub mod synthetic; @@ -36,7 +38,6 @@ pub(crate) const FPS_WINDOW: usize = 60; /// DXGI delivers no new frames. If a client is connected and we haven't sent anything for this /// long, resend the last frame so the encoder emits the pending IDR and the iPad gets a sync /// sample instead of timing out on a black screen. -#[cfg(any(windows, test))] pub(crate) const IDLE_KEEPALIVE: Duration = Duration::from_millis(750); /// With nobody connected the cached picture is refreshed this often, so a new @@ -108,6 +109,137 @@ pub(crate) fn frame_has_image(last_present_time: i64, seen_image: bool) -> bool last_present_time != 0 || seen_image } +/// Where Windows puts the pointer on a captured output. +#[cfg(any(windows, test))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PointerPlace { + /// Apart from the picture, as Desktop Duplication reports it. + Apart, + /// Windows draws it into the picture. + InPicture, + /// The second screen's driver can't be told (`DriverCursor::Unknown`). + /// The pointer is handled as in 0.3.4: no pointer messages, and capture + /// draws the reported pointer when Desktop Duplication reports it + /// visible with an image. + Unknown, +} + +/// Where Windows puts the pointer: apart, except on the second screen, +/// where the running driver decides (`vdd_settings::running_cursor`). +/// With its hardware cursor, an image larger than its caps goes into the +/// picture; without one, every image does. Measured on the reference PC, +/// a pointer in the picture made every move a new image, and Desktop +/// Duplication kept reporting a stale position. +#[cfg(any(windows, test))] +pub(crate) fn pointer_place( + second_screen: bool, + driver: crate::vdd_settings::DriverCursor, + shape: Option<(u32, u32)>, +) -> PointerPlace { + use crate::vdd_settings::DriverCursor; + match driver { + _ if !second_screen => PointerPlace::Apart, + DriverCursor::Hardware(caps) if shape.is_none_or(|size| caps.takes(size)) => { + PointerPlace::Apart + } + DriverCursor::Hardware(_) | DriverCursor::Software => PointerPlace::InPicture, + DriverCursor::Unknown => PointerPlace::Unknown, + } +} + +/// Whether capture draws the pointer into the video: for a client that +/// doesn't draw it, unless Windows already drew it there. Drawing the +/// reported pointer over Windows' own put a second, frozen pointer at the +/// stale position. Where that can't be told, as 0.3.4 did: whenever +/// Desktop Duplication reports it visible with an image, which it doesn't +/// for a pointer Windows drew in (on the reference PC's second screen +/// without the driver's cursor it reported it hidden from the first frame). +#[cfg(any(windows, test))] +pub(crate) fn composite_pointer( + client_draws_pointer: bool, + place: PointerPlace, + visible: bool, + has_shape: bool, +) -> bool { + let drawn_elsewhere = match place { + PointerPlace::Apart => client_draws_pointer, + PointerPlace::InPicture => true, + PointerPlace::Unknown => false, + }; + !drawn_elsewhere && visible && has_shape +} + +/// Whether the pointer goes apart from the video for this capture: to an +/// iPad that draws it, when Windows keeps it apart. Otherwise it stays in +/// the video, as before, and the iPad is told it is hidden. +#[cfg(any(windows, test))] +pub(crate) fn pointer_apart(client_draws_pointer: bool, place: PointerPlace) -> bool { + client_draws_pointer && place == PointerPlace::Apart +} + +/// What the capture loop does with an acquired frame. +#[cfg(any(windows, test))] +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Acquired { + /// Read the image back for the encoder. + ReadBack, + /// Release it: there is no image yet ([`frame_has_image`]). + Skip, + /// Release it: only the pointer moved, and the iPad draws the pointer. + /// The last picture still goes out as on a still desktop + /// ([`keepalive_due`]): a moving pointer keeps the duplication from ever + /// timing out, and without it the watchdog took the silence for a stuck + /// capture and restarted the stream, and keyframe requests went + /// unanswered. + PointerOnly, +} + +#[cfg(any(windows, test))] +pub(crate) fn acquired(last_present_time: i64, seen_image: bool, draws_pointer: bool) -> Acquired { + if !frame_has_image(last_present_time, seen_image) { + Acquired::Skip + } else if draws_pointer && last_present_time == 0 { + Acquired::PointerOnly + } else { + Acquired::ReadBack + } +} + +/// What the cached-frame rebuild for a changed pointer answer does with its +/// copy of the staging texture. +#[cfg(any(windows, test))] +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Redraw { + /// Copied: draw the pointer as the answer wants and keep the frame. + Copied, + /// A lost duplication is waiting for the loop's handler: rebuild after + /// it, with the cache kept as it is. + Later, + /// The copy hit ACCESS_LOST: hand it to the loop's handler, which + /// re-creates the duplication and keeps the cached picture. + Lost(E), +} + +/// Copies the staging texture for the rebuild (`copy`), unless a loss is +/// pending. ACCESS_LOST (`access_lost`) goes to the loop's handler as one +/// after an acquire does; any other error, such as DEVICE_REMOVED, ends +/// the capture loop so the pipeline re-creates the device. +#[cfg(any(windows, test))] +pub(crate) fn redraw_cached( + pending_loss: bool, + copy: impl FnOnce() -> Result<(), E>, + access_lost: impl FnOnce(&E) -> bool, +) -> Result, E> { + if pending_loss { + return Ok(Redraw::Later); + } + match copy() { + Ok(()) => Ok(Redraw::Copied), + Err(error) if access_lost(&error) => Ok(Redraw::Lost(error)), + Err(error) => Err(error), + } +} + pub(crate) fn frame_budget_for(target_fps: u32) -> Duration { let fps = target_fps.max(1) as u64; Duration::from_micros(1_000_000 / fps) @@ -126,6 +258,12 @@ pub struct RawFrame { pub data: Arc>, pub width: u32, pub height: u32, + /// A picture capture already had, sent again because the desktop gave + /// no new one: the keepalive, a keyframe request's resend, the cached + /// picture redrawn with or without the pointer, or a resend during + /// duplication recovery. Also a mirror's start picture read through GDI + /// (see `StartPicture`): no new picture is due after it either. + pub cached: bool, } /// Readable bytes after a frame's last row: FFmpeg's @@ -151,6 +289,10 @@ pub struct FrameSlot { struct SlotInner { state: Mutex, condvar: Condvar, + /// Runs after [`FrameSlot::take_within`] took a frame and let go of the + /// mailbox, where a taker can be descheduled. + #[cfg(test)] + after_take: Mutex>>, } #[derive(Default)] @@ -165,6 +307,8 @@ impl FrameSlot { inner: Arc::new(SlotInner { state: Mutex::new(SlotState::default()), condvar: Condvar::new(), + #[cfg(test)] + after_take: Mutex::new(None), }), } } @@ -200,6 +344,56 @@ impl FrameSlot { self.inner.condvar.wait(&mut state); } } + + /// As [`Self::blocking_take`], but gives up after `timeout`. A frame + /// taken replaces `kept`, a frame the caller still holds: `kept` is let + /// go under the mailbox lock before the frame leaves it, so capture, + /// which recycles a few buffers, never finds them all held at once. + pub fn take_within(&self, timeout: Duration, kept: &mut Option) -> Taken { + let deadline = Instant::now() + timeout; + let taken = { + let mut state = self.inner.state.lock(); + let mut timed_out = false; + loop { + if state.frame.is_some() { + *kept = None; + break Taken::Frame(state.frame.take().expect("checked above")); + } + if state.closed { + break Taken::Closed; + } + if timed_out { + break Taken::Nothing; + } + timed_out = self + .inner + .condvar + .wait_until(&mut state, deadline) + .timed_out(); + } + }; + #[cfg(test)] + if matches!(taken, Taken::Frame(_)) { + if let Some(hook) = self.inner.after_take.lock().as_mut() { + hook(); + } + } + taken + } + + #[cfg(test)] + pub(crate) fn after_take(&self, hook: impl FnMut() + Send + 'static) { + *self.inner.after_take.lock() = Some(Box::new(hook)); + } +} + +/// What [`FrameSlot::take_within`] found. +#[derive(Debug)] +pub enum Taken { + Frame(RawFrame), + /// No frame arrived in time. + Nothing, + Closed, } /// Stamp a watchdog heartbeat (ms on the process clock). @@ -222,19 +416,9 @@ pub struct OutputInfo { pub top: i32, pub is_primary: bool, pub adapter_name: String, -} - -/// Enumerate every desktop-attached display output. Windows-only in substance; -/// other platforms have no selectable outputs (the synthetic source ignores them). -pub fn enumerate_outputs() -> Vec { - #[cfg(windows)] - { - dxgi::enumerate_outputs() - } - #[cfg(not(windows))] - { - Vec::new() - } + /// The adapter's LUID. Ordinals change when an adapter comes or goes; + /// the LUID stays until the adapter is restarted. + pub adapter_id: crate::gpu_preference::AdapterId, } /// Resolve which output (index into `outputs`) the capture loop should duplicate for @@ -270,6 +454,35 @@ pub(crate) fn resolve_target( } } +/// After the duplication was lost: whether `target` now resolves to another +/// output than `current`, the one being duplicated, by device name and +/// adapter LUID. A display unplugged or back, a dock or a driver restart +/// also surface as a lost duplication. Re-creating it on the same output +/// kept capturing the primary display after the chosen one came back, or an +/// output of an adapter that had gone; the stage opens the target afresh. +/// +/// It looks the target up as [`resolve_target`] does, without its warning: +/// it runs at every loss while the chosen display is missing. +#[cfg(any(windows, test))] +pub(crate) fn target_moved( + outputs: &[OutputInfo], + target: &crate::control::CaptureTarget, + current: &OutputInfo, +) -> bool { + use crate::control::CaptureTarget; + let primary = || outputs.iter().find(|o| o.is_primary).or(outputs.first()); + let wanted = match target { + CaptureTarget::Output(name) => outputs + .iter() + .find(|o| &o.device_name == name) + .or_else(primary), + CaptureTarget::PrimaryAuto | CaptureTarget::VirtualExtended => primary(), + }; + wanted.is_none_or(|output| { + output.device_name != current.device_name || output.adapter_id != current.adapter_id + }) +} + /// How long to wait for the virtual display to attach after enabling its driver. Overridable /// via `ETERNAL_VDD_TIMEOUT_SECS`, or within 30–60 s by the update manifest. The iPad mirrors /// meanwhile, so a longer wait costs the user nothing; it was 10 s while the iPad gave up 10 s @@ -283,6 +496,8 @@ const VDD_POLL_INTERVAL: Duration = Duration::from_millis(120); enum AttachResult { Attached(OutputInfo), TimedOut, + /// The driver runs, but Windows added no monitor for it, and won't. + NoMonitor, Cancelled, } @@ -295,6 +510,7 @@ fn wait_for_virtual_output( mut poll: impl FnMut() -> Option>, mut now: impl FnMut() -> Instant, mut wait: impl FnMut(Duration), + mut no_monitor: impl FnMut() -> bool, ) -> AttachResult { let deadline = now() + timeout; loop { @@ -307,6 +523,9 @@ fn wait_for_virtual_output( if now() >= deadline { return AttachResult::TimedOut; } + if no_monitor() { + return AttachResult::NoMonitor; + } wait(VDD_POLL_INTERVAL); } } @@ -372,6 +591,12 @@ struct VddRequest { mode: Option, /// The adapters' LUIDs before the Enable, to find the driver's own. before: Vec, + /// The driver's start before the Enable, if it was running, and what + /// an Enable that starts it gives Windows for the pointer, from the + /// file it reads (see `vdd_settings::record_after_enable`). + instance: crate::vdd_settings::DriverRun, + settings: Option, + cursor: crate::vdd_settings::DriverCursor, } /// Before each enable: name the main screen's graphics card for the driver @@ -457,18 +682,64 @@ fn prepare_vdd_settings(shared: &SharedControl, outputs: &[OutputInfo]) -> VddRe } Err(_) => {} } + // The driver reads this file when it starts; a driver already running + // keeps what it read then. + let settings = std::fs::read_to_string(path).ok(); + let reads_cursor = owned + || crate::vdd_settings::driver_version() + .is_some_and(|version| crate::vdd_settings::driver_reads_cursor(&version)); + let overridden = crate::vdd_settings::registry_overrides_cursor(); + let cursor = + crate::vdd_settings::starting_cursor(settings.as_deref(), reads_cursor, overridden); + let instance = crate::vdd_device::run_state(); + info!( + ?instance, + reads_cursor, + overridden, + ?cursor, + "The virtual display's pointer settings" + ); VddRequest { pin: crate::vdd_settings::pin_result(wanted, &applied), mode, before, + instance, + settings, + cursor, } } +/// Where the record of the driver's last start is kept. +#[cfg(windows)] +pub(crate) fn cursor_record_path() -> Option { + crate::settings::app_data_dir().map(|dir| dir.join(crate::vdd_settings::CURSOR_RECORD)) +} + +/// What the running driver does with the pointer, from the record of its +/// start (`vdd_settings::running_cursor`). +#[cfg(windows)] +pub(crate) fn running_driver_cursor() -> crate::vdd_settings::DriverCursor { + let record = cursor_record_path().and_then(|path| crate::vdd_settings::load_record(&path)); + let running = crate::vdd_device::run_state(); + let cursor = crate::vdd_settings::running_cursor(record.as_ref(), running); + info!( + recorded = record.is_some(), + ?running, + settings_sha256 = record + .as_ref() + .map_or("", |record| record.settings_sha256.as_str()), + ?cursor, + "The running virtual display's pointer" + ); + cursor +} + /// Bring a freshly attached second screen to the mode asked for, when /// Windows restored another one for this set of displays, and log the mode /// it ends up in, once per attach. Only the virtual display changes. #[cfg(windows)] fn settle_vdd_mode( + shared: &SharedControl, chosen: OutputInfo, requested: Option, mut heartbeat: impl FnMut(), @@ -505,7 +776,7 @@ fn settle_vdd_mode( // Windows applies the change asynchronously to DXGI's view of the output. for _ in 0..30 { heartbeat(); - if let Some(output) = enumerate_outputs() + if let Some(output) = listing::outputs(shared) .into_iter() .find(|output| output.device_name == chosen.device_name) { @@ -689,20 +960,6 @@ pub(crate) fn client_connected(shared: &SharedControl) -> bool { shared.client_connected() } -/// The displays to capture from: [`enumerate_outputs`], or in end-to-end -/// tests the fake virtual display's. -#[cfg(any(windows, test, debug_assertions))] -fn outputs(shared: &SharedControl) -> Vec { - #[cfg(debug_assertions)] - if let Some(outputs) = second_screen::fake::outputs(shared) { - return outputs; - } - // Only the debug-build fake reads it. - #[cfg(not(debug_assertions))] - let _ = shared; - enumerate_outputs() -} - /// Reconcile the virtual display device to the requested target and return the concrete /// target to capture. `VirtualExtended` captures the virtual display once it is attached and /// a viewer is connected (or one left less than [`second_screen::RELEASE_GRACE`] ago), and @@ -744,17 +1001,24 @@ pub(crate) fn reconcile_virtual_display( // Settings and codec changes restart capture without removing the // managed monitor, and the stream moves onto it this way once it // attaches. Reuse that output rather than turning it on again. - let attached = owned.clone().filter(|name| { - outputs(shared) - .iter() - .any(|output| !output.is_primary && &output.device_name == name) + let attached = owned.as_ref().and_then(|name| { + listing::outputs(shared) + .into_iter() + .find(|output| !output.is_primary && &output.device_name == name) }); - if let Some(name) = attached { + // Gone from the desktop, the listing cleared its proof. + if let Some(output) = attached { if status == VddStatus::Active && (connected || shared.second_screen.lock().keep_without_viewer()) { - second_screen::start_proving(shared, generation, Instant::now()); - return Some(CaptureTarget::Output(name)); + let proof_output = (output.adapter_id, output.device_name.clone()); + second_screen::start_proving_on( + shared, + generation, + Instant::now(), + Some(proof_output), + ); + return Some(CaptureTarget::Output(output.device_name)); } } // The transport's next tick starts the display beside this mirror. @@ -787,6 +1051,139 @@ pub(crate) fn reconcile_virtual_display( } } +/// What a static display's capture has to show. Sending the cached picture +/// again keeps the viewer's stream alive while the desktop changes nothing. +#[cfg(any(windows, test))] +#[derive(Debug, Default)] +pub(crate) struct Picture { + /// The current duplication has delivered an image: the first frames of a + /// new one carry none (see [`frame_has_image`]). + pub seen_image: bool, + /// The cached picture is a real image, not the black start frame. + real: bool, + /// When a duplication was re-created with a real picture cached and no + /// new image since. + held_since: Option, +} + +/// How long a second screen resends its cached picture after its +/// duplication was re-created without a new image. A virtual display that +/// stopped sending frames looks the same as one that shows nothing new, and +/// the resends hid it from the watchdog; after this the stage opens it +/// again without the cached picture, and the watchdog judges what follows. +#[cfg(any(windows, test))] +pub(crate) const PICTURE_HOLD: Duration = Duration::from_secs(15); + +#[cfg(any(windows, test))] +impl Picture { + /// An image was read back into the cache. + pub fn cached(&mut self) { + self.real = true; + self.held_since = None; + } + + /// The lost duplication was re-created. The cached picture is still the + /// screen's: a static second screen shows nothing new after a lock screen + /// or a UAC prompt, and while keepalives waited for a new image the + /// watchdog restarted the pipeline under the viewer ("no captured frame + /// for 5039ms with a client connected"). It is resent for at most + /// [`PICTURE_HOLD`] from the first re-creation without a new image. + pub fn recreated(&mut self, now: Instant) { + self.seen_image = false; + if self.real { + self.held_since.get_or_insert(now); + } + } + + /// The resolution changed: the cache is a black frame of the new size. + pub fn resized(&mut self) { + self.real = false; + self.held_since = None; + } + + /// A second screen has resent its cached picture for [`PICTURE_HOLD`] + /// without a new image: the stage opens it again. + pub fn held_too_long(&self, now: Instant, second_screen: bool) -> bool { + second_screen + && self + .held_since + .is_some_and(|since| now.duration_since(since) >= PICTURE_HOLD) + } + + pub fn real(&self) -> bool { + self.real + } +} + +/// Naps up to `total` in slices of at most 50 ms, calling `heartbeat` before +/// each, and stops as soon as `wake` says to: a viewer that connected during +/// the idle throttle's 250 ms nap waited out the rest of it before capture +/// resent the picture. +#[cfg(any(windows, test))] +pub(crate) fn nap_unless( + total: Duration, + mut wake: impl FnMut() -> bool, + mut heartbeat: impl FnMut(), + mut sleep: impl FnMut(Duration), +) { + const SLICE: Duration = Duration::from_millis(50); + let mut left = total; + while !left.is_zero() && !wake() { + heartbeat(); + let nap = left.min(SLICE); + sleep(nap); + left -= nap; + } +} + +/// How long a mirror's new duplication may go without an image before its +/// start picture is read through GDI instead (see [`StartPicture`]). +#[cfg(any(windows, test))] +pub(crate) const START_PICTURE_GRACE: Duration = Duration::from_millis(100); + +/// A mirror's start picture when its duplication gives none. Desktop +/// Duplication delivers an image only once the desktop presents one. A +/// display that went to sleep, and possibly a desktop that changes nothing, +/// presents none: the encoder then opened only at the 750 ms keepalive, which sent the +/// black start frame, and the viewer saw black until the desktop changed. +/// GDI still reads what the desktop shows. +#[cfg(any(windows, test))] +#[derive(Debug)] +pub(crate) struct StartPicture { + since: Instant, + tried: bool, +} + +#[cfg(any(windows, test))] +impl StartPicture { + pub fn new(now: Instant) -> Self { + Self { + since: now, + tried: false, + } + } + + /// A duplication was created again, or its picture size changed. + pub fn duplicated(&mut self, now: Instant) { + *self = Self::new(now); + } + + /// Whether to read the start picture through GDI now: once per + /// duplication, [`START_PICTURE_GRACE`] after it was created, while the + /// cache holds no real picture (`real`, see [`Picture::real`]). Never on + /// the second screen, whose image from the duplication is its proof. + pub fn due(&mut self, real: bool, second_screen: bool, now: Instant) -> bool { + if real || second_screen || self.tried { + return false; + } + if now.saturating_duration_since(self.since) < START_PICTURE_GRACE { + return false; + } + self.tried = true; + true + } +} + /// Whether an idle keepalive may resend the last frame. Before the second /// screen has delivered an image, that frame is the black placeholder: /// sending it would replace the iPad's mirror with black, so the switch @@ -798,7 +1195,6 @@ pub(crate) fn keepalive_allowed(seen_image: bool, second_screen: bool) -> bool { /// How long a keyframe request waits for the encoder to take the resent frame /// before the picture is resent again. -#[cfg(any(windows, test))] const KEYFRAME_RESEND: Duration = Duration::from_millis(100); /// Whether an idle display's last picture is due to be resent to the viewer. @@ -806,7 +1202,6 @@ const KEYFRAME_RESEND: Duration = Duration::from_millis(100); /// at once when there is a real picture to send: a viewer back within the /// second screen's grace period otherwise waited for the next keepalive, up /// to 750 ms, before the encoder could start its session. -#[cfg(any(windows, test))] pub(crate) fn keepalive_due( since_send: Duration, keyframe_requested: bool, @@ -838,7 +1233,6 @@ pub(crate) fn read_back_image( pub fn run_capture_stage( slot: FrameSlot, shared: SharedControl, - adapter_index: u32, generation: u64, ) -> Result<(), Box> { struct CloseOnExit(FrameSlot); @@ -851,36 +1245,105 @@ pub fn run_capture_stage( let result = recovery::run_until_unblocked( &shared, |frame_number| { - let result = run_platform_loop( - &guard.0, - shared.clone(), - adapter_index, - generation, - frame_number, - ); + let result = run_platform_loop(&guard.0, shared.clone(), generation, frame_number); if let Err(error) = &result { - if recovery::passing_error(error.as_ref()) { - second_screen::note_interruption(&shared, generation); - } else { - second_screen::note_persistent_error(&shared, generation); - } - second_screen::note_capture_error(&shared, generation); + note_stage_error(&shared, generation, error.as_ref()); } result }, - |delay| wait_while_blocked(&shared, generation, delay), + |delay| { + wait_while_blocked(&shared, generation, delay, || { + listing::any_attached(&shared) + }) + }, ); - if let Err(ref e) = result { - error!(error = %e, "Capture loop exited with error"); + stage_exit(result, stopping(&shared, generation)) +} + +/// The stream is stopping, or a newer generation has replaced this one. +fn stopping(shared: &SharedControl, generation: u64) -> bool { + !shared.running.load(std::sync::atomic::Ordering::SeqCst) + || !crate::supervisor::generation_is_current(generation) +} + +/// How the capture stage ends. An error once the stream is stopping is the +/// stop: tearing the display down can fail the call in flight ("The +/// parameter is incorrect"), and that is no capture failure for the +/// supervisor to restart or count. +fn stage_exit( + result: Result<(), Box>, + stopping: bool, +) -> Result<(), Box> { + match result { + Err(e) if stopping => { + info!(error = %e, "Capture loop stopped with the stream"); + Ok(()) + } + Err(e) => { + error!(error = %e, "Capture loop exited with error"); + Err(e) + } + Ok(()) => Ok(()), } - result } +/// What a capture attempt's error says about a second screen proving +/// itself. A [`recovery::Reopen`] is not a capture error: the stage opens +/// the display again at once, and the proof keeps its deadline. Noting it +/// as one failed the proof, and a display that moved during the first +/// proof went back to the mirror. +fn note_stage_error( + shared: &SharedControl, + generation: u64, + error: &(dyn std::error::Error + Send + Sync + 'static), +) { + if recovery::passing_error(error) { + second_screen::note_interruption(shared, generation); + } else { + second_screen::note_persistent_error(shared, generation); + } + if error.downcast_ref::().is_none() { + second_screen::note_capture_error(shared, generation); + } +} + +/// While no display is on, how often the wait looks for one to come back. +const DISPLAY_LOOK: Duration = Duration::from_millis(250); + /// Sleeps between blocked-capture retries. The watchdog has nothing to fix /// while Windows withholds the desktop, so the stage keeps its heartbeats. -fn wait_while_blocked(shared: &SharedControl, generation: u64, delay: Duration) -> bool { +/// While no display is on, `display_back` is asked every [`DISPLAY_LOOK`] +/// and a display that came back ends the wait: the retries settle 5 s +/// apart, and the picture came back up to 5 s after the screen did. +fn wait_while_blocked( + shared: &SharedControl, + generation: u64, + delay: Duration, + display_back: impl FnMut() -> bool, +) -> bool { + wait_while_blocked_on( + shared, + generation, + delay, + display_back, + Instant::now, + std::thread::sleep, + ) +} + +/// [`wait_while_blocked`] on the clock `now`, sleeping with `sleep`. +fn wait_while_blocked_on( + shared: &SharedControl, + generation: u64, + delay: Duration, + mut display_back: impl FnMut() -> bool, + mut now: impl FnMut() -> Instant, + mut sleep: impl FnMut(Duration), +) -> bool { use std::sync::atomic::Ordering; - let deadline = Instant::now() + delay; + let start = now(); + let deadline = start + delay; + let mut next_look = start + DISPLAY_LOOK; loop { if !shared.running.load(Ordering::SeqCst) || !crate::supervisor::generation_is_current(generation) @@ -888,11 +1351,19 @@ fn wait_while_blocked(shared: &SharedControl, generation: u64, delay: Duration) return false; } hold_heartbeats(shared); - let now = Instant::now(); - if now >= deadline { + let at = now(); + if at >= deadline { return true; } - std::thread::sleep((deadline - now).min(Duration::from_millis(50))); + if at >= next_look { + next_look = at + DISPLAY_LOOK; + if *shared.capture_block.lock() == Some(recovery::CaptureBlock::NoDisplay) + && display_back() + { + return true; + } + } + sleep((deadline - at).min(Duration::from_millis(50))); } } @@ -934,14 +1405,13 @@ pub(crate) fn synthetic_requested() -> bool { fn run_platform_loop( slot: &FrameSlot, shared: SharedControl, - adapter_index: u32, generation: u64, frame_number: &mut u64, ) -> Result<(), Box> { if synthetic_requested() { synthetic::run_capture_loop(slot, shared, generation, frame_number) } else { - dxgi::run_capture_loop(slot, shared, adapter_index, generation, frame_number) + dxgi::run_capture_loop(slot, shared, generation, frame_number) } } @@ -949,7 +1419,6 @@ fn run_platform_loop( fn run_platform_loop( slot: &FrameSlot, shared: SharedControl, - _adapter_index: u32, generation: u64, frame_number: &mut u64, ) -> Result<(), Box> { @@ -969,6 +1438,223 @@ fn pick_new_virtual_output<'a>(before: &[String], now: &'a [OutputInfo]) -> Opti #[cfg(test)] mod tests { + use std::sync::Arc; + + use parking_lot::Mutex; + + use crate::control::SharedControl; + + /// Log lines `run` writes on this thread. + fn logged(run: impl FnOnce()) -> String { + #[derive(Clone)] + struct Captured(Arc>>); + impl std::io::Write for Captured { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let buffer = Arc::new(Mutex::new(Vec::new())); + let writer = Captured(buffer.clone()); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, run); + let text = String::from_utf8_lossy(&buffer.lock()).into_owned(); + text + } + + fn failed() -> Result<(), Box> { + Err("The parameter is incorrect. (0x80070057)".into()) + } + + /// A call that fails as the stream stops is the stop: logged at info, + /// and the stage ends as completed, so the supervisor neither restarts + /// nor counts it. + #[test] + fn an_error_as_the_stream_stops_is_a_stop() { + let mut result = None; + let text = logged(|| result = Some(super::stage_exit(failed(), true))); + assert!(result.unwrap().is_ok()); + assert!(text.contains(" INFO ") && !text.contains("ERROR"), "{text}"); + assert!( + text.contains("Capture loop stopped with the stream"), + "{text}" + ); + } + + /// While the stream runs, the same error is a capture failure, as before. + #[test] + fn an_error_while_streaming_is_still_a_failure() { + let mut result = None; + let text = logged(|| result = Some(super::stage_exit(failed(), false))); + assert!(result.unwrap().is_err()); + assert!( + text.contains("ERROR") && text.contains("Capture loop exited with error"), + "{text}" + ); + let text = logged(|| assert!(super::stage_exit(Ok(()), true).is_ok())); + assert_eq!(text, ""); + } + + /// Stopping is `running` off, or a newer generation. + #[test] + fn a_stage_is_stopping_once_its_stream_stops_or_is_replaced() { + use std::sync::atomic::Ordering; + let shared = SharedControl::new(0, 1); + let live = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + shared.running.store(false, Ordering::SeqCst); + assert!(super::stopping(&shared, live)); + shared.running.store(true, Ordering::SeqCst); + assert!(super::stopping(&shared, live.wrapping_add(1_000_000))); + } + + #[test] + fn a_timed_take_returns_a_frame_nothing_or_the_close() { + let (producer, consumer) = super::frame_slot(); + let frame = |frame_number| super::RawFrame { + frame_number, + timestamp: Instant::now(), + data: Arc::new(Vec::new()), + width: 2, + height: 2, + cached: false, + }; + let short = Duration::from_millis(5); + assert!(matches!( + consumer.take_within(short, &mut None), + super::Taken::Nothing + )); + producer.publish(frame(1)); + assert!(matches!( + consumer.take_within(short, &mut None), + super::Taken::Frame(taken) if taken.frame_number == 1 + )); + let waiting = + std::thread::spawn(move || consumer.take_within(Duration::from_secs(10), &mut None)); + std::thread::sleep(Duration::from_millis(20)); + producer.publish(frame(2)); + assert!(matches!( + waiting.join().unwrap(), + super::Taken::Frame(taken) if taken.frame_number == 2 + )); + let (producer, consumer) = super::frame_slot(); + producer.close(); + assert!(matches!( + consumer.take_within(short, &mut None), + super::Taken::Closed + )); + } + + /// The synthetic source recycles three buffers. A taker that still held + /// a kept picture next to the frame it took, while capture published a + /// third, left capture no buffer to render its next frame into. + #[test] + fn a_frame_taken_in_place_of_a_kept_one_leaves_capture_a_free_buffer() { + let (producer, consumer) = super::frame_slot(); + let pool: Vec>> = (0..3).map(|_| Arc::new(vec![0u8; 16])).collect(); + let frame = |n: usize| super::RawFrame { + frame_number: n as u64, + timestamp: Instant::now(), + data: Arc::clone(&pool[n]), + width: 2, + height: 2, + cached: false, + }; + let mut kept = Some(frame(0)); + producer.publish(frame(1)); + let third = frame(2); + let capture = producer.handle(); + // The taker is descheduled right after the take: capture publishes + // its third buffer and looks for a free one, as the synthetic source + // does before it renders. + consumer.after_take(move || { + capture.publish(third.clone()); + assert!( + pool.iter().any(|buffer| Arc::strong_count(buffer) == 1), + "capture has no free buffer for its next frame" + ); + }); + let taken = consumer.take_within(Duration::from_secs(1), &mut kept); + assert!(matches!(taken, super::Taken::Frame(frame) if frame.frame_number == 1)); + assert!(kept.is_none()); + } + + #[test] + fn the_idle_nap_ends_when_a_viewer_connects() { + let slept = std::cell::RefCell::new(Vec::new()); + let checks = std::cell::Cell::new(0); + super::nap_unless( + Duration::from_millis(250), + || { + checks.set(checks.get() + 1); + checks.get() > 2 + }, + || {}, + |nap| slept.borrow_mut().push(nap), + ); + let total: Duration = slept.borrow().iter().sum(); + assert!( + total <= Duration::from_millis(100), + "napped {total:?} after the viewer connected" + ); + assert!(slept + .borrow() + .iter() + .all(|nap| *nap <= Duration::from_millis(50))); + // Nobody connects: the whole nap. + let slept = std::cell::RefCell::new(Vec::new()); + super::nap_unless( + Duration::from_millis(250), + || false, + || {}, + |nap| slept.borrow_mut().push(nap), + ); + assert_eq!( + slept.borrow().iter().sum::(), + Duration::from_millis(250) + ); + } + + #[test] + fn a_mirror_without_an_image_reads_its_start_picture_once_after_the_grace() { + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let mut gdi = super::StartPicture::new(start); + assert!( + !gdi.due(false, false, at(50)), + "the duplication may still give one" + ); + assert!(gdi.due(false, false, at(100))); + assert!(!gdi.due(false, false, at(200)), "once per duplication"); + // A new duplication (after access was lost, or a size change) whose + // cache is black tries again. + gdi.duplicated(at(1000)); + assert!(!gdi.due(false, false, at(1050))); + assert!(gdi.due(false, false, at(1100))); + } + + #[test] + fn a_real_picture_or_a_second_screen_never_reads_the_start_picture() { + let start = Instant::now(); + let later = start + Duration::from_secs(5); + let mut gdi = super::StartPicture::new(start); + assert!( + !gdi.due(true, false, later), + "the duplication gave an image" + ); + assert!( + !gdi.due(false, true, later), + "a second screen proves itself" + ); + // Neither used up the one try. + assert!(gdi.due(false, false, later)); + } #[test] fn pointer_only_frames_are_not_read_back_before_the_first_image() { @@ -978,6 +1664,166 @@ mod tests { assert!(frame_has_image(0, true)); } + /// While only the pointer moves and the iPad draws it, the duplication + /// returns pointer updates (here 600 a second) and never times out. The + /// last picture must still go out every keepalive, and a keyframe request + /// be answered within its resend time, as on a still desktop. + #[test] + fn pointer_only_updates_keep_the_picture_alive_for_an_ipad_that_draws_it() { + assert_eq!(acquired(0, false, true), Acquired::Skip, "no image yet"); + assert_eq!(acquired(0, true, false), Acquired::ReadBack, "older iPad"); + assert_eq!( + acquired(42, true, true), + Acquired::ReadBack, + "the desktop changed" + ); + let mut since_send = Duration::ZERO; + let mut resends = Vec::new(); + let step = Duration::from_micros(1_600); + let mut now = Duration::ZERO; + let (mut asked, mut pending) = (false, false); + while now < Duration::from_secs(3) { + now += step; + since_send += step; + if !asked && now >= Duration::from_millis(2_000) { + (asked, pending) = (true, true); + } + assert_eq!(acquired(0, true, true), Acquired::PointerOnly); + if keepalive_due(since_send, pending, true) { + resends.push(now); + since_send = Duration::ZERO; + pending = false; + } + } + assert!(resends.len() >= 4, "{resends:?}"); + for pair in resends.windows(2) { + assert!(pair[1] - pair[0] <= IDLE_KEEPALIVE + step, "{resends:?}"); + } + assert!( + resends.iter().any(|at| *at >= Duration::from_millis(2_000) + && *at <= Duration::from_millis(2_000) + KEYFRAME_RESEND + step), + "the keyframe request waited: {resends:?}" + ); + } + + /// #183 is a second screen report: with the driver's hardware cursor + /// the second screen sends the pointer apart, as Mirror does. Review + /// r4: an image goes into the picture when it is larger than the caps + /// the running driver started with, the user's own (64, 256) as well as + /// this host's 128. + #[test] + fn the_second_screen_places_the_pointer_by_the_running_drivers_caps() { + use crate::vdd_settings::{CursorCaps, DriverCursor}; + use PointerPlace::{Apart, InPicture, Unknown}; + let caps = |max| { + DriverCursor::Hardware(CursorCaps { + max_x: max, + max_y: max, + alpha: true, + xor: 2, + }) + }; + for (second_screen, driver, shape, place) in [ + (false, DriverCursor::Software, Some((300, 300)), Apart), + (false, DriverCursor::Unknown, Some((32, 32)), Apart), + (true, caps(128), Some((32, 32)), Apart), + (true, caps(128), None, Apart), + (true, caps(128), Some((128, 128)), Apart), + (true, caps(128), Some((129, 32)), InPicture), + (true, caps(128), Some((32, 129)), InPicture), + (true, caps(64), Some((64, 64)), Apart), + (true, caps(64), Some((65, 64)), InPicture), + (true, caps(64), Some((96, 96)), InPicture), + (true, caps(256), Some((160, 160)), Apart), + (true, caps(256), Some((256, 256)), Apart), + (true, caps(256), Some((257, 32)), InPicture), + (true, DriverCursor::Software, Some((32, 32)), InPicture), + (true, DriverCursor::Software, None, InPicture), + (true, DriverCursor::Unknown, Some((32, 32)), Unknown), + ] { + assert_eq!( + pointer_place(second_screen, driver, shape), + place, + "second_screen={second_screen} driver={driver:?} shape={shape:?}" + ); + assert_eq!(pointer_apart(true, place), place == Apart); + assert!(!pointer_apart(false, place), "an older iPad"); + } + } + + /// Review r7: an ACCESS_LOST from the rebuild's copy goes to the loop's + /// handler, which keeps the cached picture, instead of ending the + /// capture loop; a pending loss is handled before the rebuild is tried + /// again; any other error still ends the loop. + #[test] + fn a_lost_duplication_during_the_pointer_rebuild_goes_to_the_handler() { + const ACCESS_LOST: u32 = 0x887A_0026; + const DEVICE_REMOVED: u32 = 0x887A_0005; + let lost = |code: &u32| *code == ACCESS_LOST; + assert_eq!(redraw_cached(false, || Ok(()), lost), Ok(Redraw::Copied)); + assert_eq!( + redraw_cached(false, || Err(ACCESS_LOST), lost), + Ok(Redraw::Lost(ACCESS_LOST)) + ); + assert_eq!( + redraw_cached(false, || Err(DEVICE_REMOVED), lost), + Err(DEVICE_REMOVED), + "the device itself goes: the pipeline re-creates it" + ); + let mut copied = false; + assert_eq!( + redraw_cached( + true, + || { + copied = true; + Ok::<(), u32>(()) + }, + lost + ), + Ok(Redraw::Later) + ); + assert!(!copied, "not tried while a loss waits for the handler"); + } + + /// One pointer, never none and never two. Review r3: with the pointer + /// in the picture Desktop Duplication can still report it visible, + /// with a shape, at a stale spot, and drawing that over Windows' own + /// left a second, frozen one. Review r4: where the driver can't be + /// told, capture draws what Desktop Duplication reports, as 0.3.4 did, + /// whoever the client is. + #[test] + fn capture_draws_the_pointer_only_when_nobody_else_does() { + use PointerPlace::{Apart, InPicture, Unknown}; + assert!( + composite_pointer(false, Apart, true, true), + "Mirror, an older iPad" + ); + assert!( + !composite_pointer(true, Apart, true, true), + "the iPad draws it" + ); + for client_draws in [false, true] { + assert!( + !composite_pointer(client_draws, InPicture, true, true), + "Windows drew it into the picture" + ); + assert!( + composite_pointer(client_draws, Unknown, true, true), + "a driver that can't be told: as 0.3.4" + ); + for place in [Apart, InPicture, Unknown] { + assert!( + !composite_pointer(client_draws, place, false, true), + "hidden" + ); + assert!( + !composite_pointer(client_draws, place, true, false), + "no shape yet" + ); + } + } + } + pub(super) fn connect_viewer(shared: &SharedControl) { use crate::transport::link::PeerId; use crate::transport::session::ConfigSource; @@ -1029,7 +1875,7 @@ mod tests { fn a_viewer_arriving_after_idle_is_not_taken_for_a_wedge() { use crate::supervisor::{wedge_reason_at, Machine}; use std::sync::atomic::Ordering; - let machine = Machine::new(1); + let mut machine = Machine::new(1); let start_ms = 1_000_000; let arrival_ms = start_ms + 17_458; @@ -1041,14 +1887,14 @@ mod tests { .hb_capture_loop_ms .store(arrival_ms, Ordering::Relaxed); connect_viewer(&stale); - let reason = wedge_reason_at(&stale, &machine, arrival_ms).unwrap(); + let reason = wedge_reason_at(&stale, &mut machine, arrival_ms).unwrap(); assert!(reason.starts_with("no captured frame"), "{reason}"); // The first keepalive refreshes the frame stamp before the encoder // has output anything. stale .hb_capture_frame_ms .store(arrival_ms, Ordering::Relaxed); - let reason = wedge_reason_at(&stale, &machine, arrival_ms).unwrap(); + let reason = wedge_reason_at(&stale, &mut machine, arrival_ms).unwrap(); assert!(reason.starts_with("encoder silent"), "{reason}"); // With idle heartbeats the arrival is healthy at every step. @@ -1060,16 +1906,16 @@ mod tests { idle_heartbeat(&idle, now); } connect_viewer(&idle); - assert_eq!(wedge_reason_at(&idle, &machine, arrival_ms), None); + assert_eq!(wedge_reason_at(&idle, &mut machine, arrival_ms), None); idle.hb_capture_frame_ms .store(arrival_ms + 10, Ordering::Relaxed); - assert_eq!(wedge_reason_at(&idle, &machine, arrival_ms + 10), None); + assert_eq!(wedge_reason_at(&idle, &mut machine, arrival_ms + 10), None); } #[test] fn a_viewer_arriving_during_access_lost_recovery_is_not_taken_for_a_wedge() { use crate::supervisor::{wedge_reason_at, Machine}; - let machine = Machine::new(1); + let mut machine = Machine::new(1); let shared = SharedControl::new(0, 1); let start_ms = 1_000_000; startup_heartbeat(&shared, start_ms); @@ -1081,7 +1927,7 @@ mod tests { } // A viewer registers 30 ms after a poll, before the next one. connect_viewer(&shared); - assert_eq!(wedge_reason_at(&shared, &machine, now + 30), None); + assert_eq!(wedge_reason_at(&shared, &mut machine, now + 30), None); // The next poll resends the last picture to it. assert!(recovery_poll(&shared, now + 50, Duration::from_secs(6))); } @@ -1327,7 +2173,7 @@ mod tests { #[test] fn slow_virtual_attach_keeps_watchdog_alive_without_hiding_a_stall() { let shared = SharedControl::new(9876, 15_000_000); - let machine = crate::supervisor::Machine::new(1); + let mut machine = crate::supervisor::Machine::new(1); let start = Instant::now(); let elapsed = Cell::new(Duration::ZERO); let before = vec!["primary".to_string(), "physical-secondary".to_string()]; @@ -1348,26 +2194,31 @@ mod tests { assert!( crate::supervisor::wedge_reason_at( &shared, - &machine, + &mut machine, 1_000 + elapsed.get().as_millis() as u64 ) .is_none(), "driver startup must not trip the 3s/5s watchdogs" ); }, + || false, ); let AttachResult::Attached(output) = result else { panic!("virtual display did not attach") }; assert_eq!(output.device_name, "virtual"); + // The driver stalls now. The supervisor looks every 250 ms; a longer + // gap between looks is a sleeping PC, not a stall. + let stalled = 1_000 + elapsed.get().as_millis() as u64; + let mut now = stalled; + while now < stalled + 3_000 { + now += 250; + assert!(crate::supervisor::wedge_reason_at(&shared, &mut machine, now).is_none()); + } assert!( - crate::supervisor::wedge_reason_at( - &shared, - &machine, - 1_000 + elapsed.get().as_millis() as u64 + 3_001 - ) - .unwrap() - .contains("capture loop silent"), + crate::supervisor::wedge_reason_at(&shared, &mut machine, now + 250) + .unwrap() + .contains("capture loop silent"), "the normal watchdog must still detect a stalled driver" ); } @@ -1383,6 +2234,7 @@ mod tests { || Some(Vec::new()), || start + elapsed.get(), |duration| elapsed.set(elapsed.get() + duration), + || false, ); assert!(matches!(result, AttachResult::TimedOut)); assert!(elapsed.get() >= timeout && elapsed.get() < timeout + VDD_POLL_INTERVAL); @@ -1403,6 +2255,7 @@ mod tests { elapsed.set(elapsed.get() + duration); cancelled_at.set(true); }, + || false, ); assert!(matches!(result, AttachResult::Cancelled)); } @@ -1432,7 +2285,235 @@ mod tests { top, is_primary: left == 0 && top == 0, adapter_name: "Test GPU".to_string(), + adapter_id: (0, 1), + } + } + + /// A lost duplication after a display was unplugged or plugged back in, + /// or after its adapter restarted, is opened afresh from the stage. + #[test] + fn a_lost_duplication_reopens_when_the_target_moved() { + let (one, two) = (r"\\.\DISPLAY1", r"\\.\DISPLAY2"); + let both = vec![out(one, 0, 0), out(two, 1920, 0)]; + let chosen = CaptureTarget::Output(two.to_string()); + // A UAC prompt or a mode change: same output, same adapter. + assert!(!target_moved(&both, &chosen, &both[1])); + assert!(!target_moved(&both, &CaptureTarget::PrimaryAuto, &both[0])); + // The chosen display was unplugged: capture falls back to the primary. + let unplugged = vec![out(one, 0, 0)]; + assert!(target_moved(&unplugged, &chosen, &both[1])); + // It is back while the primary is captured: return to it. + assert!(target_moved(&both, &chosen, &unplugged[0])); + // Same name on an adapter that restarted, or moved to the other card + // of a dock or hybrid laptop. + let mut restarted = both.clone(); + restarted[1].adapter_id = (0, 2); + assert!(target_moved(&restarted, &chosen, &both[1])); + // Nothing listed at all: no display is on. + assert!(target_moved(&[], &CaptureTarget::PrimaryAuto, &both[0])); + } + + /// Review r1 of #205: while the chosen display is missing, capture runs + /// on the primary and checks at every loss whether it is back. The + /// check logged resolve_target's warning each time. + #[test] + fn checking_for_a_missing_display_at_each_loss_logs_nothing() { + use std::sync::{Arc, Mutex}; + struct Sink(Arc>>); + impl std::io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let out_log = Arc::new(Mutex::new(Vec::::new())); + let writer = Arc::clone(&out_log); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || Sink(Arc::clone(&writer))) + .finish(); + let primary = out(r"\\.\DISPLAY1", 0, 0); + let chosen = CaptureTarget::Output(r"\\.\DISPLAY2".to_string()); + let unplugged = vec![primary.clone()]; + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, || { + for _ in 0..20 { + assert!(!target_moved(&unplugged, &chosen, &primary)); + } + }); + let log = String::from_utf8(out_log.lock().unwrap().clone()).unwrap(); + assert!(log.is_empty(), "{log}"); + } + + /// Field, 0.3.4 and the 0.3.5 beta: a static second screen lost its + /// duplication, had it back in 3 ms, and 5 s later the watchdog restarted + /// the pipeline under the viewer ("no captured frame for 5039ms with a + /// client connected"). A re-created duplication delivers no image while + /// nothing on the screen changes, and keepalives waited for one. + #[test] + fn a_static_second_screen_keeps_its_picture_after_a_lost_duplication() { + use std::sync::atomic::Ordering::Relaxed; + let shared = SharedControl::new(0, 1); + connect_viewer(&shared); + let mut machine = crate::supervisor::Machine::new(1); + let mut picture = Picture::default(); + // Before its first image the second screen sends nothing. + assert!(!keepalive_allowed(picture.real(), true)); + picture.seen_image = true; + picture.cached(); + // Lost, and re-created at the same size. + let lost = Instant::now(); + picture.recreated(lost); + assert!(!frame_has_image(0, picture.seen_image)); + let start = 100_000; + let mut sent = start; + shared.hb_capture_frame_ms.store(start, Relaxed); + shared.hb_encode_frame_ms.store(start, Relaxed); + let mut resends = 0; + // Ten seconds of acquire timeouts on the unchanged screen. + for now in (start..start + 10_000).step_by(50) { + shared.hb_capture_loop_ms.store(now, Relaxed); + let since_send = Duration::from_millis(now - sent); + if keepalive_due(since_send, false, picture.real()) + && keepalive_allowed(picture.real(), true) + { + sent = now; + resends += 1; + shared.hb_capture_frame_ms.store(now, Relaxed); + shared.hb_encode_frame_ms.store(now, Relaxed); + } + assert_eq!( + crate::supervisor::wedge_reason_at(&shared, &mut machine, now), + None + ); + let elapsed = Duration::from_millis(now - start); + assert!(!picture.held_too_long(lost + elapsed, true)); } + assert!(resends >= 12, "{resends} keepalives in 10 s"); + // A new image ends the hold. + picture.cached(); + assert!(!picture.held_too_long(lost + PICTURE_HOLD, true)); + // A new resolution leaves only black to resend: the second screen + // waits for the new duplication's first image again. + picture.recreated(lost); + picture.resized(); + assert!(!keepalive_allowed(picture.real(), true)); + assert!(keepalive_allowed(picture.real(), false), "the mirror"); + picture.seen_image = true; + picture.cached(); + assert!(keepalive_allowed(picture.real(), true)); + } + + /// Review r1 of #205: resending the cached picture hid a virtual display + /// that sends no frames after its duplication was re-created, and the + /// picture would stay frozen for good. After [`PICTURE_HOLD`] the stage + /// opens it again without the cached picture, and the watchdog restarts + /// the pipeline 5 s later if it is still dark, as before. The mirror + /// keeps resending: it is never dark. + #[test] + fn a_second_screen_dark_after_a_lost_duplication_is_reopened_then_judged() { + use std::sync::atomic::Ordering::Relaxed; + let shared = SharedControl::new(0, 1); + connect_viewer(&shared); + let mut machine = crate::supervisor::Machine::new(1); + let mut picture = Picture { + seen_image: true, + ..Picture::default() + }; + picture.cached(); + let lost = Instant::now(); + picture.recreated(lost); + // Lost again before any new image: the hold runs from the first. + picture.recreated(lost + Duration::from_secs(5)); + let start = 100_000; + let mut sent = start; + shared.hb_capture_frame_ms.store(start, Relaxed); + shared.hb_encode_frame_ms.store(start, Relaxed); + let mut reopened_at = None; + let mut wedge = None; + for now in (start..start + 30_000).step_by(50) { + shared.hb_capture_loop_ms.store(now, Relaxed); + let at = lost + Duration::from_millis(now - start); + assert!(!picture.held_too_long(at, false), "the mirror"); + if reopened_at.is_none() && picture.held_too_long(at, true) { + // The stage opens it again: a fresh loop, nothing cached. + reopened_at = Some(now); + picture = Picture::default(); + } + let since_send = Duration::from_millis(now - sent); + if keepalive_due(since_send, false, picture.real()) + && keepalive_allowed(picture.real(), true) + { + sent = now; + shared.hb_capture_frame_ms.store(now, Relaxed); + shared.hb_encode_frame_ms.store(now, Relaxed); + } + if let Some(reason) = crate::supervisor::wedge_reason_at(&shared, &mut machine, now) { + wedge = Some((now, reason)); + break; + } + } + let reopened_at = reopened_at.expect("reopened"); + assert_eq!(reopened_at - start, PICTURE_HOLD.as_millis() as u64); + let (at, reason) = wedge.expect("the watchdog judged the dark screen"); + assert!(reason.starts_with("no captured frame"), "{reason}"); + assert!( + at - reopened_at <= 5_500, + "{} ms after reopening", + at - reopened_at + ); + } + + /// The screens came back on 2 s into a 5 s retry wait. The picture comes + /// back at the next look, not when the wait ends. + #[test] + fn a_display_coming_back_ends_the_no_display_wait() { + use std::sync::atomic::Ordering::SeqCst; + let shared = SharedControl::new(0, 1); + shared.running.store(true, SeqCst); + let generation = crate::supervisor::CURRENT_GENERATION.load(SeqCst); + *shared.capture_block.lock() = Some(recovery::CaptureBlock::NoDisplay); + // A paused clock: the wait's sleeps move it. + let start = Instant::now(); + let elapsed = Cell::new(Duration::ZERO); + let now = || start + elapsed.get(); + let sleep = |by: Duration| elapsed.set(elapsed.get() + by); + let looks = Cell::new(0); + let back = || { + looks.set(looks.get() + 1); + elapsed.get() >= Duration::from_millis(600) + }; + assert!(wait_while_blocked_on( + &shared, + generation, + Duration::from_secs(5), + back, + now, + sleep + )); + // Looks at 250, 500 and 750 ms: back at the first look after 600. + assert_eq!(elapsed.get(), Duration::from_millis(750)); + assert_eq!(looks.get(), 3); + // Other blocked states clear by the user's hand; nothing is polled. + *shared.capture_block.lock() = Some(recovery::CaptureBlock::SecureDesktop); + elapsed.set(Duration::ZERO); + looks.set(0); + assert!(wait_while_blocked_on( + &shared, + generation, + Duration::from_millis(600), + || { + looks.set(looks.get() + 1); + true + }, + now, + sleep + )); + assert_eq!(elapsed.get(), Duration::from_millis(600)); + assert_eq!(looks.get(), 0); } #[test] diff --git a/host/src/capture/pointer.rs b/host/src/capture/pointer.rs new file mode 100644 index 00000000..25747084 --- /dev/null +++ b/host/src/capture/pointer.rs @@ -0,0 +1,425 @@ +//! The pointer, sent apart from the video to an iPad that draws it itself +//! (`FEATURE_DRAWS_POINTER`). Capture converts each new Windows pointer +//! image once ([`convert_shape`]) and records every position Windows +//! reports in the [`PointerFeed`]; the transport sends them at most once a +//! frame budget (`transport::pointer`). For any other client capture keeps +//! drawing the pointer into the video, as before. +//! +//! Positions stay in the captured output's pixels, where the video has +//! always drawn the pointer: Desktop Duplication reports them relative to +//! that output, so another monitor's offset never enters, and the iPad +//! scales them with the picture. + +use std::sync::Arc; + +use eternal_wire::v2::control::{pointer_shape_len, POINTER_MAX_SIDE, POINTER_SHAPE_INVERT}; +use parking_lot::Mutex; + +// DXGI_OUTDUPL_POINTER_SHAPE_TYPE_*. +pub const SHAPE_MONOCHROME: u32 = 1; +pub const SHAPE_COLOR: u32 = 2; +pub const SHAPE_MASKED_COLOR: u32 = 4; + +/// A pointer image in the wire's form: premultiplied BGRA, then an invert +/// mask when any pixel inverts the picture under it (see +/// `eternal_wire::v2::control::pointer_shape_len`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PointerImage { + pub source_type: u8, + pub flags: u8, + pub width: u16, + pub height: u16, + pub hot: (u16, u16), + pub bytes: Vec, +} + +/// Converts a shape as `GetFramePointerShape` returns it. `height` is the +/// buffer's: a monochrome shape stacks its AND mask over its XOR mask. +/// +/// What the iPad can draw of each kind: +/// - Color: alpha-blended, as the video drew it. +/// - Monochrome: exact. AND 0 is opaque black (XOR 0) or white (XOR 1); +/// AND 1 is clear (XOR 0) or inverts the picture (XOR 1), as the I-beam +/// over text does. +/// - Masked color: mask 0 pixels are opaque. Mask 1 pixels XOR the picture +/// with their color; black changes nothing, so it is clear, and every +/// other color inverts the picture. That is exact for white, the color +/// Windows uses, and a close match otherwise. +/// +/// A side past `POINTER_MAX_SIDE` keeps its top-left part. `None` for an +/// unknown kind or an empty image. +pub fn convert_shape( + source_type: u32, + width: u32, + height: u32, + pitch: u32, + hot: (u32, u32), + raw: &[u8], +) -> Option { + let rows = match source_type { + SHAPE_MONOCHROME => height / 2, + SHAPE_COLOR | SHAPE_MASKED_COLOR => height, + _ => return None, + }; + let max = u32::from(POINTER_MAX_SIDE); + let (w, h) = (width.min(max), rows.min(max)); + if w == 0 || h == 0 { + return None; + } + let (wu, hu, pitch) = (w as usize, h as usize, pitch as usize); + let mut image = vec![0u8; wu * hu * 4]; + let mut mask = vec![0u8; wu.div_ceil(8) * hu]; + let mut inverts = false; + let byte = |index: usize| raw.get(index).copied(); + for y in 0..hu { + for x in 0..wu { + let invert = match source_type { + SHAPE_MONOCHROME => { + let (at, bit) = (x / 8, 7 - (x % 8) as u8); + let (Some(and), Some(xor)) = + (byte(y * pitch + at), byte((y + rows as usize) * pitch + at)) + else { + continue; + }; + match ((and >> bit) & 1, (xor >> bit) & 1) { + (0, xor) => { + let v = if xor == 1 { 255 } else { 0 }; + image[(y * wu + x) * 4..][..4].copy_from_slice(&[v, v, v, 255]); + false + } + (_, xor) => xor == 1, + } + } + _ => { + let at = y * pitch + x * 4; + let Some(px) = raw.get(at..at + 4) else { + continue; + }; + let (b, g, r, a) = (px[0], px[1], px[2], px[3]); + let out = &mut image[(y * wu + x) * 4..][..4]; + if source_type == SHAPE_COLOR { + let premultiply = |c: u8| ((u32::from(c) * u32::from(a) + 127) / 255) as u8; + out.copy_from_slice(&[premultiply(b), premultiply(g), premultiply(r), a]); + false + } else if a == 0 { + out.copy_from_slice(&[b, g, r, 255]); + false + } else { + (b, g, r) != (0, 0, 0) + } + } + }; + if invert { + mask[y * wu.div_ceil(8) + x / 8] |= 0x80 >> (x % 8); + inverts = true; + } + } + } + let flags = if inverts { POINTER_SHAPE_INVERT } else { 0 }; + if inverts { + image.extend_from_slice(&mask); + } + let (width, height) = (w as u16, h as u16); + debug_assert_eq!(image.len(), pointer_shape_len(width, height, flags)); + Some(PointerImage { + source_type: source_type as u8, + flags, + width, + height, + hot: ( + hot.0.min(u32::from(u16::MAX)) as u16, + hot.1.min(u32::from(u16::MAX)) as u16, + ), + bytes: image, + }) +} + +/// The current pointer image with the id the wire names it by. +#[derive(Debug, PartialEq, Eq)] +pub struct Shape { + pub id: u32, + pub image: PointerImage, +} + +/// Where Windows last put the pointer, in the captured output's pixels. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Position { + pub visible: bool, + /// The image's top-left corner. + pub x: i32, + pub y: i32, + pub host_time_us: u64, +} + +/// The latest pointer image and position, written by capture and read by +/// the transport, which [`PointerFeed::changed`] wakes. +#[derive(Default)] +pub struct PointerFeed { + state: Mutex, + changed: tokio::sync::Notify, +} + +#[derive(Default)] +struct FeedState { + shape: Option>, + position: Option, + last_id: u32, + withheld: bool, +} + +impl PointerFeed { + /// A new image gets a new id; the same image again (as a re-created + /// duplication reports it) keeps its id, so it is not sent again. + pub fn set_shape(&self, image: PointerImage) { + let mut state = self.state.lock(); + if state + .shape + .as_ref() + .is_some_and(|shape| shape.image == image) + { + return; + } + state.last_id = state.last_id.wrapping_add(1).max(1); + state.shape = Some(Arc::new(Shape { + id: state.last_id, + image, + })); + drop(state); + self.changed.notify_one(); + } + + pub fn set_position(&self, position: Position) { + self.state.lock().position = Some(position); + self.changed.notify_one(); + } + + /// Capture can't tell where Windows puts the pointer + /// (`capture::PointerPlace::Unknown`): it stays in the video, and no + /// pointer messages go to the iPad but a hidden state to clear one it + /// was drawing. + pub fn set_withheld(&self, withheld: bool) { + self.state.lock().withheld = withheld; + self.changed.notify_one(); + } + + pub fn withheld(&self) -> bool { + self.state.lock().withheld + } + + pub fn latest(&self) -> (Option>, Option) { + let state = self.state.lock(); + (state.shape.clone(), state.position) + } + + /// Resolves after the next [`Self::set_shape`] or [`Self::set_position`], + /// or at once when one came since the last wait. + pub async fn changed(&self) { + self.changed.notified().await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// One monochrome row per string, AND mask over XOR mask, 1 byte pitch. + fn monochrome(and: &[u8], xor: &[u8]) -> Vec { + [and, xor].concat() + } + + fn pixel(image: &PointerImage, x: usize, y: usize) -> [u8; 4] { + let at = (y * usize::from(image.width) + x) * 4; + image.bytes[at..at + 4].try_into().unwrap() + } + + fn inverts(image: &PointerImage, x: usize, y: usize) -> bool { + if image.flags & POINTER_SHAPE_INVERT == 0 { + return false; + } + let (w, h) = (usize::from(image.width), usize::from(image.height)); + let mask = &image.bytes[w * h * 4..]; + mask[y * w.div_ceil(8) + x / 8] & (0x80 >> (x % 8)) != 0 + } + + #[test] + fn monochrome_draws_black_white_clear_and_inverts_with_its_xor_mask() { + // Four pixels on one row: (AND, XOR) = (0,0) (0,1) (1,0) (1,1). + let raw = monochrome(&[0b0011_0000], &[0b0101_0000]); + let image = convert_shape(SHAPE_MONOCHROME, 4, 2, 1, (1, 0), &raw).unwrap(); + assert_eq!((image.width, image.height, image.hot), (4, 1, (1, 0))); + assert_eq!(image.source_type, 1); + assert_eq!(pixel(&image, 0, 0), [0, 0, 0, 255], "opaque black"); + assert_eq!(pixel(&image, 1, 0), [255, 255, 255, 255], "opaque white"); + assert_eq!(pixel(&image, 2, 0), [0, 0, 0, 0], "clear"); + assert_eq!( + pixel(&image, 3, 0), + [0, 0, 0, 0], + "inverting pixels draw nothing" + ); + assert_eq!( + (0..4).map(|x| inverts(&image, x, 0)).collect::>(), + [false, false, false, true], + ); + assert_eq!( + image.bytes.len(), + pointer_shape_len(4, 1, POINTER_SHAPE_INVERT) + ); + } + + #[test] + fn monochrome_without_inverting_pixels_sends_no_mask() { + let raw = monochrome(&[0b0100_0000], &[0b1000_0000]); + let image = convert_shape(SHAPE_MONOCHROME, 2, 2, 1, (0, 0), &raw).unwrap(); + assert_eq!(image.flags, 0); + assert_eq!(image.bytes, [255, 255, 255, 255, 0, 0, 0, 0]); + } + + #[test] + fn color_is_premultiplied_and_keeps_its_alpha() { + // BGRA rows with 4 bytes of pitch padding. + let raw = [ + 200, 100, 50, 255, 200, 100, 50, 128, 0, 0, 0, 0, // + 10, 20, 30, 0, 255, 255, 255, 64, 0, 0, 0, 0, + ]; + let image = convert_shape(SHAPE_COLOR, 2, 2, 12, (0, 0), &raw).unwrap(); + assert_eq!(image.flags, 0); + assert_eq!(pixel(&image, 0, 0), [200, 100, 50, 255]); + assert_eq!(pixel(&image, 1, 0), [100, 50, 25, 128]); + assert_eq!( + pixel(&image, 0, 1), + [0, 0, 0, 0], + "fully clear has no color" + ); + assert_eq!(pixel(&image, 1, 1), [64, 64, 64, 64]); + } + + #[test] + fn masked_color_is_opaque_clear_or_inverting() { + let raw = [ + 10, 20, 30, 0, // mask 0: opaque color + 0, 0, 0, 0xFF, // XOR with black: no change + 255, 255, 255, 0xFF, // XOR with white: invert + 0, 0, 200, 0xFF, // XOR with red: drawn as invert + ]; + let image = convert_shape(SHAPE_MASKED_COLOR, 4, 1, 16, (0, 0), &raw).unwrap(); + assert_eq!(image.flags, POINTER_SHAPE_INVERT); + assert_eq!(pixel(&image, 0, 0), [10, 20, 30, 255]); + for x in 1..4 { + assert_eq!(pixel(&image, x, 0), [0, 0, 0, 0]); + } + assert_eq!( + (0..4).map(|x| inverts(&image, x, 0)).collect::>(), + [false, false, true, true], + ); + } + + #[test] + fn short_buffers_unknown_kinds_and_huge_shapes_are_safe() { + // Rows the buffer doesn't hold stay clear. + let image = convert_shape(SHAPE_COLOR, 2, 2, 8, (0, 0), &[1, 2, 3, 255]).unwrap(); + assert_eq!(pixel(&image, 0, 0), [1, 2, 3, 255]); + assert_eq!(pixel(&image, 1, 1), [0, 0, 0, 0]); + assert_eq!(convert_shape(3, 2, 2, 8, (0, 0), &[0; 16]), None); + assert_eq!(convert_shape(SHAPE_MONOCHROME, 8, 1, 1, (0, 0), &[0]), None); + let side = u32::from(POINTER_MAX_SIDE) + 8; + let raw = vec![0xFF; (side * 4 * side) as usize]; + let image = convert_shape(SHAPE_COLOR, side, side, side * 4, (0, 0), &raw).unwrap(); + assert_eq!( + (image.width, image.height), + (POINTER_MAX_SIDE, POINTER_MAX_SIDE) + ); + } + + /// Desktop Duplication reports the pointer relative to the captured + /// output, which is where it is drawn in the video. A tap on the drawn + /// hotspot, normalized as the iPad does, lands on the desktop pixel + /// Windows has the hotspot at, whatever the output's offset: a screen + /// left of the main one, a second screen right of it, the main one. + #[test] + fn a_tap_on_the_drawn_hotspot_lands_where_windows_has_it() { + use crate::input::{norm_to_output_pixel, CaptureGeometry}; + let outputs = [ + CaptureGeometry { + left: -1920, + top: 120, + width: 1920, + height: 1080, + }, + CaptureGeometry { + left: 3440, + top: 0, + width: 2388, + height: 1668, + }, + CaptureGeometry { + left: 0, + top: 0, + width: 3440, + height: 1440, + }, + ]; + let hot = (6, 9); + for output in outputs { + for (x, y) in [ + (0, 0), + (1003, 517), + (output.width as i32 - 7, output.height as i32 - 10), + ] { + // Windows: the hotspot at this desktop pixel. + let desktop = (output.left + x + hot.0, output.top + y + hot.1); + // Duplication: the image's corner, relative to the output. + let drawn = Position { + visible: true, + x, + y, + host_time_us: 0, + }; + // The iPad: the centre of the hotspot's video pixel. + let (w, h) = (output.width as f64, output.height as f64); + let norm = |p: i32, n: f64| (((f64::from(p) + 0.5) / n) * 65535.0).round() as u16; + let tapped = norm_to_output_pixel( + norm(drawn.x + hot.0, w), + norm(drawn.y + hot.1, h), + output, + ); + // Within the input mapping's own rounding: it spreads the + // video's pixels over one step fewer. + assert!( + (tapped.0 - desktop.0).abs() <= 1 && (tapped.1 - desktop.1).abs() <= 1, + "{output:?} at {x},{y}: tapped {tapped:?}, Windows {desktop:?}" + ); + } + } + } + + #[test] + fn the_feed_names_a_new_image_once_and_keeps_the_latest_position() { + let feed = PointerFeed::default(); + let image = |v: u8| PointerImage { + source_type: 2, + flags: 0, + width: 1, + height: 1, + hot: (0, 0), + bytes: vec![v; 4], + }; + feed.set_shape(image(1)); + feed.set_shape(image(1)); + assert_eq!( + feed.latest().0.unwrap().id, + 1, + "the same image keeps its id" + ); + feed.set_shape(image(2)); + assert_eq!(feed.latest().0.unwrap().id, 2); + for x in 0..3 { + feed.set_position(Position { + visible: true, + x, + y: 0, + host_time_us: 0, + }); + } + assert_eq!(feed.latest().1.unwrap().x, 2); + } +} diff --git a/host/src/capture/recovery.rs b/host/src/capture/recovery.rs index 56beb0a8..aaeeabf2 100644 --- a/host/src/capture/recovery.rs +++ b/host/src/capture/recovery.rs @@ -38,6 +38,20 @@ pub enum CaptureBlock { } impl CaptureBlock { + /// How an iPad that asks after the storm brake is told this block. + pub fn failed_reason(self) -> eternal_wire::v2::control::FailedReason { + use eternal_wire::v2::control::FailedReason; + match self { + Self::SecureDesktop => FailedReason::SecureDesktop, + Self::DisplayChanging => FailedReason::DisplayChanging, + Self::NoDisplay => FailedReason::NoDisplay, + Self::UnsupportedOnThisGpu => FailedReason::UnsupportedOnThisGpu, + Self::TooManyCaptureApps => FailedReason::TooManyCaptureApps, + Self::SessionDisconnected => FailedReason::SessionDisconnected, + Self::DriverNotResponding => FailedReason::DriverNotResponding, + } + } + /// Maps a DXGI/Win32 HRESULT from DuplicateOutput or AcquireNextFrame. /// `None` means a real failure that the supervisor should handle. pub fn from_hresult(code: i32) -> Option { @@ -112,6 +126,9 @@ pub fn passing_error(error: &(dyn std::error::Error + Send + Sync + 'static)) -> if let Some(unavailable) = error.downcast_ref::() { return unavailable.block != CaptureBlock::UnsupportedOnThisGpu; } + if error.downcast_ref::().is_some() { + return true; + } if error.downcast_ref::().is_some() { return true; } @@ -190,6 +207,27 @@ impl std::fmt::Display for CaptureUnavailable { impl std::error::Error for CaptureUnavailable {} +/// The capture loop ended to open its display afresh: after a lost +/// duplication the target is now another output or adapter, or a re-created +/// duplication showed no new image for too long. The stage opens it again at +/// once, not at the blocked pace. +#[derive(Debug)] +pub struct Reopen { + pub detail: String, +} + +impl std::fmt::Display for Reopen { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "capture reopens: {}", self.detail) + } +} + +impl std::error::Error for Reopen {} + +/// A [`Reopen`] within this long of the last one waits at the blocked pace: +/// a layout that keeps changing must not reopen the display in a loop. +const REOPEN_SPACING: Duration = Duration::from_secs(1); + /// Delays between duplication re-creations after ACCESS_LOST. The first retry /// is immediate, as before; a duplication that is lost again right after it /// was re-created waits longer each time instead of spinning. @@ -209,7 +247,17 @@ pub enum LossStep { /// has worked this long. A UAC prompt, the lock screen or a full-screen game /// can take the desktop again right after it returns; starting the ladder /// over at each return re-created the duplication at full speed in a loop. -pub const LOSS_LADDER_RESET: Duration = Duration::from_secs(5); +/// It was 5 s, and on a PC whose desktop was taken again every 1 to 4 s the +/// ladder never came down from 2 s: each loss cost 2 s without a picture, +/// and the iPad got 3 to 17 fps for an hour. A duplication that worked for +/// a quarter of a second is re-created at once, so a steady stream of quick +/// losses costs tens of milliseconds each, and re-creations still stay +/// under four a second. +pub const LOSS_LADDER_RESET: Duration = Duration::from_millis(250); + +/// A loss that is back within this long is routine (the lock screen, a UAC +/// prompt, a mode change) and is logged at info; a longer one is a warning. +pub const LOSS_QUIET: Duration = Duration::from_secs(1); /// One loss episode lasts from the first ACCESS_LOST until a frame or an idle /// timeout proves the new duplication works. @@ -223,6 +271,8 @@ pub struct LossRecovery { step: usize, /// When the last episode ended, while `step` is kept. back_at: Option, + /// This episode outlasted [`LOSS_QUIET`] and said so. + overdue: bool, } impl LossRecovery { @@ -230,6 +280,18 @@ impl LossRecovery { self.since.is_some() } + /// True once per episode, when it has lasted [`LOSS_QUIET`]. + pub fn overdue(&mut self, now: Instant) -> bool { + let Some(since) = self.since else { + return false; + }; + if self.overdue || now.duration_since(since) < LOSS_QUIET { + return false; + } + self.overdue = true; + true + } + pub fn lost(&mut self, now: Instant) -> LossStep { if let Some(back_at) = self.back_at.take() { if now.duration_since(back_at) >= LOSS_LADDER_RESET { @@ -262,6 +324,7 @@ impl LossRecovery { return None; }; self.back_at = Some(now); + self.overdue = false; let losses = std::mem::take(&mut self.losses); Some((losses, now.duration_since(since))) } @@ -292,6 +355,14 @@ impl BlockedRetry { self.attempts += 1; (Duration::from_millis(BLOCKED_RETRY_MS[index]), changed) } + + /// Capture worked since the last blocked attempt. The next block is + /// retried at the quick pace again but is not logged again when it is + /// the same state: the pace climbed over a whole app run, and a PC + /// locked for the fourth time waited up to 5 s after it was unlocked. + pub fn resumed(&mut self) { + self.attempts = 0; + } } /// Runs `capture` until it ends for a reason other than a blocked desktop. @@ -305,18 +376,45 @@ impl BlockedRetry { /// stream epoch: an attempt that started again at 1 sent sequence 1 after /// 200, and the iPad dropped every frame until the count passed 200 again. pub fn run_until_unblocked( + shared: &SharedControl, + capture: impl FnMut(&mut u64) -> Result<(), Box>, + wait: impl FnMut(Duration) -> bool, +) -> Result<(), Box> { + run_until_unblocked_on(shared, capture, wait, Instant::now) +} + +/// [`run_until_unblocked`] on the clock `clock`, which times the +/// [`REOPEN_SPACING`]. +pub(crate) fn run_until_unblocked_on( shared: &SharedControl, mut capture: impl FnMut(&mut u64) -> Result<(), Box>, mut wait: impl FnMut(Duration) -> bool, + mut clock: impl FnMut() -> Instant, ) -> Result<(), Box> { let mut retry = BlockedRetry::default(); let mut stalled = 0; let mut frame_number = 0; + let mut last_reopen: Option = None; loop { let error = match capture(&mut frame_number) { Ok(()) => return Ok(()), Err(error) => error, }; + let error: Box = match error.downcast::() { + Ok(reopen) => { + let now = clock(); + if last_reopen.is_none_or(|at| now.duration_since(at) >= REOPEN_SPACING) { + last_reopen = Some(now); + info!(detail = %reopen.detail, "Opening the display to capture again"); + continue; + } + Box::new(CaptureUnavailable { + block: CaptureBlock::DisplayChanging, + detail: reopen.detail, + }) + } + Err(error) => error, + }; // A driver call that did not return in time, or was refused while an // earlier one is stuck. Restarting the pipeline cannot free the // driver, and refusals would spend the restart budget until the @@ -341,10 +439,16 @@ pub fn run_until_unblocked( continue; } stalled = 0; + #[cfg(windows)] + let error = access_lost_is_blocked(error); let Some(unavailable) = error.downcast_ref::() else { return Err(error); }; - *shared.capture_block.lock() = Some(unavailable.block); + // The first working frame clears the block (see + // [`note_capture_resumed`]). + if note_capture_blocked(shared, unavailable.block).is_none() { + retry.resumed(); + } let (delay, changed) = retry.blocked(unavailable.block); if changed { warn!( @@ -359,9 +463,86 @@ pub fn run_until_unblocked( } } +/// DXGI_ERROR_ACCESS_LOST that left the capture loop instead of reaching its +/// handler: a display change the stage waits out, never a pipeline failure. +/// The loop hands ACCESS_LOST from every call it knows to its handler; this +/// is for any call it does not. +#[cfg(windows)] +fn access_lost_is_blocked( + error: Box, +) -> Box { + match error.downcast_ref::() { + Some(lost) if lost.code().0 as u32 == 0x887A_0026 => Box::new(CaptureUnavailable { + block: CaptureBlock::DisplayChanging, + detail: format!("desktop duplication lost outside its handler: {lost}"), + }), + _ => error, + } +} + +/// How long Windows has to refuse the screen from this graphics card before +/// the iPad is told. The same refusal comes and goes for a moment while the +/// desktop changes mode, and the iPad's first-picture wait is 10 s. +pub const GPU_NOTICE_AFTER: Duration = Duration::from_secs(2); + +/// The heartbeat's notice while capture is blocked. A secure screen says the +/// PC is locked, and no display attached says so, so the iPad doesn't show a +/// frozen picture without a word; it clears with the block when capture +/// works again. A screen Windows won't share from this graphics card says so +/// once the refusal has lasted [`GPU_NOTICE_AFTER`] (`gpu_since` keeps when +/// it began: [`note_capture_blocked`] sets it, and a heartbeat that finds it +/// unset starts it), naming the failed second screen when +/// `second_screen_failed`. +/// The other blocks keep the held picture as before. +pub fn heartbeat_notice( + block: Option, + second_screen_failed: bool, + gpu_since: &mut Option, + now: Instant, +) -> u8 { + use eternal_wire::v2::control::{ + NOTICE_NONE, NOTICE_NO_DISPLAY, NOTICE_PC_LOCKED, NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, + NOTICE_UNSUPPORTED_GPU, + }; + if block != Some(CaptureBlock::UnsupportedOnThisGpu) { + *gpu_since = None; + } + match block { + Some(CaptureBlock::SecureDesktop) => NOTICE_PC_LOCKED, + Some(CaptureBlock::NoDisplay) => NOTICE_NO_DISPLAY, + Some(CaptureBlock::UnsupportedOnThisGpu) + if now.saturating_duration_since(*gpu_since.get_or_insert(now)) >= GPU_NOTICE_AFTER => + { + if second_screen_failed { + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU + } else { + NOTICE_UNSUPPORTED_GPU + } + } + _ => NOTICE_NONE, + } +} + +/// Records that Windows blocks capture with `block`, returning the block +/// before. A refusal from this graphics card that begins now, after capture +/// worked or another block, starts its own wait for the iPad's notice, so a +/// refusal that came back between two heartbeats never inherits an earlier +/// one's. +pub fn note_capture_blocked(shared: &SharedControl, block: CaptureBlock) -> Option { + let mut current = shared.capture_block.lock(); + let before = current.replace(block); + if before != Some(block) { + *shared.gpu_refused_since.lock() = + (block == CaptureBlock::UnsupportedOnThisGpu).then(Instant::now); + } + before +} + /// Clears the blocked state once frames flow again. pub fn note_capture_resumed(shared: &SharedControl) { - if let Some(block) = shared.capture_block.lock().take() { + let resumed = shared.capture_block.lock().take(); + *shared.gpu_refused_since.lock() = None; + if let Some(block) = resumed { info!(block = ?block, "Screen capture resumed"); } } @@ -479,6 +660,146 @@ mod tests { assert!(shared.gpu_switch.lock().as_ref().unwrap().failed); } + /// The notice for `block`, read once with no refusal before. + fn notice_now(block: Option) -> u8 { + heartbeat_notice(block, false, &mut None, Instant::now()) + } + + #[test] + fn only_a_secure_screen_no_display_or_the_wrong_card_has_a_notice_for_the_ipad() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_NO_DISPLAY, NOTICE_PC_LOCKED}; + assert_eq!( + notice_now(Some(CaptureBlock::SecureDesktop)), + NOTICE_PC_LOCKED + ); + // A closed lid or screens that are off: the field's PCs showed a + // frozen picture without a word until a display came back. + assert_eq!(notice_now(Some(CaptureBlock::NoDisplay)), NOTICE_NO_DISPLAY); + assert_eq!(notice_now(None), NOTICE_NONE); + for block in [ + CaptureBlock::DisplayChanging, + CaptureBlock::TooManyCaptureApps, + CaptureBlock::SessionDisconnected, + CaptureBlock::DriverNotResponding, + ] { + assert_eq!(notice_now(Some(block)), NOTICE_NONE, "{block:?}"); + } + } + + /// The hybrid laptops whose iPad saw 10 s of black, then "no picture": + /// Windows refused the screen from the card EternalMonitor runs on, and + /// the PC's window said so, but the iPad got no word. Now it does, once + /// the refusal has lasted 2 s (it comes and goes for a moment while the + /// desktop changes mode), naming the second screen when it failed too. + /// It clears with the refusal, and a new one waits again. + /// Review r1: capture worked for a moment and Windows refused the card + /// again, all between two heartbeats. The heartbeat never saw the gap, + /// so the new refusal took the old one's start and the iPad was told at + /// once. Now the refusal's start is kept where the block is set and + /// cleared: a new refusal waits its own 2 s, after capture worked or + /// after another block, while the same refusal going on keeps its start. + #[test] + fn a_refusal_that_comes_back_between_heartbeats_waits_its_own_2_s() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_UNSUPPORTED_GPU}; + let shared = SharedControl::new(9876, 15_000_000); + let notice = |shared: &SharedControl| { + let block = *shared.capture_block.lock(); + heartbeat_notice( + block, + false, + &mut shared.gpu_refused_since.lock(), + Instant::now(), + ) + }; + let long_ago = || Some(Instant::now() - Duration::from_secs(3)); + note_capture_blocked(&shared, CaptureBlock::UnsupportedOnThisGpu); + assert_eq!(notice(&shared), NOTICE_NONE, "a refusal that just began"); + // The same refusal, retried 3 s on: its start stays. + *shared.gpu_refused_since.lock() = long_ago(); + note_capture_blocked(&shared, CaptureBlock::UnsupportedOnThisGpu); + assert_eq!(notice(&shared), NOTICE_UNSUPPORTED_GPU); + // Capture works, then is refused again, with no heartbeat between. + note_capture_resumed(&shared); + note_capture_blocked(&shared, CaptureBlock::UnsupportedOnThisGpu); + assert_eq!(notice(&shared), NOTICE_NONE, "a fresh refusal waits again"); + // Another block in between restarts it too. + *shared.gpu_refused_since.lock() = long_ago(); + note_capture_blocked(&shared, CaptureBlock::DisplayChanging); + note_capture_blocked(&shared, CaptureBlock::UnsupportedOnThisGpu); + assert_eq!(notice(&shared), NOTICE_NONE); + // Through the capture stage's own retries: worked, then refused. + *shared.gpu_refused_since.lock() = long_ago(); + note_capture_resumed(&shared); + let mut attempts = 0; + let result = run_until_unblocked( + &shared, + |_| { + attempts += 1; + if attempts == 1 { + Err(blocked(CaptureBlock::UnsupportedOnThisGpu)) + } else { + Ok(()) + } + }, + |_| { + assert_eq!(notice(&shared), NOTICE_NONE); + true + }, + ); + assert!(result.is_ok()); + } + + #[test] + fn a_screen_refused_from_this_graphics_card_is_told_to_the_ipad_after_2_s() { + use eternal_wire::v2::control::{ + NOTICE_NONE, NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, NOTICE_UNSUPPORTED_GPU, + }; + let refused = Some(CaptureBlock::UnsupportedOnThisGpu); + let start = Instant::now(); + let at = |s: f64| start + Duration::from_secs_f64(s); + let mut since = None; + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(0.0)), + NOTICE_NONE + ); + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(1.9)), + NOTICE_NONE + ); + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(2.0)), + NOTICE_UNSUPPORTED_GPU + ); + // Second screen mode, after the second screen failed to start. + assert_eq!( + heartbeat_notice(refused, true, &mut since, at(3.0)), + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU + ); + // Capture works: no notice, and the next refusal waits again. + assert_eq!( + heartbeat_notice(None, false, &mut since, at(4.0)), + NOTICE_NONE + ); + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(5.0)), + NOTICE_NONE + ); + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(7.0)), + NOTICE_UNSUPPORTED_GPU + ); + // A display change in between restarts the wait too. + let changing = Some(CaptureBlock::DisplayChanging); + assert_eq!( + heartbeat_notice(changing, false, &mut since, at(8.0)), + NOTICE_NONE + ); + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(9.0)), + NOTICE_NONE + ); + } + #[test] fn a_stuck_driver_call_is_waited_out_in_the_stage_not_restarted() { use crate::stall::{no_hooks, run_in, Call, Lanes, Stall, Wait}; @@ -603,7 +924,7 @@ mod tests { } #[test] - fn a_working_frame_ends_the_episode_and_five_healthy_seconds_reset_the_ladder() { + fn a_working_frame_ends_the_episode_and_a_healthy_quarter_second_resets_the_ladder() { let ms = Duration::from_millis; let mut recovery = LossRecovery::default(); let start = Instant::now(); @@ -613,7 +934,8 @@ mod tests { let ended = recovery.healthy(start + ms(300)); assert_eq!(ended, Some((2, ms(300)))); assert!(!recovery.in_episode()); - // Frames keep coming for 5 s: the next loss is retried at once. + // Frames keep coming for a quarter of a second: the next loss is + // retried at once. let mut now = start + ms(300); while now < start + ms(300) + LOSS_LADDER_RESET { now += ms(16); @@ -629,8 +951,8 @@ mod tests { /// A UAC prompt or a full-screen game takes the desktop again right after /// each return. The first retry is as quick as before; the returns that - /// follow within 5 s keep climbing the ladder instead of starting over - /// at 0 ms each time. + /// follow within a quarter of a second keep climbing the ladder instead of + /// starting over at 0 ms each time. #[test] fn a_duplication_lost_again_soon_after_it_returned_keeps_its_backoff() { let ms = Duration::from_millis; @@ -638,30 +960,33 @@ mod tests { let start = Instant::now(); let mut now = start; let mut delays = Vec::new(); - for _ in 0..7 { + for _ in 0..10 { let LossStep::Retry(delay) = recovery.lost(now) else { panic!("gave up") }; delays.push(delay.as_millis() as u64); - // The new duplication works for a second, then is lost again. + // The new duplication works for 200 ms, then is lost again. now += delay + ms(1); assert!( recovery.healthy(now).is_some(), "each loss is its own episode" ); - now += ms(1_000); + now += ms(200); assert_eq!(recovery.healthy(now), None); } - assert_eq!(delays, [0, 100, 250, 500, 1000, 2000, 2000]); - // The last return was a second ago. Four healthy seconds are not - // enough; five are. - let back = now - ms(1_000); - assert_eq!(recovery.healthy(back + ms(4_000)), None); - assert_eq!(recovery.lost(back + ms(4_000)), LossStep::Retry(ms(2000))); - recovery.healthy(back + ms(6_000)); - assert_eq!(recovery.healthy(back + ms(11_000)), None); assert_eq!( - recovery.lost(back + ms(11_000)), + delays, + [0, 100, 250, 500, 1000, 2000, 2000, 2000, 2000, 2000] + ); + // The last return was 200 ms ago. 240 healthy ms are not enough; + // 250 are. + let back = now - ms(200); + assert_eq!(recovery.healthy(back + ms(240)), None); + assert_eq!(recovery.lost(back + ms(240)), LossStep::Retry(ms(2000))); + recovery.healthy(back + ms(2_300)); + assert_eq!(recovery.healthy(back + ms(2_550)), None); + assert_eq!( + recovery.lost(back + ms(2_550)), LossStep::Retry(Duration::ZERO) ); // Each return still ends its episode: the give-up window measures @@ -669,6 +994,107 @@ mod tests { assert!(now.duration_since(start) > LOSS_GIVE_UP); } + /// The field's looping PC (0.3.4): the desktop was taken again 1 to 4 s + /// after each return, for about an hour. With the ladder held for 5 s + /// every retry waited 2 s, and the iPad got 3 to 17 fps. A duplication + /// that worked for a quarter of a second is re-created at once. + #[test] + fn a_desktop_taken_every_few_seconds_is_retried_at_once() { + let ms = Duration::from_millis; + // How long each re-created duplication worked, from that PC's log, + // and two quicker losses. + let lives = [3006, 1789, 650, 1227, 2008, 300, 2260, 1005].map(ms); + let mut recovery = LossRecovery::default(); + let start = Instant::now(); + let mut now = start; + let mut without_picture = Duration::ZERO; + let mut losses = 0; + while now < start + Duration::from_secs(60) { + let LossStep::Retry(delay) = recovery.lost(now) else { + panic!("gave up") + }; + losses += 1; + assert_eq!(delay, Duration::ZERO, "loss {losses} waited"); + without_picture += delay; + now += delay; + assert!(recovery.healthy(now).is_some()); + now += lives[losses % lives.len()]; + } + assert!( + without_picture < ms(100), + "{without_picture:?} without a picture in a minute" + ); + // Still never a spin: at most four re-creations a second. + assert!(losses <= 240, "{losses} re-creations in a minute"); + } + + /// ACCESS_LOST from a call the capture loop does not hand to its handler + /// (on 0.3.4, ReleaseFrame after an acquire: "Capture loop exited with + /// error" seven times in an hour, each a pipeline restart). The stage + /// waits it out like any display change, however often it comes. + #[cfg(windows)] + #[test] + fn access_lost_that_leaves_the_capture_loop_never_fails_the_stage() { + let shared = SharedControl::new(0, 1); + let mut attempts = 0; + let mut blocks = Vec::new(); + let result = run_until_unblocked( + &shared, + |_| { + attempts += 1; + if attempts <= 7 { + Err(windows::core::Error::from_hresult(windows::core::HRESULT( + 0x887A_0026_u32 as i32, + )) + .into()) + } else { + Ok(()) + } + }, + |_| { + blocks.push(*shared.capture_block.lock()); + true + }, + ); + assert!(result.is_ok(), "{result:?}"); + assert_eq!(attempts, 8); + assert_eq!(blocks, [Some(CaptureBlock::DisplayChanging); 7]); + // Other Windows errors still reach the supervisor. + let failed = run_until_unblocked( + &shared, + |_| { + Err(windows::core::Error::from_hresult(windows::core::HRESULT( + 0x887A_0005_u32 as i32, + )) + .into()) + }, + |_| panic!("a removed device was retried in the stage"), + ); + assert!(failed.is_err()); + } + + #[test] + fn only_a_loss_not_back_within_a_second_is_a_warning() { + let ms = Duration::from_millis; + let mut recovery = LossRecovery::default(); + let start = Instant::now(); + assert!(!recovery.overdue(start), "no episode yet"); + recovery.lost(start); + assert!(!recovery.overdue(start + ms(999))); + recovery.lost(start + ms(500)); + assert!(recovery.overdue(start + LOSS_QUIET)); + assert!(!recovery.overdue(start + ms(5_000)), "said once"); + recovery.healthy(start + ms(5_000)); + // The next episode is judged on its own. + let next = start + ms(9_000); + recovery.lost(next); + assert!(recovery.healthy(next + ms(300)).is_some()); + assert!(!recovery.overdue(next + ms(2_000))); + recovery.lost(next + ms(3_000)); + assert!(!recovery.overdue(next + ms(3_999))); + assert!(recovery.overdue(next + ms(4_000))); + } + /// Review r1 of #141: "No display is on" told a desktop PC to open the /// laptop lid while a display was unplugged or a driver updated. Only /// an enumeration that found no output at all is a display that's off. @@ -744,6 +1170,7 @@ mod tests { let waits = RefCell::new(Vec::new()); let warnings = Arc::new(AtomicUsize::new(0)); let subscriber = tracing_subscriber::registry().with(Warnings(warnings.clone())); + crate::logging::prepare_log_capture(); let result = tracing::subscriber::with_default(subscriber, || { run_until_unblocked( &shared, @@ -1004,6 +1431,96 @@ mod tests { assert!(result.is_ok(), "a stopped generation exits cleanly"); } + /// The lock screen, four times in one app run. Each unlock is retried + /// as quickly as the first, not 5 s later. + #[test] + fn a_block_after_capture_worked_again_is_retried_at_the_quick_pace() { + let shared = SharedControl::new(0, 1); + let attempts = RefCell::new(0); + let waits = RefCell::new(Vec::new()); + let result = run_until_unblocked( + &shared, + |_| { + *attempts.borrow_mut() += 1; + let attempt = *attempts.borrow(); + if attempt > 12 { + return Ok(()); + } + // Three blocked attempts, then the user unlocks and capture + // works until the next lock. + if attempt % 3 == 0 { + note_capture_resumed(&shared); + } + Err(blocked(CaptureBlock::SecureDesktop)) + }, + |delay| { + waits.borrow_mut().push(delay.as_millis()); + true + }, + ); + assert!(result.is_ok()); + assert_eq!( + *waits.borrow(), + [500, 1000, 500, 1000, 2000, 500, 1000, 2000, 500, 1000, 2000, 500] + ); + } + + /// Review r1 of #205: a display that moved was reopened after the + /// blocked pace's 500 ms. It is reopened at once; a second move right + /// after waits at that pace instead of reopening in a loop. + #[test] + fn a_moved_display_is_reopened_at_once_but_not_in_a_loop() { + use std::cell::Cell; + let ms = Duration::from_millis; + let shared = SharedControl::new(0, 1); + // A paused clock: each attempt and each wait moves it. + let start = Instant::now(); + let elapsed = Cell::new(Duration::ZERO); + // Before each attempt, how far the clock moves; every attempt but + // the last asks to reopen. + let steps = [ms(0), ms(999), ms(0), ms(1_000), ms(0)]; + let mut attempt = 0; + let mut waits = Vec::new(); + let result = run_until_unblocked_on( + &shared, + |_| { + elapsed.set(elapsed.get() + steps[attempt]); + attempt += 1; + if attempt < steps.len() { + Err(Reopen { + detail: "the display layout changed".into(), + } + .into()) + } else { + Ok(()) + } + }, + |delay| { + waits.push(delay); + elapsed.set(elapsed.get() + delay); + true + }, + || start + elapsed.get(), + ); + assert!(result.is_ok()); + assert_eq!(attempt, steps.len()); + // At 0 ms: at once. At 999 ms, under a second after it: the blocked + // pace, 500 ms. At 1,499 ms, a second and more after the last one + // opened at once: at once. At 2,499 ms, exactly a second after + // that: at once. + assert_eq!(waits, [ms(500)]); + assert_eq!(elapsed.get(), ms(2_499)); + assert_eq!( + *shared.capture_block.lock(), + Some(CaptureBlock::DisplayChanging) + ); + // Not a failure of the second screen. + let reopen: Box = Box::new(Reopen { + detail: String::new(), + }); + assert!(passing_error(reopen.as_ref())); + } + #[test] fn retry_pace_resets_when_the_reason_changes() { let mut retry = BlockedRetry::default(); diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 785d4fdd..9cbf50d3 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -40,6 +40,11 @@ use tracing::warn; use crate::control::{CaptureTarget, SharedControl, VddStatus}; use crate::gpu_preference::AdapterId; +pub(crate) use memory::PANEL_LOST; + +/// The Stream page's reason for a lost panel from before the last restart: +/// the panel is back, and Second screen only waits to be chosen again. +pub(crate) const PANEL_LOST_BEFORE: &str = "panel_lost_before"; use memory::{Hold, Memory}; #[cfg(any(windows, test, debug_assertions))] @@ -98,6 +103,44 @@ pub(crate) const EXTEND_TIMEOUT: Duration = Duration::from_secs(30); #[cfg(any(windows, test, debug_assertions))] pub(crate) const BLOCKED_EXTEND_RETRY: Duration = Duration::from_secs(120); +/// How long the device may run with no monitor, while the attach waits, +/// before the attempt ends. Where the second screen works, the monitor is +/// there by the time the Enable task returns; where it never came, waiting +/// out the whole attach timeout only delayed the fallback. +#[cfg(any(windows, test, debug_assertions))] +pub(crate) const NO_MONITOR_WAIT: Duration = Duration::from_secs(10); + +/// How often the attach wait reads the device's state. +#[cfg(any(windows, test, debug_assertions))] +const DEVICE_READ_EVERY: Duration = Duration::from_secs(1); + +/// Why Second screen mirrors on this PC without trying: Windows is older +/// than the display driver needs ([`driver_supports_windows`]). Not saved: +/// each app run reads Windows again. +pub(crate) const WINDOWS_10: &str = "windows_10"; + +/// The first Windows 11 build. The display driver Setup installs is built +/// for Windows 11's display driver framework: on Windows 10 it starts, but +/// Windows never adds its screen. +#[cfg(any(windows, test))] +const FIRST_WINDOWS_11_BUILD: u32 = 22000; + +/// Set to `1` to try Second screen on Windows 10 anyway, for testing. +pub const WINDOWS_10_OVERRIDE: &str = "ETERNAL_SECOND_SCREEN_ON_WINDOWS_10"; + +/// How long the iPad says that Second screen needs Windows 11, from the +/// connection that mirrors for it. The mirror works, so the notice doesn't +/// stay over it. +pub(crate) const WINDOWS_10_IPAD_NOTICE: Duration = Duration::from_secs(10); + +/// Whether the bundled display driver can show a second screen on Windows +/// `build`, or a tester asked to try anyway (`forced`). An unknown build is +/// tried, as before. A driver that works on Windows 10 lifts the limit here. +#[cfg(any(windows, test))] +pub(crate) fn driver_supports_windows(build: Option, forced: bool) -> bool { + forced || build.is_none_or(|build| build >= FIRST_WINDOWS_11_BUILD) +} + /// How long quitting waits for the worker to exit after the last Enable. const QUIT_JOIN: Duration = Duration::from_secs(5); @@ -145,6 +188,9 @@ struct Proving { /// A time-limited driver call of this generation is outstanding (see /// [`driver_call`]): a failure now may be the driver's, not the PC's. call_pending: bool, + /// Windows lost the desktop during this proof while it was away (a + /// secure screen, the displays off), even if it is back by the failure. + away_seen: bool, } /// A second screen that loses the user's desktop more often than this in one @@ -154,6 +200,20 @@ struct Proving { /// a secure screen or with the displays off don't count. const LOSS_LIMIT: u32 = 8; +/// The output a second screen proves itself on: its adapter's LUID and its +/// device name. A display back under the same name on another adapter is +/// another display. +pub(crate) type ProofOutput = (crate::gpu_preference::AdapterId, String); + +/// How often [`State::retry_if_back`] asks Windows whether the desktop is +/// back. +const DESKTOP_CHECK: Duration = Duration::from_secs(1); + +/// How long after a second screen fails, having lost the desktop, Windows +/// may still say a secure screen took it or the displays went off, and be +/// taken as the cause. Windows can say so many seconds after the loss. +const LATE_AWAY_WINDOW: Duration = Duration::from_secs(30); + #[derive(Debug, Default)] pub(crate) struct State { /// The display attached; the stream has not moved onto it yet. @@ -167,10 +227,21 @@ pub(crate) struct State { /// A start is claimed: its Attach is queued or running. attaching: bool, proving: Option, + /// The second screen that captured its first image: the capture + /// generation and the output. The stage opening that same output again + /// in that generation (a recovery reopen) does not prove it again: a new + /// proof hid a display gone dark from the watchdog and then fell back + /// to the mirror. Another output, a new attachment or a detach seen in + /// between clears it, and the display proves itself again. + proven: Option<(u64, ProofOutput)>, + /// The output the running proof is for, when the reconciliation named it. + proving_output: Option, /// Why the second screen last failed, for the log and the backend. failure: Option<&'static str>, /// The failure saved for later app runs. memory: Memory, + /// This app run's Enable took the built-in panel away. + panel_lost_this_run: bool, /// Windows refused the mirror's screen from the card EternalMonitor /// captures on, which the display configuration then wired to another /// card: see [`note_mirror_refused`]. @@ -180,9 +251,24 @@ pub(crate) struct State { /// `blocked_retry_at` when that is set. retry_blocked: bool, blocked_retry_at: Option, + /// The last failure came while the user's desktop was away (see + /// [`desktop_away`]): once it is back, a connected viewer gets the + /// second screen again. When to look at the desktop next. + retry_when_back: Option, + /// The last failure lost the desktop with the user's desktop there: until + /// then, Windows saying it was away after all (see [`LATE_AWAY_WINDOW`]) + /// tries the second screen again once it is back. When to ask next. + late_away_until: Option, + late_away_check: Option, /// The graphics cards and the screens' wiring, as last read off the /// transport loop: see [`facts`]. facts: facts::Cache, + /// Windows is too old for the display driver: Second screen mirrors + /// without turning it on ([`note_windows`]). + windows_unsupported: bool, + /// When a connection last began mirroring for that, for the iPad's + /// notice ([`windows_notice`]). + windows_held_at: Option, /// Whether the user's desktop is away, as an end-to-end test's fake /// display says in place of Windows: see [`desktop_away`]. #[cfg(debug_assertions)] @@ -192,6 +278,10 @@ pub(crate) struct State { impl State { /// `blocked`: [`mirror_blocked_now`], read for this decision when /// [`State::decides_on_mirror`] says it needs it. + /// `secure`: capture says a secure screen holds the desktop now + /// ([`CaptureBlock::SecureDesktop`]). + /// + /// [`CaptureBlock::SecureDesktop`]: crate::capture::recovery::CaptureBlock::SecureDesktop pub(crate) fn step( &mut self, selected: bool, @@ -199,6 +289,7 @@ impl State { connected: bool, now: Instant, blocked: bool, + secure: bool, ) -> Step { self.memory.retry_clear(now); if !selected { @@ -214,6 +305,7 @@ impl State { memory: std::mem::take(&mut self.memory), mirror_refused: self.mirror_refused.take(), facts: std::mem::take(&mut self.facts), + windows_unsupported: self.windows_unsupported, #[cfg(debug_assertions)] fake_desktop_away: self.fake_desktop_away.take(), ..State::default() @@ -223,10 +315,24 @@ impl State { if let Some(reason) = self.fail_unproven(status, now, blocked) { return Step::Fallback(reason); } + self.watch_late_away(*status, now, secure); + self.retry_if_back(status, connected, now, secure); match *status { + // Before any saved failure: trying again can't help. + VddStatus::WaitingForClient if connected && self.windows_unsupported => { + *status = VddStatus::Failed; + self.failure = Some(WINDOWS_10); + self.windows_held_at = Some(now); + log_windows_hold(); + Step::Nothing + } VddStatus::WaitingForClient if connected => { - let hold = if blocked { - if let Some(hold) = self.memory.hold(now) { + let held = self.memory.hold(now); + // A blocked mirror is no reason to take the panel again. + let hold = if held.is_some_and(|hold| hold.until_chosen) { + held + } else if blocked { + if let Some(hold) = held { tracing::info!( reason = hold.reason, failures = hold.failures, @@ -236,9 +342,7 @@ impl State { } None } else { - self.memory - .hold(now) - .or_else(|| self.memory.hold_for_rule(now)) + held.or_else(|| self.memory.hold_for_rule(now)) }; if let Some(hold) = hold { *status = VddStatus::Failed; @@ -322,6 +426,7 @@ impl State { memory: std::mem::take(&mut self.memory), mirror_refused: self.mirror_refused.take(), facts: std::mem::take(&mut self.facts), + windows_unsupported: self.windows_unsupported, #[cfg(debug_assertions)] fake_desktop_away: self.fake_desktop_away.take(), ..State::default() @@ -329,16 +434,115 @@ impl State { *status = VddStatus::Failed; let passing = proving.interrupted || proving.call_pending || proving.losses > LOSS_LIMIT; self.remember(reason, now, passing, blocked); + self.after_desktop_loss(reason, proving, now, blocked); Some(reason) } + /// A second screen that showed no picture after Windows took its + /// desktop away: the lock screen, a UAC prompt or Ctrl+Alt+Del can take + /// the input before Windows says so, so the desktop may still read as + /// the user's at the failure. When the loss came while the desktop was + /// away, try again once it is back, as for a failure while it is away. + /// Otherwise watch for Windows to say so within [`LATE_AWAY_WINDOW`]. + /// Neither for a driver call that didn't return, a display that keeps + /// dropping out, or a failure [`State::remember`] already set a retry + /// for. + fn after_desktop_loss( + &mut self, + reason: &'static str, + proving: Proving, + now: Instant, + blocked: bool, + ) { + let desktop_loss = reason == "no_frames" + && proving.interrupted + && !proving.call_pending + && proving.losses <= LOSS_LIMIT; + if !desktop_loss || blocked || self.retry_when_back.is_some() { + return; + } + if proving.away_seen { + tracing::info!( + reason, + "The second screen lost the desktop while Windows had it away — trying the \ + second screen again now that the desktop is back" + ); + self.retry_when_back = Some(now); + } else { + self.late_away_until = Some(now + LATE_AWAY_WINDOW); + self.late_away_check = None; + } + } + + /// While a failure that lost the desktop waits (see + /// [`State::after_desktop_loss`]): a secure screen or the displays off + /// now means that was the cause, and the second screen is tried again + /// once the desktop is back. Once per failure, and only while it is still + /// the failure: Try again, another mode or a retry ends the wait. + fn watch_late_away(&mut self, status: VddStatus, now: Instant, secure: bool) { + let Some(until) = self.late_away_until else { + return; + }; + if status != VddStatus::Failed || now >= until { + self.late_away_until = None; + self.late_away_check = None; + return; + } + if !secure && self.late_away_check.is_some_and(|at| now < at) { + return; + } + self.late_away_check = Some(now + DESKTOP_CHECK); + if secure || self.desktop_away() { + self.late_away_until = None; + self.late_away_check = None; + tracing::info!( + reason = self.failure.unwrap_or("no_frames"), + "Windows took the desktop away just after the second screen failed — trying \ + the second screen again once the desktop is back" + ); + self.retry_when_back = Some(now + DESKTOP_CHECK); + } + } + + /// After a failure while the desktop was away, try the second screen + /// again for a connected viewer once the desktop is back. Without this + /// the app run stayed on the mirror after the PC was unlocked or its + /// displays woke, until the user clicked Try again. Windows is asked at + /// most once a second: this runs on the transport loop. + fn retry_if_back( + &mut self, + status: &mut VddStatus, + connected: bool, + now: Instant, + secure: bool, + ) { + let due = self.retry_when_back.is_some_and(|at| now >= at); + if !due || !connected || *status != VddStatus::Failed { + return; + } + if secure || self.desktop_away() { + self.retry_when_back = Some(now + DESKTOP_CHECK); + return; + } + self.retry_when_back = None; + tracing::info!( + reason = self.failure.unwrap_or("no_frames"), + "The desktop is back after the second screen failed while it was away — trying \ + the second screen again" + ); + *status = VddStatus::WaitingForClient; + self.failure = None; + } + /// Save the failure for later connections and app runs, unless /// something that passes may explain it: then only this app run keeps - /// mirroring, as before. Nor while the mirror is `blocked`, as + /// mirroring, as before, or, when the desktop is away now, until it is + /// back. Nor while the mirror is `blocked`, as /// [`mirror_blocked_now`] read it for this failure: then the next /// connection tries the second screen again. fn remember(&mut self, reason: &'static str, now: Instant, interrupted: bool, blocked: bool) { self.clear_blocked_retry(); + self.retry_when_back = None; if blocked { tracing::info!( reason, @@ -348,13 +552,18 @@ impl State { self.retry_blocked = true; return; } - if interrupted || self.desktop_away() { + let away = self.desktop_away(); + if interrupted || away { tracing::info!( reason, + away, "The second screen failed while Windows withheld the desktop or the display may \ have been asleep — not remembering it beyond this app run" ); self.memory.excuse(); + if away { + self.retry_when_back = Some(now); + } return; } if let Some(hold) = self.memory.note_failure(reason, now) { @@ -368,6 +577,22 @@ impl State { } } + /// The display took the built-in panel away: save that until Second + /// screen is chosen again. Neither a blocked mirror nor a withheld + /// desktop excuses it, since trying again takes the panel again. + #[cfg(any(windows, test, debug_assertions))] + fn remember_panel_lost(&mut self, now: Instant) { + self.clear_blocked_retry(); + if let Some(hold) = self.memory.note_failure(PANEL_LOST, now) { + tracing::info!( + reason = PANEL_LOST, + failures = hold.failures, + "Remembering that the second screen took the built-in screen away on this PC — \ + Second screen sessions mirror until Second screen is chosen again" + ); + } + } + /// [`desktop_away`], or what the fake display an end-to-end test /// installed says. fn desktop_away(&self) -> bool { @@ -412,11 +637,13 @@ pub(crate) fn next_step(shared: &SharedControl, now: Instant) -> Step { let blocked = mirror_blocked_now(shared, |status, state| { state.decides_on_mirror(selected, status, connected, now) }); + let secure = + *shared.capture_block.lock() == Some(crate::capture::recovery::CaptureBlock::SecureDesktop); let mut status = shared.vdd_status.lock(); shared .second_screen .lock() - .step(selected, &mut status, connected, now, blocked) + .step(selected, &mut status, connected, now, blocked, secure) } /// A new capture generation ends a second screen that already failed to @@ -443,6 +670,16 @@ pub(crate) fn proving(shared: &SharedControl) -> bool { /// The one log line for a connection that mirrors because the second screen /// failed on this PC before. Diagnostics count `second_screen_fallback=remembered`. fn log_hold(hold: Hold) { + if hold.until_chosen { + tracing::info!( + second_screen_fallback = "remembered", + reason = hold.reason, + failures = hold.failures, + "The second screen took the built-in screen away on this PC before — mirroring the \ + primary display until Second screen is chosen again, or the user clicks Try again" + ); + return; + } tracing::info!( second_screen_fallback = "remembered", reason = hold.reason, @@ -453,6 +690,80 @@ fn log_hold(hold: Hold) { ); } +/// The one log line for a connection that mirrors because Windows is too +/// old for the display driver. It is expected, not a failure, so it has no +/// `second_screen_fallback` field: the daily check counts those. The +/// session summary still names it `no_driver`: there is no display driver +/// for this Windows. +pub(crate) fn log_windows_hold() { + crate::diagnostics::note_fallback("no_driver"); + tracing::info!( + reason = WINDOWS_10, + "Second screen held: needs Windows 11 — mirroring the primary display without turning \ + the display driver on" + ); +} + +/// Records whether this Windows can run the display driver (see +/// [`driver_supports_windows`]), at startup. +#[cfg(any(windows, test))] +pub(crate) fn note_windows(shared: &SharedControl, build: Option, forced: bool) { + let supported = driver_supports_windows(build, forced); + if !driver_supports_windows(build, false) { + if forced { + tracing::info!( + "Windows is older than Windows 11, but {WINDOWS_10_OVERRIDE} asks to try Second \ + screen anyway" + ); + } else { + tracing::info!( + "Windows is older than Windows 11 — Second screen mirrors the primary display \ + without turning the display driver on" + ); + } + } + shared.second_screen.lock().windows_unsupported = !supported; +} + +/// [`note_windows`] for this PC: its build, and the tester's override. +#[cfg(windows)] +pub fn check_windows(shared: &SharedControl) { + let forced = std::env::var(WINDOWS_10_OVERRIDE).is_ok_and(|value| value.trim() == "1"); + note_windows(shared, windows_build(), forced); +} + +/// This PC's Windows build, as the registry names it. Windows 11 keeps +/// "Windows 10" as its product name, so only the build tells them apart. +#[cfg(windows)] +fn windows_build() -> Option { + crate::diagnostics::registry_string( + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion", + "CurrentBuildNumber", + ) + .trim() + .parse() + .ok() +} + +/// Whether Second screen mirrors on this PC because Windows is too old for +/// the display driver, for the Stream page and Settings. +pub(crate) fn windows_unsupported(shared: &SharedControl) -> bool { + shared.second_screen.lock().windows_unsupported +} + +/// Whether the heartbeat tells the iPad now that Second screen needs +/// Windows 11: for [`WINDOWS_10_IPAD_NOTICE`] after a connection began +/// mirroring for it. +pub(crate) fn windows_notice(shared: &SharedControl, now: Instant) -> bool { + let status = *shared.vdd_status.lock(); + let state = shared.second_screen.lock(); + status == VddStatus::Failed + && state.failure == Some(WINDOWS_10) + && state + .windows_held_at + .is_some_and(|at| now.saturating_duration_since(at) < WINDOWS_10_IPAD_NOTICE) +} + /// Read the failure an earlier app run saved, for this PC as it is now. #[cfg(windows)] pub fn load_memory(shared: &SharedControl) { @@ -473,6 +784,8 @@ pub fn load_memory(shared: &SharedControl) { Instant::now(), ); memory.set_rule(rule, dir.join(memory::RULE_FILE_NAME)); + let uptime = unsafe { windows::Win32::System::SystemInformation::GetTickCount64() } / 1000; + memory.set_boot(wall.saturating_sub(uptime)); shared.second_screen.lock().memory = memory; #[cfg(not(test))] display_power::watch(); @@ -488,8 +801,16 @@ pub(crate) fn note_connected(shared: &SharedControl, now: Instant) { let blocked = mirror_blocked_now(shared, |status, _| status == VddStatus::Failed); let mut status = shared.vdd_status.lock(); let mut state = shared.second_screen.lock(); + // Nothing to try on this Windows, even when the mirror shows nothing. + if *status == VddStatus::Failed && state.failure == Some(WINDOWS_10) { + state.windows_held_at = Some(now); + drop((state, status)); + log_windows_hold(); + return; + } // The mirror would show nothing: the second screen is the only picture. - if *status == VddStatus::Failed && blocked { + // Not after a lost panel: the next try takes it again. + if *status == VddStatus::Failed && blocked && state.failure != Some(PANEL_LOST) { let reason = state.failure.unwrap_or("attach_timeout"); if let Some(at) = state.blocked_retry_at.filter(|at| now < *at) { tracing::info!( @@ -529,7 +850,15 @@ pub(crate) fn note_connected(shared: &SharedControl, now: Instant) { } return; } + let retrying = *status == VddStatus::Failed && state.retry_when_back.is_some(); drop((state, status)); + if retrying { + tracing::info!( + "Client connected — the second screen failed while the desktop was away, and is \ + tried again once the desktop is back" + ); + return; + } if let Some(reason) = remembered_failure(shared) { tracing::info!( second_screen_fallback = reason, @@ -543,7 +872,9 @@ pub(crate) fn note_connected(shared: &SharedControl, now: Instant) { pub(crate) fn retries_later(shared: &SharedControl) -> bool { let status = *shared.vdd_status.lock(); let state = shared.second_screen.lock(); - status == VddStatus::Failed && (state.memory.saved() || state.retry_blocked) + status == VddStatus::Failed + && state.failure != Some(PANEL_LOST) + && (state.memory.saved() || state.retry_blocked) } /// Windows refused to duplicate the mirror's `screen` from `card`, the card @@ -670,6 +1001,18 @@ pub(crate) fn remembered_failure(shared: &SharedControl) -> Option<&'static str> } } +/// [`remembered_failure`] for the Stream page's notice, where a lost panel +/// from before the last restart is [`PANEL_LOST_BEFORE`]: only one that is +/// still off asks for a restart. +pub(crate) fn notice_failure(shared: &SharedControl) -> Option<&'static str> { + let failure = remembered_failure(shared); + let state = shared.second_screen.lock(); + if failure == Some(PANEL_LOST) && !state.panel_lost_this_run && !state.memory.panel_lost_now() { + return Some(PANEL_LOST_BEFORE); + } + failure +} + /// The user asked to try again: a failed start, or one whose thread is gone, /// is tried once more at the next connection, and a saved failure is /// forgotten. @@ -695,6 +1038,50 @@ pub fn note_attached(shared: &SharedControl, device_name: String) { let mut state = shared.second_screen.lock(); state.switch_pending = true; state.proving = None; + state.proven = None; + state.proving_output = None; +} + +/// The managed display was seen off the desktop: one that comes back, even +/// under the same name, proves itself again. +pub fn note_detached(shared: &SharedControl) { + shared.second_screen.lock().proven = None; +} + +/// A listing of the outputs (see [`super::listing`]): a second screen whose +/// output it does not show as it was (absent, or its name under another +/// adapter) proves itself again when it is back, as after +/// [`note_detached`]. +pub(crate) fn observe_listing(shared: &SharedControl, outputs: &[super::OutputInfo]) { + let shows = |(adapter, name): &ProofOutput| { + outputs + .iter() + .any(|output| output.adapter_id == *adapter && &output.device_name == name) + }; + let mut state = shared.second_screen.lock(); + if state + .proven + .as_ref() + .is_some_and(|(_, output)| !shows(output)) + { + state.proven = None; + } + if state + .proving_output + .as_ref() + .is_some_and(|output| !shows(output)) + { + state.proving_output = None; + } +} + +/// The driver's outputs, judged as every listing is (see +/// [`observe_listing`]). +#[cfg(any(windows, test, debug_assertions))] +fn listed(shared: &SharedControl, driver: &impl Driver) -> Vec { + let outputs = driver.outputs(); + observe_listing(shared, &outputs); + outputs } /// [`note_attached`], unless the mode changed or quitting began. The mode @@ -715,7 +1102,31 @@ fn note_attached_if_selected(shared: &SharedControl, device_name: String) -> boo /// generation after the switch starts the [`PROVE_TIMEOUT`] deadline; a /// later one inherits it, and any capture failure, until an image arrives. pub fn start_proving(shared: &SharedControl, generation: u64, now: Instant) { + start_proving_on(shared, generation, now, None); +} + +/// [`start_proving`] for the second screen on `output`. The generation that +/// proved that output is not asked again. +pub(crate) fn start_proving_on( + shared: &SharedControl, + generation: u64, + now: Instant, + output: Option, +) { let mut state = shared.second_screen.lock(); + if state.proving.is_none() { + let proven = output.as_ref().is_some_and(|output| { + state + .proven + .as_ref() + .is_some_and(|(by, on)| *by == generation && on == output) + }); + if proven { + return; + } + state.proven = None; + } + state.proving_output = output; match state.proving.as_mut() { // A call the earlier generation still waits on is not this one's, // and its guard can no longer clear the flag. @@ -734,6 +1145,7 @@ pub fn start_proving(shared: &SharedControl, generation: u64, now: Instant) { losses: 0, persistent: false, call_pending: false, + away_seen: false, }) } } @@ -773,6 +1185,10 @@ pub fn note_image(shared: &SharedControl, generation: u64) { let mut state = shared.second_screen.lock(); if let Some(proving) = state.proving.filter(|p| p.generation == generation) { state.proving = None; + state.proven = state + .proving_output + .take() + .map(|output| (generation, output)); tracing::info!( after_ms = proving.since.elapsed().as_millis() as u64, "Second screen captured its first image — streaming it" @@ -818,6 +1234,7 @@ fn note_interruption_while(shared: &SharedControl, generation: u64, away: bool) if let Some(proving) = shared.second_screen.lock().proving.as_mut() { if proving.generation == generation { proving.interrupted = true; + proving.away_seen |= away; if !away { proving.losses += 1; proving.capture_failed |= proving.losses > LOSS_LIMIT; @@ -1296,13 +1713,39 @@ pub(crate) trait Driver { fn settle(&self, chosen: OutputInfo) -> OutputInfo { chosen } + /// After an Enable that worked: whether Windows put the desktop on the + /// display alone, so it will not attach beside the PC's screens. The + /// display guard has already tried to bring them back, unless the + /// display took the built-in panel away. + fn turned_on(&self) -> TurnedOn { + TurnedOn::Beside + } /// Say why the display did not appear or showed no picture, while it is /// still on. fn report(&self, _failure: Failure) {} + /// The device's state while the attach waits. `None` when it can't be + /// read, or a monitor may still be arriving. + fn device(&self) -> Option { + None + } fn now(&self) -> Instant; fn sleep(&self, duration: Duration); } +/// How an Enable that worked left the PC's screens, for [`Driver::turned_on`]. +#[cfg(any(windows, test, debug_assertions))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TurnedOn { + /// As usual: the display can attach beside them. + Beside, + /// Windows put the desktop on the display alone. + Alone, + /// On a hybrid laptop, the display took the built-in panel away until + /// the PC restarts (`crate::display_guard`). The desktop is on the + /// display, and turning it off would leave the PC with no screen. + PanelLost, +} + /// Why a second screen failed, for [`Driver::report`]. #[cfg(any(windows, test, debug_assertions))] #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1318,6 +1761,33 @@ pub(crate) enum Failure { Unproven(&'static str), } +/// Whether the device has run with no monitor for [`NO_MONITOR_WAIT`], +/// read every [`DEVICE_READ_EVERY`] while the attach waits. Any other state, +/// or one that can't be read, starts the count again. +#[cfg(any(windows, test, debug_assertions))] +#[derive(Default)] +struct NoMonitorWatch { + since: Option, + next_read: Option, +} + +#[cfg(any(windows, test, debug_assertions))] +impl NoMonitorWatch { + fn gave_up(&mut self, now: Instant, read: impl FnOnce() -> Option) -> bool { + if self.next_read.is_some_and(|at| now < at) { + return false; + } + self.next_read = Some(now + DEVICE_READ_EVERY); + if read() == Some(DeviceState::NoMonitor) { + let since = *self.since.get_or_insert(now); + now.duration_since(since) >= NO_MONITOR_WAIT + } else { + self.since = None; + false + } + } +} + /// However [`attach`] ends, its claim ends with it: a start that did not /// finish leaves the status for the next tick to claim again. #[cfg(any(windows, test, debug_assertions))] @@ -1347,7 +1817,7 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura } // Still on from before (its release saw this start coming): reuse it, // with no change to wait for. - let outputs = driver.outputs(); + let outputs = listed(shared, driver); let owned = shared.vdd_output.lock().clone(); if let Some(name) = owned.filter(|name| { outputs @@ -1373,7 +1843,7 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura } driver.sleep(WAIT_STEP); } - let outputs = driver.outputs(); + let outputs = listed(shared, driver); driver.prepare(&outputs); let before: Vec = outputs .into_iter() @@ -1407,13 +1877,21 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura return; } let waiting_since = driver.now(); - let result = wait_for_virtual_output( - &before, - timeout, - || selected().then(|| driver.outputs()), - || driver.now(), - |duration| driver.sleep(duration), - ); + // No wait for a display that can't attach. + let turned_on = driver.turned_on(); + let result = if turned_on != TurnedOn::Beside { + AttachResult::TimedOut + } else { + let mut watch = NoMonitorWatch::default(); + wait_for_virtual_output( + &before, + timeout, + || selected().then(|| listed(shared, driver)), + || driver.now(), + |duration| driver.sleep(duration), + || watch.gave_up(driver.now(), || driver.device()), + ) + }; #[cfg(windows)] let result = match result { AttachResult::Attached(_) if super::simulated_attach_timeout() => AttachResult::TimedOut, @@ -1453,13 +1931,45 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura disable(shared, driver); } } - AttachResult::TimedOut => { + AttachResult::TimedOut if turned_on == TurnedOn::PanelLost => { warn!( - second_screen_fallback = "attach_timeout", - timeout_ms = timeout.as_millis() as u64, - "Virtual display did not attach in time — capturing the primary display instead" + second_screen_fallback = "panel_lost", + "The laptop's built-in screen went dark when the virtual display turned on — \ + leaving the display on and mirroring it, so the iPad shows the desktop" ); driver.report(Failure::AttachTimeout); + // No turn off: it would leave Windows with no screen at all. + let now = driver.now(); + let mut status = shared.vdd_status.lock(); + *status = VddStatus::Failed; + let mut state = shared.second_screen.lock(); + state.failure = Some(PANEL_LOST); + state.panel_lost_this_run = true; + state.remember_panel_lost(now); + } + ended @ (AttachResult::TimedOut | AttachResult::NoMonitor) => { + if matches!(ended, AttachResult::NoMonitor) { + warn!( + second_screen_fallback = "attach_timeout", + vdd_device = "no_monitor", + waited_ms = driver.now().duration_since(waiting_since).as_millis() as u64, + "Windows added no monitor for the running virtual display — capturing the \ + primary display instead" + ); + } else if turned_on == TurnedOn::Alone { + warn!( + second_screen_fallback = "vdd_only", + "Windows put the desktop on the virtual display alone — capturing the \ + primary display instead" + ); + } else { + warn!( + second_screen_fallback = "attach_timeout", + timeout_ms = timeout.as_millis() as u64, + "Virtual display did not attach in time — capturing the primary display instead" + ); + } + driver.report(Failure::AttachTimeout); // Enable succeeded but nothing attached: turn it back off so no // half-enabled device lingers as a phantom monitor. disable(shared, driver); @@ -1469,6 +1979,8 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura let mut status = shared.vdd_status.lock(); *status = VddStatus::Failed; let mut state = shared.second_screen.lock(); + // Alone too: the same notice and wait, in a record every + // version reads. state.failure = Some("attach_timeout"); state.remember("attach_timeout", now, false, blocked); if blocked && copied { @@ -1564,10 +2076,44 @@ struct WindowsDriver<'a> { request: std::cell::RefCell>, } +#[cfg(windows)] +impl WindowsDriver<'_> { + /// After an Enable that succeeded: a driver seen off before it and + /// running after it started in it, with the file `prepare` wrote, so + /// keep what that file gives the pointer + /// (`vdd_settings::record_after_enable`). One already running, or one + /// that couldn't be read, keeps the record of its own start. + fn note_start(&self) { + let request = self.request.borrow(); + let Some(request) = request.as_ref() else { + return; + }; + let after = crate::vdd_device::run_state(); + let record = crate::vdd_settings::record_after_enable( + request.instance, + after, + request.settings.as_deref(), + request.cursor, + ); + info!( + started = record.is_some(), + before = ?request.instance, + ?after, + "The virtual display's driver after the Enable" + ); + let (Some(record), Some(path)) = (record, super::cursor_record_path()) else { + return; + }; + if let Err(error) = crate::vdd_settings::save_record(&path, &record) { + warn!(%error, "Could not keep the virtual display's pointer record"); + } + } +} + #[cfg(windows)] impl Driver for WindowsDriver<'_> { fn outputs(&self) -> Vec { - super::enumerate_outputs() + super::listing::outputs(self.shared) } fn prepare(&self, outputs: &[OutputInfo]) { @@ -1575,16 +2121,29 @@ impl Driver for WindowsDriver<'_> { } fn set_enabled(&self, enabled: bool) -> bool { - crate::vdd::set_enabled(enabled, || {}) + let done = crate::vdd::set_enabled(enabled, || {}); + if enabled && done { + self.note_start(); + } + done } fn enable(&self) -> Result<(), EnableReport> { - crate::vdd::enable(|| {}) + let result = crate::vdd::enable(|| {}); + if result.is_ok() { + self.note_start(); + } + result } fn settle(&self, chosen: OutputInfo) -> OutputInfo { let request = self.request.borrow(); - let chosen = super::settle_vdd_mode(chosen, request.as_ref().and_then(|r| r.mode), || {}); + let chosen = super::settle_vdd_mode( + self.shared, + chosen, + request.as_ref().and_then(|r| r.mode), + || {}, + ); let before = request.as_ref().map_or(&[][..], |r| &r.before[..]); let render = crate::vdd_snapshot::render_card_now(before); info!( @@ -1595,6 +2154,20 @@ impl Driver for WindowsDriver<'_> { chosen } + fn device(&self) -> Option { + crate::vdd_device::query_confirmed() + } + + fn turned_on(&self) -> TurnedOn { + if crate::display_guard::last_turn_on_lost_panel() { + TurnedOn::PanelLost + } else if crate::display_guard::last_turn_on_alone() { + TurnedOn::Alone + } else { + TurnedOn::Beside + } + } + fn report(&self, failure: Failure) { let request = self.request.borrow(); let device = match failure { @@ -1643,6 +2216,30 @@ mod tests { use std::cell::{Cell, RefCell}; use std::sync::atomic::Ordering; + /// Every Windows has a build number to read. + #[cfg(windows)] + #[test] + fn this_windows_build_reads_from_the_registry() { + let build = windows_build().expect("a build number"); + assert!(build >= 10240, "{build}"); + } + + /// Native, on a Windows 11 PC: the startup check reads the real build, + /// and a Second screen connection starts the second screen as before. + #[cfg(windows)] + #[test] + #[ignore = "needs Windows 11"] + fn this_windows_11_pc_tries_the_second_screen() { + let shared = second_screen_selected(); + check_windows(&shared); + assert!(!windows_unsupported(&shared), "{:?}", windows_build()); + connect_viewer(&shared); + let now = Instant::now(); + note_connected(&shared, now); + assert_eq!(next_step(&shared, now), Step::Attach); + assert!(!windows_notice(&shared, now)); + } + /// The iPad's connect timeout in 0.3.1 and 0.3.2: it disconnects when no /// frame has decoded this long after the handshake. const IPAD_TIMEOUT: Duration = Duration::from_secs(10); @@ -1702,6 +2299,7 @@ mod tests { top: 0, is_primary: primary, adapter_name: "Test GPU".to_string(), + adapter_id: (0, 1), } } @@ -1719,7 +2317,8 @@ mod tests { /// display attaches `attach_after` later, or never. `on_sleep` sees the /// clock each time the host waits, as the transport's ticks would. /// With `enable_fails`, the Enable task reports that instead. Setting - /// the mode after the attach takes `settle_takes`. + /// the mode after the attach takes `settle_takes`. `turned_on` says how the + /// Enable left the PC's screens. struct FakeDriver<'a> { start: Instant, elapsed: Cell, @@ -1730,6 +2329,13 @@ mod tests { toggles: RefCell>, on_sleep: &'a dyn Fn(Duration), enable_fails: Option, + turned_on: TurnedOn, + /// What the device reads while it is on and no display has attached + /// yet; `None` when it can't be read. Attached, it reads ready. + device_until_attached: Option, + /// When set, what the device reads that long after it was turned on, + /// in place of `device_until_attached`: the last entry not later. + device_since_on: Vec<(Duration, Option)>, calls: RefCell>, } @@ -1749,6 +2355,9 @@ mod tests { toggles: RefCell::new(Vec::new()), on_sleep, enable_fails: None, + turned_on: TurnedOn::Beside, + device_until_attached: None, + device_since_on: Vec::new(), calls: RefCell::new(Vec::new()), } } @@ -1759,6 +2368,26 @@ mod tests { } impl Driver for FakeDriver<'_> { + fn device(&self) -> Option { + if self.on_since.get().is_none() { + return Some(DeviceState::Problem(22)); + } + if self.outputs().iter().any(|output| !output.is_primary) { + return Some(DeviceState::Ready); + } + let since_on = self.elapsed.get() - self.on_since.get().unwrap_or_default(); + match self + .device_since_on + .iter() + .rev() + .find(|(at, _)| *at <= since_on) + { + Some((_, device)) => *device, + None if self.device_since_on.is_empty() => self.device_until_attached, + None => None, + } + } + fn outputs(&self) -> Vec { let mut outputs = vec![output("primary", true)]; let attached = self @@ -1796,6 +2425,10 @@ mod tests { chosen } + fn turned_on(&self) -> TurnedOn { + self.turned_on + } + fn report(&self, failure: Failure) { self.calls.borrow_mut().push(Call::Report(failure)); } @@ -1829,6 +2462,136 @@ mod tests { } } + /// Field data (Windows 10 and 11): the Enable task succeeds, the device + /// runs, and Windows never adds its monitor. The attempt waited out the + /// whole attach timeout every time. It now ends once the device has run + /// with no monitor for NO_MONITOR_WAIT, as an attach timeout, with one + /// Enable and one Disable, and is remembered as before. + #[test] + fn a_device_that_runs_with_no_monitor_ends_the_wait_early() { + let shared = second_screen_selected(); + connect_viewer(&shared); + let start = Instant::now(); + assert_eq!(next_step(&shared, start), Step::Attach); + let on_sleep = |_: Duration| {}; + let mut driver = FakeDriver::new(secs(2.0), None, &on_sleep); + driver.device_until_attached = Some(DeviceState::NoMonitor); + attach(&shared, &driver, ATTACH_TIMEOUT); + + let toggles = driver.toggles.borrow().clone(); + assert_eq!(toggles.len(), 2, "{toggles:?}"); + let waited = toggles[1].0 - toggles[0].0; + assert!(!toggles[1].1, "{toggles:?}"); + assert!( + waited >= NO_MONITOR_WAIT && waited < NO_MONITOR_WAIT + DEVICE_READ_EVERY + secs(0.5), + "waited {waited:?}" + ); + assert!(driver + .calls() + .contains(&Call::Report(Failure::AttachTimeout))); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + assert_eq!(remembered_failure(&shared), Some("attach_timeout")); + } + + /// The early end needs a device that runs with no monitor all along. A + /// monitor that is there (Windows kept the desktop off it, and the user + /// may still choose Extend), a device that can't be read, or one that + /// attaches before NO_MONITOR_WAIT keeps the whole wait, or attaches. + #[test] + fn only_a_device_with_no_monitor_all_along_ends_the_wait_early() { + for device in [ + Some(DeviceState::Ready), + Some(DeviceState::NotStarted), + None, + ] { + let shared = second_screen_selected(); + connect_viewer(&shared); + assert_eq!(next_step(&shared, Instant::now()), Step::Attach); + let on_sleep = |_: Duration| {}; + let mut driver = FakeDriver::new(secs(2.0), None, &on_sleep); + driver.device_until_attached = device; + attach(&shared, &driver, ATTACH_TIMEOUT); + let toggles = driver.toggles.borrow().clone(); + let waited = toggles[1].0 - toggles[0].0; + assert!(waited >= ATTACH_TIMEOUT, "{device:?}: waited {waited:?}"); + } + // No monitor for a while, then the display attaches in time. + let shared = second_screen_selected(); + connect_viewer(&shared); + assert_eq!(next_step(&shared, Instant::now()), Step::Attach); + let on_sleep = |_: Duration| {}; + let mut driver = FakeDriver::new(secs(2.0), Some(NO_MONITOR_WAIT - secs(1.0)), &on_sleep); + driver.device_until_attached = Some(DeviceState::NoMonitor); + attach(&shared, &driver, ATTACH_TIMEOUT); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Active); + assert_eq!(remembered_failure(&shared), None); + } + + /// Review r1: only a confirmed absence counts. A monitor that hasn't + /// started, or a read that isn't certain (`None`), never ends the wait, + /// however long it lasts; the 10 s count starts at the first confirmed + /// absence and starts over when it stops. + #[test] + fn the_early_end_counts_only_confirmed_absence_and_starts_over() { + let run = |schedule: Vec<(Duration, Option)>, + attach_after: Option| { + let shared = second_screen_selected(); + connect_viewer(&shared); + assert_eq!(next_step(&shared, Instant::now()), Step::Attach); + let on_sleep = |_: Duration| {}; + let mut driver = FakeDriver::new(secs(2.0), attach_after, &on_sleep); + driver.device_since_on = schedule; + attach(&shared, &driver, ATTACH_TIMEOUT); + let toggles = driver.toggles.borrow().clone(); + let waited = toggles.get(1).map(|(at, _)| *at - toggles[0].0); + let status = *shared.vdd_status.lock(); + (status, waited) + }; + // A monitor still on its way for longer than 10 s, then attached. + let (status, _) = run(vec![(secs(0.0), None)], Some(secs(14.0))); + assert_eq!( + status, + VddStatus::Active, + "a pending monitor ended the wait" + ); + // Uncertain all along: the whole wait, as before. + let (status, waited) = run(vec![(secs(0.0), None)], None); + assert_eq!(status, VddStatus::Failed); + assert!(waited.unwrap() >= ATTACH_TIMEOUT, "{waited:?}"); + // Uncertain for 5 s, then a confirmed absence: 10 s from then. Reads + // come once a second, so a change is seen up to one read late, and + // the count ends up to one read late. + let (_, waited) = run( + vec![(secs(0.0), None), (secs(5.0), Some(DeviceState::NoMonitor))], + None, + ); + let waited = waited.unwrap(); + assert!( + waited >= secs(15.0) && waited < secs(15.0) + DEVICE_READ_EVERY * 2 + secs(0.5), + "{waited:?}" + ); + // A confirmed absence broken by an uncertain read starts over. + let (_, waited) = run( + vec![ + (secs(0.0), Some(DeviceState::NoMonitor)), + (secs(8.0), None), + (secs(9.0), Some(DeviceState::NoMonitor)), + ], + None, + ); + let waited = waited.unwrap(); + assert!( + waited >= secs(19.0) && waited < secs(19.0) + DEVICE_READ_EVERY * 2 + secs(0.5), + "{waited:?}" + ); + // No monitor, then a monitor arrives before 10 s. + let (status, _) = run( + vec![(secs(0.0), Some(DeviceState::NoMonitor))], + Some(secs(8.0)), + ); + assert_eq!(status, VddStatus::Active); + } + /// A 0.3.2 log and the 0.3.1 field data: the display never attaches, /// the iPad sends its goodbye at 10 s, and USB dials again 1 s later. /// Here no transport step restarts the pipeline while the display @@ -2292,6 +3055,212 @@ mod tests { ); } + /// The PC was locked, or its displays went off, while the second screen + /// proved itself. The viewer stayed on the mirror for the rest of the app + /// run, even after the user came back. + #[test] + fn a_second_screen_that_failed_while_the_desktop_was_away_is_tried_again_once_it_is_back() { + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + let shared = second_screen_selected(); + let away = Arc::new(AtomicBool::new(false)); + shared.second_screen.lock().fake_desktop_away = Some(away.clone()); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + let start = Instant::now(); + assert_eq!(next_step(&shared, start), Step::Switch); + start_proving(&shared, 7, start); + away.store(true, Ordering::Relaxed); + assert_eq!( + next_step(&shared, start + PROVE_TIMEOUT), + Step::Fallback("no_frames") + ); + assert!(!shared.second_screen.lock().memory.saved()); + // Still away: the mirror stays, and the driver is left alone. + let mut elapsed = PROVE_TIMEOUT; + while elapsed < secs(60.0) { + elapsed += secs(0.25); + assert_eq!(next_step(&shared, start + elapsed), Step::Nothing); + } + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + away.store(false, Ordering::Relaxed); + let back = start + elapsed + DESKTOP_CHECK; + assert_eq!(next_step(&shared, back), Step::Attach); + assert_eq!(remembered_failure(&shared), None); + // Once only: the next failure decides afresh. + assert!(shared.second_screen.lock().retry_when_back.is_none()); + } + + /// A second screen proving itself, with a viewer and a fake desktop, + /// whose capture then lost the desktop (`away` at that moment) and showed + /// no picture. Returns the shared state, the fake desktop and when it + /// fell back. + fn fell_back_after_a_desktop_loss( + away_at_loss: bool, + away_at_failure: bool, + ) -> ( + SharedControl, + std::sync::Arc, + Instant, + ) { + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + let shared = second_screen_selected(); + let away = Arc::new(AtomicBool::new(false)); + shared.second_screen.lock().fake_desktop_away = Some(away.clone()); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + let start = Instant::now(); + assert_eq!(next_step(&shared, start), Step::Switch); + start_proving(&shared, 7, start); + note_interruption_while(&shared, 7, away_at_loss); + away.store(away_at_failure, Ordering::Relaxed); + let failed = start + PROVE_TIMEOUT; + assert_eq!(next_step(&shared, failed), Step::Fallback("no_frames")); + (shared, away, failed) + } + + /// Steps every quarter second from `from` for `span`, expecting nothing. + fn nothing_for(shared: &SharedControl, from: Instant, span: Duration) -> Instant { + let mut now = from; + while now < from + span { + now += secs(0.25); + assert_eq!(next_step(shared, now), Step::Nothing, "{:?}", now - from); + } + now + } + + /// Windows said a secure screen had the desktop 13 s after the second + /// screen failed, having lost it: on 0.3.4 the app run stayed on the + /// mirror until Try again. It is tried again once the desktop is back. + #[test] + fn a_lock_windows_reports_after_the_fallback_brings_the_second_screen_back() { + let (shared, away, failed) = fell_back_after_a_desktop_loss(false, false); + let locked = nothing_for(&shared, failed, secs(13.0)); + away.store(true, Ordering::Relaxed); + // Locked: the mirror stays, and the driver is left alone. + let unlocked = nothing_for(&shared, locked, secs(20.0)); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + away.store(false, Ordering::Relaxed); + assert_eq!(next_step(&shared, unlocked + DESKTOP_CHECK), Step::Attach); + assert_eq!(remembered_failure(&shared), None); + let state = shared.second_screen.lock(); + assert!(state.retry_when_back.is_none() && state.late_away_until.is_none()); + } + + /// Capture saying a secure screen holds the desktop counts as well, and + /// the second screen waits until capture has the desktop again. + #[test] + fn a_secure_screen_capture_reports_after_the_fallback_brings_the_second_screen_back() { + use crate::capture::recovery::CaptureBlock; + let (shared, _away, failed) = fell_back_after_a_desktop_loss(false, false); + let blocked = nothing_for(&shared, failed, secs(13.0)); + *shared.capture_block.lock() = Some(CaptureBlock::SecureDesktop); + let resumed = nothing_for(&shared, blocked, secs(10.0)); + *shared.capture_block.lock() = None; + assert_eq!(next_step(&shared, resumed + DESKTOP_CHECK), Step::Attach); + } + + /// A loss while the desktop was away, back by the failure (Windows said + /// so before the fallback): tried again at once. + #[test] + fn a_loss_while_the_desktop_was_away_is_tried_again_once_it_is_back() { + let (shared, _away, failed) = fell_back_after_a_desktop_loss(true, false); + assert_eq!(next_step(&shared, failed + secs(0.25)), Step::Attach); + } + + /// Away at the failure itself: #207's retry, unchanged. + #[test] + fn a_failure_while_the_desktop_is_away_still_waits_for_it() { + let (shared, away, failed) = fell_back_after_a_desktop_loss(false, true); + let back = nothing_for(&shared, failed, secs(20.0)); + away.store(false, Ordering::Relaxed); + assert_eq!(next_step(&shared, back + DESKTOP_CHECK), Step::Attach); + } + + /// No secure screen, or one only after the window: the mirror stays + /// until Try again, as before. + #[test] + fn a_desktop_loss_with_no_lock_is_not_tried_again() { + let (shared, away, failed) = fell_back_after_a_desktop_loss(false, false); + let later = nothing_for(&shared, failed, LATE_AWAY_WINDOW + secs(1.0)); + away.store(true, Ordering::Relaxed); + let back = nothing_for(&shared, later, secs(5.0)); + away.store(false, Ordering::Relaxed); + nothing_for(&shared, back, secs(30.0)); + assert_eq!(remembered_failure(&shared), Some("no_frames")); + } + + /// One retry per lock: a retried second screen that fails the same way + /// with no new lock stays on the mirror. + #[test] + fn a_late_lock_brings_the_second_screen_back_once() { + let (shared, away, failed) = fell_back_after_a_desktop_loss(false, false); + away.store(true, Ordering::Relaxed); + let locked = nothing_for(&shared, failed, secs(2.0)); + away.store(false, Ordering::Relaxed); + let retry = locked + DESKTOP_CHECK; + assert_eq!(next_step(&shared, retry), Step::Attach); + note_attached(&shared, "virtual".into()); + assert_eq!(next_step(&shared, retry), Step::Switch); + start_proving(&shared, 8, retry); + note_interruption_while(&shared, 8, false); + let failed_again = retry + PROVE_TIMEOUT; + assert_eq!( + next_step(&shared, failed_again), + Step::Fallback("no_frames") + ); + nothing_for(&shared, failed_again, LATE_AWAY_WINDOW + secs(5.0)); + } + + /// A driver call that didn't return, or a display that keeps dropping + /// out, is not a desktop transition: a lock afterwards changes nothing. + #[test] + fn a_driver_failure_is_not_tried_again_after_a_lock() { + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + let shared = second_screen_selected(); + let away = Arc::new(AtomicBool::new(false)); + shared.second_screen.lock().fake_desktop_away = Some(away.clone()); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + let start = Instant::now(); + assert_eq!(next_step(&shared, start), Step::Switch); + start_proving(&shared, 7, start); + note_interruption_while(&shared, 7, false); + let _call = driver_call(&shared, 7); + let failed = start + PROVE_TIMEOUT; + assert_eq!(next_step(&shared, failed), Step::Fallback("no_frames")); + away.store(true, Ordering::Relaxed); + let back = nothing_for(&shared, failed, secs(5.0)); + away.store(false, Ordering::Relaxed); + nothing_for(&shared, back, secs(10.0)); + assert!(shared.second_screen.lock().late_away_until.is_none()); + } + + /// A proof that lost the desktop while the user was there is excused + /// too, but tried again only at Try again: retrying at once could turn + /// a display that keeps dropping out on and off forever. + #[test] + fn an_interrupted_proof_with_the_desktop_there_is_not_tried_again_by_itself() { + let shared = second_screen_selected(); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + let start = Instant::now(); + assert_eq!(next_step(&shared, start), Step::Switch); + start_proving(&shared, 7, start); + note_interruption_while(&shared, 7, false); + assert_eq!( + next_step(&shared, start + PROVE_TIMEOUT), + Step::Fallback("no_frames") + ); + for second in 1..60 { + let now = start + PROVE_TIMEOUT + Duration::from_secs(second); + assert_eq!(next_step(&shared, now), Step::Nothing); + } + assert_eq!(remembered_failure(&shared), Some("no_frames")); + } + #[test] fn a_duplication_error_on_the_second_screen_goes_back_to_the_mirror_at_once() { let shared = second_screen_selected(); @@ -2312,6 +3281,293 @@ mod tests { assert_eq!(remembered_failure(&shared), Some("duplication_failed")); } + /// Review r2 of #205 (P1): a second screen that showed no new image for + /// a while after its duplication came back is opened again by the stage. + /// Reconciling it again started a new proof: the watchdog skipped its + /// missing-frame check while the display proved itself, and 5 s later + /// the proof failed back to the mirror. In the generation that proved + /// it, it stays proven, so the watchdog judges a display gone dark. + #[test] + fn a_recovery_reopen_does_not_prove_a_proven_second_screen_again() { + use std::sync::atomic::Ordering::Relaxed; + let dir = std::env::temp_dir().join(format!( + "eternal-capture-recovery-proven-{}", + std::process::id() + )); + let shared = second_screen_selected(); + let display = fake::FakeDisplay::new(Some(Duration::ZERO), secs(10.0)); + display.install(&shared, &dir, std::time::SystemTime::now()); + assert!(Driver::set_enabled(&display, true)); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let streamed = Some(CaptureTarget::Output("virtual".into())); + let reconcile = + || reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation); + assert_eq!(reconcile(), streamed); + assert!(proving(&shared)); + note_image(&shared, generation); + assert!(!proving(&shared)); + // The loop ended with a reopen; the stage reconciles again. + assert_eq!(reconcile(), streamed); + assert!(!proving(&shared), "proven in this generation"); + // Nothing comes from it: the watchdog restarts the pipeline. + let mut machine = crate::supervisor::Machine::new(1); + let now_ms = 100_000; + shared.hb_capture_loop_ms.store(now_ms, Relaxed); + shared.hb_capture_frame_ms.store(now_ms - 5_100, Relaxed); + shared.hb_encode_frame_ms.store(now_ms - 5_100, Relaxed); + let reason = crate::supervisor::wedge_reason_at(&shared, &mut machine, now_ms); + assert!( + reason + .as_deref() + .is_some_and(|reason| reason.starts_with("no captured frame")), + "{reason:?}" + ); + // The next generation, after that restart, proves it again; so does + // a display attached anew. + start_proving(&shared, generation.wrapping_add(1), Instant::now()); + assert!(proving(&shared)); + note_attached(&shared, "virtual".into()); + assert_eq!(reconcile(), streamed); + assert!(proving(&shared)); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A second screen on the fake display, streamed and proven in the + /// current generation. Returns the display, its memory folder and the + /// generation. + fn proven_fake_second_screen( + shared: &SharedControl, + label: &str, + ) -> (fake::FakeDisplay, std::path::PathBuf, u64) { + let dir = std::env::temp_dir().join(format!( + "eternal-capture-recovery-{label}-{}", + std::process::id() + )); + let display = fake::FakeDisplay::new(Some(Duration::ZERO), secs(10.0)); + display.install(shared, &dir, std::time::SystemTime::now()); + assert!(Driver::set_enabled(&display, true)); + connect_viewer(shared); + note_attached(shared, "virtual".into()); + let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(shared)); + note_image(shared, generation); + assert!(!proving(shared)); + (display, dir, generation) + } + + /// Review r3 of #205: the virtual display came back under the same name + /// on another adapter, and capture reopened it. Reconciling matched it by + /// name, and the new display skipped its proof: dark, it would have cost + /// a watchdog restart. Another adapter is another display. + #[test] + fn a_virtual_display_back_on_another_adapter_proves_itself_again() { + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, "moved-luid"); + display.move_virtual_display(true); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(&shared), "same name, another adapter"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Review r3 of #205: the virtual display left the desktop and came back + /// within one generation, without a new attachment from the worker. Seen + /// gone by the reconciliation, or by capture's check of its output, it + /// proves itself again when it is back. + #[test] + fn a_virtual_display_detached_and_back_proves_itself_again() { + // The reconciliation sees it gone. + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, "detached"); + assert!(Driver::set_enabled(&display, false)); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::PrimaryAuto) + ); + assert!(Driver::set_enabled(&display, true)); + *shared.vdd_status.lock() = VddStatus::Active; + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(&shared), "back after a detach"); + let _ = std::fs::remove_dir_all(&dir); + + // Capture's check of the duplicated output sees it off the desktop. + let shared = second_screen_selected(); + let (_display, dir, generation) = proven_fake_second_screen(&shared, "detached-capture"); + note_detached(&shared); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(&shared), "back after a detach capture saw"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Review r4 of #205: capture's own description of the output still said + /// attached, a re-creation failed, and the listing that followed found + /// the virtual display gone. The proof was kept, so the same display back + /// before the next reconciliation inherited it. A listing without the + /// output as it was clears the proof; one with it keeps the proof. + #[test] + fn a_listing_without_the_output_makes_it_prove_itself_again() { + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, "listing"); + let target = CaptureTarget::Output("virtual".into()); + let streamed = Some(target.clone()); + let current = super::super::listing::outputs(&shared) + .into_iter() + .find(|output| output.device_name == "virtual") + .unwrap(); + // The output is still there as it was: no move, and the proof stays. + let listed = super::super::listing::outputs(&shared); + assert!(!super::super::target_moved(&listed, &target, ¤t)); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + streamed + ); + assert!(!proving(&shared), "the same output, still attached"); + // The listing finds it gone; it is back, the same, before the stage + // reconciles again. + assert!(Driver::set_enabled(&display, false)); + let listed = super::super::listing::outputs(&shared); + assert!(super::super::target_moved(&listed, &target, ¤t)); + assert!(Driver::set_enabled(&display, true)); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + streamed + ); + assert!(proving(&shared), "back after the listing saw it gone"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Review r5 of #205: reconciliation saw the proven display attached, the + /// reopen's first listing saw no output at all (no_output_listed), and + /// the same display was back before the next reconciliation. Neither + /// that path nor the stage's error handling cleared the proof. Every + /// listing goes through the judge now: the one capture opens from, the + /// no-display wait's, and the driver's. + #[test] + fn a_reopen_listing_with_no_outputs_makes_it_prove_itself_again() { + // What the reopen's DXGI listing hands the judge: nothing attached. + let shared = second_screen_selected(); + let (_display, dir, generation) = proven_fake_second_screen(&shared, "no-outputs"); + observe_listing(&shared, &[]); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(&shared), "back after a listing with no outputs"); + let _ = std::fs::remove_dir_all(&dir); + + // Each listing entry point, with the display gone while it lists. + type Look = fn(&SharedControl); + let looks: [(&str, Look); 2] = [ + ("outputs", |shared| { + super::super::listing::outputs(shared); + }), + ("any_attached", |shared| { + super::super::listing::any_attached(shared); + }), + ]; + for (name, look) in looks { + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, name); + assert!(Driver::set_enabled(&display, false)); + look(&shared); + assert!(Driver::set_enabled(&display, true)); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())), + "{name}" + ); + assert!(proving(&shared), "{name}: back before reconciling"); + let _ = std::fs::remove_dir_all(&dir); + } + + // The worker's listing through its driver. + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, "worker"); + assert!(Driver::set_enabled(&display, false)); + listed(&shared, &display); + assert!(Driver::set_enabled(&display, true)); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(&shared), "worker: back before reconciling"); + let _ = std::fs::remove_dir_all(&dir); + + // A listing that still shows it keeps the proof, whichever lists. + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, "kept"); + super::super::listing::outputs(&shared); + super::super::listing::any_attached(&shared); + listed(&shared, &display); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(!proving(&shared), "still attached throughout"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Review r3 of #205: the same output reopened in the generation that + /// proved it keeps its proof, so a display gone dark reaches the + /// watchdog (see the test above for the restart itself). + #[test] + fn a_reopen_of_the_same_output_keeps_its_proof() { + let shared = second_screen_selected(); + let (_display, dir, generation) = proven_fake_second_screen(&shared, "same-output"); + for _ in 0..3 { + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(!proving(&shared)); + } + let _ = std::fs::remove_dir_all(&dir); + } + + /// Review r2 of #205 (P2): a display that moved during the first proof + /// ends the capture loop with a reopen, and the stage noted it as a + /// capture error: the proof failed, and the second screen went back to + /// the mirror at once. A reopen keeps the proof, its deadline and the + /// interruption. + #[test] + fn a_reopen_during_the_first_proof_keeps_it_going() { + let start = Instant::now(); + let shared = second_screen_selected(); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + assert_eq!(next_step(&shared, start), Step::Switch); + start_proving(&shared, 7, start); + let reopen = crate::capture::recovery::Reopen { + detail: "the display layout changed".into(), + }; + super::super::note_stage_error(&shared, 7, &reopen); + let proving = shared.second_screen.lock().proving.unwrap(); + assert!(proving.interrupted && !proving.capture_failed); + assert_eq!(proving.since, start); + assert_eq!(next_step(&shared, start + secs(4.9)), Step::Nothing); + // Any other capture error still ends the proof at once. + let failed = std::io::Error::other("DuplicateOutput failed"); + super::super::note_stage_error(&shared, 7, &failed); + assert_eq!( + next_step(&shared, start + secs(4.9)), + Step::Fallback("duplication_failed") + ); + } + #[test] fn a_second_screen_that_shows_an_image_stays() { let shared = second_screen_selected(); @@ -2500,7 +3756,7 @@ mod tests { fn the_watchdog_leaves_a_second_screen_proving_itself_alone() { let shared = second_screen_selected(); connect_viewer(&shared); - let machine = crate::supervisor::Machine::new(1); + let mut machine = crate::supervisor::Machine::new(1); let now_ms = 100_000; shared.hb_capture_loop_ms.store(now_ms, Ordering::Relaxed); shared @@ -2512,12 +3768,12 @@ mod tests { note_attached(&shared, "virtual".into()); start_proving(&shared, 7, Instant::now()); assert_eq!( - crate::supervisor::wedge_reason_at(&shared, &machine, now_ms), + crate::supervisor::wedge_reason_at(&shared, &mut machine, now_ms), None ); // After its first image a stalled second screen is a wedge again. note_image(&shared, 7); - assert!(crate::supervisor::wedge_reason_at(&shared, &machine, now_ms).is_some()); + assert!(crate::supervisor::wedge_reason_at(&shared, &mut machine, now_ms).is_some()); } /// Review round 2, defect 1: when no tick ended the proof before the @@ -2948,6 +4204,7 @@ mod tests { .with_ansi(false) .with_writer(move || Sink(Arc::clone(&writer))) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { attach(&shared, &driver, ATTACH_TIMEOUT) }); @@ -2978,6 +4235,68 @@ mod tests { } } + /// The hybrid laptop: the Enable left the virtual display as the only + /// screen, and the display guard gave the panel back without it. The + /// start falls back to the mirror at once, with its own reason, instead + /// of waiting 30 s for a display that cannot attach; the failure is + /// remembered as a timeout's, with the same notice and wait. + #[test] + fn a_display_windows_left_alone_falls_back_at_once() { + use std::sync::Arc; + struct Sink(Arc>>); + impl std::io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let shared = second_screen_selected(); + connect_viewer(&shared); + assert_eq!(next_step(&shared, Instant::now()), Step::Attach); + let on_sleep = |_| {}; + // It would attach later, if it were waited for. + let driver = FakeDriver { + turned_on: TurnedOn::Alone, + ..FakeDriver::new(secs(7.0), Some(secs(1.0)), &on_sleep) + }; + let out = Arc::new(parking_lot::Mutex::new(Vec::::new())); + let writer = Arc::clone(&out); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || Sink(Arc::clone(&writer))) + .finish(); + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, || attach(&shared, &driver, ATTACH_TIMEOUT)); + let lines = String::from_utf8(out.lock().clone()).unwrap(); + assert!( + lines.contains( + "Windows put the desktop on the virtual display alone — capturing the primary \ + display instead second_screen_fallback=\"vdd_only\"" + ), + "{lines}" + ); + assert!(!lines.contains("did not attach in time"), "{lines}"); + assert_eq!( + driver.calls(), + [ + Call::Prepare, + Call::Enable, + Call::Report(Failure::AttachTimeout), + Call::Disable + ] + ); + // Turned off after the gap, not after the attach wait. + let toggles = driver.toggles.borrow().clone(); + assert_eq!(toggles[1].0 - toggles[0].0, MIN_TOGGLE_GAP, "{toggles:?}"); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + assert_eq!(shared.vdd_output.lock().as_deref(), None); + assert_eq!(remembered_failure(&shared), Some("attach_timeout")); + assert_eq!(next_step(&shared, driver.now()), Step::Nothing); + } + /// "Windows added the second screen but kept your desktop off it": the /// iPad keeps the mirror, so after Try again the start waits 30 s for /// the user to choose Extend, as long as the usual wait and never added diff --git a/host/src/capture/second_screen/fake.rs b/host/src/capture/second_screen/fake.rs index 4ab29bcd..c640cb66 100644 --- a/host/src/capture/second_screen/fake.rs +++ b/host/src/capture/second_screen/fake.rs @@ -55,6 +55,8 @@ struct Inner { desktop_away: Arc, /// The display configuration: see [`FakeDisplay::move_screen`]. facts: Arc>>, + /// See [`FakeDisplay::move_virtual_display`]. + virtual_on_other_card: AtomicBool, } impl FakeDisplay { @@ -72,6 +74,7 @@ impl FakeDisplay { toggles: Mutex::new(Vec::new()), desktop_away: Arc::new(AtomicBool::new(false)), facts: Arc::new(Mutex::new(Some(screen_on(None)))), + virtual_on_other_card: AtomicBool::new(false), })) } @@ -131,6 +134,15 @@ impl FakeDisplay { *self.0.facts.lock() = Some(screen_on(card)); } + /// List the virtual display, under the same name, on the PC's other + /// card (true) or on [`APP_GPU`] (false, at first), as when its driver + /// is restarted on another adapter. + pub fn move_virtual_display(&self, other_card: bool) { + self.0 + .virtual_on_other_card + .store(other_card, Ordering::Relaxed); + } + /// Each time it was turned on (true) or off, on its clock, since it was /// made. pub fn toggles(&self) -> Vec<(Duration, bool)> { @@ -146,6 +158,11 @@ impl Driver for FakeDisplay { .lock() .zip(*self.0.attach_after.lock()) .is_some_and(|(on, after)| self.now() >= on + after); + let virtual_card = if self.0.virtual_on_other_card.load(Ordering::Relaxed) { + OTHER_CARD + } else { + APP_CARD + }; let output = |name: &str, primary: bool| OutputInfo { adapter_index: 0, output_index: u32::from(!primary), @@ -156,6 +173,7 @@ impl Driver for FakeDisplay { top: 0, is_primary: primary, adapter_name: APP_GPU.to_string(), + adapter_id: if primary { APP_CARD } else { virtual_card }, }; let mut outputs = vec![output("primary", true)]; if attached { @@ -270,7 +288,10 @@ pub(crate) fn mirror(shared: &SharedControl) -> Result<(), CaptureUnavailable> { } /// The installed fake display's outputs, beside the primary one. -pub(crate) fn outputs(shared: &SharedControl) -> Option> { +pub(in crate::capture) fn outputs( + shared: &SharedControl, + _: crate::capture::listing::Judged, +) -> Option> { installed(shared).map(|display| display.outputs()) } diff --git a/host/src/capture/second_screen/memory.rs b/host/src/capture/second_screen/memory.rs index 72773103..675f0596 100644 --- a/host/src/capture/second_screen/memory.rs +++ b/host/src/capture/second_screen/memory.rs @@ -19,6 +19,12 @@ //! picture, or where Try again was clicked, is exempt from every rule for //! good: a rule never moves a PC that works into the mirror. //! +//! One failure has no wait that ends by itself: [`PANEL_LOST`], a hybrid +//! laptop whose built-in panel the virtual display took away until a +//! restart. Each try costs the user a black screen and a restart, so only +//! choosing Second screen again (Try again, or another mode and back) tries +//! it again, also after an update changes the fingerprint. +//! //! The files are written by a [`Saver`] thread, never by the caller: the //! caller holds the locks the transport takes at every pass, and a write on //! a slow disk took 3 s, long enough for the iPad to give up. The thread @@ -44,6 +50,12 @@ pub(super) const RULE_FILE_NAME: &str = "second-screen-rule.json"; /// Saved for a manifest rule: nothing was tried. const HARDWARE_RULE: &str = "hardware_rule"; +/// The virtual display took the built-in panel away until the PC restarts. +/// Saved as an `attach_timeout` with `panel_lost` set: an older version +/// reads that as a timeout and waits, where it would drop a reason it +/// doesn't know and try again at once. +pub(crate) const PANEL_LOST: &str = "panel_lost"; + /// The file's layout. Another value is ignored, like an unreadable file. const FORMAT: u32 = 1; @@ -72,11 +84,12 @@ const BACKOFF: [Duration; 5] = [ /// no usable display. A failed Enable task is not one: its notice already /// says how to repair the install, and it costs no wait. Nor is a manifest /// rule, but it is saved the same way. -const REMEMBERED: [&str; 4] = [ +const REMEMBERED: [&str; 5] = [ "attach_timeout", "no_frames", "duplication_failed", HARDWARE_RULE, + PANEL_LOST, ]; /// What decides whether the second screen can work on this PC. Every value @@ -189,6 +202,9 @@ struct Record { failures: u32, /// When the last one failed, in seconds since 1970. failed_at: u64, + /// The failure is [`PANEL_LOST`]. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + panel_lost: bool, } /// Connections mirror without trying: why, and when the host tries again. @@ -197,6 +213,9 @@ pub(crate) struct Hold { pub reason: &'static str, pub failures: u32, pub retry_in: Duration, + /// No wait ends it: only choosing Second screen again ([`PANEL_LOST`]). + /// `retry_in` is then zero. + pub until_chosen: bool, } #[derive(Debug, Clone, Copy)] @@ -204,6 +223,8 @@ struct Failed { reason: &'static str, failures: u32, failed_at: u64, + /// Noted by this app run, not read from the file. + this_run: bool, } /// Where the failure is saved and what it is for. Without one, as in unit @@ -250,6 +271,8 @@ pub(crate) struct Memory { /// failure and its count stay for the next app run. excused: bool, rule: Option, + /// When Windows last started, in seconds since 1970, once known. + booted_at: Option, } impl Memory { @@ -277,7 +300,10 @@ impl Memory { let _ = remove_file(&path); None } - Ok(Some((record, _))) if record.fingerprint != fingerprint => { + // A lost panel stays saved through an update: see [`PANEL_LOST`]. + Ok(Some((record, reason))) + if record.fingerprint != fingerprint && reason != PANEL_LOST => + { tracing::info!( reason = %record.reason, changed = %record.fingerprint.changes(&fingerprint).join(","), @@ -291,6 +317,7 @@ impl Memory { reason, failures: record.failures, failed_at: record.failed_at, + this_run: false, }), }; let saver = Arc::new(Saver::new(spawn)); @@ -309,12 +336,15 @@ impl Memory { failed, excused: false, rule: None, + booted_at: None, }; if let Some(failed) = memory.failed { + let hold = memory.hold(now); tracing::info!( reason = failed.reason, failures = failed.failures, - retry_in_s = memory.hold(now).map_or(0, |hold| hold.retry_in.as_secs()), + retry_in_s = hold.map_or(0, |hold| hold.retry_in.as_secs()), + until_chosen = hold.is_some_and(|hold| hold.until_chosen), "The second screen failed on this PC before" ); } @@ -403,6 +433,14 @@ impl Memory { /// waiting longer. pub(crate) fn hold(&self, now: Instant) -> Option { let failed = self.failed?; + if failed.reason == PANEL_LOST { + return Some(Hold { + reason: failed.reason, + failures: failed.failures, + retry_in: Duration::ZERO, + until_chosen: true, + }); + } let wall = self.wall(now)?; let retry_at = failed .failed_at @@ -411,6 +449,27 @@ impl Memory { reason: failed.reason, failures: failed.failures, retry_in: Duration::from_secs(retry_at - wall), + until_chosen: false, + }) + } + + /// When Windows last started, in seconds since 1970: a lost panel saved + /// before it was mended by that restart. + #[cfg_attr(not(any(windows, test)), allow(dead_code))] // Windows reads the start. + pub(crate) fn set_boot(&mut self, booted_at: u64) { + self.booted_at = Some(booted_at); + } + + /// Whether the saved failure is a lost panel that is still off: noted by + /// this app run, or saved since Windows last started. Not one from before + /// a restart, nor any while the start is unknown. + pub(crate) fn panel_lost_now(&self) -> bool { + self.failed.is_some_and(|failed| { + failed.reason == PANEL_LOST + && (failed.this_run + || self + .booted_at + .is_some_and(|booted_at| failed.failed_at >= booted_at)) }) } @@ -433,11 +492,13 @@ impl Memory { let failed_at = self.wall(now)?; let store = self.store.as_ref()?; let failures = self.failed.map_or(0, |f| f.failures).saturating_add(1); + let panel_lost = reason == PANEL_LOST; match encode(Some(Record { fingerprint: store.fingerprint.clone(), - reason: reason.to_string(), + reason: if panel_lost { "attach_timeout" } else { reason }.to_string(), failures, failed_at, + panel_lost, })) { Ok(data) => store.save(store.path.clone(), Save::Failure(data), now), Err(error) => warn!(%error, "{}", FAILURE_NOT_SAVED), @@ -447,6 +508,7 @@ impl Memory { reason, failures, failed_at, + this_run: true, }); self.hold(now) } @@ -861,6 +923,11 @@ fn read(path: &Path) -> Result, String> { return Err("no failures".to_string()); } let reason = remembered(&record.reason).ok_or_else(|| format!("reason {}", record.reason))?; + let reason = if record.panel_lost { + PANEL_LOST + } else { + reason + }; Ok(Some((record, reason))) } @@ -891,9 +958,10 @@ fn overwrite_file(path: &Path, data: &[u8]) -> std::io::Result<()> { #[cfg(test)] mod tests { use super::super::{ - attach, next_step, note_attached, note_connected, note_image, remembered_failure, - retries_later, set_facts, start_proving, Driver, Failure, Step, BLOCKED_EXTEND_RETRY, - PROVE_TIMEOUT, + attach, driver_supports_windows, next_step, note_attached, note_connected, note_image, + note_windows, notice_failure, remembered_failure, retries_later, set_facts, start_proving, + windows_notice, Driver, Failure, Step, TurnedOn, BLOCKED_EXTEND_RETRY, PANEL_LOST_BEFORE, + PROVE_TIMEOUT, WINDOWS_10, WINDOWS_10_IPAD_NOTICE, }; use super::*; use crate::capture::tests::{connect_viewer, hybrid_laptop, refuse_moved_mirror, NVIDIA}; @@ -1011,11 +1079,14 @@ mod tests { store.saver.saving.lock().clear_failure_logged } - /// A display that never attaches, on a paused clock. + /// A display that never attaches, on a paused clock. `turned_on` says + /// how its Enable left the PC's screens. struct NeverAttaches { start: Instant, elapsed: Cell, enables: Cell, + disables: Cell, + turned_on: TurnedOn, } impl NeverAttaches { @@ -1024,6 +1095,8 @@ mod tests { start, elapsed: Cell::new(Duration::ZERO), enables: Cell::new(0), + disables: Cell::new(0), + turned_on: TurnedOn::Beside, } } } @@ -1040,18 +1113,26 @@ mod tests { top: 0, is_primary: true, adapter_name: "Test GPU".to_string(), + adapter_id: (0, 1), }] } fn prepare(&self, _outputs: &[OutputInfo]) {} fn set_enabled(&self, enabled: bool) -> bool { - if enabled { - self.enables.set(self.enables.get() + 1); - } + let count = if enabled { + &self.enables + } else { + &self.disables + }; + count.set(count.get() + 1); true } + fn turned_on(&self) -> TurnedOn { + self.turned_on + } + fn now(&self) -> Instant { self.start + self.elapsed.get() } @@ -1061,6 +1142,116 @@ mod tests { } } + /// The hybrid laptop: the Enable took the built-in panel away until a + /// restart. The display stays on, so the desktop stays on the iPad, and + /// no wait ends the hold: not the next app run a week later, not an + /// update, not a blocked mirror. Only choosing Second screen again tries + /// it. The record reads as a timeout to an older version, which then + /// waits instead of trying at once. + #[test] + fn a_lost_panel_is_held_until_second_screen_is_chosen_again() { + let path = temp_file("panel-lost"); + let (memory, start) = app_run(&path, Duration::ZERO); + let shared = second_screen_with(memory); + connect_viewer(&shared); + assert_eq!(next_step(&shared, start), Step::Attach); + let driver = NeverAttaches { + turned_on: TurnedOn::PanelLost, + ..NeverAttaches::new(start) + }; + attach(&shared, &driver, ATTACH_TIMEOUT); + assert_eq!((driver.enables.get(), driver.disables.get()), (1, 0)); + assert_eq!(driver.now(), start, "no attach wait, no toggle gap"); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + assert_eq!(remembered_failure(&shared), Some(PANEL_LOST)); + assert!(!retries_later(&shared), "the notice says how to try again"); + let record = &saved(&path)["failure"]; + assert_eq!( + ( + &record["reason"], + &record["panel_lost"], + &record["failures"] + ), + ( + &serde_json::json!("attach_timeout"), + &serde_json::json!(true), + &serde_json::json!(1) + ) + ); + + // A week later, after an update, with Windows refusing the mirror. + written(&path); + let later = Instant::now(); + let updated = Fingerprint { + app: "0.3.6".to_string(), + ..laptop() + }; + let memory = Memory::load(path.clone(), updated, T0 + 7 * DAY, later); + let shared = second_screen_with(memory); + refuse_moved_mirror(&shared); + connect_viewer(&shared); + note_connected(&shared, later); + for day in [0, 1, 30] { + let now = later + Duration::from_secs(day * DAY); + reconnect(&shared, now); + assert_eq!(next_step(&shared, now), Step::Nothing, "day {day}"); + } + assert_eq!(remembered_failure(&shared), Some(PANEL_LOST)); + let hold = shared.second_screen.lock().memory.hold(later).unwrap(); + assert!(hold.until_chosen); + + // Choosing it again (another mode, then Second screen) tries it. + *shared.capture_target.lock() = CaptureTarget::PrimaryAuto; + assert_eq!(next_step(&shared, later), Step::Nothing); + assert!(!exists(&path)); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + *shared.vdd_status.lock() = VddStatus::WaitingForClient; + assert_eq!(next_step(&shared, later), Step::Attach); + remove_folder(&path); + } + + /// The Stream page asks for a restart only while the panel is off: in + /// the app run that lost it, and in a later one before the PC restarts. + /// After the restart the panel is back, and the notice only says that + /// Second screen waits to be chosen again. + #[test] + fn a_lost_panel_asks_for_a_restart_only_until_the_pc_restarted() { + let path = temp_file("panel-lost-notice"); + let (memory, start) = app_run(&path, Duration::ZERO); + let shared = second_screen_with(memory); + connect_viewer(&shared); + assert_eq!(next_step(&shared, start), Step::Attach); + let driver = NeverAttaches { + turned_on: TurnedOn::PanelLost, + ..NeverAttaches::new(start) + }; + attach(&shared, &driver, ATTACH_TIMEOUT); + assert_eq!(notice_failure(&shared), Some(PANEL_LOST)); + // The user quits and starts the app again, with no restart between: + // Windows started an hour before the failure. + for (booted_at, notice) in [ + (T0 - HOUR, PANEL_LOST), + // The restart, ten minutes after it. + (T0 + 600, PANEL_LOST_BEFORE), + ] { + let (mut memory, later) = app_run(&path, Duration::from_secs(1200)); + memory.set_boot(booted_at); + let shared = second_screen_with(memory); + connect_viewer(&shared); + note_connected(&shared, later); + assert_eq!(next_step(&shared, later), Step::Nothing, "held"); + assert_eq!(remembered_failure(&shared), Some(PANEL_LOST)); + assert_eq!(notice_failure(&shared), Some(notice), "{booted_at}"); + } + // Without the start known, a saved one counts as from before. + let (memory, later) = app_run(&path, Duration::from_secs(1200)); + let shared = second_screen_with(memory); + connect_viewer(&shared); + assert_eq!(next_step(&shared, later), Step::Nothing); + assert_eq!(notice_failure(&shared), Some(PANEL_LOST_BEFORE)); + remove_folder(&path); + } + /// The field case: the display never attaches. The next app run mirrors /// at once, without the driver or the 30 s wait, says why once per /// connection, and tries again after the wait. @@ -2378,6 +2569,111 @@ mod tests { ))); } + /// Windows 10: a Second screen connection mirrors at once, with no + /// Enable, and says why: its own log line, and the iPad's + /// notice lasts a few seconds from each connection. Neither a failure + /// an earlier version saved, Try again, nor choosing Second screen again + /// turns the driver on. + #[test] + fn on_windows_10_second_screen_mirrors_without_turning_the_driver_on() { + let path = temp_file("windows-10"); + let (mut memory, start) = app_run(&path, Duration::ZERO); + memory.note_failure("attach_timeout", start); + let shared = second_screen_with(memory); + note_windows(&shared, Some(19045), false); + connect_viewer(&shared); + note_connected(&shared, start); + let (log, step) = logs(|| next_step(&shared, start)); + assert_eq!(step, Step::Nothing, "no Enable"); + assert!( + log.contains("Second screen held: needs Windows 11") && log.contains(WINDOWS_10), + "{log}" + ); + assert!( + !log.contains("second_screen_fallback") && !log.contains("remembered"), + "an expected hold is no fallback for the daily check: {log}" + ); + assert!(shared.vdd_worker.queue.lock().commands.is_empty()); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + assert_eq!(remembered_failure(&shared), Some(WINDOWS_10)); + assert_eq!(notice_failure(&shared), Some(WINDOWS_10)); + let generation = + crate::supervisor::CURRENT_GENERATION.load(std::sync::atomic::Ordering::SeqCst); + assert_eq!( + crate::capture::reconcile_virtual_display( + &CaptureTarget::VirtualExtended, + &shared, + generation + ), + Some(CaptureTarget::PrimaryAuto), + "the mirror" + ); + // The iPad's notice, for a while from the connection. + assert!(windows_notice(&shared, start)); + assert!(windows_notice( + &shared, + start + WINDOWS_10_IPAD_NOTICE - secs(1) + )); + assert!(!windows_notice(&shared, start + WINDOWS_10_IPAD_NOTICE)); + + // The next connection says it again, without the saved failure. + let later = start + Duration::from_secs(60); + let (log, step) = logs(|| { + reconnect(&shared, later); + next_step(&shared, later) + }); + assert_eq!(step, Step::Nothing); + assert!( + log.contains("Second screen held: needs Windows 11") + && !log.contains("second_screen_fallback") + && !log.contains("remembered"), + "{log}" + ); + assert!(windows_notice(&shared, later)); + + shared.retry_second_screen(); + assert_eq!(next_step(&shared, later), Step::Nothing, "Try again"); + *shared.capture_target.lock() = CaptureTarget::PrimaryAuto; + assert_eq!(next_step(&shared, later), Step::Nothing); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + *shared.vdd_status.lock() = VddStatus::WaitingForClient; + assert_eq!(next_step(&shared, later), Step::Nothing, "chosen again"); + assert_eq!(remembered_failure(&shared), Some(WINDOWS_10)); + assert!(shared.vdd_worker.queue.lock().commands.is_empty()); + } + + /// Windows 11, an unknown build and the tester's override try the + /// second screen as before, with no Windows 10 notice. + #[test] + fn windows_11_and_the_windows_10_override_try_the_second_screen() { + assert!(!driver_supports_windows(Some(19045), false)); + assert!(!driver_supports_windows(Some(21999), false)); + assert!(driver_supports_windows(Some(22000), false)); + assert!(driver_supports_windows(Some(26200), false)); + assert!(driver_supports_windows(None, false)); + assert!(driver_supports_windows(Some(19045), true)); + for (build, forced) in [ + (Some(26200), false), + (Some(22000), false), + (None, false), + (Some(19045), true), + ] { + let path = temp_file("windows-11"); + let (memory, start) = app_run(&path, Duration::ZERO); + let shared = second_screen_with(memory); + note_windows(&shared, build, forced); + connect_viewer(&shared); + note_connected(&shared, start); + assert_eq!( + next_step(&shared, start), + Step::Attach, + "{build:?} {forced}" + ); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Attaching); + assert!(!windows_notice(&shared, start)); + } + } + /// The manifest names this PC: Second screen sessions start in the /// mirror with the usual notice, and Try again works and is kept for /// this PC's cards, through driver and Windows updates. @@ -3002,6 +3298,7 @@ mod tests { .with_ansi(false) .with_writer(move || sink.clone()) .finish(); + crate::logging::prepare_log_capture(); let value = tracing::subscriber::with_default(subscriber, act); let log = String::from_utf8(out.lock().clone()).unwrap(); (log, value) diff --git a/host/src/capture/synthetic.rs b/host/src/capture/synthetic.rs index fadc35bb..050af609 100644 --- a/host/src/capture/synthetic.rs +++ b/host/src/capture/synthetic.rs @@ -9,6 +9,10 @@ //! Every frame carries its frame number encoded as a row of black/white //! squares that survives lossy H.264 encoding, so end-to-end tests can decode //! frames and assert real progression through the entire stack. +//! +//! `ETERNAL_SYNTH_POINTER=1` instead holds a still picture and moves a +//! pointer over it every frame, as when only the mouse moves on a still +//! desktop (see [`PointerScene`]). use std::sync::atomic::Ordering; use std::sync::Arc; @@ -16,7 +20,8 @@ use std::time::Instant; use tracing::info; -use super::{frame_budget_for, heartbeat, FrameSlot, RawFrame}; +use super::pointer::{convert_shape, PointerImage, Position, SHAPE_COLOR}; +use super::{frame_budget_for, heartbeat, keepalive_due, FrameSlot, RawFrame, IDLE_KEEPALIVE}; use crate::control::SharedControl; use crate::stats::PIPELINE_STATS; @@ -44,9 +49,25 @@ fn synthetic_size() -> (u32, u32) { (DEFAULT_WIDTH, DEFAULT_HEIGHT) } +/// `ETERNAL_SYNTH_STILL=1`: a desktop that never changes. After its first +/// picture the source resends it only as DXGI capture does on a still +/// desktop, for the idle keepalive or a keyframe request (see +/// [`keepalive_due`]), so a test can time a new viewer's first frame there. +fn still_desktop() -> bool { + std::env::var("ETERNAL_SYNTH_STILL").is_ok_and(|v| v.trim() == "1") +} + +/// `ETERNAL_SYNTH_REFUSE_GPU=1`, debug builds only: Windows refuses the +/// screen from the graphics card EternalMonitor runs on, as on the hybrid +/// laptops, for as long as it is set. Each attempt reads it again. +#[cfg(debug_assertions)] +fn gpu_refused() -> bool { + std::env::var("ETERNAL_SYNTH_REFUSE_GPU").is_ok_and(|v| v.trim() == "1") +} + /// Produce frames until `shared.running` clears or the channel closes. /// Mirrors the DXGI loop's contract: paced by `target_fps`, records capture -/// stats, ships BGRA `RawFrame`s. +/// stats, ships BGRA `RawFrame`s, and clears a block with its first one. pub fn run_capture_loop( slot: &FrameSlot, shared: SharedControl, @@ -63,8 +84,16 @@ pub fn run_capture_loop( if !proving { super::second_screen::fake::mirror(&shared)?; } + #[cfg(debug_assertions)] + if gpu_refused() { + return Err(Box::new(super::recovery::CaptureUnavailable { + block: super::recovery::CaptureBlock::UnsupportedOnThisGpu, + detail: "simulated: DXGI_ERROR_UNSUPPORTED (0x887A0004)".into(), + })); + } let (width, height) = synthetic_size(); let mut buffers = capture_buffers(width, height); + let mut resumed = false; info!(width, height, "Synthetic capture source active"); *shared.capture_geometry.lock() = Some(crate::input::CaptureGeometry { @@ -78,6 +107,14 @@ pub fn run_capture_loop( // avoids normal sleep coalescing without burning an entire CPU core. let mut timer = super::timing::FrameTimer::new(); let mut next_deadline = Instant::now(); + let mut pointer = std::env::var("ETERNAL_SYNTH_POINTER") + .is_ok_and(|v| v.trim() == "1") + .then(|| PointerScene::new(&shared, width, height)); + // `ETERNAL_SYNTH_STILL=1`: the first picture, then only what DXGI sends + // for a desktop that never changes. + let mut still: Option>> = None; + let mut last_send = Instant::now(); + let still_mode = still_desktop(); loop { if !shared.running.load(Ordering::SeqCst) @@ -92,22 +129,57 @@ pub fn run_capture_loop( heartbeat(&shared.hb_capture_loop_ms); - *frame_number += 1; - let data = render_next_frame(&mut buffers, width, height, *frame_number); - - PIPELINE_STATS.lock().record_capture(width, height); - heartbeat(&shared.hb_capture_frame_ms); - - slot.publish(RawFrame { - frame_number: *frame_number, - timestamp: frame_start, - data, - width, - height, - }); - #[cfg(debug_assertions)] - if std::mem::take(&mut proving) { - super::second_screen::note_image(&shared, generation); + // A still picture sent again, as DXGI capture marks its resends. + let mut cached = false; + let data = if let Some(scene) = pointer.as_mut() { + cached = shared.client_draws_pointer(); + scene.tick(&shared, &mut buffers, frame_start, frame_number) + } else { + let data = match still.as_ref() { + Some(picture) + if shared.client_connected() + && keepalive_due( + frame_start.duration_since(last_send), + shared.force_next_idr.load(Ordering::SeqCst), + true, + ) => + { + cached = true; + Arc::clone(picture) + } + Some(_) => { + timer.wait_until(frame_start + frame_budget); + continue; + } + None => render_next_frame(&mut buffers, width, height, *frame_number + 1), + }; + *frame_number += 1; + last_send = frame_start; + if still_mode && still.is_none() { + still = Some(Arc::clone(&data)); + } + Some(data) + }; + + if let Some(data) = data { + PIPELINE_STATS.lock().record_capture(width, height); + heartbeat(&shared.hb_capture_frame_ms); + + slot.publish(RawFrame { + frame_number: *frame_number, + timestamp: frame_start, + data, + width, + height, + cached, + }); + if !std::mem::replace(&mut resumed, true) { + super::recovery::note_capture_resumed(&shared); + } + #[cfg(debug_assertions)] + if std::mem::take(&mut proving) { + super::second_screen::note_image(&shared, generation); + } } next_deadline += frame_budget; @@ -123,6 +195,146 @@ pub fn run_capture_loop( Ok(()) } +/// A still picture with a pointer that moves every frame. For a client that +/// draws the pointer it goes to the pointer feed only, and a frame is sent +/// only for an IDR or the idle keepalive, as the DXGI loop does. For any +/// other client each move is a new frame with the pointer drawn in. +struct PointerScene { + width: u32, + height: u32, + picture: Vec, + image: PointerImage, + step: u32, + last_publish: Option, +} + +impl PointerScene { + fn new(shared: &SharedControl, width: u32, height: u32) -> Self { + let mut picture = super::frame_buffer((width * 4) as usize * height as usize); + render_synthetic_frame(&mut picture, width, height, 0); + let image = arrow(); + shared.pointer.set_shape(image.clone()); + info!(width, height, "Synthetic pointer over a still picture"); + Self { + width, + height, + picture, + image, + step: 0, + last_publish: None, + } + } + + fn tick( + &mut self, + shared: &SharedControl, + buffers: &mut [Arc>; 3], + now: Instant, + frame_number: &mut u64, + ) -> Option>> { + self.step = self.step.wrapping_add(1); + let (x, y) = pointer_path(self.step, self.width, self.height); + shared.pointer.set_position(Position { + visible: true, + x, + y, + host_time_us: crate::clock::host_now_us(), + }); + let draws = shared.client_draws_pointer(); + let due = !draws + || self + .last_publish + .is_none_or(|at| now - at >= IDLE_KEEPALIVE) + || shared.force_next_idr.load(Ordering::SeqCst); + if !due { + return None; + } + self.last_publish = Some(now); + *frame_number += 1; + let buffer = buffers + .iter_mut() + .find(|buffer| Arc::strong_count(buffer) == 1) + .expect("encoder and latest-frame slot retain at most two buffers"); + let buf = Arc::get_mut(buffer).unwrap(); + buf.copy_from_slice(&self.picture); + draw_frame_counter(buf, self.width, *frame_number); + if !draws { + draw_pointer(buf, self.width, self.height, &self.image, x, y); + } + Some(Arc::clone(buffer)) + } +} + +/// Where the synthetic pointer is at `step`: across the picture and back, +/// a new position every step. +pub fn pointer_path(step: u32, width: u32, height: u32) -> (i32, i32) { + let span = width.saturating_sub(64).max(2); + let along = (step * 5) % (2 * span); + let x = 16 + + if along < span { + along + } else { + 2 * span - along + } as i32; + let y = (height / 2) as i32 + ((step % 40) as i32 - 20); + (x, y) +} + +/// A 12x19 white arrow with a black edge, as Windows' color pointer is. +fn arrow() -> PointerImage { + let (w, h) = (12u32, 19u32); + let mut raw = vec![0u8; (w * h * 4) as usize]; + for y in 0..h { + for x in 0..=y.min(w - 1) { + let edge = x == 0 || x == y.min(w - 1) || y == h - 1; + let v = if edge { 0 } else { 255 }; + raw[((y * w + x) * 4) as usize..][..4].copy_from_slice(&[v, v, v, 255]); + } + } + convert_shape(SHAPE_COLOR, w, h, w * 4, (0, 0), &raw).expect("a valid arrow") +} + +/// Draws a converted pointer image over a BGRA frame: "over" for its +/// premultiplied pixels, then inverting where its mask says. +pub fn draw_pointer( + buf: &mut [u8], + width: u32, + height: u32, + image: &PointerImage, + px: i32, + py: i32, +) { + let (iw, ih) = (usize::from(image.width), usize::from(image.height)); + let mask = (image.flags & eternal_wire::v2::control::POINTER_SHAPE_INVERT != 0) + .then(|| &image.bytes[iw * ih * 4..]); + for row in 0..ih { + let sy = py + row as i32; + if sy < 0 || sy >= height as i32 { + continue; + } + for col in 0..iw { + let sx = px + col as i32; + if sx < 0 || sx >= width as i32 { + continue; + } + let src = &image.bytes[(row * iw + col) * 4..][..4]; + let at = (sy as usize * width as usize + sx as usize) * 4; + let dst = &mut buf[at..at + 3]; + let keep = 255 - u32::from(src[3]); + for c in 0..3 { + dst[c] = (u32::from(src[c]) + u32::from(dst[c]) * keep / 255).min(255) as u8; + } + if mask + .is_some_and(|mask| mask[row * iw.div_ceil(8) + col / 8] & (0x80 >> (col % 8)) != 0) + { + for c in dst.iter_mut() { + *c = !*c; + } + } + } + } +} + /// The encoder and latest-frame slot can each retain one buffer. Keep three /// allocations so a delayed consumer never discards a busy spare and causes /// a fresh full-frame allocation on the next tick. diff --git a/host/src/control.rs b/host/src/control.rs index 30ac372c..37fd1d09 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -17,6 +17,10 @@ pub struct SharedControl { /// always <= `bitrate_bps`. The encoder reopens its session when this /// changes for hardware encoders. libx264 reconfigures the open context. pub abr_current_bps: Arc, + /// The open encoder session takes a new bitrate only by reopening (AMF, + /// libx265, or a session that refused a change in place), which costs a + /// stutter and an IDR, so ABR steps up less often for it. + pub encoder_reopens_for_bitrate: Arc, pub target_fps: Arc, pub client_preferred_fps: Arc, pub encoder_input: Arc>, @@ -30,6 +34,11 @@ pub struct SharedControl { /// Encoder name override from the Settings tab, applied on next pipeline restart. /// `None` means use auto-detected encoder from GpuInfo. pub encoder_override: Arc>>, + /// The DXGI adapter of the encoder that streams, for waking it when an + /// iPad looks for PCs (see `gpu_wake`): set at each encoder open and + /// when a late detection is adopted, None for a CPU encoder or one on + /// a card detection didn't choose. + pub wake_adapter: Arc>>, /// Which display output the capture loop should duplicate, applied on next pipeline /// restart. `PrimaryAuto` mirrors the primary monitor (today's behavior). pub capture_target: Arc>, @@ -42,6 +51,14 @@ pub struct SharedControl { /// Set while Windows blocks screen capture (lock screen, UAC, a display /// it will not duplicate); the GUI explains it instead of a frozen picture. pub capture_block: Arc>>, + /// Since when the heartbeats have seen Windows refuse the screen from + /// this graphics card, for their notice (`recovery::heartbeat_notice`). + pub gpu_refused_since: Arc>>, + /// The stream port between two owners (a generation's transport and the + /// standby responder), bound and with its queue, so a HELLO2 sent while + /// it changes hands waits for the next one. The supervisor closes it + /// when it shuts down. + pub stream_socket: Arc>>, /// With `UnsupportedOnThisGpu`: the card the captured screen is wired to /// and the switch that would move EternalMonitor onto it. pub gpu_diagnosis: Arc>>, @@ -76,11 +93,20 @@ pub struct SharedControl { pub hb_capture_loop_ms: Arc, pub hb_capture_frame_ms: Arc, pub hb_encode_frame_ms: Arc, + /// The transport loop's heartbeat and the step it was taking, which the + /// supervisor watches (`transport::beat`). + pub transport_beat: Arc, + /// Wakes the transport to send a goodbye the PC going down left, and + /// counts the goodbyes it sent (`crate::power`). + pub power: Arc, /// Desktop-space rectangle of the output currently being captured, set by /// the capture stage at (re)init. The input relay maps the client's /// normalized touch coordinates onto this rect. `None` until the first /// capture stage comes up. pub capture_geometry: Arc>>, + /// The one thread that hands iPad input to Windows, for every stream + /// generation of this app. + pub input: Arc, /// The user's "prefer HEVC" setting. The encoder switches to an H.265 /// session only when this is on AND the connected client advertised HEVC /// decode in its HELLO2. @@ -100,6 +126,11 @@ pub struct SharedControl { pub pairing: Arc>, /// Which network the next stream listens on. pub listen_on: Arc>, + /// The pointer for an iPad that draws it (see `capture::pointer`). + pub pointer: Arc, + /// The session whose iPad reported it holds the pointer image whole + /// (see [`SharedControl::client_draws_pointer`]). + pub(crate) pointer_confirmed: Arc, } /// Runtime state of the managed virtual extended display. @@ -162,9 +193,12 @@ impl SharedControl { .expect("OS entropy for pairing code"), )), listen_on: Arc::new(Mutex::new(Default::default())), + pointer: Arc::new(Default::default()), + pointer_confirmed: Arc::new(AtomicU32::new(0)), running: Arc::new(AtomicBool::new(false)), bitrate_bps: Arc::new(AtomicU32::new(initial_bitrate_bps)), abr_current_bps: Arc::new(AtomicU32::new(initial_bitrate_bps)), + encoder_reopens_for_bitrate: Arc::new(AtomicBool::new(false)), target_fps: Arc::new(AtomicU32::new(DEFAULT_TARGET_FPS)), client_preferred_fps: Arc::new(AtomicU32::new(0)), encoder_input: Arc::new(Mutex::new(Default::default())), @@ -175,10 +209,13 @@ impl SharedControl { ))))), force_next_idr: Arc::new(AtomicBool::new(false)), encoder_override: Arc::new(Mutex::new(None)), + wake_adapter: Arc::new(Mutex::new(None)), capture_target: Arc::new(Mutex::new(CaptureTarget::PrimaryAuto)), vdd_status: Arc::new(Mutex::new(VddStatus::Inactive)), vdd_problem: Arc::new(Mutex::new(None)), capture_block: Arc::new(Mutex::new(None)), + gpu_refused_since: Arc::new(Mutex::new(None)), + stream_socket: Arc::new(Mutex::new(None)), gpu_diagnosis: Arc::new(Mutex::new(None)), screen_watch: Arc::new(Mutex::new(crate::gpu_preference::ScreenWatch::new())), gpu_switch: Arc::new(Mutex::new(None)), @@ -191,7 +228,10 @@ impl SharedControl { hb_capture_loop_ms: Arc::new(AtomicU64::new(0)), hb_capture_frame_ms: Arc::new(AtomicU64::new(0)), hb_encode_frame_ms: Arc::new(AtomicU64::new(0)), + transport_beat: Default::default(), + power: Default::default(), capture_geometry: Arc::new(Mutex::new(None)), + input: Arc::default(), hevc_enabled: Arc::new(AtomicBool::new(false)), audio_enabled: Arc::new(AtomicBool::new(true)), active_codec: Arc::new(AtomicU8::new(eternal_wire::v2::control::CODEC_H264)), @@ -228,6 +268,26 @@ impl SharedControl { /// A client is connected and heard from: one whose session is on hold /// after a silence counts as gone for the second screen, so its display /// still goes the same time after the iPad's last packet. + /// The connected iPad draws the pointer, so capture leaves it out of + /// the video. From when its reports say it holds the current pointer + /// image whole: until then it has nothing to draw, so the pointer stays + /// in the video and the positions sent say hidden. + pub fn client_draws_pointer(&self) -> bool { + let session = self.session.lock().pointer_session_id(); + session.is_some_and(|id| self.pointer_confirmed.load(Ordering::Relaxed) == id) + } + + /// A receiver report named the pointer image its iPad holds. + pub(crate) fn note_pointer_report(&self, shape_id: u32) { + let Some(session) = self.session.lock().pointer_session_id() else { + return; + }; + let current = self.pointer.latest().0.map(|shape| shape.id); + if shape_id != 0 && current == Some(shape_id) { + self.pointer_confirmed.store(session, Ordering::Relaxed); + } + } + pub(crate) fn client_heard(&self) -> bool { let session = self.session.lock(); session.is_active() && !session.on_hold() @@ -252,6 +312,13 @@ pub fn merge_fps(host: u32, client: u32) -> u32 { pub enum SupervisorCommand { /// Tear the current pipeline down and start a fresh generation. Restart, + /// The stream of `generation` asks to start again: its chosen network + /// changed, or its virtual display has to go. Only while that stream + /// still runs: one the user stopped, or one a newer stream replaced, + /// stays as it is. + RestartStream { + generation: u64, + }, /// Stop streaming (pipeline down, GUI stays). `Start` resumes. Stop, /// Start a fresh pipeline from Stopped/Failed. @@ -316,6 +383,79 @@ mod tests { } } + /// Capture asks this before leaving the pointer out of the video. + #[test] + fn capture_draws_the_pointer_unless_the_connected_ipad_does() { + use eternal_wire::v2::control::{ControlMessage, Hello2, FEATURE_DRAWS_POINTER}; + let shared = SharedControl::new(9876, 15_000_000); + assert!(!shared.client_draws_pointer(), "nobody connected"); + for (nonce, caps, draws) in [(1, 0, false), (2, FEATURE_DRAWS_POINTER, true)] { + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: nonce, + listen_port: 50000, + decoder_caps: 1, + feature_caps: caps, + screen_px_w: 2420, + screen_px_h: 1668, + screen_pt_w: 1210, + screen_pt_h: 834, + refresh_hz: 120, + device_name: "Test iPad".into(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + struct Config; + impl crate::transport::session::ConfigSource for Config { + fn stream_config(&self) -> eternal_wire::v2::control::StreamConfig { + Default::default() + } + fn host_name(&self) -> String { + "TEST".into() + } + } + shared.session.lock().handle_control( + PeerId::udp("10.0.0.5:50000".parse().unwrap()), + 0, + hello, + &Config, + std::time::Instant::now(), + ); + assert!(shared.client_connected()); + assert!(!shared.client_draws_pointer(), "not before it answers"); + let session_id = shared.session.lock().session_id().unwrap(); + shared.session.lock().handle_control( + PeerId::udp("10.0.0.5:50000".parse().unwrap()), + session_id, + ControlMessage::ReceiverReport(Default::default()), + &Config, + std::time::Instant::now(), + ); + assert!( + !shared.client_draws_pointer(), + "not before it holds the image" + ); + shared + .pointer + .set_shape(crate::capture::pointer::PointerImage { + source_type: 2, + flags: 0, + width: 1, + height: 1, + hot: (0, 0), + bytes: vec![nonce as u8; 4], + }); + let current = shared.pointer.latest().0.unwrap().id; + shared.note_pointer_report(current + 1); + assert!(!shared.client_draws_pointer(), "another image"); + shared.note_pointer_report(current); + assert_eq!(shared.client_draws_pointer(), draws, "caps {caps:#x}"); + } + } + #[test] fn client_connected_requires_a_session_not_just_an_address() { let shared = SharedControl::new(9876, 15_000_000); diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 02ccf0d6..b5ef117e 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -5,7 +5,7 @@ use std::time::Duration; use crate::control::{CaptureTarget, SharedControl}; use crate::transport::link::LinkId; -use crate::transport::session::{EndReason, SessionEnd}; +use crate::transport::session::{EndReason, SessionEnd, StageFailure, StreamStop}; use tracing::info; static SYSTEM_REPORT: OnceLock = OnceLock::new(); @@ -34,6 +34,7 @@ pub fn log_system_report() { extern "C" { fn eternal_install_ffmpeg_logger(sink: unsafe extern "C" fn(i32, *const libc::c_char)); + fn eternal_ffmpeg_verbose(wanted: libc::c_int); } /// Install once, before encoder threads start. The C shim owns the platform's va_list ABI. @@ -45,6 +46,53 @@ pub fn install_ffmpeg_logging() { thread_local! { /// Set while this thread opens an encoder that is one of several to try. static PROBING: std::cell::Cell = const { std::cell::Cell::new(false) }; + /// FFmpeg's lines from this thread while it watches them (see + /// [`FfmpegWatch`]), verbose ones included. + static WATCHED: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; +} + +/// Threads watching FFmpeg's verbose lines. The shim forwards them while +/// any watches; FFmpeg's own level never changes after it is installed. +static WATCHERS: std::sync::Mutex = std::sync::Mutex::new(0); + +/// Keeps FFmpeg's lines from this thread, its verbose ones included, from +/// [`FfmpegWatch::start`] to [`FfmpegWatch::finish`]. Warnings and errors +/// are logged as always; verbose lines are only kept, never logged. Lines +/// from other threads meanwhile are not kept, and their verbose lines are +/// dropped as before. +pub struct FfmpegWatch(()); + +impl FfmpegWatch { + pub fn start() -> Self { + install_ffmpeg_logging(); + WATCHED.with(|watched| *watched.borrow_mut() = Some(Vec::new())); + let mut watchers = WATCHERS.lock().unwrap_or_else(|e| e.into_inner()); + *watchers += 1; + if *watchers == 1 { + unsafe { eternal_ffmpeg_verbose(1) }; + } + Self(()) + } + + /// The lines this thread logged since the watch started. + pub fn finish(self) -> Vec { + let lines = WATCHED + .with(|watched| watched.borrow_mut().take()) + .unwrap_or_default(); + drop(self); + lines + } +} + +impl Drop for FfmpegWatch { + fn drop(&mut self) { + WATCHED.with(|watched| watched.borrow_mut().take()); + let mut watchers = WATCHERS.lock().unwrap_or_else(|e| e.into_inner()); + *watchers = watchers.saturating_sub(1); + if *watchers == 0 { + unsafe { eternal_ffmpeg_verbose(0) }; + } + } } /// Runs `open`, an encoder open with another encoder to try if it fails, @@ -74,6 +122,15 @@ unsafe extern "C" fn ffmpeg_message(level: i32, message: *const libc::c_char) { if message.is_empty() { return; } + WATCHED.with(|watched| { + if let Some(lines) = watched.borrow_mut().as_mut() { + lines.push(message.to_string()); + } + }); + // Only a watching thread wants verbose lines (see `FfmpegWatch`). + if level > ffmpeg_next::ffi::AV_LOG_WARNING { + return; + } // `ffmpeg_level` keeps FFmpeg's own level either way. if level <= ffmpeg_next::ffi::AV_LOG_ERROR && !PROBING.with(std::cell::Cell::get) { tracing::error!(target: "ffmpeg", ffmpeg_level = level, "{message}"); @@ -126,6 +183,7 @@ fn log_stream_summary(shared: &SharedControl) { target_fps = shared.effective_fps(), max_dgram = shared.max_dgram.load(Relaxed), input = ?*shared.encoder_input.lock(), hevc = shared.hevc_enabled.load(Relaxed), retransmits = sample.transport_retransmits, usb_drops = sample.usb_frames_dropped, + pointer_sent = sample.pointer_states_sent, pointer_shapes = sample.pointer_shapes_sent, usb_state = %sample.usb_link_state, usb_devices = sample.usb_devices, capture_heartbeat_age_ms = now_ms.saturating_sub(shared.hb_capture_frame_ms.load(Relaxed)), encode_heartbeat_age_ms = now_ms.saturating_sub(shared.hb_encode_frame_ms.load(Relaxed)), @@ -247,6 +305,13 @@ static SESSION_FALLBACK: parking_lot::Mutex = /// `session_id` started: fallbacks logged from now on are its own. pub fn note_session_start(session_id: u32, shared: &SharedControl) { + // The slot is the process's, so a test that starts a session would + // replace another test's. + #[cfg(test)] + assert!( + tests::FALLBACK_SLOT.is_locked(), + "a test that starts a session holds diagnostics::tests::FALLBACK_SLOT" + ); *SESSION_FALLBACK.lock() = SessionFallback { session_id: Some(session_id), mode: Some(Arc::clone(&shared.capture_target)), @@ -254,7 +319,11 @@ pub fn note_session_start(session_id: u32, shared: &SharedControl) { }; } -fn note_fallback(reason: &str) { +/// Notes `reason` for the live session's summary, as a +/// `second_screen_fallback` log field does. Also called directly for a +/// Second screen that is held by design rather than failed, so the field +/// (which the daily check counts as a failure) isn't logged for it. +pub(crate) fn note_fallback(reason: &str) { let (session_id, mode) = { let slot = SESSION_FALLBACK.lock(); match (slot.session_id, &slot.mode) { @@ -285,7 +354,9 @@ fn take_fallback(session_id: u32) -> Option<&'static str> { } /// A `second_screen_fallback` reason as the summary names it. `remembered` -/// is a failure saved by an earlier app run. +/// is a failure saved by an earlier app run. The backend's check of the +/// summary lists each token (`copy_session_summaries`), so a new reason goes +/// as `other` until it lists that one too. fn fallback_token(reason: &str) -> &'static str { match reason { "attach_timeout" => "attach_timeout", @@ -294,6 +365,8 @@ fn fallback_token(reason: &str) -> &'static str { "no_frames" => "no_frames", "duplication_failed" => "duplication_failed", "remembered" => "remembered", + "vdd_only" => "vdd_only", + "panel_lost" => "panel_lost", _ => "other", } } @@ -331,30 +404,77 @@ impl tracing_subscriber::Layer for FallbackWatch { /// Logs the summary of a session, or of a HELLO2 the host turned down. The /// upload sends it as category `summary` (`crate::telemetry`). pub fn log_session_end(end: &SessionEnd, shared: &SharedControl) { + info!("{}", summary_line(end, shared)); +} + +/// The summary of `end`, with what the PC knows at the end, for +/// [`log_session_end`] and [`log_session_end_now`] alike. +fn summary_line(end: &SessionEnd, shared: &SharedControl) -> String { let second_screen = *shared.capture_target.lock() == CaptureTarget::VirtualExtended; // `mode=` is the mode at the end; the fallback was counted in Second // screen mode, perhaps before a switch to Mirror. let fallback = end.session_id.and_then(take_fallback); - info!( - "{}", - session_summary(end, second_screen, fallback, host_facts()) - ); + let refused = *shared.capture_block.lock() + == Some(crate::capture::recovery::CaptureBlock::UnsupportedOnThisGpu); + session_summary(end, second_screen, fallback, refused, host_facts()) } -/// The app is quitting: the summary of the stream it stops. +/// The app is quitting: the summary of the stream it stops. Its iPad is +/// told the PC ended the session, or, when a restart or an update hands over +/// to a new copy, to connect again. pub fn log_session_end_at_quit(shared: &SharedControl) { - let end = shared.session.lock().shut_down(std::time::Instant::now()); - if let Some(end) = end { + stop_session(shared, quit_stop(shared)); +} + +/// Why the app quits: for good, or for a restart or an update that hands +/// over to a new copy. +pub fn quit_stop(shared: &SharedControl) -> StreamStop { + if shared.handoff.holder().is_some() { + StreamStop::Handoff + } else { + StreamStop::Quit + } +} + +/// The stream stops for good: ends the session, which the transport tells +/// why on its way out, stops the stream, and logs the summary. +pub fn stop_session(shared: &SharedControl, why: StreamStop) { + if let Some(end) = end_stream(shared, why) { log_session_end(&end, shared); } } +/// [`stop_session`] without the summary, which the caller writes. The +/// goodbye and `running` change under one session lock, so a transport +/// starting meanwhile never takes the goodbye for a stale one +/// (`Session::transport_resumed`). +pub fn end_stream(shared: &SharedControl, why: StreamStop) -> Option { + let mut session = shared.session.lock(); + let end = session.shut_down(std::time::Instant::now(), why); + shared.stop(); + end +} + +/// [`log_session_end`] for a PC about to go down: the line goes straight to +/// the session log and the upload outbox on disk, so it survives the +/// process ending a moment later, and the next launch uploads it. +pub fn log_session_end_now(end: &SessionEnd, shared: &SharedControl) { + let line = crate::startup::info_line(&summary_line(end, shared)); + let written = crate::logging::append_now(&line).and(crate::telemetry::queue_now(&line, "info")); + if let Err(error) = written { + tracing::warn!(%error, "Could not save the session summary before the PC went down"); + } +} + /// The `session_summary v=1` line (b-session-summary contract): enumerated -/// tokens, bounded numbers and the facts above, nothing else. +/// tokens, bounded numbers and the facts above, nothing else. `gpu_refused`: +/// Windows refuses the screen from the graphics card EternalMonitor runs on, +/// so a session that gets no picture failed in capture, not on the network. pub fn session_summary( end: &SessionEnd, second_screen: bool, fallback: Option<&'static str>, + gpu_refused: bool, host: &HostFacts, ) -> String { use eternal_wire::v2::control::{ByeReason, HelloStatus}; @@ -379,18 +499,36 @@ pub fn session_summary( _ if streamed => "streamed", EndReason::Goodbye(ByeReason::UserDisconnect) => "user_ended", EndReason::Goodbye(ByeReason::Error) | EndReason::Liveness => "no_picture", - EndReason::Goodbye(_) | EndReason::Superseded | EndReason::Shutdown => "app_ended", + EndReason::Goodbye(_) + | EndReason::Superseded + | EndReason::Shutdown + | EndReason::Stopped => "app_ended", EndReason::Revoked => "pairing", - EndReason::LinkLost => "error", + EndReason::LinkLost | EndReason::Failed(_) => "error", }; let failure = match end.reason { EndReason::Refused(status) if pairing(status) => "none", EndReason::Refused(_) => "other", EndReason::LinkLost if matches!(end.link, LinkId::Usb { .. }) => "usb_lost", EndReason::LinkLost => "other", + EndReason::Liveness | EndReason::Goodbye(ByeReason::Error) if !streamed && gpu_refused => { + "capture_failed" + } EndReason::Liveness | EndReason::Goodbye(ByeReason::Error) if !streamed => "no_video", EndReason::Liveness => "signal_lost", + // The iPad gave up on video that went out: none of it arrived, or + // no frame of it completed. + EndReason::Goodbye(ByeReason::Error) if !end.frames_received => { + if end.frags_received { + "no_frame" + } else { + "no_video" + } + } EndReason::Goodbye(ByeReason::Error) => "other", + EndReason::Failed(StageFailure::Capture) => "capture_failed", + EndReason::Failed(StageFailure::Encoder) => "encoder_failed", + EndReason::Failed(StageFailure::Other) => "other", _ => "none", }; let end_token = match end.reason { @@ -398,7 +536,10 @@ pub fn session_summary( EndReason::Liveness => "liveness", EndReason::Superseded => "superseded", EndReason::Shutdown => "shutdown", - EndReason::Refused(_) | EndReason::LinkLost | EndReason::Revoked => "error", + EndReason::Stopped => "user", + EndReason::Refused(_) | EndReason::LinkLost | EndReason::Revoked | EndReason::Failed(_) => { + "error" + } }; let session_id = end.session_id.map_or("-".to_string(), |id| id.to_string()); let transport = match end.link { @@ -590,7 +731,7 @@ pub(crate) mod tests { r" stage=(started|hello_sent|accepted|video_sent|first_frame|steady)", r" outcome=(streamed|no_reply|no_picture|pairing|user_ended|app_ended|error)", r" failure=(none|no_reply|no_video|no_frame|vdd_enable_failed|vdd_attach_timeout|vdd_not_attached|capture_failed|encoder_failed|usb_lost|signal_lost|other)", - r" fallback=(none|attach_timeout|enable_failed|no_driver|no_frames|duplication_failed|remembered|other)", + r" fallback=(none|attach_timeout|enable_failed|no_driver|no_frames|duplication_failed|remembered|vdd_only|panel_lost|other)", r" end=(goodbye|liveness|superseded|shutdown|timeout|user|background|error)", r" ttff_ms=(0|[1-9][0-9]{0,5}|-) duration_ms=(0|[1-9][0-9]{0,9})", r" os=(win:[0-9]{1,6}\.[0-9]{1,6}|ipados:[0-9]{1,3}\.[0-9]{1,3}(\.[0-9]{1,3})?) hw=(pc|iPad[0-9]{1,2},[0-9]{1,2})", @@ -671,6 +812,7 @@ pub(crate) mod tests { reason, first_video: first_video_ms.map(Duration::from_millis), frames_received: first_video_ms.is_some(), + frags_received: first_video_ms.is_some(), duration: Duration::from_millis(duration_ms), } } @@ -694,6 +836,7 @@ pub(crate) mod tests { ), true, Some("attach_timeout"), + false, &pc(), ); assert_eq!( @@ -743,6 +886,24 @@ pub(crate) mod tests { end(Goodbye(ByeReason::Error), Some(500), 30_000), ["steady", "streamed", "other", "goodbye"], ), + // The iPad gave up on video that went out: none of it arrived, + // or no frame of it completed. + ( + SessionEnd { + frames_received: false, + frags_received: false, + ..end(Goodbye(ByeReason::Error), Some(300), 10_000) + }, + ["video_sent", "streamed", "no_video", "goodbye"], + ), + ( + SessionEnd { + frames_received: false, + frags_received: true, + ..end(Goodbye(ByeReason::Error), Some(300), 10_000) + }, + ["video_sent", "streamed", "no_frame", "goodbye"], + ), ( end(Liveness, None, 3_000), ["accepted", "no_picture", "no_video", "liveness"], @@ -767,6 +928,27 @@ pub(crate) mod tests { end(Shutdown, None, 20), ["accepted", "app_ended", "none", "shutdown"], ), + // Stop on the PC and the storm brake are not a clean quit. + ( + end(Stopped, Some(250), 20_000), + ["steady", "streamed", "none", "user"], + ), + ( + end(Stopped, None, 20), + ["accepted", "app_ended", "none", "user"], + ), + ( + end(Failed(StageFailure::Capture), Some(250), 20_000), + ["steady", "streamed", "capture_failed", "error"], + ), + ( + end(Failed(StageFailure::Encoder), None, 20), + ["accepted", "error", "encoder_failed", "error"], + ), + ( + end(Failed(StageFailure::Other), Some(250), 9_000), + ["first_frame", "streamed", "other", "error"], + ), (usb, ["first_frame", "streamed", "usb_lost", "error"]), ( end(LinkLost, None, 100), @@ -798,7 +980,7 @@ pub(crate) mod tests { ), ]; for (end, expected) in cases { - let line = session_summary(&end, false, None, &pc()); + let line = session_summary(&end, false, None, false, &pc()); assert_eq!(tokens(&line), expected, "{end:?}"); assert_contract(&line); } @@ -806,6 +988,7 @@ pub(crate) mod tests { &end(Refused(HelloStatus::Busy), None, 0), true, Some("no_frames"), + false, &pc(), ); assert!( @@ -816,6 +999,47 @@ pub(crate) mod tests { ); } + /// A hybrid laptop whose screen Windows won't share from the card + /// EternalMonitor runs on: the iPad gave up with no picture. That is a + /// capture failure, which the backend's check lists, not lost video. A + /// session that got its picture, or that the user ended, keeps its own + /// tokens. + #[test] + fn no_picture_from_a_refused_graphics_card_is_a_capture_failure() { + use EndReason::{Goodbye, Liveness}; + for (end, expected) in [ + ( + end(Goodbye(ByeReason::Error), None, 10_000), + ["accepted", "no_picture", "capture_failed", "goodbye"], + ), + ( + end(Liveness, None, 10_000), + ["accepted", "no_picture", "capture_failed", "liveness"], + ), + ( + end(Goodbye(ByeReason::UserDisconnect), None, 4_000), + ["accepted", "user_ended", "none", "goodbye"], + ), + ( + end(Liveness, Some(400), 60_000), + ["steady", "streamed", "signal_lost", "liveness"], + ), + ] { + let line = session_summary(&end, false, None, true, &pc()); + assert_eq!(tokens(&line), expected, "{end:?}"); + assert_contract(&line); + } + // Without the refusal, the same end is lost video, as before. + let line = session_summary( + &end(Goodbye(ByeReason::Error), None, 10_000), + false, + None, + false, + &pc(), + ); + assert_eq!(tokens(&line)[2], "no_video"); + } + #[test] fn summary_numbers_stay_in_the_contracts_range() { let long = SessionEnd { @@ -825,7 +1049,7 @@ pub(crate) mod tests { duration: Duration::from_secs(40 * 86_400), ..end(EndReason::Liveness, None, 0) }; - let line = session_summary(&long, false, None, &pc()); + let line = session_summary(&long, false, None, false, &pc()); assert!(line.contains(" nonce=00000000ffffffff "), "{line}"); assert!( line.contains(" ttff_ms=600000 duration_ms=2592000000 "), @@ -837,7 +1061,13 @@ pub(crate) mod tests { gpu: gpu_token(&[]), hybrid: None, }; - let line = session_summary(&end(EndReason::Shutdown, None, 0), false, None, &unknown); + let line = session_summary( + &end(EndReason::Shutdown, None, 0), + false, + None, + false, + &unknown, + ); assert!(line.ends_with(" os=win:0.0 hw=pc gpu=- hybrid=-"), "{line}"); assert_contract(&line); } @@ -883,6 +1113,8 @@ pub(crate) mod tests { ("no_frames", "no_frames"), ("duplication_failed", "duplication_failed"), ("remembered", "remembered"), + ("vdd_only", "vdd_only"), + ("panel_lost", "panel_lost"), ("C:\\Users\\someone", "other"), ] { assert_eq!(fallback_token(reason), token); @@ -891,6 +1123,7 @@ pub(crate) mod tests { let shared = SharedControl::new(9876, 15_000_000); *shared.capture_target.lock() = CaptureTarget::VirtualExtended; let subscriber = tracing_subscriber::registry().with(FallbackWatch); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { tracing::info!(second_screen_fallback = "remembered", "before the session"); note_session_start(1, &shared); @@ -911,6 +1144,54 @@ pub(crate) mod tests { }); } + /// The Windows 10 Second screen hold names `no_driver` in the session + /// summary without logging a `second_screen_fallback` field, which the + /// daily check counts as a failure. A failed start without a driver + /// still logs the field. + #[test] + fn a_windows_10_hold_is_summarised_as_no_driver_without_a_fallback_line() { + use tracing_subscriber::prelude::*; + struct Fields(std::sync::Arc>>); + impl tracing_subscriber::Layer for Fields { + fn on_event( + &self, + event: &tracing::Event<'_>, + _: tracing_subscriber::layer::Context<'_, S>, + ) { + let names = event + .metadata() + .fields() + .iter() + .map(|f| f.name().to_string()); + self.0.lock().extend(names); + } + } + let _slot = FALLBACK_SLOT.lock(); + let shared = SharedControl::new(9876, 15_000_000); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + let fields = std::sync::Arc::new(parking_lot::Mutex::new(Vec::::new())); + let subscriber = tracing_subscriber::registry() + .with(FallbackWatch) + .with(Fields(std::sync::Arc::clone(&fields))); + crate::logging::prepare_log_capture(); + let has_field = || { + fields + .lock() + .iter() + .any(|name| name == "second_screen_fallback") + }; + tracing::subscriber::with_default(subscriber, || { + note_session_start(1, &shared); + crate::capture::second_screen::log_windows_hold(); + assert!(!has_field(), "{:?}", fields.lock()); + assert_eq!(take_fallback(1), Some("no_driver")); + note_session_start(2, &shared); + tracing::info!(second_screen_fallback = "no_driver", "no driver here"); + assert!(has_field()); + assert_eq!(take_fallback(2), Some("no_driver")); + }); + } + /// A fallback counts for the live session in the mode it has when the /// fallback is logged. Session 1 starts the virtual display and leaves, /// and its start fails late: neither the gap before session 2 nor a @@ -930,6 +1211,7 @@ pub(crate) mod tests { session_id: Some(id), ..end(EndReason::Goodbye(ByeReason::UserDisconnect), None, 1_000) }; + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { *shared.capture_target.lock() = CaptureTarget::VirtualExtended; note_session_start(1, &shared); @@ -990,6 +1272,49 @@ pub(crate) mod tests { ); } + /// The hybrid laptop's two fallbacks along the whole path: the session's + /// summary names each (the backend's check lists both), passes that + /// check, and uploads as written, in its own category. + #[test] + fn a_lone_display_or_lost_panel_reaches_the_uploaded_summary() { + use tracing_subscriber::prelude::*; + let _slot = FALLBACK_SLOT.lock(); + let summaries = std::sync::Arc::new(parking_lot::Mutex::new(Vec::::new())); + let subscriber = tracing_subscriber::registry() + .with(FallbackWatch) + .with(SummaryCapture(summaries.clone())); + let shared = SharedControl::new(9876, 15_000_000); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, || { + for (id, reason) in [(1, "vdd_only"), (2, "panel_lost")] { + note_session_start(id, &shared); + tracing::warn!(second_screen_fallback = reason, "fell back"); + let ended = SessionEnd { + session_id: Some(id), + ..end( + EndReason::Goodbye(ByeReason::UserDisconnect), + Some(400), + 60_000, + ) + }; + log_session_end(&ended, &shared); + } + }); + let lines = summaries.lock().clone(); + assert_eq!(lines.len(), 2, "{lines:?}"); + for (line, token) in lines.iter().zip(["vdd_only", "panel_lost"]) { + assert_contract(line); + assert!(line.contains(&format!(" fallback={token} ")), "{line}"); + let logged = format!("2026-11-07T01:02:03.123456Z INFO {line}"); + assert_eq!( + crate::telemetry::session_summary(&logged), + Some(line.as_str()) + ); + assert_eq!(crate::telemetry::uploaded(&logged), *line); + } + } + #[test] fn windows_11_is_named_from_its_build_not_its_registry_label() { // The customer's PC: Windows 11 26H2 with the legacy label. diff --git a/host/src/discovery.rs b/host/src/discovery.rs index ed39b782..e33eeb5d 100644 --- a/host/src/discovery.rs +++ b/host/src/discovery.rs @@ -1,5 +1,6 @@ +use std::net::IpAddr; use std::thread; -use std::time::Duration; +use std::time::{Duration, Instant}; use mdns_sd::{ServiceDaemon, ServiceInfo}; use tracing::{error, info, warn}; @@ -9,6 +10,33 @@ use crate::stats::PIPELINE_STATS; const SERVICE_TYPE: &str = "_eternaldisplay._udp.local."; const INSTANCE_NAME: &str = "EternalMonitor"; const RE_ADVERT_INTERVAL: Duration = Duration::from_secs(60); +/// A re-advertisement with the addresses of the last logged one is logged +/// once in this long, with how many there were. Every minute, it was one of +/// the most common lines in the uploads. +const RE_ADVERT_LOG_INTERVAL: Duration = Duration::from_secs(3600); + +/// Which re-advertisements get a log line. +#[derive(Default)] +struct ReAdvertLog { + last: Option<(Vec, Instant)>, + unchanged: u32, +} + +impl ReAdvertLog { + /// For a re-advertisement of `addrs` at `now`: `Some(n)` to log it, where + /// `n` is how many unchanged ones went unlogged before it. + fn note(&mut self, mut addrs: Vec, now: Instant) -> Option { + addrs.sort(); + if let Some((last, logged)) = &self.last { + if *last == addrs && now.duration_since(*logged) < RE_ADVERT_LOG_INTERVAL { + self.unchanged += 1; + return None; + } + } + self.last = Some((addrs, now)); + Some(std::mem::take(&mut self.unchanged)) + } +} /// Registers the EternalMonitor service on the local network via mDNS/DNS-SD /// so the iOS app's NetworkScanner (NWBrowser) can discover it. @@ -18,8 +46,15 @@ const RE_ADVERT_INTERVAL: Duration = Duration::from_secs(60); /// documented upsert in mdns-sd, so the refresh never takes the service off /// the air (the old unregister-first cycle opened a ~1s discovery hole). /// +/// `on_browse` runs each time another device on the network looks for PCs +/// running EternalMonitor, as an iPad does on its connect screen (see +/// [`watch_browsers`] and [`crate::gpu_wake`]). +/// /// Returns the `ServiceDaemon` handle — call [`say_goodbye`] on it at exit. -pub fn advertise_service(port: u16) -> Option { +pub fn advertise_service( + port: u16, + on_browse: impl FnMut() + Send + 'static, +) -> Option { let mdns = match ServiceDaemon::new() { Ok(d) => d, Err(e) => { @@ -50,11 +85,13 @@ pub fn advertise_service(port: u16) -> Option { } info!("mDNS service registered — iPad can now discover this host"); PIPELINE_STATS.lock().mdns_active = true; + watch_browsers(on_browse); let mdns_clone = mdns.clone(); let host_fqdn_for_thread = host_fqdn.clone(); let instance_for_thread = instance_name.clone(); let _ = full_name; // fullname only mattered for the old unregister cycle + let mut log = ReAdvertLog::default(); thread::spawn(move || loop { thread::sleep(RE_ADVERT_INTERVAL); let Some(info) = build_service_info(port, &host_fqdn_for_thread, &instance_for_thread) @@ -66,7 +103,11 @@ pub fn advertise_service(port: u16) -> Option { // Plain upsert: the service stays continuously advertised (addresses // may have changed, e.g. DHCP renew or interface flap). match mdns_clone.register(info) { - Ok(_) => info!(addrs = ?addrs, "mDNS re-advertisement"), + Ok(_) => { + if let Some(unchanged) = log.note(addrs.clone(), Instant::now()) { + info!(addrs = ?addrs, unchanged, "mDNS re-advertisement"); + } + } Err(error) => warn!(error = %error, "mDNS re-advertisement register failed"), } }); @@ -74,6 +115,173 @@ pub fn advertise_service(port: u16) -> Option { Some(mdns) } +/// Calls `on_browse` on a thread of its own each time a device that isn't +/// this PC asks the network for the service: a PTR or ANY question for +/// [`SERVICE_TYPE`]. The daemon's own events can't tell: it answers its own +/// probes too, at each registration and re-advertisement. Reads alongside +/// the daemon on the shared mDNS port and sends nothing. +#[cfg(windows)] +fn watch_browsers(mut on_browse: impl FnMut() + Send + 'static) { + use socket2::{Domain, Protocol, Socket, Type}; + let group = std::net::Ipv4Addr::new(224, 0, 0, 251); + let socket = (|| -> std::io::Result { + let socket = Socket::new(Domain::IPV4, Type::DGRAM, Some(Protocol::UDP))?; + socket.set_reuse_address(true)?; + socket.bind(&std::net::SocketAddr::from(([0, 0, 0, 0], 5353)).into())?; + let mut joined = 0; + for address in own_ipv4() { + joined += usize::from(socket.join_multicast_v4(&group, &address).is_ok()); + } + if joined == 0 { + socket.join_multicast_v4(&group, &std::net::Ipv4Addr::UNSPECIFIED)?; + } + Ok(socket.into()) + })(); + let socket = match socket { + Ok(socket) => socket, + Err(error) => { + info!(%error, "Not watching for iPads looking for this PC"); + return; + } + }; + let _ = thread::Builder::new() + .name("mdns-browsers".into()) + .spawn(move || { + let mut packet = vec![0u8; MDNS_PACKET_MAX]; + let mut errors = 0u32; + loop { + match socket.recv_from(&mut packet) { + Ok((len, from)) => { + errors = 0; + if asks_for_service(&packet[..len]) && !is_own(from.ip()) { + on_browse(); + } + } + Err(error) => { + let pause = after_receive_error(&error, errors); + errors = errors.saturating_add(1); + if errors.is_power_of_two() { + tracing::debug!(%error, errors, "mDNS browse watch: receive failed"); + } + thread::sleep(pause); + } + } + } + }); +} + +/// The largest mDNS message (RFC 6762 section 17). A longer datagram fails +/// the receive with WSAEMSGSIZE, and is dropped. +#[cfg(windows)] +const MDNS_PACKET_MAX: usize = 9000; + +/// How long the browse watch pauses after a receive error, the `in_a_row`th +/// since the last packet. It never stops: one oversized or reset datagram +/// had ended it until the app restarted. Errors that pass (an oversized +/// datagram, a reset from an ICMP reply, an interruption) go on at once, a +/// few in a row; anything else, or more, waits longer each time, up to a +/// minute. +#[cfg(any(windows, test))] +fn after_receive_error(error: &std::io::Error, in_a_row: u32) -> Duration { + use std::io::ErrorKind; + const WSAEMSGSIZE: i32 = 10040; + let passing = matches!( + error.kind(), + ErrorKind::Interrupted + | ErrorKind::WouldBlock + | ErrorKind::TimedOut + | ErrorKind::ConnectionReset + ) || error.raw_os_error() == Some(WSAEMSGSIZE); + if passing && in_a_row < 8 { + return Duration::ZERO; + } + (Duration::from_millis(100) * 2u32.pow(in_a_row.min(10))).min(Duration::from_secs(60)) +} + +#[cfg(not(windows))] +fn watch_browsers(_on_browse: impl FnMut() + Send + 'static) {} + +/// This PC's IPv4 addresses. +#[cfg(windows)] +fn own_ipv4() -> Vec { + crate::network::adapters() + .unwrap_or_default() + .into_iter() + .filter(|adapter| adapter.up) + .flat_map(|adapter| adapter.ipv4) + .collect() +} + +/// Whether a query came from this PC: its own daemon, or another program on it. +#[cfg(windows)] +fn is_own(address: IpAddr) -> bool { + address.is_loopback() + || match address { + IpAddr::V4(address) => own_ipv4().contains(&address), + IpAddr::V6(_) => false, + } +} + +/// Whether `packet` is a DNS query (not a response) with a PTR or ANY +/// question for [`SERVICE_TYPE`], as a device browsing for EternalMonitor +/// PCs sends. +#[cfg(any(windows, test))] +pub(crate) fn asks_for_service(packet: &[u8]) -> bool { + const PTR: u16 = 12; + const ANY: u16 = 255; + let u16_at = |at: usize| -> Option { + Some(u16::from_be_bytes([*packet.get(at)?, *packet.get(at + 1)?])) + }; + let (Some(flags), Some(questions)) = (u16_at(2), u16_at(4)) else { + return false; + }; + if flags & 0x8000 != 0 { + return false; + } + let wanted = SERVICE_TYPE.trim_end_matches('.'); + let mut at = 12; + for _ in 0..questions { + let Some((name, next)) = dns_name(packet, at) else { + return false; + }; + let Some(kind) = u16_at(next) else { + return false; + }; + if (kind == PTR || kind == ANY) && name.eq_ignore_ascii_case(wanted) { + return true; + } + at = next + 4; + } + false +} + +/// The dotted name at `at`, following compression pointers, and where the +/// record goes on after it. None if it is malformed or loops. +#[cfg(any(windows, test))] +fn dns_name(packet: &[u8], mut at: usize) -> Option<(String, usize)> { + let mut name = String::new(); + let mut after = None; + for _ in 0..32 { + let len = *packet.get(at)? as usize; + if len == 0 { + return Some((name, after.unwrap_or(at + 1))); + } + if len & 0xC0 == 0xC0 { + let target = ((len & 0x3F) << 8) | *packet.get(at + 1)? as usize; + after.get_or_insert(at + 2); + at = target; + continue; + } + let label = packet.get(at + 1..at + 1 + len)?; + if !name.is_empty() { + name.push('.'); + } + name.push_str(std::str::from_utf8(label).ok()?); + at += 1 + len; + } + None +} + /// Send mDNS goodbye packets and stop the daemon. Called on the way out so /// iPads drop the host from their scan list promptly instead of waiting for /// the TTL to lapse. The short grace period lets the goodbyes hit the wire — @@ -120,6 +328,92 @@ fn hostname() -> Option { mod tests { use super::*; + /// A DNS packet with one question per `(name, type)`. + fn query(flags: u16, questions: &[(&str, u16)]) -> Vec { + let mut packet = vec![0, 0]; + packet.extend_from_slice(&flags.to_be_bytes()); + packet.extend_from_slice(&(questions.len() as u16).to_be_bytes()); + packet.extend_from_slice(&[0; 6]); + for (name, kind) in questions { + for label in name.split('.') { + packet.push(label.len() as u8); + packet.extend_from_slice(label.as_bytes()); + } + packet.push(0); + packet.extend_from_slice(&kind.to_be_bytes()); + packet.extend_from_slice(&[0x80, 1]); + } + packet + } + + #[test] + fn a_browse_for_the_service_is_told_from_other_mdns_traffic() { + let ours = "_eternaldisplay._udp.local"; + assert!(asks_for_service(&query(0, &[(ours, 12)]))); + assert!(asks_for_service(&query( + 0, + &[("_airplay._tcp.local", 12), (ours, 12)] + ))); + assert!(asks_for_service(&query( + 0, + &[("_EternalDisplay._UDP.local", 255)] + ))); + assert!(!asks_for_service(&query(0, &[("_airplay._tcp.local", 12)]))); + assert!(!asks_for_service(&query(0, &[(ours, 1)])), "an A question"); + assert!( + !asks_for_service(&query(0x8400, &[(ours, 12)])), + "a response" + ); + // A second question that points back into the first name. + let mut packet = query(0, &[("_airplay._tcp.local", 12)]); + packet[5] = 2; + packet.extend_from_slice(&[15]); + packet.extend_from_slice(b"_eternaldisplay"); + packet.extend_from_slice(&[4]); + packet.extend_from_slice(b"_udp"); + // "local" is at offset 12 + 1 + 8 + 1 + 4 = 26. + packet.extend_from_slice(&[0xC0, 26, 0, 12, 0, 1]); + assert!(asks_for_service(&packet)); + } + + #[test] + fn a_receive_error_never_ends_the_browse_watch() { + use std::io::{Error, ErrorKind}; + let oversized = Error::from_raw_os_error(10040); + let reset = Error::from(ErrorKind::ConnectionReset); + let interrupted = Error::from(ErrorKind::Interrupted); + for error in [&oversized, &reset, &interrupted] { + assert_eq!(after_receive_error(error, 0), Duration::ZERO, "{error}"); + assert_eq!(after_receive_error(error, 7), Duration::ZERO, "{error}"); + assert!( + after_receive_error(error, 8) > Duration::ZERO, + "{error} keeps coming" + ); + } + let other = Error::from(ErrorKind::PermissionDenied); + let pauses: Vec = (0..20).map(|n| after_receive_error(&other, n)).collect(); + assert!(pauses[0] > Duration::ZERO); + assert!( + pauses.windows(2).all(|pair| pair[0] <= pair[1]), + "{pauses:?}" + ); + assert_eq!(*pauses.last().unwrap(), Duration::from_secs(60)); + } + + #[test] + fn a_malformed_or_looping_packet_is_not_a_browse() { + assert!(!asks_for_service(&[])); + assert!(!asks_for_service(&[0; 11])); + let full = query(0, &[("_eternaldisplay._udp.local", 12)]); + for cut in 12..full.len() - 4 { + assert!(!asks_for_service(&full[..cut]), "cut at {cut}"); + } + // A pointer to itself. + let mut looping = vec![0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0]; + looping.extend_from_slice(&[0xC0, 12, 0, 12, 0, 1]); + assert!(!asks_for_service(&looping)); + } + /// The service pins no address: mdns-sd advertises every interface's, /// Wi-Fi and Tailscale alike. The Address row's ranking never reaches it. #[test] @@ -128,4 +422,24 @@ mod tests { assert!(info.is_addr_auto()); assert!(info.get_addresses().is_empty()); } + + /// The minute's re-advertisement is logged when its addresses change and + /// otherwise once an hour, with the count since the last line. + #[test] + fn unchanged_re_advertisements_are_logged_hourly() { + let start = Instant::now(); + let minute = |n: u32| start + RE_ADVERT_INTERVAL * n; + let wifi: IpAddr = "192.0.2.10".parse().unwrap(); + let usb: IpAddr = "198.51.100.1".parse().unwrap(); + let mut log = ReAdvertLog::default(); + assert_eq!(log.note(vec![], minute(1)), Some(0)); + let logged: Vec<_> = (2..=121) + .filter_map(|n| log.note(vec![], minute(n)).map(|c| (n, c))) + .collect(); + assert_eq!(logged, [(61, 59), (121, 59)]); + // A new address list is logged at once; the order doesn't matter. + assert_eq!(log.note(vec![wifi, usb], minute(122)), Some(0)); + assert_eq!(log.note(vec![usb, wifi], minute(123)), None); + assert_eq!(log.note(vec![wifi], minute(124)), Some(1)); + } } diff --git a/host/src/display_guard.rs b/host/src/display_guard.rs index 3bdb16a6..fe5b839d 100644 --- a/host/src/display_guard.rs +++ b/host/src/display_guard.rs @@ -13,11 +13,35 @@ //! turned on, the guard re-applies that layout, or else switches Windows to //! "PC screen only" when the built-in panel was on, and logs the result. //! Nothing else ever changes the layout, which stays the user's: a PC that -//! keeps any real screen sees no change at all. A turn on is only logged: -//! a layout with the virtual display alone cannot attach, so the attempt -//! times out and its turn off brings the guard in. One attempt per change. -//! Right before "PC screen only" the guard reads the screens again, and -//! stops if one came back or they can't be read. +//! keeps any real screen sees no change at all. +//! +//! Turning the display off while it is the only screen on is what loses the +//! panel for good: on a hybrid laptop, Windows then stops reporting any +//! screen, and refuses every layout until the PC restarts. So the guard acts +//! while the display is still on. A turn on that leaves the virtual display +//! alone, where the PC had a real screen before, switches Windows to +//! "Extend", which keeps the second screen, or else puts the saved layout +//! back, or "PC screen only". A turn off that finds the virtual display +//! alone first puts the saved layout back, "PC screen only", or "Extend", +//! and if none brings a real screen back it leaves the display on: the +//! desktop stays on it, which the iPad and Windows+P can still reach. +//! One attempt per change. Between two calls the guard reads the screens +//! again, and stops if one came back or they can't be read. +//! +//! On a hybrid laptop (AMD integrated and discrete graphics), the virtual +//! display's arrival also takes the built-in panel away below the layout: +//! Windows lists the panel's target as unavailable, every layout that names +//! it is refused, and only a restart gives it back. The guard tells that +//! case apart, on a PC with more than one graphics card, by the panel's +//! target: the virtual display is on, the panel is not, no built-in target +//! is offered any more, and Windows has said the lid is open (a closed lid +//! withdraws the panel on purpose). It then makes no `SetDisplayConfig` +//! call, and while the host runs it doesn't turn the display off: the +//! desktop stays on it, which the iPad still shows. A turn off that nothing +//! streams after, at start-up, at quit, in the failure reporter or once +//! Windows ends the session, runs, so the restart that brings the panel +//! back starts without the display: its enabled state would outlast the +//! restart. //! //! The layout from before a turn on is also kept on disk, so the failure //! reporter, a new process, compares with it when the host that turned @@ -131,6 +155,20 @@ pub trait Displays { fn reapply(&mut self, saved: &Self::Saved) -> Result<(), i32>; /// `SetDisplayConfig` with "PC screen only". fn internal_only(&mut self) -> Result<(), i32>; + /// `SetDisplayConfig` with "Extend". + fn extend(&mut self) -> Result<(), i32>; + /// What Windows offers of the built-in panel's targets now, active or + /// not. `None` when it can't say. + fn built_in(&mut self) -> Option; + /// The PC has more than one graphics card. + fn hybrid(&self) -> bool; + /// The laptop's lid, as Windows last said. + fn lid(&self) -> Lid; + /// Windows is ending the session: shutting down, restarting or signing out. + fn session_ending(&self) -> bool; + /// This turn off is a cleanup, after which nothing streams the display: + /// at start-up, at quit, or in the failure reporter once the host ended. + fn cleanup(&self) -> bool; fn sleep(&mut self, duration: Duration); fn log(&mut self, warn: bool, text: &str); /// The host whose layout from before its turn on is kept on disk: this @@ -152,11 +190,73 @@ pub trait Saved: Sized { fn decode(text: &str) -> Option; } -/// How a turn off ended. +/// The laptop's lid, from Windows' GUID_LIDSWITCH_STATE_CHANGE +/// notifications. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Lid { + /// Windows hasn't said: before its first notification, which comes once + /// it knows the lid device and its state with no bound on when, after a + /// failed registration, and on a PC without a lid. + #[default] + Unknown, + Open, + Closed, +} + +/// What the lid notifications said so far: [`Lid::Unknown`] until the +/// first one, and for good when there is none. +pub struct LidWatch(std::sync::atomic::AtomicU8); + +impl LidWatch { + const UNKNOWN: u8 = u8::MAX; + + pub const fn new() -> Self { + Self(std::sync::atomic::AtomicU8::new(Self::UNKNOWN)) + } + + /// One notification's value: 0 closed, 1 open. + pub fn note(&self, value: u32) { + let state = u8::from(value != 0); + self.0.store(state, std::sync::atomic::Ordering::Relaxed); + } + + pub fn lid(&self) -> Lid { + match self.0.load(std::sync::atomic::Ordering::Relaxed) { + Self::UNKNOWN => Lid::Unknown, + 0 => Lid::Closed, + _ => Lid::Open, + } + } +} + +impl Default for LidWatch { + fn default() -> Self { + Self::new() + } +} + +/// The built-in panel's targets, as `QDC_ALL_PATHS` lists them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BuiltIn { + /// None listed: a desktop PC. + Missing, + /// At least one is available for a path. + Offered, + /// Listed, and none available: a closed lid, or the hybrid laptop's + /// panel after the virtual display took it ([`lost_panel`] tells them + /// apart). + Withdrawn, +} + +/// How a change ended. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Outcome { /// Nothing to do, or nothing known to compare with. Left, + /// A turn on left the virtual display alone, and "Extend" put the PC's + /// screens back beside it. + Extended, + /// The PC's screens are back, without the virtual display. Restored { internal: bool, }, @@ -164,6 +264,9 @@ pub enum Outcome { /// it changed nothing more. Recovered, Failed, + /// A turn on took the built-in panel away for good: see [`lost_panel`]. + /// Nothing was tried, and the display stays on. + PanelLost, } const RECORD_HEADER: &str = "eternalmonitor-screens 1"; @@ -265,6 +368,11 @@ pub fn from_hex(text: &str) -> Option> { /// next turn off checks against it. pub struct Guard { reference: Option>, + /// The last turn on left the virtual display alone, and "Extend" did + /// not put the PC's screens beside it: it will not attach. + alone: bool, + /// The last turn on took the built-in panel away for good. + panel_lost: bool, } impl Default for Guard { @@ -275,22 +383,72 @@ impl Default for Guard { impl Guard { pub const fn new() -> Self { - Self { reference: None } + Self { + reference: None, + alone: false, + panel_lost: false, + } + } + + /// Whether the last turn on left the virtual display alone, with no + /// real screen beside it. + pub fn alone(&self) -> bool { + self.alone + } + + /// Whether the last turn on took the built-in panel away until the PC + /// restarts. + pub fn panel_lost(&self) -> bool { + self.panel_lost } } +/// A `SetDisplayConfig` call that can bring a real screen back while the +/// virtual display is on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Step { + /// "Extend": the PC's screens beside the virtual display. + Extend, + /// The layout from before the turn on, without the virtual display. + SavedLayout, + /// "PC screen only". + Internal, +} + +impl Step { + const fn name(self) -> &'static str { + match self { + Self::Extend => "extend", + Self::SavedLayout => "saved_layout", + Self::Internal => "internal", + } + } +} + +/// How [`take_back`] ended. +enum TakeBack { + /// This call brought a real screen back. + Applied(Step, Option>), + /// A real screen came back between two calls. + CameBack(Option>), + /// None did: each call's result, and the screens now. + Failed(Vec, Option>), +} + impl Guard { /// Before a driver change: log the screens. A turn on keeps them to /// compare with, also on disk for the failure reporter. A turn off /// with nothing kept, as at startup or in the reporter, compares with /// the record its host kept, else with the screens just before it. - /// `display_off`: the driver tasks know the display is off. + /// `display_off`: the driver tasks know the display is off. False when + /// the turn off must not run: the virtual display is the only screen + /// on, the PC had a real one, and nothing brought it back. pub fn before( &mut self, enabled: bool, display_off: bool, displays: &mut impl Displays, - ) { + ) -> bool { let layout = displays.read(); let text = format!( "Screens before the virtual display turns {}: {}", @@ -298,6 +456,8 @@ impl Guard { line(layout.as_ref().map(|layout| &layout.screens[..])) ); displays.log(false, &text); + let alone = layout.as_ref().is_some_and(|layout| alone(&layout.screens)); + let screens = layout.as_ref().map(|layout| layout.screens.clone()); if enabled { // A turn on can run while the display is still on: after an // Enable that finished once the host had given up on it, or a @@ -332,11 +492,110 @@ impl Guard { } self.reference = kept.or(layout); } + if let (false, true, Some(screens)) = (enabled, alone, screens) { + return self.take_back_before_off(displays, &screens); + } + true } - /// After a driver change: log the screens, and after a turn off, - /// restore them if every real screen is gone. A turn off that `done` - /// deletes the kept record once checked. + /// The virtual display is the only screen on, and is about to turn off: + /// put a real screen back first. False when none came back and the PC + /// had one before, so the turn off must not run, nor while the panel is + /// lost. Windows ending the session lets it run: the desktop goes, and + /// the next start then begins without the display. `screens`: now. + fn take_back_before_off( + &mut self, + displays: &mut impl Displays, + screens: &[Screen], + ) -> bool { + if displays.session_ending() { + displays.log( + false, + "Windows is ending the session — turning off the virtual display, the only \ + screen on, so the next start begins without it outcome=session_end", + ); + return true; + } + let withdrawn = withdrawn_panel(displays, screens); + let lid = displays.lid(); + // Also with the lid unknown: a lid in fact closed loses nothing. + if withdrawn && lid != Lid::Closed && displays.cleanup() { + let reason = match lid { + Lid::Open => "reason=panel_lost", + _ => "reason=panel_withdrawn lid=unknown", + }; + let text = format!( + "The built-in screen is still unavailable, and nothing streams the virtual \ + display after this — turning it off, so the restart that brings the screen \ + back starts without it outcome=released {reason} {}", + line(Some(screens)) + ); + displays.log(true, &text); + return true; + } + if withdrawn && lid == Lid::Open { + let text = format!( + "Leaving the virtual display on: the built-in screen is still unavailable, and \ + turning the display off would leave the PC with no screen outcome=kept_on \ + reason=panel_lost {}; restarting the PC brings the built-in screen back", + line(Some(screens)) + ); + displays.log(true, &text); + return false; + } + let reference = self.reference.as_ref(); + let had_real = reference.is_some_and(|layout| real(&layout.screens).next().is_some()); + let internal = reference.is_some_and(|layout| real(&layout.screens).any(|s| s.internal)); + displays.log( + true, + &format!( + "The virtual display is the only screen on; bringing the PC's screens back \ + before turning it off. Before: {}", + line(reference.map(|layout| &layout.screens[..])) + ), + ); + let mut steps = Vec::new(); + if had_real { + steps.push(Step::SavedLayout); + } + if internal { + steps.push(Step::Internal); + } + steps.push(Step::Extend); + match take_back(displays, &steps, reference) { + TakeBack::Applied(step, now) => { + restored(displays, step.name(), &now); + true + } + TakeBack::CameBack(now) => { + came_back(displays, &now); + true + } + TakeBack::Failed(tried, now) if had_real => { + let text = format!( + "Leaving the virtual display on: turning it off now would leave the PC with \ + no screen outcome=kept_on {}{}; Windows+P then \"PC screen only\" brings \ + the built-in screen back", + tried + .iter() + .map(|step| format!("{step} ")) + .collect::(), + line(now.as_ref().map(|layout| &layout.screens[..])) + ); + displays.log(true, &text); + false + } + TakeBack::Failed(tried, now) => { + failed(displays, &tried, &now); + true + } + } + } + + /// After a driver change: log the screens. After a turn on that left + /// the virtual display alone, put the PC's screens back; after a turn + /// off, restore them if every real screen is gone. A turn off that + /// `done` deletes the kept record once checked. pub fn after( &mut self, enabled: bool, @@ -344,13 +603,13 @@ impl Guard { displays: &mut impl Displays, ) -> Outcome { if enabled { - let layout = displays.read(); - let text = format!( - "Screens after the virtual display turned on: {}", - line(layout.as_ref().map(|layout| &layout.screens[..])) + let outcome = self.check_turn_on(displays); + self.alone = matches!( + outcome, + Outcome::Restored { .. } | Outcome::Failed | Outcome::PanelLost ); - displays.log(false, &text); - return Outcome::Left; + self.panel_lost = outcome == Outcome::PanelLost; + return outcome; } let outcome = self.check(displays); if done { @@ -359,6 +618,69 @@ impl Guard { outcome } + /// Windows may have put the desktop on the virtual display alone. A + /// turn off then would lose the PC's screens, so bring them back now, + /// while Windows still reports them: beside it with "Extend", which + /// keeps the second screen, else without it. + fn check_turn_on(&mut self, displays: &mut impl Displays) -> Outcome { + let reference = self.reference.as_ref(); + let before = reference.map_or(&[][..], |layout| &layout.screens[..]); + let layout = settle(displays, before); + let text = format!( + "Screens after the virtual display turned on: {}", + line(layout.as_ref().map(|layout| &layout.screens[..])) + ); + displays.log(false, &text); + let (Some(reference), Some(after)) = (reference, layout) else { + return Outcome::Left; + }; + // No layout can name a target Windows withdrew, so none is tried. + if real(&reference.screens).any(|screen| screen.internal) + && lost_panel(displays, &after.screens) + { + let text = format!( + "The built-in screen went dark when the virtual display turned on: Windows \ + reports it unavailable, and only a restart brings it back. Leaving the virtual \ + display on, so the desktop stays on it outcome=panel_lost {}. Before: {}", + line(Some(&after.screens)), + line(Some(&reference.screens)) + ); + displays.log(true, &text); + return Outcome::PanelLost; + } + let Plan::Restore { internal } = plan(&reference.screens, &after.screens) else { + return Outcome::Left; + }; + displays.log( + true, + &format!( + "Windows put the desktop on the virtual display alone; bringing the PC's \ + screens back while it is on. Before: {}", + line(Some(&reference.screens)) + ), + ); + let mut steps = vec![Step::Extend, Step::SavedLayout]; + if internal { + steps.push(Step::Internal); + } + match take_back(displays, &steps, Some(reference)) { + TakeBack::Applied(step, now) + if now.as_ref().is_some_and(|now| has_vdd(&now.screens)) => + { + let text = format!( + "Screens back beside the virtual display outcome=extended applied={} {}", + step.name(), + line(now.as_ref().map(|layout| &layout.screens[..])) + ); + displays.log(false, &text); + Outcome::Extended + } + TakeBack::Applied(step, now) => restored(displays, step.name(), &now), + TakeBack::CameBack(now) => came_back(displays, &now), + TakeBack::Failed(tried, now) => failed(displays, &tried, &now), + } + } + fn check(&mut self, displays: &mut impl Displays) -> Outcome { let reference = self.reference.take(); let before = reference @@ -384,52 +706,61 @@ impl Guard { line(Some(&reference.screens)) ), ); - // Each step's result, for the log when nothing worked. - let mut tried = Vec::new(); - match displays.reapply(&reference.saved) { + let mut steps = vec![Step::SavedLayout]; + // A switch to "PC screen only" would turn off an external monitor + // the user had on alone. + if internal { + steps.push(Step::Internal); + } + match take_back(displays, &steps, Some(&reference)) { + TakeBack::Applied(step, now) => restored(displays, step.name(), &now), + TakeBack::CameBack(now) => came_back(displays, &now), + TakeBack::Failed(tried, now) => failed(displays, &tried, &now), + } + } +} + +/// Make each call of `steps` in turn until a real screen is on, waiting up +/// to [`SETTLE`] after each one Windows takes. `reference` is the layout +/// from before the turn on, for [`Step::SavedLayout`]. +fn take_back( + displays: &mut D, + steps: &[Step], + reference: Option<&Layout>, +) -> TakeBack { + // Each call's result, for the log when nothing worked. + let mut tried = Vec::new(); + let mut now = None; + for &step in steps { + let result = match (step, reference) { + (Step::Extend, _) => displays.extend(), + (Step::SavedLayout, Some(reference)) => displays.reapply(&reference.saved), + (Step::SavedLayout, None) => continue, + (Step::Internal, _) => displays.internal_only(), + }; + match result { Ok(()) => { - let now = settle(displays, &reference.screens); + now = wait_for_real(displays); if has_real(&now) { - return restored(displays, "saved_layout", &now); + return TakeBack::Applied(step, now); } - tried.push("reason=saved_layout_no_screen".to_string()); + tried.push(format!("reason={}_no_screen", step.name())); } - Err(code) => tried.push(format!("reason=saved_layout_refused code={code}")), + Err(code) => tried.push(format!("reason={}_refused code={code}", step.name())), } // Windows may have changed the screens during that call. Look again // right before changing anything more: a screen that came back // stays as Windows set it. - let now = displays.read(); + now = displays.read(); if has_real(&now) { - let text = format!( - "Screens came back without a change outcome=restored applied=none {}", - line(now.as_ref().map(|layout| &layout.screens[..])) - ); - displays.log(false, &text); - return Outcome::Recovered; + return TakeBack::CameBack(now); } if now.is_none() { tried.push("reason=screens_unknown".into()); - return failed(displays, &tried, &now); - } - if !internal { - return failed(displays, &tried, &now); - } - match displays.internal_only() { - Ok(()) => { - let now = settle(displays, &reference.screens); - if has_real(&now) { - return restored(displays, "internal", &now); - } - tried.push("reason=internal_no_screen".into()); - failed(displays, &tried, &now) - } - Err(code) => { - tried.push(format!("reason=internal_refused code={code}")); - failed(displays, &tried, &now) - } + break; } } + TakeBack::Failed(tried, now) } /// The layout in the record this host kept, if there is one. @@ -453,6 +784,15 @@ fn restored( } } +fn came_back(displays: &mut D, now: &Option>) -> Outcome { + let text = format!( + "Screens came back without a change outcome=restored applied=none {}", + line(now.as_ref().map(|layout| &layout.screens[..])) + ); + displays.log(false, &text); + Outcome::Recovered +} + fn failed( displays: &mut D, tried: &[String], @@ -477,15 +817,52 @@ fn has_real(layout: &Option>) -> bool { .is_some_and(|layout| real(&layout.screens).next().is_some()) } +fn has_vdd(screens: &[Screen]) -> bool { + screens.iter().any(|screen| screen.virtual_display) +} + +/// The virtual display is on, and no real screen is. +fn alone(screens: &[Screen]) -> bool { + has_vdd(screens) && real(screens).next().is_none() +} + +/// On a PC with more than one graphics card, the virtual display is on in +/// `screens`, the built-in panel is off, and Windows no longer offers any +/// built-in target: the hybrid laptop's black panel, or a closed lid. +fn withdrawn_panel(displays: &mut D, screens: &[Screen]) -> bool { + has_vdd(screens) + && !screens.iter().any(|screen| screen.internal) + && displays.hybrid() + && displays.built_in() == Some(BuiltIn::Withdrawn) +} + +/// The hybrid laptop's black panel: [`withdrawn_panel`] with the lid known +/// to be open. After a turn on this is read once the screens settle +/// ([`SETTLE`] after the Enable, while no real screen came back). A lid the +/// user closed withdraws the panel too, and is no failure: the panel comes +/// back when it opens. Nor is a lid Windows hasn't reported yet. +fn lost_panel(displays: &mut D, screens: &[Screen]) -> bool { + displays.lid() == Lid::Open && withdrawn_panel(displays, screens) +} + /// The screens once a real one is back, or as they are after [`SETTLE`]. A /// PC that has one returns at once; only one that lost them all waits. fn settle(displays: &mut D, before: &[Screen]) -> Option> { + if real(before).next().is_none() { + return displays.read(); + } + wait_for_real(displays) +} + +/// The screens once a real one is on, or as they are after [`SETTLE`], or +/// once they can't be read. +fn wait_for_real(displays: &mut D) -> Option> { let mut waited = Duration::ZERO; loop { let layout = displays.read(); let lost = layout .as_ref() - .is_some_and(|layout| plan(before, &layout.screens) != Plan::Leave); + .is_some_and(|layout| real(&layout.screens).next().is_none()); if !lost || waited >= SETTLE { return layout; } @@ -498,38 +875,76 @@ fn settle(displays: &mut D, before: &[Screen]) -> Option> = parking_lot::Mutex::new(Guard::new()); -/// Run a driver change for `owner`'s display with the guard around it. -/// `log` writes one line; `display_off` says the driver tasks know the -/// display is off; `done` tells a change that succeeded. +/// One driver change, for [`around`]. +#[cfg(windows)] +pub(crate) struct Change { + /// A turn on, else a turn off. + pub enabled: bool, + /// The host whose display it is. + pub owner: crate::startup::ProcessId, + /// Writes one line. + pub log: fn(bool, &str), + /// The driver tasks know the display is off. + pub display_off: bool, + /// Nothing streams the display after this turn off. + pub cleanup: bool, +} + +/// Run `change` with the guard around it: `run` makes it, `done` tells one +/// that succeeded. A turn off that would leave the PC with no screen does +/// not run: `kept_on` is its result. #[cfg(windows)] pub(crate) fn around( - enabled: bool, - owner: crate::startup::ProcessId, - log: fn(bool, &str), - display_off: bool, - change: impl FnOnce() -> T, + change: Change, + run: impl FnOnce() -> T, done: impl FnOnce(&T) -> bool, + kept_on: impl FnOnce() -> T, ) -> T { - let mut displays = windows_impl::Windows::new(owner, log); - GUARD.lock().before(enabled, display_off, &mut displays); - let result = change(); + let Change { + enabled, + owner, + log, + display_off, + cleanup, + } = change; + let mut displays = windows_impl::Windows::new(owner, log, cleanup); + if !GUARD.lock().before(enabled, display_off, &mut displays) { + return kept_on(); + } + let result = run(); let done = done(&result); GUARD.lock().after(enabled, done, &mut displays); result } +/// Whether the last turn on left the virtual display alone, with no real +/// screen beside it: then it does not attach as a second screen. +#[cfg(windows)] +pub(crate) fn last_turn_on_alone() -> bool { + GUARD.lock().alone() +} + +/// Whether the last turn on took the built-in panel away until the PC +/// restarts. +#[cfg(windows)] +pub(crate) fn last_turn_on_lost_panel() -> bool { + GUARD.lock().panel_lost() +} + #[cfg(windows)] mod windows_impl { - use super::{Layout, Screen}; + use super::{BuiltIn, Layout, Screen}; use std::time::Duration; use windows::Win32::Devices::Display::{ SetDisplayConfig, DISPLAYCONFIG_MODE_INFO, DISPLAYCONFIG_MODE_INFO_TYPE_SOURCE, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_DISPLAYPORT_EMBEDDED, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_INTERNAL, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_LVDS, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_UDI_EMBEDDED, DISPLAYCONFIG_PATH_INFO, - QDC_ONLY_ACTIVE_PATHS, SDC_ALLOW_CHANGES, SDC_APPLY, SDC_TOPOLOGY_INTERNAL, + DISPLAYCONFIG_VIDEO_OUTPUT_TECHNOLOGY, QDC_ALL_PATHS, QDC_ONLY_ACTIVE_PATHS, + SDC_ALLOW_CHANGES, SDC_APPLY, SDC_TOPOLOGY_EXTEND, SDC_TOPOLOGY_INTERNAL, SDC_USE_SUPPLIED_DISPLAY_CONFIG, SDC_VALIDATE, }; + use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_SHUTTINGDOWN}; pub type Saved = (Vec, Vec); @@ -538,14 +953,17 @@ mod windows_impl { pub struct Windows { log: fn(bool, &str), + cleanup: bool, owner: Option, file: Option, } impl Windows { - pub fn new(owner: crate::startup::ProcessId, log: fn(bool, &str)) -> Self { + pub fn new(owner: crate::startup::ProcessId, log: fn(bool, &str), cleanup: bool) -> Self { + lid::watch(); Self { log, + cleanup, // Without its start time a process id could be a later process's. owner: (owner.start != 0).then(|| owner.to_string()), file: crate::settings::app_data_dir().map(|directory| directory.join(RECORD_FILE)), @@ -588,6 +1006,17 @@ mod windows_impl { } } + /// A laptop's built-in panel. + fn is_built_in(technology: DISPLAYCONFIG_VIDEO_OUTPUT_TECHNOLOGY) -> bool { + [ + DISPLAYCONFIG_OUTPUT_TECHNOLOGY_INTERNAL, + DISPLAYCONFIG_OUTPUT_TECHNOLOGY_LVDS, + DISPLAYCONFIG_OUTPUT_TECHNOLOGY_DISPLAYPORT_EMBEDDED, + DISPLAYCONFIG_OUTPUT_TECHNOLOGY_UDI_EMBEDDED, + ] + .contains(&technology) + } + fn screen(path: &DISPLAYCONFIG_PATH_INFO, modes: &[DISPLAYCONFIG_MODE_INFO]) -> Screen { let technology = path.targetInfo.outputTechnology; // Without QDC_VIRTUAL_MODE_AWARE the index is the whole field. @@ -606,13 +1035,7 @@ mod windows_impl { Screen { adapter: crate::vdd_snapshot::luid_value(path.targetInfo.adapterId), target: path.targetInfo.id, - internal: [ - DISPLAYCONFIG_OUTPUT_TECHNOLOGY_INTERNAL, - DISPLAYCONFIG_OUTPUT_TECHNOLOGY_LVDS, - DISPLAYCONFIG_OUTPUT_TECHNOLOGY_DISPLAYPORT_EMBEDDED, - DISPLAYCONFIG_OUTPUT_TECHNOLOGY_UDI_EMBEDDED, - ] - .contains(&technology), + internal: is_built_in(technology), virtual_display: crate::vdd_snapshot::is_vdd_monitor( &crate::vdd_snapshot::monitor_path(path), ), @@ -620,6 +1043,76 @@ mod windows_impl { } } + /// The lid from Windows' GUID_LIDSWITCH_STATE_CHANGE notifications: the + /// current state comes once Windows knows the lid device and its state, + /// then each change. + mod lid { + use std::sync::Once; + + use windows::core::GUID; + use windows::Win32::Foundation::{ERROR_SUCCESS, HANDLE}; + use windows::Win32::System::Power::{ + PowerSettingRegisterNotification, DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS, + POWERBROADCAST_SETTING, + }; + use windows::Win32::UI::WindowsAndMessaging::{ + DEVICE_NOTIFY_CALLBACK, PBT_POWERSETTINGCHANGE, + }; + + const LIDSWITCH_STATE_CHANGE: GUID = + GUID::from_u128(0xba3e0f4d_b817_4094_a2d1_d56379e6a0f3); + static WATCH: super::super::LidWatch = super::super::LidWatch::new(); + + unsafe extern "system" fn changed( + _context: *const core::ffi::c_void, + kind: u32, + setting: *const core::ffi::c_void, + ) -> u32 { + if kind == PBT_POWERSETTINGCHANGE && !setting.is_null() { + let setting = setting.cast::(); + let header = unsafe { std::ptr::read_unaligned(setting) }; + if header.PowerSetting == LIDSWITCH_STATE_CHANGE && header.DataLength >= 4 { + let data = unsafe { std::ptr::addr_of!((*setting).Data) }.cast::(); + WATCH.note(unsafe { std::ptr::read_unaligned(data) }); + } + } + 0 + } + + /// Start listening. Later calls do nothing. + pub(super) fn watch() { + static REGISTER: Once = Once::new(); + REGISTER.call_once(|| { + // Windows keeps both for as long as the process runs. + let parameters = Box::leak(Box::new(DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS { + Callback: Some(changed), + Context: std::ptr::null_mut(), + })); + let mut registration = std::ptr::null_mut(); + let result = unsafe { + PowerSettingRegisterNotification( + &LIDSWITCH_STATE_CHANGE, + DEVICE_NOTIFY_CALLBACK, + HANDLE((parameters as *mut DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS).cast()), + &mut registration, + ) + }; + if result != ERROR_SUCCESS { + tracing::warn!( + error = result.0, + "Could not watch the lid; a built-in screen Windows withdraws is not \ + taken for lost" + ); + } + }); + } + + pub(super) fn lid() -> super::super::Lid { + watch(); + WATCH.lid() + } + } + impl super::Displays for Windows { type Saved = Saved; @@ -656,6 +1149,44 @@ mod windows_impl { } } + fn extend(&mut self) -> Result<(), i32> { + match unsafe { SetDisplayConfig(None, None, SDC_APPLY | SDC_TOPOLOGY_EXTEND) } { + 0 => Ok(()), + code => Err(code), + } + } + + fn built_in(&mut self) -> Option { + let (paths, _) = crate::vdd_snapshot::display_config(QDC_ALL_PATHS)?; + let mut panels = paths + .iter() + .filter(|path| is_built_in(path.targetInfo.outputTechnology)) + .peekable(); + Some(if panels.peek().is_none() { + BuiltIn::Missing + } else if panels.any(|path| path.targetInfo.targetAvailable.as_bool()) { + BuiltIn::Offered + } else { + BuiltIn::Withdrawn + }) + } + + fn hybrid(&self) -> bool { + crate::diagnostics::host_facts().hybrid == Some(true) + } + + fn lid(&self) -> super::Lid { + lid::lid() + } + + fn session_ending(&self) -> bool { + unsafe { GetSystemMetrics(SM_SHUTTINGDOWN) != 0 } + } + + fn cleanup(&self) -> bool { + self.cleanup + } + fn sleep(&mut self, duration: Duration) { std::thread::sleep(duration); } @@ -756,13 +1287,22 @@ mod tests { /// After an accepted call, reads return this. reapplied: Option>, internal: Option>, + extended: Option>, refuse_reapply: bool, refuse_internal: bool, + refuse_extend: bool, /// What reads return after a refused call: Windows changed the /// screens meanwhile. `Some(None)`: they can't be read. after_refused: Option>>, reapply_calls: Vec, internal_calls: usize, + extend_calls: usize, + /// What `QDC_ALL_PATHS` says of the panel; `None` (unread) by default. + built_in: Option, + hybrid: bool, + lid: Lid, + session_ending: bool, + cleanup: bool, slept: Duration, logs: Vec<(bool, String)>, owner: Option, @@ -789,7 +1329,7 @@ mod tests { } fn set_display_config_calls(&self) -> usize { - self.reapply_calls.len() + self.internal_calls + self.reapply_calls.len() + self.internal_calls + self.extend_calls } fn text(&self) -> String { @@ -840,6 +1380,38 @@ mod tests { Ok(()) } + fn extend(&mut self) -> Result<(), i32> { + self.extend_calls += 1; + if self.refuse_extend { + return Err(1610); + } + if let Some(screens) = self.extended.clone() { + self.reads.clear(); + self.last = Some(screens); + } + Ok(()) + } + + fn built_in(&mut self) -> Option { + self.built_in + } + + fn hybrid(&self) -> bool { + self.hybrid + } + + fn lid(&self) -> Lid { + self.lid + } + + fn session_ending(&self) -> bool { + self.session_ending + } + + fn cleanup(&self) -> bool { + self.cleanup + } + fn sleep(&mut self, duration: Duration) { self.slept += duration; } @@ -901,6 +1473,478 @@ mod tests { assert_eq!(plan(&[], &[]), Plan::Leave); } + /// The hybrid laptop where the panel went black: its panel, on the + /// integrated card, was the only screen before the turn on. + fn laptop_panel() -> Screen { + Screen { + adapter: 0x1298b, + target: 512, + internal: true, + virtual_display: false, + mode: Some((1920, 1200, 165_000)), + } + } + + /// After its turn on, Windows showed the desktop on the virtual display + /// alone, at 800x600. + fn lone_vdd() -> Screen { + Screen { + mode: Some((800, 600, 60_000)), + ..vdd() + } + } + + /// The guard's check before a turn off: whether it may run. + fn turn_off_allowed(guard: &mut Guard, fake: &mut Fake) -> bool { + guard.before(false, false, fake) + } + + /// The laptop's log: `paths=1 [internal …] vdd=0` before the turn on, + /// `paths=0 [] vdd=1` after it. The 0.3.4 guard only logged that, the + /// attempt timed out, and its turn off left Windows reporting no screen + /// at all (`paths=unknown`), which nothing but a restart repaired. Now + /// "Extend" puts the panel back beside the virtual display at once, + /// and the second screen can still attach. + #[test] + fn a_turn_on_that_leaves_the_virtual_display_alone_extends_at_once() { + let mut fake = Fake::reading([vec![laptop_panel()], vec![lone_vdd()]]); + fake.extended = Some(vec![laptop_panel(), lone_vdd()]); + let mut guard = Guard::new(); + assert!(guard.before(true, false, &mut fake)); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Extended); + assert_eq!( + ( + fake.extend_calls, + fake.reapply_calls.len(), + fake.internal_calls + ), + (1, 0, 0) + ); + assert!(!guard.alone()); + let text = fake.text(); + assert!( + text.contains( + "Screens after the virtual display turned on: paths=0 [] vdd=1\n\ + Windows put the desktop on the virtual display alone; bringing the PC's \ + screens back while it is on. Before: paths=1 [internal tgt=1298b/512 \ + 1920x1200@165Hz] vdd=0\n\ + Screens back beside the virtual display outcome=extended applied=extend \ + paths=1 [internal tgt=1298b/512 1920x1200@165Hz] vdd=1" + ), + "{text}" + ); + // The session's turn off then finds the panel on, as on any PC. + assert!(turn_off_allowed(&mut guard, &mut fake)); + fake.reads.push_back(Some(vec![laptop_panel()])); + assert_eq!(guard.after(false, true, &mut fake), Outcome::Left); + assert_eq!(fake.set_display_config_calls(), 1); + } + + /// Where "Extend" does not bring the panel back, the layout from before + /// the turn on does, without the virtual display: the start falls back + /// to the mirror, and its turn off finds the panel on. + #[test] + fn a_lone_virtual_display_that_extend_cannot_fix_gives_the_panel_back() { + let mut fake = Fake::reading([vec![laptop_panel()], vec![lone_vdd()]]); + fake.refuse_extend = true; + fake.reapplied = Some(vec![laptop_panel()]); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!( + guard.after(true, true, &mut fake), + Outcome::Restored { internal: false } + ); + assert_eq!((fake.extend_calls, &fake.reapply_calls[..]), (1, &[1][..])); + assert!(guard.alone(), "the display will not attach"); + assert!( + fake.text().ends_with( + "Screens restored outcome=restored applied=saved_layout paths=1 \ + [internal tgt=1298b/512 1920x1200@165Hz] vdd=0" + ), + "{}", + fake.text() + ); + assert!(turn_off_allowed(&mut guard, &mut fake)); + // Even a turn off after which Windows reports nothing, as on the + // laptop, changes nothing more: the panel was on before it. + fake.reads.push_back(None); + assert_eq!(guard.after(false, true, &mut fake), Outcome::Left); + assert_eq!(fake.set_display_config_calls(), 2); + + // Nothing works: the display stays alone and will not attach. + let mut fake = Fake::reading([vec![laptop_panel()], vec![lone_vdd()]]); + (fake.refuse_extend, fake.refuse_reapply) = (true, true); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); + assert_eq!((fake.extend_calls, fake.reapply_calls.len()), (1, 1)); + // "PC screen only" too: the panel is the built-in one. + assert_eq!(fake.internal_calls, 1); + assert!(guard.alone()); + // The next turn on that extends clears it. + let mut fake = Fake::reading([vec![laptop_panel()], vec![laptop_panel(), lone_vdd()]]); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); + assert!(!guard.alone()); + } + + /// The virtual display ended up alone during the session, Windows+P + /// "Second screen only" say. Its turn off would leave Windows with no + /// screen to report, so the guard puts the panel back first. + #[test] + fn a_turn_off_with_the_virtual_display_alone_puts_a_real_screen_back_first() { + let mut fake = Fake::reading([ + vec![laptop_panel()], + vec![laptop_panel(), lone_vdd()], + vec![lone_vdd()], + ]); + fake.reapplied = Some(vec![laptop_panel()]); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); + assert!(turn_off_allowed(&mut guard, &mut fake)); + // The layout from before the turn on, before the turn off runs. + assert_eq!(fake.reapply_calls, [1]); + let text = fake.text(); + assert!( + text.ends_with( + "Screens before the virtual display turns off: paths=0 [] vdd=1\n\ + The virtual display is the only screen on; bringing the PC's screens back \ + before turning it off. Before: paths=1 [internal tgt=1298b/512 \ + 1920x1200@165Hz] vdd=0\n\ + Screens restored outcome=restored applied=saved_layout paths=1 \ + [internal tgt=1298b/512 1920x1200@165Hz] vdd=0" + ), + "{text}" + ); + assert_eq!(guard.after(false, true, &mut fake), Outcome::Left); + assert_eq!(fake.set_display_config_calls(), 1); + } + + /// When nothing brings a real screen back, the turn off does not run: + /// the desktop stays on the virtual display, where the iPad and + /// Windows+P still reach it, rather than on no screen at all. + #[test] + fn the_virtual_display_is_never_turned_off_while_it_is_the_only_screen() { + let mut fake = Fake::reading([ + vec![laptop_panel()], + vec![laptop_panel(), lone_vdd()], + vec![lone_vdd()], + ]); + fake.owner = Some(HOST.into()); + (fake.refuse_reapply, fake.refuse_extend) = (true, true); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + guard.after(true, true, &mut fake); + assert!(!turn_off_allowed(&mut guard, &mut fake)); + // Every call, once: the layout, "PC screen only", "Extend". + assert_eq!( + ( + fake.reapply_calls.len(), + fake.internal_calls, + fake.extend_calls + ), + (1, 1, 1) + ); + let (warn, text) = fake.logs.last().unwrap(); + assert!(*warn); + assert_eq!( + text, + "Leaving the virtual display on: turning it off now would leave the PC with no \ + screen outcome=kept_on reason=saved_layout_refused code=87 \ + reason=internal_no_screen reason=extend_refused code=1610 paths=0 [] vdd=1; \ + Windows+P then \"PC screen only\" brings the built-in screen back" + ); + // The layout stays kept, in memory and for the failure reporter. + assert!(fake.kept.is_some()); + // The next turn off, the quit's say, tries again with it. + fake.refuse_reapply = false; + fake.reapplied = Some(vec![laptop_panel()]); + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.reapply_calls, [1, 1]); + guard.after(false, true, &mut fake); + assert_eq!(fake.kept, None); + + // A PC that had no real screen before turns it off as it did. + let mut fake = Fake::reading([vec![], vec![lone_vdd()], vec![lone_vdd()]]); + fake.refuse_extend = true; + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!( + ( + fake.reapply_calls.len(), + fake.internal_calls, + fake.extend_calls + ), + (0, 0, 1) + ); + } + + /// The hybrid laptop as `QDC_ALL_PATHS` showed it on the second run: + /// after the turn on, the panel's target was listed for every source + /// but `available=False`, so "Extend" was refused (31), the saved layout + /// was refused (1610) and "PC screen only" brought nothing back. Each + /// later turn off was held back after the same three calls. + fn hybrid_laptop() -> Fake { + let mut fake = Fake::reading([vec![laptop_panel()], vec![lone_vdd()]]); + fake.owner = Some(HOST.into()); + (fake.hybrid, fake.built_in, fake.lid) = (true, Some(BuiltIn::Withdrawn), Lid::Open); + (fake.refuse_extend, fake.refuse_reapply) = (true, true); + fake + } + + /// Its signature, the panel's target withdrawn while the virtual + /// display is on, ends the turn on with no `SetDisplayConfig` call, and + /// keeps the display on: the desktop stays on it, which the iPad shows. + #[test] + fn a_turn_on_that_takes_the_hybrid_laptops_panel_away_changes_nothing() { + let mut fake = hybrid_laptop(); + let mut guard = Guard::new(); + assert!(guard.before(true, false, &mut fake)); + assert_eq!(guard.after(true, true, &mut fake), Outcome::PanelLost); + assert_eq!(fake.set_display_config_calls(), 0); + assert!(guard.alone() && guard.panel_lost()); + let (warn, text) = fake.logs.last().unwrap(); + assert!(*warn); + assert_eq!( + text, + "The built-in screen went dark when the virtual display turned on: Windows reports \ + it unavailable, and only a restart brings it back. Leaving the virtual display on, \ + so the desktop stays on it outcome=panel_lost paths=0 [] vdd=1. Before: paths=1 \ + [internal tgt=1298b/512 1920x1200@165Hz] vdd=0" + ); + // Every turn off, the quit's too, stays held, still without a call. + for _ in 0..2 { + assert!(!turn_off_allowed(&mut guard, &mut fake)); + } + assert_eq!(fake.set_display_config_calls(), 0); + assert_eq!( + fake.logs.last().unwrap().1, + "Leaving the virtual display on: the built-in screen is still unavailable, and \ + turning the display off would leave the PC with no screen outcome=kept_on \ + reason=panel_lost paths=0 [] vdd=1; restarting the PC brings the built-in screen \ + back" + ); + assert!(fake.kept.is_some(), "kept for the failure reporter"); + // Once Windows ends the session, the turn off runs, so the restart + // starts without the display. + fake.session_ending = true; + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.set_display_config_calls(), 0); + assert!(fake.logs.last().unwrap().1.ends_with("outcome=session_end")); + // A panel that comes back (the lid opened, say) lets it run as on any PC. + let mut fake = hybrid_laptop(); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + guard.after(true, true, &mut fake); + fake.built_in = Some(BuiltIn::Offered); + fake.refuse_reapply = false; + fake.reapplied = Some(vec![laptop_panel()]); + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.reapply_calls, [1]); + } + + /// Quitting before the restart. Once the host quits, nothing streams the + /// display: the iPad's stream ends with it. The quit's turn off, a + /// cleanup, runs then, with no `SetDisplayConfig` call, so the restart + /// starts without the display, whose enabled state would outlast it. So + /// does the failure reporter's after a host that ended on its own, + /// comparing with the layout that host kept. + #[test] + fn quitting_with_the_panel_lost_turns_the_display_off_for_the_restart() { + let mut fake = hybrid_laptop(); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::PanelLost); + // While the host runs, held. + assert!(!turn_off_allowed(&mut guard, &mut fake)); + // The quit. + fake.cleanup = true; + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.set_display_config_calls(), 0); + let (warn, text) = fake.logs.last().unwrap(); + assert!(*warn); + assert_eq!( + text, + "The built-in screen is still unavailable, and nothing streams the virtual display \ + after this — turning it off, so the restart that brings the screen back starts \ + without it outcome=released reason=panel_lost paths=0 [] vdd=1" + ); + // The failure reporter, a new process, after the host ended. + let mut reporter = Guard::new(); + assert!(fake.kept.is_some()); + assert!(turn_off_allowed(&mut reporter, &mut fake)); + assert!(fake.text().contains( + "Comparing with the screens kept from before the virtual display turned on: paths=1" + )); + assert_eq!(fake.set_display_config_calls(), 0); + } + + /// The host that starts after a restart with the display still on (a + /// power cut ended the last one) has no layout from before. Its + /// start-up turn off is a cleanup too: it runs, with no call. + #[test] + fn a_start_up_turn_off_with_the_panel_withdrawn_turns_it_off() { + let mut fake = hybrid_laptop(); + fake.cleanup = true; + fake.reads = VecDeque::from([Some(vec![lone_vdd()])]); + let mut guard = Guard::new(); + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.set_display_config_calls(), 0); + assert!(fake.text().contains("outcome=released reason=panel_lost")); + // On a PC with one graphics card, it goes as before. + fake.hybrid = false; + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.extend_calls, 1); + } + + /// The other ways a turn on leaves the virtual display alone keep the + /// guard's calls: one graphics card, a panel Windows still offers, a + /// desktop PC, or a target list Windows can't give. + #[test] + fn only_a_withdrawn_panel_on_a_hybrid_pc_is_the_lost_panel() { + for (hybrid, built_in) in [ + (false, Some(BuiltIn::Withdrawn)), + (true, Some(BuiltIn::Offered)), + (true, Some(BuiltIn::Missing)), + (true, None), + ] { + let mut fake = hybrid_laptop(); + (fake.hybrid, fake.built_in) = (hybrid, built_in); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!( + guard.after(true, true, &mut fake), + Outcome::Failed, + "{hybrid} {built_in:?}" + ); + assert!(!guard.panel_lost()); + assert_eq!(fake.set_display_config_calls(), 3, "{hybrid} {built_in:?}"); + } + // The panel still on beside the display: nothing happened to it. + let mut fake = hybrid_laptop(); + fake.reads = VecDeque::from([ + Some(vec![laptop_panel()]), + Some(vec![laptop_panel(), lone_vdd()]), + ]); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); + assert!(!guard.panel_lost()); + // A closed laptop on an external monitor never had the panel on. + let mut fake = hybrid_laptop(); + fake.reads = VecDeque::from([Some(vec![monitor()]), Some(vec![lone_vdd()])]); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); + } + + /// A lid the user closes while the virtual display turns on withdraws + /// the panel too, and is no failure. With an external monitor on, the + /// turn on goes as on any PC; with the panel alone before, as #185's + /// steps go. Neither is a lost panel. + #[test] + fn a_lid_closed_during_the_turn_on_is_not_a_lost_panel() { + let mut fake = hybrid_laptop(); + fake.lid = Lid::Closed; + fake.reads = VecDeque::from([ + Some(vec![laptop_panel(), monitor()]), + Some(vec![monitor(), lone_vdd()]), + ]); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); + assert!(!guard.panel_lost() && !guard.alone()); + assert_eq!(fake.set_display_config_calls(), 0); + assert!(!fake.text().contains("panel_lost"), "{}", fake.text()); + + let mut fake = hybrid_laptop(); + fake.lid = Lid::Closed; + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); + assert!(!guard.panel_lost()); + assert!(!fake.text().contains("panel_lost"), "{}", fake.text()); + } + + /// Windows sends the lid's first state once it knows the lid device, + /// with no bound on when. Until then the lid is unknown, and a + /// withdrawn panel is not taken for lost: it goes through #185's steps, + /// and nothing records a lost panel. Once Windows says the lid is open, + /// the next turn on is. + #[test] + fn a_lid_state_windows_has_not_sent_yet_is_no_lost_panel() { + let watch = LidWatch::new(); + let mut fake = hybrid_laptop(); + fake.lid = watch.lid(); + assert_eq!(fake.lid, Lid::Unknown); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); + assert!(!guard.panel_lost()); + assert!(!fake.text().contains("panel_lost"), "{}", fake.text()); + // The first notification, late: open. + watch.note(1); + let mut fake = hybrid_laptop(); + fake.lid = watch.lid(); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::PanelLost); + watch.note(0); + assert_eq!(watch.lid(), Lid::Closed); + } + + /// With the lid watch's registration failed, the lid stays unknown for + /// good. No lost panel is ever recorded, so a lid closed on purpose + /// can't become one; while the host runs, a turn off goes as #185's do. + /// The cleanup at quit still turns the display off, so the restart + /// starts without it whatever the lid was. + #[test] + fn a_failed_lid_registration_records_no_lost_panel_yet_quitting_turns_it_off() { + let mut fake = hybrid_laptop(); + fake.lid = LidWatch::new().lid(); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); + assert!(!guard.panel_lost()); + let calls = fake.set_display_config_calls(); + // While the host runs: #185's steps, then held. + assert!(!turn_off_allowed(&mut guard, &mut fake)); + assert!(fake.set_display_config_calls() > calls); + assert!( + !fake.text().contains("reason=panel_lost"), + "{}", + fake.text() + ); + // The quit. + let calls = fake.set_display_config_calls(); + fake.cleanup = true; + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.set_display_config_calls(), calls); + assert!(fake + .logs + .last() + .unwrap() + .1 + .contains("outcome=released reason=panel_withdrawn lid=unknown")); + } + + /// The start-up turn off of a closed laptop whose virtual display is + /// still on from an earlier run, with no lost panel ever: it goes as + /// before, and the display is turned off. + #[test] + fn a_closed_lid_start_up_turn_off_runs_as_before() { + let mut fake = hybrid_laptop(); + fake.lid = Lid::Closed; + fake.reads = VecDeque::from([Some(vec![lone_vdd()])]); + let mut guard = Guard::new(); + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.extend_calls, 1); + assert!(!fake.text().contains("panel_lost"), "{}", fake.text()); + } + #[test] fn a_healthy_pc_sees_no_set_display_config_call() { // Panel and monitor stay through the session; the monitor as the @@ -925,13 +1969,6 @@ mod tests { ]); assert_eq!(session(&mut fake).1, Outcome::Left); assert_eq!(fake.set_display_config_calls(), 0); - // A turn on that leaves the virtual display alone is only logged: - // the layout stays the user's until the turn off. - let mut fake = Fake::reading([vec![panel()], vec![vdd()]]); - let mut guard = Guard::new(); - guard.before(true, false, &mut fake); - assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); - assert_eq!(fake.set_display_config_calls(), 0); } #[test] @@ -975,8 +2012,13 @@ mod tests { #[test] fn every_real_screen_lost_reapplies_the_layout_from_before_the_turn_on() { - // The desktop moved to the virtual display alone, then nothing. - let mut fake = Fake::reading([vec![panel(), monitor()], vec![vdd()], vec![vdd()], vec![]]); + // Both screens went with the virtual display's turn off. + let mut fake = Fake::reading([ + vec![panel(), monitor()], + vec![panel(), monitor(), vdd()], + vec![panel(), monitor(), vdd()], + vec![], + ]); fake.reapplied = Some(vec![panel(), monitor()]); let (_, outcome) = session(&mut fake); assert_eq!(outcome, Outcome::Restored { internal: false }); @@ -1006,7 +2048,13 @@ mod tests { #[test] fn a_screen_windows_brings_back_by_itself_is_left_alone() { - let mut fake = Fake::reading([vec![panel()], vec![vdd()], vec![vdd()], vec![], vec![]]); + let mut fake = Fake::reading([ + vec![panel()], + vec![panel(), vdd()], + vec![panel(), vdd()], + vec![], + vec![], + ]); fake.reads.push_back(Some(vec![panel()])); assert_eq!(session(&mut fake).1, Outcome::Left); assert_eq!(fake.set_display_config_calls(), 0); @@ -1015,7 +2063,12 @@ mod tests { #[test] fn a_refused_layout_falls_back_to_pc_screen_only_when_the_panel_was_on() { - let mut fake = Fake::reading([vec![panel(), monitor()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel(), monitor()], + vec![panel(), monitor(), vdd()], + vec![panel(), monitor(), vdd()], + vec![], + ]); fake.refuse_reapply = true; fake.internal = Some(vec![panel()]); assert_eq!(session(&mut fake).1, Outcome::Restored { internal: true }); @@ -1029,7 +2082,12 @@ mod tests { fake.text() ); // A layout Windows took that brought no screen back counts as failed too. - let mut fake = Fake::reading([vec![panel()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel()], + vec![panel(), vdd()], + vec![panel(), vdd()], + vec![], + ]); fake.internal = Some(vec![panel()]); assert_eq!(session(&mut fake).1, Outcome::Restored { internal: true }); assert_eq!((fake.reapply_calls.len(), fake.internal_calls), (1, 1)); @@ -1038,7 +2096,12 @@ mod tests { #[test] fn without_the_panel_before_there_is_no_switch_to_it() { // A closed laptop on an external monitor that went dark. - let mut fake = Fake::reading([vec![monitor()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![monitor()], + vec![monitor(), vdd()], + vec![monitor(), vdd()], + vec![], + ]); fake.refuse_reapply = true; assert_eq!(session(&mut fake).1, Outcome::Failed); assert_eq!((fake.reapply_calls.len(), fake.internal_calls), (1, 0)); @@ -1054,7 +2117,12 @@ mod tests { #[test] fn one_attempt_per_change() { - let mut fake = Fake::reading([vec![panel()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel()], + vec![panel(), vdd()], + vec![panel(), vdd()], + vec![], + ]); (fake.refuse_reapply, fake.refuse_internal) = (true, true); let (mut guard, outcome) = session(&mut fake); assert_eq!(outcome, Outcome::Failed); @@ -1104,7 +2172,7 @@ mod tests { fake.text() ); // Known before, unknown after. - let mut fake = Fake::reading([vec![panel()], vec![vdd()], vec![vdd()]]); + let mut fake = Fake::reading([vec![panel()], vec![panel(), vdd()], vec![panel(), vdd()]]); fake.reads.push_back(None); assert_eq!(session(&mut fake).1, Outcome::Left); assert_eq!(fake.set_display_config_calls(), 0); @@ -1113,7 +2181,12 @@ mod tests { #[test] fn the_fallback_looks_again_and_leaves_screens_windows_brought_back() { // Windows put both screens back while it refused the saved layout. - let mut fake = Fake::reading([vec![panel(), monitor()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel(), monitor()], + vec![panel(), monitor(), vdd()], + vec![panel(), monitor(), vdd()], + vec![], + ]); fake.refuse_reapply = true; fake.after_refused = Some(Some(vec![panel(), monitor()])); fake.internal = Some(vec![panel()]); @@ -1130,7 +2203,12 @@ mod tests { fake.text() ); // The same after a layout Windows took that showed nothing at first. - let mut fake = Fake::reading([vec![panel(), monitor()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel(), monitor()], + vec![panel(), monitor(), vdd()], + vec![panel(), monitor(), vdd()], + vec![], + ]); fake.internal = Some(vec![panel()]); let mut guard = Guard::new(); guard.before(true, false, &mut fake); @@ -1163,6 +2241,24 @@ mod tests { fn internal_only(&mut self) -> Result<(), i32> { self.0.internal_only() } + fn extend(&mut self) -> Result<(), i32> { + self.0.extend() + } + fn built_in(&mut self) -> Option { + self.0.built_in() + } + fn hybrid(&self) -> bool { + self.0.hybrid() + } + fn lid(&self) -> Lid { + self.0.lid() + } + fn session_ending(&self) -> bool { + self.0.session_ending() + } + fn cleanup(&self) -> bool { + self.0.cleanup() + } fn sleep(&mut self, duration: Duration) { self.0.sleep(duration) } @@ -1185,7 +2281,12 @@ mod tests { #[test] fn the_fallback_stops_when_the_screens_cannot_be_read() { - let mut fake = Fake::reading([vec![panel()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel()], + vec![panel(), vdd()], + vec![panel(), vdd()], + vec![], + ]); fake.refuse_reapply = true; fake.after_refused = Some(None); fake.internal = Some(vec![panel()]); @@ -1203,19 +2304,19 @@ mod tests { const HOST: &str = "4242-133456789012345678"; - /// The host turned the display on, which left the virtual display alone, - /// then crashed. Its failure reporter, a new process with nothing in - /// memory, turns the display off and the PC has no screen. + /// The host turned the display on, then crashed. Its failure reporter, + /// a new process with nothing in memory, turns the display off, which + /// takes the PC's screens with it. #[test] fn the_failure_reporter_restores_the_layout_its_host_kept() { - let mut host = Fake::reading([vec![panel(), monitor()], vec![vdd()]]); + let mut host = Fake::reading([vec![panel(), monitor()], vec![panel(), monitor(), vdd()]]); host.owner = Some(HOST.into()); let mut guard = Guard::new(); guard.before(true, false, &mut host); guard.after(true, true, &mut host); let kept = host.kept.clone().expect("the host kept its screens"); - let mut reporter = Fake::reading([vec![vdd()], vec![]]); + let mut reporter = Fake::reading([vec![panel(), monitor(), vdd()], vec![]]); reporter.read_count = 100; reporter.owner = Some(HOST.into()); reporter.kept = Some(kept.clone()); @@ -1239,16 +2340,31 @@ mod tests { // Checked after a turn off that worked: deleted. assert_eq!(reporter.kept, None); - // Without the record, the reporter only sees the virtual display before: - // nothing to compare with, the screen stays black. This is what the - // record fixes. + // The host's display was left on alone: the reporter puts that + // layout back before its turn off. + let mut reporter = Fake::reading([vec![vdd()]]); + reporter.read_count = 100; + reporter.owner = Some(HOST.into()); + reporter.kept = Some(kept); + reporter.reapplied = Some(vec![panel(), monitor()]); + let mut guard = Guard::new(); + assert!(guard.before(false, false, &mut reporter)); + assert_eq!(reporter.reapply_calls, [1]); + assert_eq!(guard.after(false, true, &mut reporter), Outcome::Left); + + // Without the record, the reporter only sees the virtual display + // before: no layout to put back, only "Extend", which found no + // screen here. This is what the record fixes. let mut reporter = Fake::reading([vec![vdd()], vec![]]); reporter.owner = Some(HOST.into()); reporter.reapplied = Some(vec![panel()]); let mut guard = Guard::new(); - guard.before(false, false, &mut reporter); + assert!(guard.before(false, false, &mut reporter)); assert_eq!(guard.after(false, true, &mut reporter), Outcome::Left); - assert_eq!(reporter.set_display_config_calls(), 0); + assert_eq!( + (reporter.reapply_calls.len(), reporter.extend_calls), + (0, 1) + ); } #[test] @@ -1282,7 +2398,9 @@ mod tests { Outcome::Left, "{record}" ); - assert_eq!(reporter.set_display_config_calls(), 0, "{record}"); + // Only "Extend", which needs no layout. + assert!(reporter.reapply_calls.is_empty(), "{record}"); + assert_eq!(reporter.internal_calls, 0, "{record}"); // Another host's record stays, for its own reporter. if owner != Some(HOST) { assert_eq!(reporter.kept.as_deref(), Some(record.as_str())); @@ -1391,14 +2509,12 @@ mod tests { // The record on disk still holds the first read, for the reporter. let kept = parse::(HOST, fake.kept.as_deref().unwrap()).unwrap(); assert_eq!((kept.screens.len(), kept.saved), (2, 1)); - guard.after(true, true, &mut fake); - guard.before(false, false, &mut fake); - fake.reads.push_back(Some(vec![])); + // "Extend" brings nothing back here; the first read's layout does. assert_eq!( - guard.after(false, true, &mut fake), + guard.after(true, true, &mut fake), Outcome::Restored { internal: false } ); - assert_eq!(fake.reapply_calls, [1]); + assert_eq!((fake.extend_calls, &fake.reapply_calls[..]), (1, &[1][..])); // The same retry in a host whose layout is only on disk, as after a // failed Disable that left nothing in memory. @@ -1436,7 +2552,7 @@ mod tests { let mut host = Fake::reading([vec![panel()]]); host.owner = Some(HOST.into()); Guard::new().before(true, false, &mut host); - let mut reporter = Fake::reading([vec![vdd()], vec![]]); + let mut reporter = Fake::reading([vec![panel(), vdd()], vec![]]); reporter.owner = Some(HOST.into()); reporter.kept = host.kept.clone(); reporter.reapplied = Some(vec![panel()]); diff --git a/host/src/encoder/input.rs b/host/src/encoder/input.rs index de34d4a2..43960c40 100644 --- a/host/src/encoder/input.rs +++ b/host/src/encoder/input.rs @@ -1,5 +1,5 @@ use ffmpeg_next::format::Pixel; -use tracing::info; +use tracing::{info, warn}; use crate::capture::FRAME_PADDING; @@ -96,77 +96,239 @@ pub(super) fn retries_as_yuv(mode: EncoderInput, format: Pixel) -> bool { mode == EncoderInput::Automatic && matches!(format, Pixel::BGRA | Pixel::BGRZ) } +/// swscale converting BGRA to `format` on `threads` threads, or nothing +/// when the encoder takes BGRA itself. pub(super) fn conversion_context( format: Pixel, width: u32, height: u32, -) -> Result, ffmpeg_next::Error> { + threads: usize, +) -> Result, ffmpeg_next::Error> { if !matches!(format, Pixel::YUV420P | Pixel::NV12) { return Ok(None); } - ffmpeg_next::software::scaling::Context::get( - Pixel::BGRA, - width, - height, - format, - width, - height, - ffmpeg_next::software::scaling::Flags::FAST_BILINEAR, - ) + with_one_thread_fallback(threads, |threads| { + Scaler::new(format, width, height, threads) + }) .map(Some) } -/// Converts capture's tightly packed BGRA rows into `output` where they lie: -/// swscale reads the capture buffer itself, with no copy into a BGRA frame -/// first. The call returns when the conversion is done, so the buffer can -/// go back to capture straight after. +/// Opens a scaler with `open` on `threads` threads, or on one when that +/// fails: swscale can fail to start its threads, and the single-threaded +/// conversion, which needs none, must still open wherever it did before. +fn with_one_thread_fallback( + threads: usize, + mut open: impl FnMut(usize) -> Result, +) -> Result { + match open(threads) { + Err(error) if threads > 1 => { + warn!(%error, threads, "swscale could not start its threads; converting on one"); + open(1) + } + opened => opened, + } +} + +/// Threads one conversion is split across: half the logical processors, at +/// most four. A 2732x2048 frame takes about 14 ms on one thread of a desktop +/// CPU, most of a 60 fps frame's 16.7 ms. +pub(super) fn scaler_threads() -> usize { + threads_for(std::thread::available_parallelism().map_or(1, |count| count.get())) +} + +fn threads_for(processors: usize) -> usize { + (processors / 2).clamp(1, 4) +} + +/// swscale converting BGRA to `output` at one size, FAST_BILINEAR, on +/// `threads` threads. FFmpeg 7.1 splits a conversion across threads only in +/// `sws_scale_frame`; `sws_scale`, which ffmpeg-next's `Context::run` calls, +/// always runs on one. +pub(super) struct Scaler { + context: *mut ffmpeg_next::ffi::SwsContext, + width: u32, + height: u32, + output: Pixel, + threads: usize, +} + +impl Scaler { + fn new( + output: Pixel, + width: u32, + height: u32, + threads: usize, + ) -> Result { + use ffmpeg_next::ffi::{ + av_opt_set_int, sws_alloc_context, sws_init_context, AVPixelFormat, SWS_FAST_BILINEAR, + }; + let context = unsafe { sws_alloc_context() }; + if context.is_null() { + return Err(ffmpeg_next::Error::Other { + errno: libc::ENOMEM, + }); + } + // Freed by Drop from here on, including on the errors below. + let scaler = Self { + context, + width, + height, + output, + threads, + }; + let options = [ + (c"srcw", i64::from(width)), + (c"srch", i64::from(height)), + (c"src_format", AVPixelFormat::from(Pixel::BGRA) as i64), + (c"dstw", i64::from(width)), + (c"dsth", i64::from(height)), + (c"dst_format", AVPixelFormat::from(output) as i64), + (c"sws_flags", i64::from(SWS_FAST_BILINEAR)), + (c"threads", threads as i64), + ]; + for (name, value) in options { + let result = unsafe { av_opt_set_int(context.cast(), name.as_ptr(), value, 0) }; + if result < 0 { + return Err(ffmpeg_next::Error::from(result)); + } + } + let result = + unsafe { sws_init_context(context, std::ptr::null_mut(), std::ptr::null_mut()) }; + if result < 0 { + return Err(ffmpeg_next::Error::from(result)); + } + Ok(scaler) + } + + pub(super) fn threads(&self) -> usize { + self.threads + } + + /// Converts `input`, a BGRA frame of the scaler's size, into `output`. + pub(super) fn run( + &mut self, + input: &ffmpeg_next::frame::Video, + output: &mut ffmpeg_next::frame::Video, + ) -> Result<(), ffmpeg_next::Error> { + if input.format() != Pixel::BGRA + || input.width() != self.width + || input.height() != self.height + { + return Err(ffmpeg_next::Error::InputChanged); + } + self.check_output(output)?; + // SAFETY: both frames own their buffers and have the sizes checked. + let result = unsafe { self.scale_frame(output.as_mut_ptr(), input.as_ptr()) }; + if result < 0 { + return Err(ffmpeg_next::Error::from(result)); + } + Ok(()) + } + + /// `sws_scale_frame`, after which swscale holds no reference to either + /// frame. It drops them itself once it gets as far as converting. In + /// FFmpeg 7.1 `sws_frame_start` can fail after taking the source, which + /// would leave it held until the next conversion, so an error ends the + /// frame here too. + /// + /// # Safety + /// Both frames must be valid, and `output` reference-counted. + unsafe fn scale_frame( + &mut self, + output: *mut ffmpeg_next::ffi::AVFrame, + source: *const ffmpeg_next::ffi::AVFrame, + ) -> libc::c_int { + let result = unsafe { ffmpeg_next::ffi::sws_scale_frame(self.context, output, source) }; + if result < 0 { + unsafe { ffmpeg_next::ffi::sws_frame_end(self.context) }; + } + result + } + + fn check_output(&self, output: &ffmpeg_next::frame::Video) -> Result<(), ffmpeg_next::Error> { + if output.format() != self.output + || output.width() != self.width + || output.height() != self.height + { + return Err(ffmpeg_next::Error::OutputChanged); + } + Ok(()) + } +} + +impl Drop for Scaler { + fn drop(&mut self) { + unsafe { ffmpeg_next::ffi::sws_freeContext(self.context) } + } +} + +/// Converts capture's BGRA rows, `row_bytes` apart, into `output` where they +/// lie: swscale reads the capture buffer itself, with no copy into a BGRA +/// frame first. Rows wider than the scaler's input, or more of them, are a +/// capture with an odd last column or row, which is left out. The call +/// returns when the conversion is done, so the buffer can go back to +/// capture straight after. /// /// swscale can read up to [`FRAME_PADDING`] bytes past the last row, so /// `rows` without that many after the picture, as capture's /// [`frame_buffer`](crate::capture::frame_buffer) has, are copied into a /// frame FFmpeg allocated first. pub(super) fn convert_packed_bgra( - scaler: &mut ffmpeg_next::software::scaling::Context, + scaler: &mut Scaler, rows: &[u8], + row_bytes: usize, output: &mut ffmpeg_next::frame::Video, ) -> Result<(), ffmpeg_next::Error> { - let input = *scaler.input(); - let row_bytes = input.width as usize * 4; - let frame_bytes = row_bytes * input.height as usize; - if input.format != Pixel::BGRA || rows.len() < frame_bytes { + use ffmpeg_next::ffi::{ + av_buffer_create, av_frame_alloc, av_frame_free, AVPixelFormat, AV_BUFFER_FLAG_READONLY, + }; + let (width, height) = (scaler.width, scaler.height); + let picture_bytes = width as usize * 4; + // The last row ends after its picture's pixels. + let frame_bytes = row_bytes * (height as usize).saturating_sub(1) + picture_bytes; + if row_bytes < picture_bytes || rows.len() < frame_bytes { return Err(ffmpeg_next::Error::InputChanged); } - let wanted = scaler.output(); - if output.format() != wanted.format - || output.width() != wanted.width - || output.height() != wanted.height - { - return Err(ffmpeg_next::Error::OutputChanged); - } + scaler.check_output(output)?; if rows.len() < frame_bytes + FRAME_PADDING { - let copied = copied_frame(&rows[..frame_bytes], input.width, input.height); + let copied = copied_frame(&rows[..frame_bytes], row_bytes, width, height); return scaler.run(&copied, output); } - let planes = [ - rows.as_ptr(), - std::ptr::null(), - std::ptr::null(), - std::ptr::null(), - ]; - let strides = [row_bytes as libc::c_int, 0, 0, 0]; - // swscale reads `input.height` rows of `row_bytes` from `rows`, and at - // most FRAME_PADDING more, checked above, and writes the planes `output` - // allocated at the output size. + // sws_scale_frame takes reference-counted frames, so `rows` is wrapped + // in a buffer that never frees it. `scale_frame` leaves swscale holding + // no reference, whether it converted or failed, and the frame is freed + // below, so nothing refers to `rows` after this call. + unsafe extern "C" fn borrowed(_: *mut libc::c_void, _: *mut u8) {} + let mut source = unsafe { av_frame_alloc() }; + if source.is_null() { + return Err(ffmpeg_next::Error::Other { + errno: libc::ENOMEM, + }); + } + // SAFETY: swscale reads `height` rows of `picture_bytes`, `row_bytes` + // apart, from `rows`, and at most FRAME_PADDING more, checked above, and + // writes the planes `output` allocated at the output size. let result = unsafe { - ffmpeg_next::ffi::sws_scale( - scaler.as_mut_ptr(), - planes.as_ptr(), - strides.as_ptr(), - 0, - input.height as libc::c_int, - (*output.as_mut_ptr()).data.as_ptr(), - (*output.as_mut_ptr()).linesize.as_ptr(), - ) + let data = rows.as_ptr().cast_mut(); + (*source).format = AVPixelFormat::from(Pixel::BGRA) as libc::c_int; + (*source).width = width as libc::c_int; + (*source).height = height as libc::c_int; + (*source).data[0] = data; + (*source).linesize[0] = row_bytes as libc::c_int; + (*source).buf[0] = av_buffer_create( + data, + rows.len(), + Some(borrowed), + std::ptr::null_mut(), + AV_BUFFER_FLAG_READONLY, + ); + let result = if (*source).buf[0].is_null() { + -libc::ENOMEM + } else { + scaler.scale_frame(output.as_mut_ptr(), source) + }; + av_frame_free(&mut source); + result }; if result < 0 { return Err(ffmpeg_next::Error::from(result)); @@ -174,19 +336,25 @@ pub(super) fn convert_packed_bgra( Ok(()) } -/// `rows` copied into a BGRA frame FFmpeg allocated. At odd widths swscale -/// reads one pixel past each row for the last chroma sample, which lands in -/// the frame's unset stride padding, so each row's last pixel is repeated -/// there. -fn copied_frame(rows: &[u8], width: u32, height: u32) -> ffmpeg_next::frame::Video { - let row_bytes = width as usize * 4; +/// The top-left `width`x`height` of `rows`, `row_bytes` apart, copied into a +/// BGRA frame FFmpeg allocated. At odd widths swscale reads one pixel past +/// each row for the last chroma sample, which lands in the frame's unset +/// stride padding, so each row's last pixel is repeated there. +fn copied_frame( + rows: &[u8], + row_bytes: usize, + width: u32, + height: u32, +) -> ffmpeg_next::frame::Video { + let picture_bytes = width as usize * 4; let mut copied = ffmpeg_next::frame::Video::new(Pixel::BGRA, width, height); let stride = copied.stride(0); - for (y, row) in rows.chunks_exact(row_bytes).enumerate() { + for (y, row) in rows.chunks(row_bytes).take(height as usize).enumerate() { + let row = &row[..picture_bytes]; let line = &mut copied.data_mut(0)[y * stride..(y + 1) * stride]; - line[..row_bytes].copy_from_slice(row); - if let Some(next) = line.get_mut(row_bytes..row_bytes + 4) { - next.copy_from_slice(&row[row_bytes - 4..]); + line[..picture_bytes].copy_from_slice(row); + if let Some(next) = line.get_mut(picture_bytes..picture_bytes + 4) { + next.copy_from_slice(&row[picture_bytes - 4..]); } } copied @@ -385,18 +553,29 @@ mod tests { } } - /// What the copy into an aligned BGRA frame gives. + /// What the copy into an aligned BGRA frame gives, converted by + /// ffmpeg-next's single-threaded `sws_scale`. fn converted_from_copy( - scaler: &mut ffmpeg_next::software::scaling::Context, format: Pixel, rows: &[u8], width: u32, height: u32, ) -> ffmpeg_next::frame::Video { let frame_bytes = width as usize * 4 * height as usize; - let copied = copied_frame(&rows[..frame_bytes], width, height); + let copied = copied_frame(&rows[..frame_bytes], width as usize * 4, width, height); let mut converted = ffmpeg_next::frame::Video::new(format, width, height); - scaler.run(&copied, &mut converted).unwrap(); + ffmpeg_next::software::scaling::Context::get( + Pixel::BGRA, + width, + height, + format, + width, + height, + ffmpeg_next::software::scaling::Flags::FAST_BILINEAR, + ) + .unwrap() + .run(&copied, &mut converted) + .unwrap(); converted } @@ -422,37 +601,146 @@ mod tests { } } - /// swscale reading a padded capture buffer gives the same picture as the - /// copy into an aligned BGRA frame it replaced, at widths whose rows are - /// and aren't 64-byte multiples. + /// swscale reading a padded capture buffer, on one thread or several, + /// gives the same picture as single-threaded swscale reading the copy into + /// an aligned BGRA frame it replaced, at widths whose rows are (1680) and + /// aren't (2732, 1366) 64-byte multiples, and whole iPad screens. #[test] fn converting_capture_rows_in_place_matches_the_copied_frame() { ffmpeg_next::init().unwrap(); - for (width, height) in [(2732u32, 64u32), (96, 64), (1366, 32), (102, 18)] { + for (width, height) in [ + (2732u32, 2048u32), + (2732, 64), + (1680, 1050), + (96, 64), + (1366, 32), + (102, 18), + ] { let row_bytes = width as usize * 4; let frame_bytes = row_bytes * height as usize; let mut rows = crate::capture::frame_buffer(frame_bytes); fill_pattern(&mut rows[..frame_bytes], row_bytes); for format in [Pixel::YUV420P, Pixel::NV12] { - let mut scaler = conversion_context(format, width, height).unwrap().unwrap(); - let before = converted_from_copy(&mut scaler, format, &rows, width, height); + let before = converted_from_copy(format, &rows, width, height); + for threads in [1, 4] { + let mut scaler = Scaler::new(format, width, height, threads).unwrap(); + let mut after = ffmpeg_next::frame::Video::new(format, width, height); + convert_packed_bgra(&mut scaler, &rows, row_bytes, &mut after).unwrap(); + let label = format!("{width}x{height} {format:?} threads={threads}"); + assert_same_picture(&before, &after, &label); + } + let mut scaler = conversion_context(format, width, height, scaler_threads()) + .unwrap() + .unwrap(); let mut after = ffmpeg_next::frame::Video::new(format, width, height); - convert_packed_bgra(&mut scaler, &rows, &mut after).unwrap(); - assert_same_picture(&before, &after, &format!("{width}x{height} {format:?}")); // A short buffer or another size is refused, never read past. assert_eq!( - convert_packed_bgra(&mut scaler, &rows[..frame_bytes - 1], &mut after), + convert_packed_bgra( + &mut scaler, + &rows[..frame_bytes - 1], + row_bytes, + &mut after + ), + Err(ffmpeg_next::Error::InputChanged) + ); + assert_eq!( + convert_packed_bgra(&mut scaler, &rows, row_bytes - 4, &mut after), Err(ffmpeg_next::Error::InputChanged) ); let mut other = ffmpeg_next::frame::Video::new(format, width + 2, height); assert_eq!( - convert_packed_bgra(&mut scaler, &rows, &mut other), + convert_packed_bgra(&mut scaler, &rows, row_bytes, &mut other), Err(ffmpeg_next::Error::OutputChanged) ); } } } + #[test] + fn conversions_use_half_the_processors_up_to_four_threads() { + ffmpeg_next::init().unwrap(); + for (processors, threads) in [(1, 1), (2, 1), (4, 2), (6, 3), (8, 4), (16, 4), (64, 4)] { + assert_eq!(threads_for(processors), threads, "{processors} processors"); + } + let scaler = conversion_context(Pixel::YUV420P, 2732, 2048, scaler_threads()) + .unwrap() + .unwrap(); + assert_eq!(scaler.threads(), scaler_threads()); + let mut set = 0; + let result = unsafe { + ffmpeg_next::ffi::av_opt_get_int( + scaler.context.cast(), + c"threads".as_ptr(), + 0, + &mut set, + ) + }; + assert!(result >= 0); + assert_eq!(set, scaler_threads() as i64); + } + + /// A conversion swscale refuses still leaves it holding no reference to + /// the source, so a borrowed capture buffer is free when the call ends. + /// This error comes after swscale holds both frames. The one that needed + /// `scale_frame`'s own `sws_frame_end`, in FFmpeg 7.1's `sws_frame_start` + /// after it took the source, only happens when memory runs out. + #[test] + fn a_failed_conversion_keeps_no_reference_to_the_source() { + ffmpeg_next::init().unwrap(); + let (width, height) = (96, 64); + let source = ffmpeg_next::frame::Video::new(Pixel::BGRA, width, height); + for threads in [1, 4] { + let mut scaler = Scaler::new(Pixel::YUV420P, width, height, threads).unwrap(); + // An output larger than the scaler's makes swscale fail once it + // holds both frames. + let mut output = ffmpeg_next::frame::Video::new(Pixel::YUV420P, width, height * 2); + let result = unsafe { scaler.scale_frame(output.as_mut_ptr(), source.as_ptr()) }; + assert!(result < 0, "threads={threads}"); + let held = + unsafe { ffmpeg_next::ffi::av_buffer_get_ref_count((*source.as_ptr()).buf[0]) }; + assert_eq!(held, 1, "threads={threads}"); + } + } + + /// A scaler whose threads fail to start opens on one thread instead; a + /// failure on one thread is the encoder's, as before threading. + #[test] + fn a_scaler_whose_threads_fail_converts_on_one() { + ffmpeg_next::init().unwrap(); + let refused = || ffmpeg_next::Error::Other { + errno: libc::ENOMEM, + }; + let mut tried = Vec::new(); + let scaler = with_one_thread_fallback(4, |threads| { + tried.push(threads); + if threads > 1 { + return Err(refused()); + } + Scaler::new(Pixel::YUV420P, 96, 64, threads) + }) + .unwrap(); + assert_eq!(tried, [4, 1]); + assert_eq!(scaler.threads(), 1); + + for threads in [4, 1] { + let mut tried = Vec::new(); + let opened = with_one_thread_fallback(threads, |threads| { + tried.push(threads); + Err(refused()) + }); + assert!(opened.is_err()); + assert_eq!(tried, if threads > 1 { vec![4, 1] } else { vec![1] }); + } + + let mut tried = Vec::new(); + let scaler = with_one_thread_fallback(4, |threads| { + tried.push(threads); + Scaler::new(Pixel::YUV420P, 96, 64, threads) + }) + .unwrap(); + assert_eq!((tried, scaler.threads()), (vec![4], 4)); + } + #[test] fn capture_padding_is_ffmpegs() { assert_eq!( @@ -461,21 +749,22 @@ mod tests { ); } - /// Rows with no readable bytes after them, ending where an unreadable - /// page starts, convert to the same picture without a crash: swscale - /// can read past the last row, so they take the copy. + /// `len` bytes that end where a page that faults on any access starts. #[cfg(unix)] - #[test] - fn rows_without_padding_are_never_read_past() { - ffmpeg_next::init().unwrap(); - let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize; - for (width, height) in [(2733u32, 2048u32), (1365, 32), (17, 3)] { - let row_bytes = width as usize * 4; - let frame_bytes = row_bytes * height as usize; - let readable = frame_bytes.div_ceil(page) * page; + struct Guarded { + base: *mut libc::c_void, + mapped: usize, + len: usize, + } + + #[cfg(unix)] + impl Guarded { + fn new(len: usize) -> Self { + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize; + let readable = len.div_ceil(page) * page; // SAFETY: a fresh private mapping, `readable` bytes then one - // page that faults on any access; unmapped below. - let rows = unsafe { + // page that faults on any access; unmapped on drop. + unsafe { let base = libc::mmap( std::ptr::null_mut(), readable + page, @@ -487,26 +776,91 @@ mod tests { assert_ne!(base, libc::MAP_FAILED); let guard = base.cast::().add(readable); assert_eq!(libc::mprotect(guard.cast(), page, libc::PROT_NONE), 0); - std::slice::from_raw_parts_mut(guard.sub(frame_bytes), frame_bytes) - }; + Self { + base, + mapped: readable + page, + len, + } + } + } + + fn bytes(&mut self) -> &mut [u8] { + let start = self.mapped - unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize; + // SAFETY: the last `len` readable bytes of the mapping. + unsafe { + std::slice::from_raw_parts_mut( + self.base.cast::().add(start - self.len), + self.len, + ) + } + } + } + + #[cfg(unix)] + impl Drop for Guarded { + fn drop(&mut self) { + assert_eq!(unsafe { libc::munmap(self.base, self.mapped) }, 0); + } + } + + /// Rows with no readable bytes after them, ending where an unreadable + /// page starts, convert to the same picture without a crash: swscale + /// can read past the last row, so they take the copy. + #[cfg(unix)] + #[test] + fn rows_without_padding_are_never_read_past() { + ffmpeg_next::init().unwrap(); + for (width, height) in [(2733u32, 2048u32), (1365, 32), (17, 3)] { + let row_bytes = width as usize * 4; + let frame_bytes = row_bytes * height as usize; + let mut mapping = Guarded::new(frame_bytes); + let rows = mapping.bytes(); fill_pattern(rows, row_bytes); // The pixel swscale reads after each odd-width row is set. - let copied = copied_frame(rows, width, height); + let copied = copied_frame(rows, row_bytes, width, height); let stride = copied.stride(0); for y in 0..height as usize { let line = &copied.data(0)[y * stride..y * stride + row_bytes + 4]; assert_eq!(line[row_bytes..], line[row_bytes - 4..row_bytes]); } for format in [Pixel::YUV420P, Pixel::NV12] { - let mut scaler = conversion_context(format, width, height).unwrap().unwrap(); - let mut after = ffmpeg_next::frame::Video::new(format, width, height); - convert_packed_bgra(&mut scaler, rows, &mut after).unwrap(); - let before = converted_from_copy(&mut scaler, format, rows, width, height); - assert_same_picture(&before, &after, &format!("{width}x{height} {format:?}")); + let before = converted_from_copy(format, rows, width, height); + for threads in [1, 4] { + let mut scaler = Scaler::new(format, width, height, threads).unwrap(); + let mut after = ffmpeg_next::frame::Video::new(format, width, height); + convert_packed_bgra(&mut scaler, rows, row_bytes, &mut after).unwrap(); + let label = format!("{width}x{height} {format:?} threads={threads}"); + assert_same_picture(&before, &after, &label); + } } - unsafe { - let base = rows.as_mut_ptr().add(frame_bytes).sub(readable); - assert_eq!(libc::munmap(base.cast(), readable + page), 0); + } + } + + /// Capture's padded rows are converted where they lie, on one thread or + /// several, without reading past the padding: an unreadable page starts + /// right after it. Even widths give the copy's picture; at odd widths the + /// last chroma column takes the next row's first pixel instead. + #[cfg(unix)] + #[test] + fn padded_rows_are_read_in_place_no_further_than_the_padding() { + ffmpeg_next::init().unwrap(); + for (width, height) in [(2732u32, 2048u32), (2733, 2048), (1680, 1050), (17, 3)] { + let row_bytes = width as usize * 4; + let frame_bytes = row_bytes * height as usize; + let mut mapping = Guarded::new(frame_bytes + FRAME_PADDING); + let rows = mapping.bytes(); + fill_pattern(&mut rows[..frame_bytes], row_bytes); + for format in [Pixel::YUV420P, Pixel::NV12] { + let before = converted_from_copy(format, rows, width, height); + for threads in [1, 4] { + let mut scaler = Scaler::new(format, width, height, threads).unwrap(); + let mut after = ffmpeg_next::frame::Video::new(format, width, height); + convert_packed_bgra(&mut scaler, rows, row_bytes, &mut after).unwrap(); + if width % 2 == 0 { + let label = format!("{width}x{height} {format:?} threads={threads}"); + assert_same_picture(&before, &after, &label); + } + } } } } @@ -531,7 +885,7 @@ mod tests { let mut planar = ffmpeg_next::frame::Video::new(Pixel::YUV420P, width, height); let mut nv12 = ffmpeg_next::frame::Video::new(Pixel::NV12, width, height); for output in [&mut planar, &mut nv12] { - conversion_context(output.format(), width, height) + conversion_context(output.format(), width, height, 1) .unwrap() .unwrap() .run(&bgra, output) @@ -561,10 +915,10 @@ mod tests { ); } } - assert!(conversion_context(Pixel::BGRA, width, height) + assert!(conversion_context(Pixel::BGRA, width, height, 1) .unwrap() .is_none()); - assert!(conversion_context(Pixel::BGRZ, width, height) + assert!(conversion_context(Pixel::BGRZ, width, height, 1) .unwrap() .is_none()); } diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index e606b783..788eb192 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -14,7 +14,7 @@ use eternal_wire::h264::{ normalize_h264_payload_with_info, parameter_sets_from_extradata, H264BitstreamState, }; -use crate::capture::FrameSlot; +use crate::capture::{FrameSlot, RawFrame, Taken}; use crate::control::SharedControl; use crate::gpu::{GpuInfo, GpuVendor}; use crate::stats::PIPELINE_STATS; @@ -29,6 +29,16 @@ const AMF_CAPTURE_PACKET_LIMIT: u64 = 120; const AMF_IDR_WARNING_PACKET: u64 = 60; const AMF_FORCED_INTRA_PERIOD: u64 = 30; +/// The size a capture of `width`x`height` streams at. 4:2:0 needs even +/// dimensions, which libx264 enforces and hardware encoders may too, so a +/// screen such as 1680x945 encodes without its odd last row (or column): +/// cropped, not scaled, so every other pixel stays exact. Input still maps +/// onto the whole capture (see [`crate::input::norm_to_output_pixel`]), so +/// the cropped row stays reachable at the picture's edge. +pub fn encoded_size(width: u32, height: u32) -> (u32, u32) { + (width & !1, height & !1) +} + /// Encoded H.264 NAL unit ready for transport. #[derive(Debug, Clone)] pub struct NALUnit { @@ -96,14 +106,26 @@ fn run_encode_loop( return Err(format!("{} codec not found in FFmpeg", encoder_name).into()); } info!(encoder = %encoder_name, "Found encoder codec"); + // The processor encodes only because startup's GPU detection did not + // return in time, and no override chose the encoder. + let detection_stalled = gpu.detection_stalled && shared.encoder_override.lock().is_none(); { + let (nvenc_failed, vendors) = { + let failures = LAUNCH_FAILURES.lock(); + (failures.any("_nvenc"), failures.vendors.clone()) + }; let mut stats = PIPELINE_STATS.lock(); stats.set_codec_name(&codec_display_name); // Fresh pipeline start: clear any stale software-fallback flag. The encode loop // re-sets it below only if a hardware encoder actually fails to open. A deliberate - // libx264 override is not a fallback, so it intentionally leaves this false. - stats.set_software_fallback(false); - stats.nvidia_driver_outdated = false; + // libx264 override is not a fallback, so it intentionally leaves this false. A + // stalled detection is one from the start. + stats.set_software_fallback(detection_stalled); + stats.gpu_detection_stalled = detection_stalled; + // A driver note stays while the failure that gave it is remembered, + // as that encoder is not tried again to give it anew. + stats.nvidia_driver_outdated &= nvenc_failed; + stats.driver_update_vendors = vendors; } let mut encoder_state: Option = None; @@ -134,6 +156,11 @@ fn run_encode_loop( let mut hevc_unavailable_warned = false; let mut hevc_open_failed = false; let mut stall_recovery = StallRecovery::default(); + // An open in the lane going on in the background while libx264 streams. + let mut background: Option = None; + // When a picture is due while no session is open: FIRST_PICTURE_WAIT + // after the first try of an encoder. + let mut first_picture_by: Option = None; let mut last_keyframe_reopen: Option = None; let mut pending_keyframe = false; let mut bitrate_pacer = BitratePacer::default(); @@ -157,7 +184,34 @@ fn run_encode_loop( } }; - while let Some(raw_frame) = frames.blocking_take() { + let mut owed_keyframe: Option = None; + let mut cadence = CaptureCadence::default(); + loop { + let (raw_frame, given_again) = match owed_keyframe.as_mut() { + None => match frames.blocking_take() { + Some(frame) => (frame, false), + None => break, + }, + Some(owed) => match owed.next(&frames) { + Next::Frame(frame) => (frame, false), + Next::Again(frame) => { + PIPELINE_STATS.lock().keyframe_resubmits += 1; + (frame, true) + } + Next::Closed => break, + Next::GaveUp => { + info!( + given = owed.given, + "The encoder still holds the session's first keyframe; waiting for the next capture" + ); + owed_keyframe = None; + continue; + } + }, + }; + if !given_again { + cadence.captured(raw_frame.timestamp, raw_frame.cached); + } if !shared.running.load(Ordering::SeqCst) || !crate::supervisor::generation_is_current(generation) { @@ -194,30 +248,28 @@ fn run_encode_loop( if !hevc_wanted { hevc_open_failed = false; } - let preferred: String = if hevc_wanted && !hevc_open_failed { - match hevc_encoder_for(&encoder_name) { - Some(hevc) if ffmpeg_next::encoder::find_by_name(hevc).is_some() => { - hevc.to_string() - } - _ => { - if !hevc_unavailable_warned { - hevc_unavailable_warned = true; - warn!( - encoder = %encoder_name, - "HEVC requested but no H.265 encoder is available — staying on H.264" - ); - } - encoder_name.clone() - } - } - } else { - encoder_name.clone() - }; - // No open given up on still holds FFmpeg's lock. Until then, an open - // in that lane would only be refused. + let preferred = preferred_encoder( + &encoder_name, + hevc_wanted, + hevc_open_failed, + &mut hevc_unavailable_warned, + ); + // An HEVC open going on in the background is of no use once this + // frame wants H.264: the stream goes on, and reopens from there. + if drop_obsolete_background(&mut background, &preferred) { + stall_recovery.trying = false; + } + // No open given up on, or going on in the background, still holds + // FFmpeg's lock. Until then, an open in that lane would only be + // refused. let lane_clear = crate::stall::abandoned(crate::stall::Call::EncoderOpen) == 0; let desired_encoder = stall_recovery.encoder_for_frame(&preferred, Instant::now(), lane_clear); + // While an encoder opens in the background, libx264 streams on. + let desired_encoder = match (&background, &encoder_state) { + (Some(_), Some(state)) => state.opened_encoder.clone(), + _ => desired_encoder, + }; forget_failed_reopen( &mut failed_reopen, &mut negotiated, @@ -257,6 +309,14 @@ fn run_encode_loop( let size_changed = encoder_state .as_ref() .is_some_and(|state| !state.accepts_frame(raw_frame.width, raw_frame.height)); + if size_changed { + if let Some(behind) = background.take() { + info!( + encoder = behind.open.encoder(), + "The capture size changed while an encoder opened in the background; that session will not be used" + ); + } + } // Latch requests before deciding whether to reopen. Requests received // during the rate limit must survive until AMF can emit an IDR. pending_keyframe |= shared.force_next_idr.swap(false, Ordering::SeqCst); @@ -270,25 +330,59 @@ fn run_encode_loop( }); if encoder_state.is_none() { - match EncoderState::new( + let picture_by = + *first_picture_by.get_or_insert_with(|| Instant::now() + FIRST_PICTURE_WAIT); + match open_session( + &LAUNCH_FAILURES, &desired_encoder, raw_frame.width, raw_frame.height, desired_bitrate, target_fps, &mut input_mode, - first_open(&desired_encoder), + first_open(&desired_encoder, picture_by), &mut opening, ) { Ok(state) => { encoder_state = Some(state); + first_picture_by = None; bitrate_pacer.opened(Instant::now()); if stall_recovery.opened(&desired_encoder, &preferred) && desired_encoder != "libx265" { PIPELINE_STATS.lock().set_software_fallback(false); } - note_encoder_opened(&shared, &desired_encoder); + note_encoder_opened(&shared, &desired_encoder, &gpu); + } + // No picture yet, and the iPad would give up waiting for + // it: libx264 streams while the open goes on. + Err(e) if is_still_opening(e.as_ref()) => { + let open = take_still_opening(e.as_ref()).expect("the open going on"); + warn!( + encoder = %desired_encoder, + waited_ms = open.pending.running_for().as_millis() as u64, + "The encoder has not opened yet; streaming on libx264 while it opens in the background" + ); + let state = open_session( + &LAUNCH_FAILURES, + "libx264", + raw_frame.width, + raw_frame.height, + desired_bitrate, + target_fps, + &mut input_mode, + Opening::First, + &mut opening, + )?; + stall_recovery.outwaited(&desired_encoder, "libx264", Instant::now()); + background = Some(Behind::first(open, desired_bitrate)); + encoder_state = Some(state); + first_picture_by = None; + bitrate_pacer.opened(Instant::now()); + PIPELINE_STATS + .lock() + .set_software_fallback(encoder_name != "libx264"); + note_encoder_opened(&shared, "libx264", &gpu); } Err(e) if is_hevc_encoder(&desired_encoder) => { warn!( @@ -310,105 +404,198 @@ fn run_encode_loop( "Preferred encoder failed to open; trying remaining encoders" ); let stalled = is_open_stall(e.as_ref()); + let mut still_opening = None; let (name, state) = open_remaining_encoders( (&desired_encoder, &e.to_string()), crate::gpu::vendors_present, |name| { - EncoderState::new( + let opened = open_session( + &LAUNCH_FAILURES, name, raw_frame.width, raw_frame.height, desired_bitrate, target_fps, &mut input_mode, - first_open(name), + first_open(name, picture_by), &mut opening, - ) + ); + if let Err(error) = &opened { + if let Some(open) = take_still_opening(error.as_ref()) { + still_opening = Some(open); + } + } + opened }, )?; - // After a stall the preferred encoder stays preferred, - // to be tried again; a real error leaves it for good. - if !stall_recovery.failed(stalled, &preferred, name, Instant::now()) { - encoder_name = name.to_string(); + if let Some(open) = still_opening { + // The preferred encoder failed for a reason of its + // own (a stalled one leaves no lane for another), + // and libx264 streams while the next one opens. + warn!( + encoder = open.encoder(), + waited_ms = open.pending.running_for().as_millis() as u64, + "The encoder has not opened yet; streaming on libx264 while it opens in the background" + ); + encoder_name = open.encoder().to_string(); + stall_recovery.opened(name, &preferred); + stall_recovery.outwaited(open.encoder(), name, Instant::now()); + background = Some(Behind::first(open, desired_bitrate)); + } else { + // After a stall the preferred encoder stays preferred, + // to be tried again; a real error leaves it for good. + if !stall_recovery.failed(stalled, &preferred, name, Instant::now()) { + encoder_name = name.to_string(); + } + stall_recovery.opened(name, &preferred); } - stall_recovery.opened(name, &preferred); encoder_state = Some(state); + first_picture_by = None; bitrate_pacer.opened(Instant::now()); PIPELINE_STATS .lock() .set_software_fallback(name == "libx264"); - note_encoder_opened(&shared, name); + note_encoder_opened(&shared, name, &gpu); } Err(e) => return Err(e), } - } else if size_changed - || encoder_state.as_ref().is_some_and(|state| { - reopen_needed( - ( - &state.opened_encoder, - state.opened_bitrate, - state.opened_fps, - ), - (&desired_encoder, wanted_bitrate, target_fps), - keyframe_reopen, - failed_reopen.as_ref(), - lane_clear, - ) - }) - { - // Bitrate changed (ABR rung or the user's slider), the - // negotiated codec changed (HEVC toggle / client capability), the - // capture size changed (display mode change), or AMF owes a - // keyframe: hardware encoders ignore parameter pokes on an open - // context, so each means a session reopen (~50-200ms hiccup, no - // transport teardown, same stream epoch) followed by a forced IDR. - // So does a try of the preferred encoder after a stalled open. - // Every reopen takes the newest bitrate target, paced or not. - if keyframe_reopen { - last_keyframe_reopen = Some(Instant::now()); - } - let (previous, current_encoder, reason) = encoder_state - .as_ref() - .map(|s| { - let reason = reopen_reason( - (&s.opened_encoder, s.opened_fps), - (&desired_encoder, target_fps), - size_changed, - keyframe_reopen, - stall_recovery - .owed - .as_ref() - .is_some_and(|owed| owed.preferred == desired_encoder), - ); - (s.opened_bitrate, s.opened_encoder.clone(), reason) + } else { + // A reopen this frame: the open that went on in the background, + // once it returned, or a new one. + let reopened = if let Some(result) = background.as_mut().and_then(|behind| { + behind.open.poll(&mut input_mode, |args| { + EncoderState::new( + &args.encoder_name, + args.size.0, + args.size.1, + args.bitrate_bps, + args.target_fps, + &mut EncoderInput::Yuv420, + Opening::Behind, + &mut || {}, + ) }) - .unwrap_or_default(); - stall_recovery.starting(&desired_encoder); - match EncoderState::new( - &desired_encoder, - raw_frame.width, - raw_frame.height, - desired_bitrate, - target_fps, - &mut input_mode, + }) { + let behind = background.take().expect("it was just polled"); + if let Err(error) = &result { + note_open_failure( + &LAUNCH_FAILURES, + behind.open.encoder(), + behind.open.args.size, + error.as_ref(), + ); + } + Some((result, behind.reopen)) + } else if background.is_none() + && (size_changed + || encoder_state.as_ref().is_some_and(|state| { + reopen_needed( + ( + &state.opened_encoder, + state.opened_bitrate, + state.opened_fps, + ), + (&desired_encoder, wanted_bitrate, target_fps), + keyframe_reopen, + failed_reopen.as_ref(), + lane_clear, + ) + })) + { + // Bitrate changed (ABR rung or the user's slider), the + // negotiated codec changed (HEVC toggle / client capability), the + // capture size changed (display mode change), or AMF owes a + // keyframe: hardware encoders ignore parameter pokes on an open + // context, so each means a session reopen (~50-200ms hiccup, no + // transport teardown, same stream epoch) followed by a forced IDR. + // So does a try of the preferred encoder after a stalled open. + // Every reopen takes the newest bitrate target, paced or not. + if keyframe_reopen { + last_keyframe_reopen = Some(Instant::now()); + } + let (previous, current_encoder, reason) = encoder_state + .as_ref() + .map(|s| { + let reason = reopen_reason( + (&s.opened_encoder, s.opened_fps), + (&desired_encoder, target_fps), + size_changed, + keyframe_reopen, + stall_recovery + .owed + .as_ref() + .is_some_and(|owed| owed.preferred == desired_encoder), + ); + (s.opened_bitrate, s.opened_encoder.clone(), reason) + }) + .unwrap_or_default(); + stall_recovery.starting(&desired_encoder); // The old session cannot take a frame of the new size. - if size_changed { + let how = if size_changed { Opening::First + } else if opens_behind(¤t_encoder, &desired_encoder) { + Opening::Behind } else { Opening::Reopen - }, - &mut opening, - ) { - Ok(state) => { + }; + let reopen = Reopening { + encoder: desired_encoder.clone(), + from: previous, + current: current_encoder, + reason, + size_changed, + keyframe: keyframe_reopen, + bitrate: desired_bitrate, + fps: target_fps, + first: false, + in_background: false, + }; + match open_session( + &LAUNCH_FAILURES, + &desired_encoder, + raw_frame.width, + raw_frame.height, + desired_bitrate, + target_fps, + &mut input_mode, + how, + &mut opening, + ) { + Err(e) if is_still_opening(e.as_ref()) => { + let open = take_still_opening(e.as_ref()).expect("the open going on"); + info!( + from = %reopen.current, + encoder = %desired_encoder, + reason, + "Opening the encoder in the background; the stream goes on meanwhile" + ); + background = Some(Behind { + open, + reopen: Reopening { + in_background: true, + ..reopen + }, + }); + None + } + result => Some((result, reopen)), + } + } else { + None + }; + match reopened { + None => {} + Some((Ok(state), reopen)) => { info!( - from = previous, - to = desired_bitrate, - encoder = %desired_encoder, - width = raw_frame.width, - height = raw_frame.height, - size_changed, - keyframe = keyframe_reopen, - reason, + from = reopen.from, + to = state.opened_bitrate, + encoder = %reopen.encoder, + width = state.frame.width(), + height = state.frame.height(), + size_changed = reopen.size_changed, + keyframe = reopen.keyframe, + reason = reopen.reason, + in_background = reopen.in_background, "Reopened encoder session" ); encoder_state = Some(state); @@ -417,57 +604,80 @@ fn run_encode_loop( pending_keyframe = true; // Back from libx264 to hardware (libx265 only replaces a // libx264 that was already preferred). - if stall_recovery.opened(&desired_encoder, &preferred) - && desired_encoder != "libx265" + if stall_recovery.opened(&reopen.encoder, &preferred) + && reopen.encoder != "libx265" { PIPELINE_STATS.lock().set_software_fallback(false); } - note_encoder_opened(&shared, &desired_encoder); + note_encoder_opened(&shared, &reopen.encoder, &gpu); } // The old session cannot take a frame of the new size. A // pipeline restart opens a fresh encoder, with the software // fallback if the hardware one rejects this size. - Err(e) if size_changed => { + Some((Err(e), reopen)) if reopen.size_changed => { return Err(format!( - "{desired_encoder} could not reopen at {}x{}: {e}", - raw_frame.width, raw_frame.height + "{} could not reopen at {}x{}: {e}", + reopen.encoder, raw_frame.width, raw_frame.height ) .into()); } - Err(e) => { + // The pipeline's first open, which went on in the background. + Some((Err(e), reopen)) if reopen.first => { + if is_open_stall(e.as_ref()) { + // Tried again once the lane is clear, as after any + // stalled open (see `StallRecovery::outwaited`). + warn!( + error = %e, + encoder = %reopen.encoder, + "The encoder did not open in the background; streaming on libx264 until it is tried again" + ); + } else { + warn!( + error = %e, + encoder = %reopen.encoder, + "The encoder failed to open in the background; staying on libx264 for this pipeline" + ); + stall_recovery.owed = None; + if is_hevc_encoder(&reopen.encoder) { + hevc_open_failed = true; + } else { + encoder_name = "libx264".to_string(); + } + } + } + Some((Err(e), reopen)) => { let stalled = is_open_stall(e.as_ref()); if stalled { warn!( error = %e, - encoder = %desired_encoder, + encoder = %reopen.encoder, "Encoder reopen stalled — keeping the current session" ); - } else if is_hevc_encoder(&desired_encoder) { + } else if is_hevc_encoder(&reopen.encoder) { warn!( error = %e, - encoder = %desired_encoder, + encoder = %reopen.encoder, "HEVC reopen failed — keeping the current session" ); } else { warn!( error = %e, - bitrate = desired_bitrate, + bitrate = reopen.bitrate, "Encoder reopen failed — keeping the current session" ); } // A stall is tried again once the lane is clear, while // tries are left. - if !stall_recovery.failed(stalled, &preferred, ¤t_encoder, Instant::now()) + if !stall_recovery.failed(stalled, &preferred, &reopen.current, Instant::now()) { give_up_reopen( - &desired_encoder, - ¤t_encoder, + &reopen.encoder, + &reopen.current, stall_recovery.owed.is_some(), &mut encoder_name, &mut hevc_open_failed, ); - failed_reopen = - Some((desired_encoder.clone(), desired_bitrate, target_fps)); + failed_reopen = Some((reopen.encoder.clone(), reopen.bitrate, reopen.fps)); } } } @@ -478,45 +688,18 @@ fn run_encode_loop( .expect("Encoder state must be initialized"); let current_bitrate = encoder.opened_bitrate; let opened_is_hevc = is_hevc_encoder(&encoder.opened_encoder); + shared.encoder_reopens_for_bitrate.store( + !takes_bitrate_in_place(&encoder.opened_encoder, refused_this_launch), + Ordering::Relaxed, + ); - let src_row_bytes = (raw_frame.width * 4) as usize; let convert_start = Instant::now(); - #[cfg(target_os = "macos")] - let converted = match encoder.native_converter.as_ref().map(|converter| { - converter.convert( - &raw_frame.data, - raw_frame.width, - raw_frame.height, - src_row_bytes, - &mut encoder.frame, - ) - }) { - Some(Ok(())) => true, - Some(Err(error)) => { - warn!(%error, "Using swscale after native color conversion failed"); - encoder.native_converter = None; - false - } - None => false, - }; - #[cfg(not(target_os = "macos"))] - let converted = false; - if !converted { - if let Some(scaler) = encoder.scaler.as_mut() { - input::convert_packed_bgra(scaler, &raw_frame.data, &mut encoder.frame)?; - } else { - // BGRA input: the encoder converts. Its frame takes a copy, - // so capture's buffer is free before the encoder can block. - let stride = encoder.frame.stride(0); - let bgra_plane = encoder.frame.data_mut(0); - for y in 0..raw_frame.height as usize { - let src_offset = y * src_row_bytes; - let dst_offset = y * stride; - bgra_plane[dst_offset..dst_offset + src_row_bytes] - .copy_from_slice(&raw_frame.data[src_offset..src_offset + src_row_bytes]); - } - } - } + encoder.convert(&raw_frame.data, (raw_frame.width * 4) as usize)?; + // The picture is kept only while a session's first keyframe may stay + // inside the encoder, to be given to it again. + let picture = (holds_last_frame(&encoder.opened_encoder) + && (encoder.frames_submitted == 0 || owed_keyframe.is_some())) + .then(|| raw_frame.clone()); // Conversion finished synchronously. Recycle capture's buffer before // the codec or sender can block, including on the direct macOS path. drop(raw_frame.data); @@ -540,9 +723,27 @@ fn run_encode_loop( ); let encode_start = Instant::now(); - encoder.send_frame(pts, raw_frame.timestamp, raw_frame.frame_number)?; + // A bitrate Quick Sync took in place applies with this frame: an + // error now is its refusal. + let qsv_change = encoder + .bitrate_change + .is_some_and(|change| change.live == LiveBitrate::QsvReset); + // NVENC reconfigures for a new bitrate while it takes this frame, + // and says so only in its verbose lines. + let nvenc_watch = encoder + .bitrate_change + .is_some_and(|change| change.live == LiveBitrate::NvencReconfig) + .then(crate::diagnostics::FfmpegWatch::start); + let mut reset_refused: Option = None; + if let Err(error) = encoder.send_frame(pts, raw_frame.timestamp, raw_frame.frame_number) { + if !qsv_change { + return Err(error.into()); + } + reset_refused = Some(error.to_string()); + } - loop { + let mut keyframe_out = false; + while reset_refused.is_none() { match encoder.encoder.receive_packet(&mut encoder.packet) { Ok(()) => { let encode_us = encode_start.elapsed().as_micros(); @@ -586,6 +787,7 @@ fn run_encode_loop( au_info.is_keyframe(packet_is_key, &encoder.opened_encoder); (nal_data, is_keyframe) }; + keyframe_out |= is_keyframe; if !opened_is_hevc { encoder.observe_amf_diagnostics( &nal_data, @@ -648,9 +850,45 @@ fn run_encode_loop( } } Err(e) if is_eagain(&e) => break, + Err(e) if qsv_change => reset_refused = Some(e.to_string()), Err(e) => return Err(e.into()), } } + if let Some(watch) = nvenc_watch { + if nvenc_reconfigured(&watch.finish()) { + encoder.bitrate_change = None; + } else if reset_refused.is_none() { + reset_refused = Some("NVENC did not reconfigure the session's bitrate".to_string()); + } + } + if let Some(error) = reset_refused { + let live = encoder + .bitrate_change + .map_or(LiveBitrate::QsvReset, |change| change.live); + live.refused().store(true, Ordering::Relaxed); + warn!( + %error, + encoder = %encoder.opened_encoder, + bitrate = encoder.opened_bitrate, + "The encoder did not take the new bitrate on the open session; reopening it, as for its bitrate changes from now on" + ); + encoder_state = None; + pending_keyframe = true; + // A kept picture belonged to that session; the new one starts + // with an IDR. + owed_keyframe = None; + continue; + } + if qsv_change { + encoder.bitrate_change = None; + } + owed_keyframe = OwedKeyframe::after_encode( + owed_keyframe.take(), + picture, + keyframe_out, + encode_start, + keyframe_flush_wait(target_fps, raw_frame.cached, cadence.pace()), + ); } info!("Capture channel closed, encoder shutting down"); @@ -662,7 +900,7 @@ struct EncoderState { native_converter: Option, encoder: ffmpeg_next::codec::encoder::video::Encoder, /// BGRA to the encoder's YUV input, reading capture's buffer directly. - scaler: Option, + scaler: Option, frame: ffmpeg_next::frame::Video, packet: ffmpeg_next::Packet, h264: H264BitstreamState, @@ -688,6 +926,93 @@ struct EncoderState { /// time_base 1/60 (escape hatch until the real-PTS path is verified on /// NVENC/AMF hardware). legacy_pts: bool, + /// A hardware bitrate change made in place that the session has not yet + /// been seen to take (see [`Unconfirmed`]). + bitrate_change: Option, +} + +/// How an encoder takes a new bitrate on the open session, without the +/// reopen that costs a stutter. FFmpeg 7.1.1 checks the context's bitrate +/// before each frame in three of its encoders: +/// - libx264 applies bitrate and VBV changes through x264_encoder_reconfig. +/// - Quick Sync resets the session's rate control to it (qsvenc.c, +/// `update_bitrate` and `MFXVideoENCODE_Reset`), with no new sequence, +/// instead of its slow open. +/// - NVENC reconfigures the session (nvenc.c, `reconfig_encoder` and +/// `nvEncReconfigureEncoder`) with an IDR, if the card can change its +/// bitrate live (NV_ENC_CAPS_SUPPORT_DYN_BITRATE_CHANGE), instead of a +/// new session. +/// +/// AMF sets its bitrate only when it opens (amfenc_h264.c and +/// amfenc_hevc.c), and libx265 has no reconfigure, so both reopen, and ABR +/// steps up less often for them (see `AbrController::encoder_reopens`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LiveBitrate { + X264, + QsvReset, + NvencReconfig, +} + +impl LiveBitrate { + fn of(encoder_name: &str) -> Option { + if encoder_name == "libx264" { + Some(Self::X264) + } else if encoder_name.ends_with("_qsv") { + Some(Self::QsvReset) + } else if encoder_name.ends_with("_nvenc") { + Some(Self::NvencReconfig) + } else { + None + } + } + + /// Set once a session refused a bitrate change in place: from then on, + /// this launch, that encoder's bitrate changes reopen it, as before. + fn refused(self) -> &'static std::sync::atomic::AtomicBool { + static QSV: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + static NVENC: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + static NEVER: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + match self { + Self::QsvReset => &QSV, + Self::NvencReconfig => &NVENC, + Self::X264 => &NEVER, + } + } +} + +/// Whether `encoder_name` takes a new bitrate on the open session, unless +/// `refused` says a session of its kind refused one. +fn takes_bitrate_in_place(encoder_name: &str, refused: impl Fn(LiveBitrate) -> bool) -> bool { + LiveBitrate::of(encoder_name).is_some_and(|live| live == LiveBitrate::X264 || !refused(live)) +} + +fn refused_this_launch(live: LiveBitrate) -> bool { + live.refused().load(Ordering::Relaxed) +} + +/// A hardware bitrate change made in place, settled by the end of the frame +/// it goes in with: then the session took it, or it is replaced by one +/// opened at the new bitrate, so the session's bitrate never outlives an +/// unconfirmed change. Quick Sync applies it with that frame, and an error +/// there is its refusal. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Unconfirmed { + live: LiveBitrate, +} + +/// Whether NVENC reconfigured its session for a new bitrate, from FFmpeg's +/// lines while it took the change's frame. FFmpeg 7.1.1's nvenc.c +/// (`reconfig_encoder`) logs "avg bitrate change" at verbose level when it +/// calls nvEncReconfigureEncoder for one, and "failed to reconfigure nvenc" +/// when that fails; it returns no error either way, and skips the change +/// with no line on a card that cannot change its bitrate live. An IDR says +/// nothing: one the frame forced, or the GOP's own, comes out either way. +/// Anything but a reconfigure that ran and did not fail is a refusal. +fn nvenc_reconfigured(lines: &[String]) -> bool { + lines.iter().any(|line| line.contains("avg bitrate change")) + && !lines + .iter() + .any(|line| line.contains("failed to reconfigure nvenc")) } impl EncoderState { @@ -697,47 +1022,78 @@ impl EncoderState { bitrate_bps: u32, target_fps: u32, ) -> bool { - if self.opened_encoder != "libx264" + if !takes_bitrate_in_place(&self.opened_encoder, refused_this_launch) || desired_encoder != self.opened_encoder || target_fps != self.opened_fps || bitrate_bps == self.opened_bitrate { return false; } - // FFmpeg 7.1's libx264 wrapper applies bitrate and VBV changes through - // x264_encoder_reconfig before the next frame. Preserve its reference - // frames, PTS epoch and buffers instead of allocating a second encoder - // for every ABR step. Other encoders retain the existing reopen path. + // Preserve the encoder's reference frames, PTS epoch and buffers + // instead of allocating a second encoder for every ABR step. Other + // encoders retain the existing reopen path. self.encoder.set_bit_rate(bitrate_bps.max(500_000) as usize); - self.encoder - .set_max_bit_rate(bitrate_bps.max(500_000) as usize); - // A half-second VBV is enabled at open, as required for x264's live - // bitrate reconfiguration. The context belongs to this encode thread. - unsafe { - (*self.encoder.as_mut_ptr()).rc_buffer_size = (bitrate_bps.max(500_000) / 2) as i32 - }; - info!( - from = self.opened_bitrate, - to = bitrate_bps, - "Updated software encoder bitrate" - ); + if self.opened_encoder == "libx264" { + self.encoder + .set_max_bit_rate(bitrate_bps.max(500_000) as usize); + // A half-second VBV is enabled at open, as required for x264's live + // bitrate reconfiguration. The context belongs to this encode thread. + unsafe { + (*self.encoder.as_mut_ptr()).rc_buffer_size = (bitrate_bps.max(500_000) / 2) as i32 + }; + info!( + from = self.opened_bitrate, + to = bitrate_bps, + "Updated software encoder bitrate" + ); + } else { + let live = LiveBitrate::of(&self.opened_encoder).expect("takes it in place"); + if live == LiveBitrate::NvencReconfig { + // NVENC sizes the VBV to two seconds of the bitrate when it + // opens, and reconfigures it with the bitrate, as a session + // opened at it would have. + unsafe { + (*self.encoder.as_mut_ptr()).rc_buffer_size = + (bitrate_bps.max(500_000) as i32).saturating_mul(2) + }; + } + // Quick Sync opened in VBR with no maximum rate or buffer size, + // so only the target changes. + self.bitrate_change = Some(Unconfirmed { live }); + info!( + from = self.opened_bitrate, + to = bitrate_bps, + encoder = %self.opened_encoder, + "Updated hardware encoder bitrate without reopening" + ); + } self.opened_bitrate = bitrate_bps; true } + /// Opens a session for captures of `capture_width`x`capture_height`, + /// at their [`encoded_size`]. #[allow(clippy::too_many_arguments)] fn new( encoder_name: &str, - width: u32, - height: u32, + capture_width: u32, + capture_height: u32, bitrate_bps: u32, target_fps: u32, input_mode: &mut EncoderInput, opening: Opening, progress: &mut dyn FnMut(), ) -> Result> { + let (width, height) = encoded_size(capture_width, capture_height); info!(encoder = encoder_name, width, height, bitrate_bps, target_fps, input = ?input_mode, "Opening encoder session"); + if (width, height) != (capture_width, capture_height) { + info!( + capture_size = %format_args!("{capture_width}x{capture_height}"), + encoded_size = %format_args!("{width}x{height}"), + "Encoding without the capture's odd last column or row" + ); + } let codec = ffmpeg_next::encoder::find_by_name(encoder_name) .ok_or_else(|| format!("{} codec not found in FFmpeg", encoder_name))?; // AVCodec owns this terminated static list. A null list means that @@ -752,7 +1108,7 @@ impl EncoderState { formats }; info!(encoder = encoder_name, advertised_formats = ?supported, input = ?input_mode, "Encoder advertised input formats"); - open_at_input(input_mode, encoder_name, &supported, |pixel_format| { + let opened = open_at_input(input_mode, encoder_name, &supported, |pixel_format| { Self::open_at( codec, encoder_name, @@ -763,7 +1119,16 @@ impl EncoderState { opening, &mut *progress, ) - }) + }); + // One that goes on in the background is tried at YUV420 there. + if let Err(error) = &opened { + if let Some(still) = error.downcast_ref::() { + if let Some(open) = still.1.lock().as_mut() { + open.yuv_retry = input::retries_as_yuv(*input_mode, open.args.pixel_format); + } + } + } + opened } #[allow(clippy::too_many_arguments)] @@ -806,7 +1171,42 @@ impl EncoderState { encoder.set_gop(30); configure_encoder_flags(&mut encoder, encoder_name); - let encoder = match open_encoder(encoder, encoder_name, opening, progress) { + let args = SessionArgs { + encoder_name: encoder_name.to_string(), + size: (width, height), + bitrate_bps, + target_fps, + pixel_format, + legacy_pts, + }; + match open_encoder(encoder, encoder_name, opening, progress) { + Opened::Now(result) => Self::opened(result, &args), + Opened::Behind(pending) => Err(Box::new(StillOpening( + encoder_name.to_string(), + parking_lot::Mutex::new(Some(BackgroundOpen { + pending, + args, + yuv_retry: false, + })), + ))), + } + } + + /// The session from an open that returned with `result`, or its error. + fn opened( + result: OpenResult, + args: &SessionArgs, + ) -> Result> { + let SessionArgs { + ref encoder_name, + size: (width, height), + bitrate_bps, + target_fps, + pixel_format, + legacy_pts, + } = *args; + let encoder_name = encoder_name.as_str(); + let encoder = match result { Ok(Ok(encoder)) => encoder, Ok(Err(error)) => { note_nvidia_driver(encoder_name, error, crate::gpu::nvidia_driver_versions); @@ -830,27 +1230,43 @@ impl EncoderState { let mut h264 = H264BitstreamState::default(); h264.refresh_parameter_sets_from_extradata(encoder_extradata(&encoder)); - let scaler = input::conversion_context(pixel_format, width, height)?; - if scaler.is_some() { - info!(output = ?pixel_format, "swscale BGRA conversion context created"); + #[cfg(target_os = "macos")] + let native_converter = if pixel_format == ffmpeg_next::format::Pixel::YUV420P { + match vimage::Converter::new() { + Ok(converter) => { + info!("Accelerate BGRA->YUV420P converter ready"); + Some(converter) + } + Err(error) => { + warn!(%error, "Using swscale color conversion"); + None + } + } + } else { + None + }; + // Where Accelerate converts, swscale is only its fallback, so it + // keeps no threads waiting. + #[cfg(target_os = "macos")] + let threads = if native_converter.is_some() { + 1 + } else { + input::scaler_threads() + }; + #[cfg(not(target_os = "macos"))] + let threads = input::scaler_threads(); + let scaler = input::conversion_context(pixel_format, width, height, threads)?; + if let Some(scaler) = &scaler { + info!( + output = ?pixel_format, + threads = scaler.threads(), + "swscale BGRA conversion context created" + ); } Ok(Self { #[cfg(target_os = "macos")] - native_converter: if pixel_format == ffmpeg_next::format::Pixel::YUV420P { - match vimage::Converter::new() { - Ok(converter) => { - info!("Accelerate BGRA->YUV420P converter ready"); - Some(converter) - } - Err(error) => { - warn!(%error, "Using swscale color conversion"); - None - } - } - } else { - None - }, + native_converter, encoder, scaler, frame: ffmpeg_next::frame::Video::new(pixel_format, width, height), @@ -866,17 +1282,20 @@ impl EncoderState { last_pts: -1, in_flight: InFlightFrames::default(), legacy_pts, + bitrate_change: None, }) } fn next_pts(&mut self, captured: Instant, frame_number: u64) -> i64 { - if self.legacy_pts { - return frame_number as i64; - } - let epoch = *self.pts_epoch.get_or_insert(captured); - let real = captured.saturating_duration_since(epoch).as_micros() as i64; - // Strict monotonicity (keepalive resends and clock quirks). - let pts = real.max(self.last_pts + 1); + let pts = if self.legacy_pts { + frame_number as i64 + } else { + let epoch = *self.pts_epoch.get_or_insert(captured); + captured.saturating_duration_since(epoch).as_micros() as i64 + }; + // Strict monotonicity (keepalive resends, a picture given again with + // its capture number and time, and clock quirks). + let pts = pts.max(self.last_pts + 1); self.last_pts = pts; pts } @@ -902,8 +1321,38 @@ impl EncoderState { .unwrap_or(just_submitted) } + /// Converts a capture whose rows lie `row_bytes` apart into `self.frame`: + /// its top-left `self.frame`-sized part, pixel for pixel, which leaves an + /// odd last column or row out (see [`encoded_size`]). + fn convert(&mut self, rows: &[u8], row_bytes: usize) -> Result<(), ffmpeg_next::Error> { + let (width, height) = (self.frame.width(), self.frame.height()); + #[cfg(target_os = "macos")] + if let Some(converter) = self.native_converter.as_ref() { + match converter.convert(rows, width, height, row_bytes, &mut self.frame) { + Ok(()) => return Ok(()), + Err(error) => { + warn!(%error, "Using swscale after native color conversion failed"); + self.native_converter = None; + } + } + } + if let Some(scaler) = self.scaler.as_mut() { + return input::convert_packed_bgra(scaler, rows, row_bytes, &mut self.frame); + } + // BGRA input: the encoder converts. Its frame takes a copy, + // so capture's buffer is free before the encoder can block. + let picture_bytes = width as usize * 4; + let stride = self.frame.stride(0); + let bgra_plane = self.frame.data_mut(0); + for y in 0..height as usize { + bgra_plane[y * stride..][..picture_bytes] + .copy_from_slice(&rows[y * row_bytes..][..picture_bytes]); + } + Ok(()) + } + fn accepts_frame(&self, width: u32, height: u32) -> bool { - self.frame.width() == width && self.frame.height() == height + encoded_size(width, height) == (self.frame.width(), self.frame.height()) } fn observe_amf_diagnostics( @@ -963,21 +1412,51 @@ type OpenResult = Result< crate::stall::Stall, >; -/// What streams while an encoder session opens, which sets its time limit. +/// What streams while an encoder session opens, which sets its time limit +/// and how long it is waited for. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Opening { - /// Nothing: the pipeline's first open and its fallbacks, or a reopen at - /// a new capture size, which the old session cannot encode. + /// Nothing: the pipeline's last resort, libx264, or a reopen at a new + /// capture size, which the old session cannot encode. First, /// As `First`, with another encoder to try if this one fails: every /// first open but libx264's. FFmpeg's errors while it opens are logged - /// as warnings (see [`crate::diagnostics::while_probing`]). - Probe, + /// as warnings (see [`crate::diagnostics::while_probing`]). One in the + /// lane is waited for only until the pipeline's first picture is due, + /// at the instant given (see [`FIRST_PICTURE_WAIT`]): then it goes on + /// opening in the background while libx264 streams. The instant holds + /// whatever came before the open: a fallback's earlier tries, a BGRA + /// open retried at YUV420, a call's thread that was slow to start. + Probe(Instant), /// The live session, which goes on if the reopen is given up on. The /// picture freezes until then, so the limit is shorter. Reopen, + /// A reopen in the lane while a session that needs none streams + /// (libx264): it opens in the background, and the stream goes on + /// meanwhile, so it has a first open's limits (see [`opens_behind`]). + Behind, } +/// How long a pipeline may go without a picture, from its first try of an +/// encoder in the lane. The iPad gives up on a session after 10 s without +/// one. A hardware open that hung was waited for up to its 15 s limit, so +/// libx264 took over too late, and the iPad showed no picture (field +/// triage, Nov 2026). Past this, libx264 streams while the open goes on in +/// the background, and the stream switches to it if it opens within its +/// limit. Above the slowest NVENC and AMF opens in the field (2.6 s and +/// 1.2 s) and Quick Sync's p99 (3.2 s): the few slower Quick Sync opens +/// start on libx264 for a moment. +const FIRST_PICTURE_WAIT: Duration = Duration::from_secs(3); + +/// An encoder open that returned, or one that goes on in the background. +enum Opened { + Now(OpenResult), + Behind(PendingOpen), +} + +type PendingOpen = + crate::stall::Pending>; + /// Open `encoder` with its options. An encoder that opens under FFmpeg's /// process-wide lock opens under its time limit (see [`crate::stall`]); one /// that returns after it is closed on the call's thread. @@ -986,7 +1465,7 @@ fn open_encoder( encoder_name: &str, opening: Opening, progress: &mut dyn FnMut(), -) -> OpenResult { +) -> Opened { open_encoder_on( crate::stall::lanes(), encoder, @@ -1001,83 +1480,254 @@ fn open_encoder_on( encoder: ffmpeg_next::codec::encoder::video::Video, encoder_name: &str, opening: Opening, - progress: &mut dyn FnMut(), -) -> OpenResult { + mut progress: &mut dyn FnMut(), +) -> Opened { let name = encoder_name.to_string(); + let probing = matches!(opening, Opening::Probe(_)); // On the thread that makes the call, where FFmpeg logs. let open = move || { let open = || encoder.open_with(encoder_options(&name)); - if opening == Opening::Probe { + if probing { crate::diagnostics::while_probing(open) } else { open() } }; if !takes_codec_lock(encoder_name) { - return Ok(open()); + return Opened::Now(Ok(open())); + } + let limits = open_limits(encoder_name, opening); + match crate::stall::start_on(lanes, crate::stall::Call::EncoderOpen, limits, open) { + Ok(pending) => match wait_or_park(pending, opening, &mut progress) { + Ok(result) => Opened::Now(result), + Err(pending) => Opened::Behind(pending), + }, + Err(stall) => Opened::Now(Err(stall)), + } +} + +/// Waits for an open in the lane as `opening` says: to its limit, until the +/// first picture is due, or not at all. Err: it goes on, parked. +fn wait_or_park Instant>( + mut pending: crate::stall::Pending, + opening: Opening, + progress: &mut impl FnMut(), +) -> Result, crate::stall::Pending> { + let waited = match opening { + Opening::Probe(by) => pending.wait_until(by, progress), + Opening::Behind => None, + Opening::First | Opening::Reopen => pending.wait_up_to(Duration::MAX, progress), + }; + if let Some(result) = waited { + return Ok(result); + } + match pending.park() { + Some(result) => Ok(result), + None => Err(pending), } - crate::stall::run_on( - lanes, - crate::stall::Call::EncoderOpen, - open_limits(encoder_name, opening), - open, - progress, - ) } -/// A pipeline's first open of `encoder_name`: anything but libx264 has -/// another encoder after it. -fn first_open(encoder_name: &str) -> Opening { +/// A pipeline's first open of `encoder_name`, whose first picture is due +/// `by`: anything but libx264 has another encoder after it. +fn first_open(encoder_name: &str, by: Instant) -> Opening { if encoder_name == "libx264" { Opening::First } else { - Opening::Probe + Opening::Probe(by) } } fn open_limits(encoder_name: &str, opening: Opening) -> crate::stall::Limits { match opening { - Opening::First | Opening::Probe => crate::stall::encoder_open_limits(encoder_name), + Opening::First | Opening::Probe(_) | Opening::Behind => { + crate::stall::encoder_open_limits(encoder_name) + } Opening::Reopen => crate::stall::encoder_reopen_limits(encoder_name), } } -/// Encoders FFmpeg 7.1 opens under its one process-wide lock -/// (FF_CODEC_CAP_NOT_INIT_THREADSAFE): the NVENC, AMF and QSV hardware -/// encoders, which wait on a graphics driver, and libx265. While one of them -/// is stuck holding the lock, any other would block behind it, so they share -/// one time-limited lane. libx264 and VideoToolbox do not take the lock. -fn takes_codec_lock(encoder_name: &str) -> bool { - encoder_name == "libx265" - || ["_nvenc", "_amf", "_qsv"] - .iter() - .any(|suffix| encoder_name.ends_with(suffix)) +/// The encoder a frame prefers: the H.265 encoder for `encoder_name` when +/// HEVC is wanted (the user's preference and the client's decoding) and has +/// not failed, if FFmpeg has it; else `encoder_name`. `warned` says once +/// that there is none. +fn preferred_encoder( + encoder_name: &str, + hevc_wanted: bool, + hevc_open_failed: bool, + warned: &mut bool, +) -> String { + if !hevc_wanted || hevc_open_failed { + return encoder_name.to_string(); + } + match hevc_encoder_for(encoder_name) { + Some(hevc) if ffmpeg_next::encoder::find_by_name(hevc).is_some() => hevc.to_string(), + _ => { + if !*warned { + *warned = true; + warn!( + encoder = %encoder_name, + "HEVC requested but no H.265 encoder is available — staying on H.264" + ); + } + encoder_name.to_string() + } + } } -/// `vendors_present` lists the graphics cards' vendors when the fallback -/// runs (see [`crate::gpu::vendors_present`]). The encoders of the vendors -/// listed are tried first, so a PC with a working one never probes the -/// others: AMF on a PC without an AMD card only logs that its DLL failed to -/// load. Every other hardware encoder is still tried after them, because -/// Windows does not list every card that can encode (an NVIDIA card in TCC -/// mode). A hybrid laptop lists both of its cards; an unknown vendor moves -/// no encoder. With no list, the order is NVENC, AMF, Quick Sync, as before. -/// -/// `failed` is the encoder that failed first, and why. If every encoder -/// fails, one error names each and its reason: their FFmpeg errors were -/// warnings while they were tried. -fn open_remaining_encoders( - (failed, why): (&str, &str), - vendors_present: impl FnOnce() -> Option>, - mut open: impl FnMut(&str) -> Result>, -) -> Result<(&'static str, T), Box> { - // Open errors already start with the encoder's name. - let mut reasons = vec![why.to_string()]; - let note = |reasons: &mut Vec, name: &str, reason: String| { - reasons.push(if reason.starts_with(name) { - reason - } else { - format!("{name}: {reason}") +/// Drops an HEVC open going on in the background once the frame prefers +/// H.264 (HEVC turned off, or a client that decodes only H.264): adopted, +/// it would replace a session the client can decode with one it may not. +/// The session that streams meanwhile goes on. True when one was dropped; +/// its result is released when it returns. +fn drop_obsolete_background(background: &mut Option, preferred: &str) -> bool { + let obsolete = background.as_ref().is_some_and(|behind| { + is_hevc_encoder(behind.open.encoder()) && !is_hevc_encoder(preferred) + }); + if let Some(behind) = background.take_if(|_| obsolete) { + info!( + encoder = behind.open.encoder(), + preferred, + "The codec changed while an encoder opened in the background; that session will not be used" + ); + } + obsolete +} + +/// Whether a reopen from the `current` session to `wanted` opens in the +/// background: libx264 needs no lane, so it streams on while an encoder in +/// the lane opens, instead of freezing for as long as the open takes. +fn opens_behind(current: &str, wanted: &str) -> bool { + !takes_codec_lock(current) && takes_codec_lock(wanted) +} + +/// What a session opens with, to finish an open that returns later. +struct SessionArgs { + encoder_name: String, + size: (u32, u32), + bitrate_bps: u32, + target_fps: u32, + pixel_format: ffmpeg_next::format::Pixel, + legacy_pts: bool, +} + +/// An encoder session that opens in the background while libx264 streams. +struct BackgroundOpen { + pending: PendingOpen, + args: SessionArgs, + /// An Automatic BGRA open: if it fails, YUV420 is tried, as + /// [`open_at_input`] tries it for an open that fails while waited for. + yuv_retry: bool, +} + +impl BackgroundOpen { + fn encoder(&self) -> &str { + &self.args.encoder_name + } + + /// The session once its open returned, or the open's error; past its + /// limit, its stall. None while it opens within its limit. A BGRA open + /// that failed is tried again at YUV420 with `retry_yuv`, and `input` + /// keeps YUV420 for the pipeline's later opens: the error comes back only + /// if that fails too, so an encoder that works at YUV420 is not left for + /// the launch. + fn poll( + &mut self, + input: &mut EncoderInput, + retry_yuv: impl FnOnce( + &SessionArgs, + ) + -> Result>, + ) -> Option>> { + let result = self.pending.poll()?; + let opened = EncoderState::opened(result, &self.args); + let Err(error) = &opened else { + return Some(opened); + }; + if !std::mem::take(&mut self.yuv_retry) || is_open_stall(error.as_ref()) { + return Some(opened); + } + warn!( + encoder = %self.args.encoder_name, + %error, + "Encoder did not open with BGRA input; using YUV420 for this stream" + ); + *input = EncoderInput::Yuv420; + match retry_yuv(&self.args) { + Err(error) if is_still_opening(error.as_ref()) => { + *self = take_still_opening(error.as_ref()).expect("the open going on"); + None + } + retried => Some(retried), + } + } +} + +/// An open in the lane that had not returned when its wait ended and goes +/// on in the background, carried out of [`EncoderState::new`] to the encode +/// loop, which streams on libx264 meanwhile. +struct StillOpening(String, parking_lot::Mutex>); + +impl std::fmt::Debug for StillOpening { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("StillOpening").field(&self.0).finish() + } +} + +impl std::fmt::Display for StillOpening { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} has not opened yet", self.0) + } +} + +impl std::error::Error for StillOpening {} + +fn is_still_opening(error: &(dyn std::error::Error + Send + Sync + 'static)) -> bool { + error.is::() +} + +/// The open that goes on in the background, from the error that carries it. +fn take_still_opening( + error: &(dyn std::error::Error + Send + Sync + 'static), +) -> Option { + error.downcast_ref::()?.1.lock().take() +} + +/// Encoders FFmpeg 7.1 opens under its one process-wide lock +/// (FF_CODEC_CAP_NOT_INIT_THREADSAFE): the NVENC, AMF and QSV hardware +/// encoders, which wait on a graphics driver, and libx265. While one of them +/// is stuck holding the lock, any other would block behind it, so they share +/// one time-limited lane. libx264 and VideoToolbox do not take the lock. +fn takes_codec_lock(encoder_name: &str) -> bool { + encoder_name == "libx265" + || ["_nvenc", "_amf", "_qsv"] + .iter() + .any(|suffix| encoder_name.ends_with(suffix)) +} + +/// `vendors_present` lists the graphics cards' vendors when the fallback +/// runs (see [`crate::gpu::vendors_present`]). The encoders of the vendors +/// listed are tried first, so a PC with a working one never probes the +/// others: AMF on a PC without an AMD card only logs that its DLL failed to +/// load. Every other hardware encoder is still tried after them, because +/// Windows does not list every card that can encode (an NVIDIA card in TCC +/// mode). A hybrid laptop lists both of its cards; an unknown vendor moves +/// no encoder. With no list, the order is NVENC, AMF, Quick Sync, as before. +/// +/// `failed` is the encoder that failed first, and why. If every encoder +/// fails, one error names each and its reason: their FFmpeg errors were +/// warnings while they were tried. +fn open_remaining_encoders( + (failed, why): (&str, &str), + vendors_present: impl FnOnce() -> Option>, + mut open: impl FnMut(&str) -> Result>, +) -> Result<(&'static str, T), Box> { + // Open errors already start with the encoder's name. + let mut reasons = vec![why.to_string()]; + let note = |reasons: &mut Vec, name: &str, reason: String| { + reasons.push(if reason.starts_with(name) { + reason + } else { + format!("{name}: {reason}") }); }; let mut hardware = if matches!(failed, "h264_nvenc" | "h264_amf" | "h264_qsv") { @@ -1114,6 +1764,14 @@ fn open_remaining_encoders( } match open(name) { Ok(state) => return Ok((name, state)), + // Every other open in the lane would wait behind it. + Err(error) if is_still_opening(error.as_ref()) => { + info!( + encoder = name, + "Hardware encoder is still opening; libx264 streams while it opens in the background" + ); + break; + } Err(error) => { warn!(encoder = name, %error, "Hardware encoder did not open"); note(&mut reasons, name, error.to_string()); @@ -1188,6 +1846,154 @@ fn note_nvidia_driver( PIPELINE_STATS.lock().nvidia_driver_outdated = true; } +/// Hardware encoders that failed to open this launch, each at the encoded +/// size it failed at. Each is not tried again at that size, so its FFmpeg +/// errors are logged once instead of at every pipeline start and reopen +/// (field triage, Nov 2026: 13-44 lines per PC, over and over, for an NVENC +/// API too old, an AMF component or a Quick Sync session that could not be +/// created). Forgotten when the user clicks Restart, as the fallback notice +/// asks after a driver update. A stall is not a failure: the driver did not +/// answer in time, which says nothing about the encoder. +#[derive(Default)] +pub(crate) struct LaunchFailures { + opens: Vec<(String, (u32, u32))>, + /// AMD and Intel cards whose encoder failed to open: the fallback + /// notice asks for their driver update. + vendors: Vec, +} + +static LAUNCH_FAILURES: parking_lot::Mutex = + parking_lot::Mutex::new(LaunchFailures { + opens: Vec::new(), + vendors: Vec::new(), + }); + +impl LaunchFailures { + /// Why `encoder` is not opened at `size`: it failed there this launch. + fn earlier(&self, encoder: &str, size: (u32, u32)) -> Option { + self.opens + .iter() + .any(|(name, at)| name == encoder && *at == size) + .then(|| { + format!( + "{encoder} failed to open at {}x{} earlier in this launch; not tried again until Restart", + size.0, size.1 + ) + }) + } + + /// Whether an encoder named with `suffix` failed this launch. + fn any(&self, suffix: &str) -> bool { + self.opens.iter().any(|(name, _)| name.ends_with(suffix)) + } + + /// `encoder` failed to open at `size`, for a reason of its own. When it + /// is AMD's or Intel's, and `vendors_present` (read only then) lists + /// that vendor's card, that card's driver could not start it. + fn failed( + &mut self, + encoder: &str, + size: (u32, u32), + vendors_present: impl FnOnce() -> Option>, + ) { + if !is_hardware_encoder(encoder) || self.earlier(encoder, size).is_some() { + return; + } + self.opens.push((encoder.to_string(), size)); + let vendor = if encoder.ends_with("_amf") { + GpuVendor::Amd + } else if encoder.ends_with("_qsv") { + GpuVendor::Intel + } else { + return; + }; + if self.vendors.contains(&vendor) + || !vendors_present().is_some_and(|present| present.contains(&vendor)) + { + return; + } + warn!( + %vendor, + encoder, + "The graphics driver could not start video encoding on this card; video is encoded on the processor unless another graphics card can, and updating the driver may enable hardware encoding" + ); + self.vendors.push(vendor); + } +} + +fn is_hardware_encoder(encoder_name: &str) -> bool { + ["_nvenc", "_amf", "_qsv"] + .iter() + .any(|suffix| encoder_name.ends_with(suffix)) +} + +/// Forget this launch's failed opens and their driver notes, so the next +/// pipeline tries every encoder again: the user clicked Restart, maybe +/// after a driver update. +pub fn forget_launch_failures() { + let forgotten = std::mem::take(&mut *LAUNCH_FAILURES.lock()); + if !forgotten.opens.is_empty() { + info!( + encoders = forgotten.opens.len(), + "Trying the encoders that failed to open again" + ); + } +} + +/// Opens a session as [`EncoderState::new`] does, unless `encoder_name` +/// failed to open at this size earlier in this launch, and remembers a +/// hardware encoder's own failure in `failures`. +#[allow(clippy::too_many_arguments)] +fn open_session( + failures: &parking_lot::Mutex, + encoder_name: &str, + capture_width: u32, + capture_height: u32, + bitrate_bps: u32, + target_fps: u32, + input_mode: &mut EncoderInput, + opening: Opening, + progress: &mut dyn FnMut(), +) -> Result> { + let size = encoded_size(capture_width, capture_height); + if let Some(why) = failures.lock().earlier(encoder_name, size) { + return Err(why.into()); + } + let opened = EncoderState::new( + encoder_name, + capture_width, + capture_height, + bitrate_bps, + target_fps, + input_mode, + opening, + progress, + ); + if let Err(error) = &opened { + note_open_failure(failures, encoder_name, size, error.as_ref()); + } + opened +} + +/// Remembers a hardware encoder's own failure to open at `size`, and shows +/// its driver note on the Stream page. +fn note_open_failure( + failures: &parking_lot::Mutex, + encoder_name: &str, + size: (u32, u32), + error: &(dyn std::error::Error + Send + Sync + 'static), +) { + if is_open_stall(error) { + return; + } + let vendors = { + let mut failures = failures.lock(); + failures.failed(encoder_name, size, crate::gpu::vendors_present); + failures.vendors.clone() + }; + PIPELINE_STATS.lock().driver_update_vendors = vendors; +} + /// An encoder open that its time limit gave up on, or refused while an /// earlier one was stuck. The driver did not answer in time, which says /// nothing about whether the encoder works. @@ -1202,8 +2008,10 @@ impl std::fmt::Display for OpenStalled { impl std::error::Error for OpenStalled {} +/// An open given up on or refused, or one still going on in the background: +/// neither says the encoder does not work. fn is_open_stall(error: &(dyn std::error::Error + Send + Sync + 'static)) -> bool { - error.downcast_ref::().is_some() + error.downcast_ref::().is_some() || is_still_opening(error) } /// Opens at the pixel format `input` picks from `supported`. When nothing @@ -1236,6 +2044,51 @@ fn open_at_input( } } +/// A reopen, for its log line and for what follows if it fails. +struct Reopening { + encoder: String, + /// The bitrate of the session it replaces. + from: u32, + /// The session that streams until it opens. + current: String, + reason: &'static str, + size_changed: bool, + keyframe: bool, + bitrate: u32, + fps: u32, + /// The pipeline's first open, which went on past its first picture's + /// wait (see [`FIRST_PICTURE_WAIT`]). + first: bool, + in_background: bool, +} + +/// An open going on in the background, and the reopen it makes once it +/// returns. +struct Behind { + open: BackgroundOpen, + reopen: Reopening, +} + +impl Behind { + /// The pipeline's first open of `open`'s encoder, while libx264 streams + /// at `bitrate`. + fn first(open: BackgroundOpen, bitrate: u32) -> Self { + let reopen = Reopening { + encoder: open.encoder().to_string(), + from: bitrate, + current: "libx264".to_string(), + reason: "recovery", + size_changed: false, + keyframe: false, + bitrate: open.args.bitrate_bps, + fps: open.args.target_fps, + first: true, + in_background: true, + }; + Self { open, reopen } + } +} + /// How long after a stalled open each new try of the preferred encoder /// waits, at the earliest: at most three tries a pipeline generation. const STALL_RETRY_WAITS: [Duration; 3] = [ @@ -1380,6 +2233,20 @@ impl StallRecovery { true } + /// The pipeline's first open of `preferred` had not returned when its + /// first picture was due, and goes on in the background while `using` + /// streams. If it opens, [`Self::opened`] ends the recovery. If it is + /// given up on at its limit, the first try is due once the lane is + /// clear, as after an open that stalled while the stream waited. + fn outwaited(&mut self, preferred: &str, using: &str, now: Instant) { + self.trying = false; + self.owed = Some(Owed { + preferred: preferred.to_string(), + using: using.to_string(), + next_try: now + STALL_RETRY_WAITS[0], + }); + } + /// No more tries this generation. fn end(&mut self) { self.owed = None; @@ -1415,14 +2282,21 @@ fn reopen_needed( /// ABR step just after an AMF keyframe reopen) reopens once. const BITRATE_DECREASE_GAP: Duration = Duration::from_secs(1); -/// Paces reopens for a bitrate decrease alone. Hardware encoders (and +/// How long after a Quick Sync session opened a bitrate change alone may +/// wait, up or down, once Quick Sync refused to take one in place (see +/// [`takes_bitrate_in_place`]): each of its reopens freezes the picture +/// for about 0.6 s, and one PC in the field reopened 148 times. +const QSV_BITRATE_REOPEN_GAP: Duration = Duration::from_secs(5); + +/// Paces reopens for a bitrate change alone. Hardware encoders (and /// libx265) take a new bitrate only by reopening, and each reopen costs an /// IDR and a visible stutter. A decrease takes effect at once, or 1 s after /// the session opened if that is later, so the stream never overruns a /// congested link more than 1 s longer than before, and a decrease right /// after another reopen does not make a second one at once. An increase /// takes effect at once, as do codec, frame rate, size, keyframe and -/// recovery reopens, and every reopen opens at the newest target. +/// recovery reopens, and every reopen opens at the newest target. Quick +/// Sync, whose reopens are slowest, waits 5 s for either. #[derive(Default)] struct BitratePacer { /// When the open session opened, for any reason. @@ -1432,14 +2306,21 @@ struct BitratePacer { impl BitratePacer { /// The bitrate this frame wants. `opened` and `wanted` are encoder, /// bitrate and frame rate. The newest target, unless only the bitrate - /// differs, downwards, within 1 s of the session's open: then the - /// session's own. + /// differs, downwards, within 1 s of the session's open, or either way + /// within 5 s of a Quick Sync session's: then the session's own. fn wanted(&self, opened: (&str, u32, u32), wanted: (&str, u32, u32), now: Instant) -> u32 { + let gap = if opened.0.ends_with("_qsv") { + QSV_BITRATE_REOPEN_GAP + } else if wanted.1 < opened.1 { + BITRATE_DECREASE_GAP + } else { + Duration::ZERO + }; let waits = (opened.0, opened.2) == (wanted.0, wanted.2) - && wanted.1 < opened.1 + && wanted.1 != opened.1 && self .opened_at - .is_some_and(|at| now.duration_since(at) < BITRATE_DECREASE_GAP); + .is_some_and(|at| now.duration_since(at) < gap); if waits { opened.1 } else { @@ -1625,6 +2506,161 @@ pub fn ignores_forced_keyframes(encoder_name: &str) -> bool { matches!(encoder_name, "h264_amf" | "hevc_amf") } +/// True for encoders that keep each frame until they are given the next. +/// FFmpeg 7.1's AMF wrapper asks AMF for output only when it submits a frame +/// (8.0 can wait for it, with `query_timeout`). On a still desktop a new +/// viewer's opening keyframe sat inside AMF until capture's next picture, +/// the 750 ms keepalive: AMD PCs were slow on 39-56% of reconnects. +pub fn holds_last_frame(encoder_name: &str) -> bool { + matches!(encoder_name, "h264_amf" | "hevc_amf") +} + +/// The shortest wait for capture's next picture before an encoder that +/// [holds its last frame](holds_last_frame) is given the last picture again, +/// to bring out a session's first keyframe: longer than AMF takes to encode +/// one. See [`keyframe_flush_wait`]. +const KEYFRAME_FLUSH_WAIT: Duration = Duration::from_millis(20); + +/// A gap between capture's real pictures shorter than this is the desktop's +/// pace while it moves; a longer one is an idle spell, which says nothing +/// about the pace (capture resends a still picture after 100 ms for a +/// keyframe request, else at its 750 ms keepalive). +const MOVING_GAP: Duration = Duration::from_millis(100); +/// After an idle spell this long, the pace seen before it is forgotten. +const PACE_FORGOTTEN: Duration = Duration::from_secs(10); + +/// How long the encode loop waits for capture's next picture, after encoding +/// `cached` (see [`RawFrame::cached`]), before it gives that picture again. +/// After a cached picture of a still desktop no picture is due, so the +/// shortest wait brings the keyframe out soonest. After a real one, a frame +/// and a half: of the target rate, or of capture's pace while the desktop +/// moves (`pace`) when that is slower, as for a 50 Hz display streamed at +/// 60 fps or 30 fps content. A flat 20 ms gave those pictures to AMF twice. +fn keyframe_flush_wait(target_fps: u32, cached: bool, pace: Option) -> Duration { + if cached { + return KEYFRAME_FLUSH_WAIT; + } + let frame = crate::capture::frame_budget_for(target_fps).max(pace.unwrap_or_default()); + (frame * 3 / 2).max(KEYFRAME_FLUSH_WAIT) +} + +/// The pace of capture's real pictures, measured only between real pictures +/// and kept across idle spells: motion that resumes keeps its pace. +#[derive(Default)] +struct CaptureCadence { + last: Option, + gaps: std::collections::VecDeque, +} + +impl CaptureCadence { + const KEPT: usize = 8; + + /// A picture from capture (not one given again). + fn captured(&mut self, at: Instant, cached: bool) { + if cached { + return; + } + if let Some(last) = self.last.replace(at) { + let gap = at.saturating_duration_since(last); + if gap < MOVING_GAP { + if self.gaps.len() == Self::KEPT { + self.gaps.pop_front(); + } + self.gaps.push_back(gap); + } else if gap >= PACE_FORGOTTEN { + self.gaps.clear(); + } + } + } + + /// The longest recent gap between real pictures while the desktop moved, + /// if any were seen. + fn pace(&self) -> Option { + self.gaps.iter().copied().max() + } +} + +/// How long after the keyframe went in the last picture is given again, +/// before the keyframe is left to capture's next picture: an AMD driver can +/// take a few hundred ms to encode a fresh session's first frame, and +/// capture's keepalive follows at 750 ms. +const KEYFRAME_FLUSH_FOR: Duration = Duration::from_millis(500); + +/// A session's first keyframe still inside an encoder that holds its last +/// frame, and the picture to give it again. +struct OwedKeyframe { + /// The last picture encoded. It pins one of capture's few buffers. + picture: Option, + /// When the picture stops being given again. + until: Instant, + /// How long to wait for capture's next picture first. + wait: Duration, + given: u32, +} + +/// What the encode loop encodes next while a keyframe is owed. +enum Next { + Frame(RawFrame), + /// The last picture, given again. + Again(RawFrame), + Closed, + /// The deadline passed: capture's next picture brings the keyframe out. + GaveUp, +} + +impl OwedKeyframe { + fn until(until: Instant) -> Self { + Self { + picture: None, + until, + wait: KEYFRAME_FLUSH_WAIT, + given: 0, + } + } + + /// What is owed after a frame was submitted at `submitted`: nothing once + /// the keyframe came out or the encoder holds no frame, else `picture` + /// to give again after `wait`, until [`KEYFRAME_FLUSH_FOR`] after the + /// first try. + fn after_encode( + owed: Option, + picture: Option, + keyframe_out: bool, + submitted: Instant, + wait: Duration, + ) -> Option { + let picture = picture.filter(|_| !keyframe_out)?; + let mut owed = owed.unwrap_or_else(|| Self::until(submitted + KEYFRAME_FLUSH_FOR)); + owed.picture = Some(picture); + owed.wait = wait; + Some(owed) + } + + /// Capture's next picture if one arrives within the wait, else the last + /// picture again, until the deadline. + fn next(&mut self, frames: &FrameSlot) -> Next { + let wait = self + .wait + .min(self.until.saturating_duration_since(Instant::now())); + match frames.take_within(wait, &mut self.picture) { + Taken::Frame(frame) => Next::Frame(frame), + Taken::Closed => Next::Closed, + Taken::Nothing => match self.picture.take() { + Some(picture) if Instant::now() < self.until => { + self.given += 1; + debug!( + frame = picture.frame_number, + given = self.given, + "Giving the encoder its last picture again to bring out the keyframe" + ); + Next::Again(picture) + } + _ => Next::GaveUp, + }, + } + } +} + /// Whether a pending keyframe request needs a session reopen: the encoder /// ignores per-frame requests, the session has already sent its opening IDR, /// and the last such reopen is older than the client's own request limit. @@ -1667,7 +2703,8 @@ fn codec_display_name_for(encoder_name: &str) -> &str { /// After every successful encoder open: report the live codec on the wire /// (HELLO_ACK/heartbeat `StreamConfig.codec`) and in the GUI. -fn note_encoder_opened(shared: &SharedControl, encoder_name: &str) { +fn note_encoder_opened(shared: &SharedControl, encoder_name: &str, gpu: &GpuInfo) { + *shared.wake_adapter.lock() = crate::gpu_wake::wake_adapter_for(encoder_name, gpu); let codec = if is_hevc_encoder(encoder_name) { eternal_wire::v2::control::CODEC_HEVC } else { @@ -1816,7 +2853,17 @@ fn prepare_frame_for_encode( } else { "period" }; - if is_amf { + // The period runs all stream long, and FFmpeg 7.1's AMF ignores it + // (see `ignores_forced_keyframes`): debug only. + if !force_hit { + debug!( + encoder = encoder_name, + frame = frame_number, + period = AMF_FORCED_INTRA_PERIOD, + reason, + "[AMF] Forced IDR" + ); + } else if is_amf { info!( encoder = encoder_name, frame = frame_number, @@ -2323,29 +3370,227 @@ mod tests { assert_eq!(labels, (480..490).map(|n| (at(n), n)).collect::>()); } + fn libx264_at(width: u32, height: u32) -> EncoderState { + EncoderState::new( + "libx264", + width, + height, + 8_000_000, + 60, + &mut EncoderInput::Yuv420, + Opening::First, + &mut || {}, + ) + .unwrap_or_else(|error| panic!("{width}x{height}: {error}")) + } + + /// swscale gets threads only where it converts every frame: with + /// Accelerate converting on macOS it is a fallback and runs on one. #[test] - fn nvenc_and_qsv_make_a_requested_keyframe_an_idr() { - for (name, option) in [ - ("h264_nvenc", "forced-idr"), - ("hevc_nvenc", "forced-idr"), - ("h264_qsv", "forced_idr"), - ("hevc_qsv", "forced_idr"), + fn swscale_gets_threads_only_where_it_converts_every_frame() { + let state = libx264_at(96, 64); + let scaler = state.scaler.as_ref().expect("x264 takes YUV420P"); + #[cfg(target_os = "macos")] + assert_eq!( + (state.native_converter.is_some(), scaler.threads()), + (true, 1) + ); + #[cfg(not(target_os = "macos"))] + assert_eq!(scaler.threads(), input::scaler_threads()); + } + + /// A synthetic capture, in a buffer as capture hands it over. + fn synthetic_capture(width: u32, height: u32, frame_number: u64) -> Vec { + let frame_bytes = width as usize * 4 * height as usize; + let mut rows = crate::capture::frame_buffer(frame_bytes); + crate::capture::synthetic::render_synthetic_frame( + &mut rows[..frame_bytes], + width, + height, + frame_number, + ); + rows + } + + /// A screen with an odd height (1680x945, from a report) or width + /// streams: libx264 refused to open at those sizes, and the pipeline + /// gave up after six restarts. The encoder opens at the even size below, + /// takes the capture's frames, and they decode at that size with the + /// picture intact. + #[test] + fn odd_screen_sizes_encode_at_the_even_size_below() { + ffmpeg_next::init().unwrap(); + for ((width, height), encoded) in [ + ((1680, 945), (1680, 944)), + ((1365, 768), (1364, 768)), + ((641, 361), (640, 360)), + ((1366, 768), (1366, 768)), ] { - assert_eq!(encoder_options(name).get(option), Some("1"), "{name}"); + let mut state = libx264_at(width, height); + assert_eq!((state.frame.width(), state.frame.height()), encoded); + // The same capture size never reopens the session; another does. + assert!(state.accepts_frame(width, height)); + assert!(!state.accepts_frame(width + 2, height)); + assert!(!state.accepts_frame(width, height + 2)); + let codec = ffmpeg_next::decoder::find(ffmpeg_next::codec::Id::H264).unwrap(); + let mut decoder = ffmpeg_next::codec::Context::new_with_codec(codec) + .decoder() + .video() + .unwrap(); + let mut decoded = ffmpeg_next::frame::Video::empty(); + let mut counters = Vec::new(); + let start = Instant::now(); + for n in 1..=10u64 { + let rows = synthetic_capture(width, height, n); + state.convert(&rows, width as usize * 4).unwrap(); + let captured = start + Duration::from_micros(n * 16_667); + let pts = state.next_pts(captured, n); + state.frame.set_pts(Some(pts)); + state.send_frame(pts, captured, n).unwrap(); + while state.encoder.receive_packet(&mut state.packet).is_ok() { + if state.h264.needs_parameter_sets() { + state + .h264 + .refresh_parameter_sets_from_extradata(encoder_extradata( + &state.encoder, + )); + } + // What the transport sends: Annex B with parameter sets. + let (annex_b, _) = normalize_h264_payload_with_info( + state.packet.data().unwrap_or_default(), + state.packet.is_key(), + &mut state.h264, + &state.opened_encoder, + ); + decoder + .send_packet(&ffmpeg_next::Packet::copy(&annex_b)) + .unwrap(); + while decoder.receive_frame(&mut decoded).is_ok() { + assert_eq!((decoded.width(), decoded.height()), encoded); + counters.push(crate::capture::synthetic::decode_frame_counter_from_luma( + decoded.data(0), + decoded.stride(0), + encoded.0, + )); + } + } + } + assert_eq!( + counters, + (1..=counters.len() as u64).collect::>(), + "{width}x{height}" + ); + assert!(counters.len() >= 5, "{width}x{height}: {counters:?}"); } - for name in ["h264_nvenc", "hevc_nvenc"] { - assert_eq!(encoder_options(name).get("delay"), Some("0"), "{name}"); + } + + /// The rows of each of `frame`'s planes that hold the picture. + fn picture(frame: &ffmpeg_next::frame::Video) -> Vec> { + use ffmpeg_next::format::Pixel; + let mut rows = Vec::new(); + for plane in 0..frame.planes() { + let bytes = match (frame.format(), plane) { + (Pixel::BGRA | Pixel::BGRZ, _) => frame.width() as usize * 4, + (Pixel::NV12, 1) => frame.plane_width(1) as usize * 2, + _ => frame.plane_width(plane) as usize, + }; + for y in 0..frame.plane_height(plane) as usize { + rows.push(frame.data(plane)[y * frame.stride(plane)..][..bytes].to_vec()); + } } + rows } - /// Encodes frame `n` as the encode loop does, and returns each packet - /// that came out: whether the transport takes it as a keyframe, and the - /// capture time and number of its frame. - fn encode( - state: &mut EncoderState, - n: u64, - captured: Instant, - force_idr: bool, + /// A capture with an odd last column or row encodes exactly what the + /// rest of it, alone, does: the crop leaves that column and row out and + /// resamples nothing, on every input path (swscale to YUV420P and NV12, + /// Accelerate on macOS, the BGRA copy NVENC reads), with or without + /// padding after the capture's last row. + #[test] + fn cropping_changes_no_pixel_of_the_picture() { + use ffmpeg_next::format::Pixel; + ffmpeg_next::init().unwrap(); + for (width, height) in [(1365u32, 945u32), (1365, 944), (1366, 945)] { + let (even_width, even_height) = encoded_size(width, height); + let capture = synthetic_capture(width, height, 77); + let row_bytes = width as usize * 4; + let unpadded = &capture[..row_bytes * height as usize]; + // The capture's top-left even_width x even_height, alone. + let even_bytes = even_width as usize * 4; + let mut alone = crate::capture::frame_buffer(even_bytes * even_height as usize); + for y in 0..even_height as usize { + alone[y * even_bytes..][..even_bytes] + .copy_from_slice(&capture[y * row_bytes..][..even_bytes]); + } + let mut paths = vec!["YUV420P", "NV12", "BGRA"]; + if cfg!(target_os = "macos") { + paths.push("Accelerate"); + } + for path in paths { + let label = format!("{width}x{height} {path}"); + let mut state = libx264_at(width, height); + // The frame and conversion an encoder that takes `path` has. + #[cfg(target_os = "macos")] + if path != "Accelerate" { + state.native_converter = None; + } + let format = match path { + "NV12" => Pixel::NV12, + "BGRA" => Pixel::BGRA, + _ => Pixel::YUV420P, + }; + state.frame = ffmpeg_next::frame::Video::new(format, even_width, even_height); + state.scaler = input::conversion_context( + format, + even_width, + even_height, + input::scaler_threads(), + ) + .unwrap(); + assert_eq!(state.scaler.is_some(), format != Pixel::BGRA); + state.convert(&alone, even_bytes).unwrap(); + let expected = picture(&state.frame); + for rows in [&capture[..], unpadded] { + state.frame = ffmpeg_next::frame::Video::new(format, even_width, even_height); + state.convert(rows, row_bytes).unwrap(); + assert_eq!(picture(&state.frame), expected, "{label}"); + } + // Accelerate converted each, with no fallback to swscale. + #[cfg(target_os = "macos")] + assert_eq!(state.native_converter.is_some(), path == "Accelerate"); + if format == Pixel::BGRA { + let rows = capture.chunks(row_bytes).take(even_height as usize); + for (y, row) in rows.enumerate() { + assert_eq!(expected[y], row[..even_bytes], "{label} row {y}"); + } + } + } + } + } + + #[test] + fn nvenc_and_qsv_make_a_requested_keyframe_an_idr() { + for (name, option) in [ + ("h264_nvenc", "forced-idr"), + ("hevc_nvenc", "forced-idr"), + ("h264_qsv", "forced_idr"), + ("hevc_qsv", "forced_idr"), + ] { + assert_eq!(encoder_options(name).get(option), Some("1"), "{name}"); + } + for name in ["h264_nvenc", "hevc_nvenc"] { + assert_eq!(encoder_options(name).get("delay"), Some("0"), "{name}"); + } + } + + /// Encodes frame `n` as the encode loop does, and returns each packet + /// that came out: whether the transport takes it as a keyframe, and the + /// capture time and number of its frame. + fn encode( + state: &mut EncoderState, + n: u64, + captured: Instant, + force_idr: bool, ) -> Vec<(bool, Instant, u64)> { let pts = state.next_pts(captured, n); state.frame.set_pts(Some(pts)); @@ -2644,11 +3889,21 @@ mod tests { .with_ansi(false) .with_writer(move || Sink(Arc::clone(&writer))) .finish(); + crate::logging::prepare_log_capture(); let result = tracing::subscriber::with_default(subscriber, run); let lines = String::from_utf8(out.lock().clone()).unwrap(); (result, lines.lines().map(str::to_string).collect()) } + /// An open that returned, as every open but a wait-limited one in the + /// lane does. + fn now(opened: Opened) -> OpenResult { + match opened { + Opened::Now(result) => result, + Opened::Behind(_) => panic!("the open went on in the background"), + } + } + /// A real FFmpeg open that fails with an FFmpeg error line: libx264, /// which opens on the calling thread, with no picture size. fn failing_open(opening: Opening) -> Vec { @@ -2662,7 +3917,13 @@ mod tests { video.set_format(ffmpeg_next::format::Pixel::YUV420P); video.set_time_base(ffmpeg_next::Rational(1, 60)); let (opened, lines) = logged(|| { - open_encoder_on(crate::stall::lanes(), video, "libx264", opening, &mut || {}) + now(open_encoder_on( + crate::stall::lanes(), + video, + "libx264", + opening, + &mut || {}, + )) }); assert!(matches!(opened, Ok(Err(_))), "the open must fail"); lines @@ -2673,7 +3934,7 @@ mod tests { #[test] fn ffmpeg_errors_are_warnings_only_while_an_encoder_is_one_of_several() { - let probed = failing_open(Opening::Probe); + let probed = failing_open(Opening::Probe(Instant::now() + FIRST_PICTURE_WAIT)); assert!(!probed.is_empty()); assert!( probed @@ -2696,14 +3957,15 @@ mod tests { ); } // The pipeline's first open probes, except the last resort. - assert_eq!(first_open("h264_nvenc"), Opening::Probe); - assert_eq!(first_open("hevc_amf"), Opening::Probe); - assert_eq!(first_open("libx265"), Opening::Probe); - assert_eq!(first_open("h264_videotoolbox"), Opening::Probe); - assert_eq!(first_open("libx264"), Opening::First); + let by = Instant::now() + FIRST_PICTURE_WAIT; + assert_eq!(first_open("h264_nvenc", by), Opening::Probe(by)); + assert_eq!(first_open("hevc_amf", by), Opening::Probe(by)); + assert_eq!(first_open("libx265", by), Opening::Probe(by)); + assert_eq!(first_open("h264_videotoolbox", by), Opening::Probe(by)); + assert_eq!(first_open("libx264", by), Opening::First); // A probe has the first open's limits. assert_eq!( - open_limits("h264_qsv", Opening::Probe), + open_limits("h264_qsv", Opening::Probe(Instant::now())), open_limits("h264_qsv", Opening::First) ); } @@ -3002,8 +4264,19 @@ mod tests { /// Encoder, bitrate and frame rate of the open session. opened: (String, u32, u32), pacer: BitratePacer, - /// Bitrate changes the open libx264 session took in place. + /// Bitrate changes the open libx264 or Quick Sync session took in + /// place. in_place: usize, + /// A Quick Sync session refused a bitrate change in place. + qsv_refused: bool, + /// An NVENC session did: a card that cannot change its bitrate live. + nvenc_refused: bool, + /// Whether FFmpeg reconfigures an NVENC session for a change made + /// in place (see `nvenc_reconfigured`). + nvenc_reconfigures: bool, + /// The session was dropped after a refused change; the next frame + /// opens one, as the encode loop's first open does. + dropped: bool, /// The user's preference. hevc_wanted: bool, bitrate: u32, @@ -3030,6 +4303,12 @@ mod tests { opened: (opened.to_string(), 15_000_000, 60), pacer: BitratePacer::default(), in_place: 0, + qsv_refused: false, + // As on a card that cannot change its bitrate live: these + // tests reopen NVENC sessions with bitrate changes. + nvenc_refused: true, + nvenc_reconfigures: true, + dropped: false, hevc_wanted: is_hevc_encoder(opened), bitrate: 15_000_000, fps: 60, @@ -3064,11 +4343,28 @@ mod tests { self.session.is_some() && !client_hevc && is_hevc_encoder(&self.opened.0); self.undecodable_retry .frame(undecodable, &mut self.failed_reopen, self.now); - // As `update_bitrate_without_reopening`. - if (self.opened.0.as_str(), self.opened.2) == ("libx264", self.fps) - && desired == "libx264" + if std::mem::take(&mut self.dropped) { + self.opened = (desired.clone(), self.bitrate, self.fps); + self.pacer.opened(self.now); + self.failed_reopen = None; + return Some(desired); + } + // As `update_bitrate_without_reopening`, settled at the end of + // the frame: a refused NVENC change drops the session. + if takes_bitrate_in_place(&self.opened.0, |live| match live { + LiveBitrate::QsvReset => self.qsv_refused, + LiveBitrate::NvencReconfig => self.nvenc_refused, + LiveBitrate::X264 => false, + }) && (self.opened.0.as_str(), self.opened.2) == (desired.as_str(), self.fps) && self.opened.1 != self.bitrate { + if LiveBitrate::of(&self.opened.0) == Some(LiveBitrate::NvencReconfig) + && !self.nvenc_reconfigures + { + self.nvenc_refused = true; + self.dropped = true; + return None; + } self.opened.1 = self.bitrate; self.failed_reopen = None; self.in_place += 1; @@ -3605,10 +4901,12 @@ mod tests { #[test] fn a_reopen_goes_ahead_as_soon_as_the_lane_clears_whenever_a_try_is_due() { - // QSV streams. ABR steps stall its reopen, twice: the next tries + // QSV streams, and reopens for a bitrate change since it refused + // one in place. ABR steps stall its reopen, twice: the next tries // are due 2 s and then 8 s later. Each time the lane clears 1 s // after the stall, and a bitrate change reopens on the next frame. let mut p = Pipeline::new("h264_qsv", "h264_qsv", Instant::now()); + p.qsv_refused = true; for (bitrate, stalls) in [(10_000_000, true), (12_000_000, true), (14_000_000, false)] { p.bitrate = bitrate; let reopen = if stalls { @@ -3719,13 +5017,9 @@ mod tests { #[test] fn a_drop_reopens_at_once_or_within_a_second_of_the_last_open() { - for name in [ - "h264_nvenc", - "h264_amf", - "h264_qsv", - "hevc_nvenc", - "libx265", - ] { + // Quick Sync takes a new bitrate in place; see + // `quick_sync_takes_every_bitrate_change_in_place_until_it_refuses_one`. + for name in ["h264_nvenc", "h264_amf", "hevc_nvenc", "libx265"] { let h264 = match name { "hevc_nvenc" => "h264_nvenc", "libx265" => "libx264", @@ -3910,175 +5204,893 @@ mod tests { } #[test] - fn the_reopen_line_says_why() { - let nvenc = ("h264_nvenc", 60); - let x264 = ("libx264", 60); - assert_eq!(reopen_reason(nvenc, nvenc, false, false, false), "bitrate"); + fn quick_sync_takes_every_bitrate_change_in_place_until_it_refuses_one() { + for name in ["libx264", "h264_qsv", "hevc_qsv"] { + assert!(takes_bitrate_in_place(name, |_| false), "{name}"); + } + for name in ["h264_amf", "hevc_amf", "libx265"] { + assert!(!takes_bitrate_in_place(name, |_| false), "{name}"); + } + assert!(!takes_bitrate_in_place("h264_qsv", |_| true)); + assert!(takes_bitrate_in_place("libx264", |_| true)); + // As libx264: rises and drops, some within a second of each other, + // each on the frame it arrives, and no reopen. The field's worst PC + // reopened Quick Sync 148 times, about 0.6 s of frozen picture each. + let t0 = Instant::now(); + let mut p = Pipeline::new("h264_qsv", "h264_qsv", t0); + for (at, bitrate) in [ + (0, 12_000_000), + (100, 10_000_000), + (5_000, 15_000_000), + (5_100, 12_000_000), + ] { + p.now = t0 + Duration::from_millis(at); + p.bitrate = bitrate; + assert_eq!(p.frame(Reopen::Opens), None, "at {at} ms"); + assert_eq!(p.opened.1, bitrate, "at {at} ms"); + } + assert_eq!(p.in_place, 4); + // A frame rate change still reopens. + p.fps = 30; + assert_eq!(p.frame(Reopen::Opens).as_deref(), Some("h264_qsv")); + // Once a session refused a change, a change reopens, at most every + // 5 s, rises too: here 5 s after that frame rate reopen, and 5 s + // after the previous bitrate reopen. + p.qsv_refused = true; + let reopens = play( + &mut p, + &[ + (0, Some(10_000_000)), + (1_000, Some(14_000_000)), + (6_000, Some(8_000_000)), + (6_500, Some(9_000_000)), + ], + 15_000, + ); + assert_eq!(reopens, [(5_008, 14_000_000), (10_016, 9_000_000)]); + assert_eq!(p.in_place, 4); + } + + #[test] + fn a_quick_sync_bitrate_reopen_waits_5_s_for_either_direction() { + let t0 = Instant::now(); + let at = |ms: u64| t0 + Duration::from_millis(ms); + let mut pacer = BitratePacer::default(); + pacer.opened(t0); + let qsv = ("h264_qsv", 10_000_000, 60); assert_eq!( - reopen_reason(nvenc, ("h264_nvenc", 30), false, false, false), - "fps" + pacer.wanted(qsv, ("h264_qsv", 12_000_000, 60), at(1_000)), + 10_000_000 ); assert_eq!( - reopen_reason(nvenc, ("hevc_nvenc", 60), false, false, false), - "codec" + pacer.wanted(qsv, ("h264_qsv", 8_000_000, 60), at(4_999)), + 10_000_000 ); - assert_eq!(reopen_reason(x264, nvenc, false, false, true), "recovery"); assert_eq!( - reopen_reason(x264, ("hevc_nvenc", 60), false, false, true), - "recovery" + pacer.wanted(qsv, ("h264_qsv", 8_000_000, 60), at(5_000)), + 8_000_000 ); - let amf = ("h264_amf", 60); - assert_eq!(reopen_reason(amf, amf, false, true, false), "keyframe"); assert_eq!( - reopen_reason(nvenc, ("hevc_nvenc", 30), true, true, false), - "size" + pacer.wanted(qsv, ("h264_qsv", 12_000_000, 60), at(5_000)), + 12_000_000 + ); + // A frame rate or codec change goes ahead at the newest target. + assert_eq!( + pacer.wanted(qsv, ("h264_qsv", 12_000_000, 30), at(10)), + 12_000_000 + ); + assert_eq!( + pacer.wanted(qsv, ("hevc_qsv", 12_000_000, 60), at(10)), + 12_000_000 + ); + // Other encoders keep #165's pacing: a rise at once, a drop 1 s on. + let nvenc = ("h264_nvenc", 10_000_000, 60); + assert_eq!( + pacer.wanted(nvenc, ("h264_nvenc", 12_000_000, 60), at(10)), + 12_000_000 + ); + assert_eq!( + pacer.wanted(nvenc, ("h264_nvenc", 8_000_000, 60), at(999)), + 10_000_000 + ); + assert_eq!( + pacer.wanted(nvenc, ("h264_nvenc", 8_000_000, 60), at(1_000)), + 8_000_000 ); } - #[test] - fn a_qsv_open_as_slow_as_the_slowest_seen_opens_on_qsv() { - use crate::stall::{encoder_open_limits, no_hooks, run_in, Call, Lanes, Wait}; - use std::sync::atomic::AtomicU64; - use std::sync::Arc; - let lanes: &'static Lanes = Box::leak(Box::new(Lanes::new())); - let out = Arc::new(parking_lot::Mutex::new(Vec::::new())); - struct Sink(Arc>>); - impl std::io::Write for Sink { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.0.lock().extend_from_slice(buf); - Ok(buf.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - let writer = Arc::clone(&out); - let subscriber = tracing_subscriber::fmt() - .with_ansi(false) - .with_writer(move || Sink(Arc::clone(&writer))) - .finish(); - let mut recovery = StallRecovery::default(); - let now = Instant::now(); - let desired = recovery.encoder_for_frame("h264_qsv", now, lane_clear(lanes)); - // The open returns once the caller's clock reads 4.4 s, the slowest - // QSV open in the field logs. + /// A clock that stands still until the test moves it: the ms since the + /// test began are in the returned atomic. + fn paused_clock() -> ( + std::sync::Arc, + impl Fn() -> Instant + Clone + Send + 'static, + ) { let start = Instant::now(); - let reads = Arc::new(AtomicU64::new(0)); - let clock_reads = Arc::clone(&reads); - let opened = tracing::subscriber::with_default(subscriber, || { - run_in( - lanes, - Call::EncoderOpen, - Wait { - limits: encoder_open_limits(&desired), - slice: Duration::from_millis(1), - now: move || { - start - + Duration::from_millis( - 100 * clock_reads.fetch_add(1, Ordering::SeqCst), - ) - }, - }, - move || { - while reads.load(Ordering::SeqCst) < 44 { - std::thread::sleep(Duration::from_millis(1)); - } - "h264_qsv" - }, - || {}, - no_hooks(), - ) - }); - assert_eq!(opened, Ok("h264_qsv")); - assert!(!recovery.opened(&desired, "h264_qsv")); - assert!(recovery.owed.is_none()); - assert!(lane_clear(lanes)); - let lines = String::from_utf8(out.lock().clone()).unwrap(); - assert!(!lines.contains("stall=encoder_open"), "{lines}"); - // Not even slow: the warning starts above the slowest open seen. - assert!(!lines.contains("slow=encoder_open"), "{lines}"); + let ms = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let read = std::sync::Arc::clone(&ms); + (ms, move || { + start + Duration::from_millis(read.load(Ordering::SeqCst)) + }) } - #[test] - fn hevc_software_does_not_block_behind_a_stuck_hardware_open() { - use crate::stall::{no_hooks, run_in, Call, Lanes, Stall, Wait}; - use std::time::Duration; - for name in ["h264_nvenc", "hevc_amf", "h264_qsv", "libx265"] { - assert!(takes_codec_lock(name), "{name}"); - } - for name in ["libx264", "h264_videotoolbox", "hevc_videotoolbox"] { - assert!(!takes_codec_lock(name), "{name}"); - } - let lanes: &'static Lanes = Box::leak(Box::new(Lanes::new())); - // An NVENC open is stuck holding FFmpeg's lock. - let start = Instant::now(); - let reads = std::sync::atomic::AtomicU64::new(0); - let (release, released) = std::sync::mpsc::channel::<()>(); - let stuck = run_in( + /// A fake hardware open of `name` in `lanes`, on `clock`, that hangs + /// until the returned sender sends, then opens. + fn hung_open Instant>( + lanes: &'static crate::stall::Lanes, + name: &'static str, + opening: Opening, + clock: N, + ) -> ( + std::sync::mpsc::Sender<()>, + crate::stall::Pending<&'static str, N>, + ) { + let (release, hang) = stuck_open(); + let pending = crate::stall::start_in( lanes, - Call::EncoderOpen, - Wait { - limits: Call::EncoderOpen.limits(), + crate::stall::Call::EncoderOpen, + crate::stall::Wait { + limits: open_limits(name, opening), slice: Duration::from_millis(1), - now: move || { - start + Duration::from_millis(100 * reads.fetch_add(1, Ordering::SeqCst)) - }, + now: clock, }, move || { - let _ = released.recv(); + hang(); + name }, - || {}, - no_hooks(), - ); - assert!(matches!(stuck, Err(Stall::TimedOut { .. }))); - let context = |name: &str| { - let codec = ffmpeg_next::encoder::find_by_name(name)?; - let mut video = ffmpeg_next::codec::Context::new_with_codec(codec) - .encoder() - .video() - .ok()?; - video.set_width(64); - video.set_height(64); - video.set_format(ffmpeg_next::format::Pixel::YUV420P); - video.set_time_base(ffmpeg_next::Rational(1, 60)); - Some(video) + crate::stall::no_hooks(), + ) + .unwrap(); + (release, pending) + } + + /// Waits for the pipeline's first open as the encode loop does, as + /// `opening` says, each 1 ms heartbeat moving the paused clock 100 ms on. + fn first_open_of Instant>( + pending: crate::stall::Pending<&'static str, N>, + opening: Opening, + ms: &std::sync::atomic::AtomicU64, + ) -> Result, crate::stall::Pending<&'static str, N>> + { + wait_or_park(pending, opening, &mut || { + ms.fetch_add(100, Ordering::SeqCst); + }) + } + + #[test] + fn time_before_the_open_starts_counts_against_the_first_pictures_wait() { + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let (ms, clock) = paused_clock(); + let opening = Opening::Probe(clock() + FIRST_PICTURE_WAIT); + // 2.2 s go by before the open's thread is running, as Windows once + // took to start one on a loaded PC. + ms.store(2_200, Ordering::SeqCst); + let (release, pending) = hung_open(lanes, "h264_nvenc", opening, clock.clone()); + let Err(parked) = first_open_of(pending, opening, &ms) else { + panic!("the hung open returned") }; - // A reconnect asks for HEVC again: libx265 is refused at once - // instead of waiting on the lock with no limit, and the encode loop - // stays on H.264. - if let Some(video) = context("libx265") { - let started = Instant::now(); - let opened = open_encoder_on(lanes, video, "libx265", Opening::First, &mut || {}); - assert!( - matches!( - opened, - Err(Stall::Refused { - call: Call::EncoderOpen - }) - ), - "{:?}", - opened.map(|r| r.map(|_| ())) - ); - assert!(started.elapsed() < Duration::from_secs(1)); - } - // libx264 opens meanwhile. - let video = context("libx264").expect("libx264 is in every build"); - assert!(matches!( - open_encoder_on(lanes, video, "libx264", Opening::First, &mut || {}), - Ok(Ok(_)) - )); - let _ = release.send(()); + // Parked when the picture was due, 3 s after it was asked for, not + // 3 s after the thread began. + assert_eq!(ms.load(Ordering::SeqCst), 3_000); + assert_eq!(parked.running_for(), Duration::from_millis(800)); + release.send(()).unwrap(); + drop(parked); + wait_for_clear_lane(lanes); + // Due before the open even starts: parked at once. + ms.store(4_000, Ordering::SeqCst); + let (release, pending) = hung_open(lanes, "h264_amf", opening, clock); + let Err(parked) = wait_or_park(pending, opening, &mut || panic!("it waited")) else { + panic!("returned") + }; + assert_eq!(parked.running_for(), Duration::ZERO); + release.send(()).unwrap(); + drop(parked); + wait_for_clear_lane(lanes); } #[test] - fn startup_uses_software_only_after_all_other_hardware_fails() { - for failed in ["h264_nvenc", "h264_amf", "h264_qsv"] { - let mut attempts = Vec::new(); - let (selected, ()) = open_remaining_encoders((failed, "test"), no_list, |name| { - attempts.push(name.to_string()); - if name == "libx264" { - Ok(()) - } else { + fn a_hung_first_open_leaves_the_picture_to_libx264_when_it_is_due() { + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let (ms, clock) = paused_clock(); + let ((parked, opened_late), lines) = logged(|| { + let opening = Opening::Probe(clock() + FIRST_PICTURE_WAIT); + let (release, pending) = hung_open(lanes, "h264_qsv", opening, clock.clone()); + let Err(mut parked) = first_open_of(pending, opening, &ms) else { + panic!("the hung open returned") + }; + // Given up waiting for when the first picture was due, 7 s + // before the iPad's timeout; not given up on. + let parked_at = parked.running_for(); + // libx264 needs no lane and opens at once. Every hardware open + // is refused at once instead of waiting on FFmpeg's lock. + assert_eq!( + open_in(lanes, "libx264", Opening::First, || {}).unwrap(), + "libx264" + ); + for name in ["h264_nvenc", "h264_amf", "hevc_qsv", "libx265"] { + let error = open_in(lanes, name, Opening::First, || {}).unwrap_err(); + assert!(is_open_stall(error.as_ref()), "{name}: {error}"); + assert!(error.to_string().contains("not started"), "{error}"); + } + // The stream: libx264 while QSV opens. + let t0 = Instant::now(); + let at = |ms: u64| t0 + Duration::from_millis(ms); + let mut recovery = StallRecovery::default(); + recovery.outwaited("h264_qsv", "libx264", at(3_000)); + for ms in [3_000, 4_000, 5_000, 14_000] { + assert_eq!( + recovery.encoder_for_frame("h264_qsv", at(ms), lane_clear(lanes)), + "libx264", + "at {ms} ms" + ); + } + // QSV opens at 4.4 s, the slowest open in the field, within + // its 15 s limit: the stream takes that very session. + ms.store(4_400, Ordering::SeqCst); + assert!(parked.poll().is_none()); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + let opened = parked.poll(); + assert!(recovery.opened("h264_qsv", "h264_qsv")); + assert!(recovery.owed.is_none()); + assert_eq!(recovery.tries, 0); + (parked_at, opened) + }); + assert_eq!(parked, FIRST_PICTURE_WAIT); + assert_eq!(opened_late, Some(Ok("h264_qsv"))); + assert!( + !lines.iter().any(|line| line.contains("stall=encoder_open")), + "{lines:?}" + ); + } + + #[test] + fn an_open_that_returns_before_the_first_picture_is_due_is_not_parked() { + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let (ms, clock) = paused_clock(); + let opening = Opening::Probe(clock() + FIRST_PICTURE_WAIT); + let (release, pending) = hung_open(lanes, "h264_nvenc", opening, clock); + // NVENC's slowest open in the field: 2.6 s. The clock stops there, + // however long the hand-over takes on a loaded machine. + let mut beats = 0; + let mut released = false; + let opened = wait_or_park(pending, opening, &mut || { + beats += 1; + if !released && ms.fetch_add(100, Ordering::SeqCst) + 100 >= 2_600 { + release.send(()).unwrap(); + released = true; + } + }); + assert!(matches!(opened, Ok(Ok("h264_nvenc")))); + assert!(lane_clear(lanes)); + // The watchdog's heartbeat ran meanwhile. + assert!(beats >= 26, "{beats}"); + // A deadline already past (an earlier encoder of the fallback used + // the wait up) parks at once; so does a background reopen. + for behind in [false, true] { + let (_ms, clock) = paused_clock(); + let opening = if behind { + Opening::Behind + } else { + Opening::Probe(clock()) + }; + let (release, pending) = hung_open(lanes, "h264_amf", opening, clock); + let Err(parked) = wait_or_park(pending, opening, &mut || panic!("it waited")) else { + panic!("returned") + }; + assert_eq!(parked.running_for(), Duration::ZERO); + release.send(()).unwrap(); + drop(parked); + wait_for_clear_lane(lanes); + } + // A reopen of a live session still waits, to its 9 s limit. + let (ms, clock) = paused_clock(); + let (release, pending) = hung_open(lanes, "h264_qsv", Opening::Reopen, clock); + let gave_up = wait_or_park(pending, Opening::Reopen, &mut || { + ms.fetch_add(100, Ordering::SeqCst); + }); + let Ok(Err(crate::stall::Stall::TimedOut { after, .. })) = gave_up else { + panic!("not given up on at the reopen limit") + }; + assert_eq!(after, Duration::from_secs(9)); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + } + + #[test] + fn a_first_open_hung_past_its_limit_is_tried_again_in_the_background() { + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let (ms, clock) = paused_clock(); + let t0 = Instant::now(); + let at = |ms: u64| t0 + Duration::from_millis(ms); + let mut recovery = StallRecovery::default(); + let opening = Opening::Probe(clock() + FIRST_PICTURE_WAIT); + let (release, pending) = hung_open(lanes, "h264_nvenc", opening, clock.clone()); + let Err(mut parked) = first_open_of(pending, opening, &ms) else { + panic!("the hung open returned") + }; + recovery.outwaited("h264_nvenc", "libx264", at(3_000)); + // Given up on at NVENC's 15 s limit, with the one stall line; the + // lane stays taken until the driver returns. + ms.store(15_000, Ordering::SeqCst); + let (result, lines) = logged(|| parked.poll()); + assert!(matches!( + result, + Some(Err(crate::stall::Stall::TimedOut { .. })) + )); + assert!( + lines + .iter() + .any(|line| line.contains("stall=encoder_open after_ms=15000")), + "{lines:?}" + ); + assert!(!lane_clear(lanes)); + assert_eq!( + recovery.encoder_for_frame("h264_nvenc", at(20_000), false), + "libx264" + ); + // The driver returns at 30 s. The try is due: it opens in the + // background, so libx264 streams on instead of freezing for it. + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + let desired = recovery.encoder_for_frame("h264_nvenc", at(30_000), true); + assert_eq!(desired, "h264_nvenc"); + assert!(opens_behind("libx264", &desired)); + assert_eq!( + open_limits("h264_nvenc", Opening::Behind), + crate::stall::encoder_open_limits("h264_nvenc") + ); + recovery.starting(&desired); + ms.store(30_000, Ordering::SeqCst); + let (release, pending) = hung_open(lanes, "h264_nvenc", Opening::Behind, clock); + let Err(mut trying) = wait_or_park(pending, Opening::Behind, &mut || { + panic!("a background open waited") + }) else { + panic!("returned") + }; + assert_eq!( + recovery.encoder_for_frame("h264_nvenc", at(30_016), lane_clear(lanes)), + "libx264" + ); + ms.store(30_200, Ordering::SeqCst); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + assert_eq!(trying.poll(), Some(Ok("h264_nvenc"))); + assert!(recovery.opened("h264_nvenc", "h264_nvenc")); + assert!(recovery.owed.is_none()); + } + + #[test] + fn only_a_switch_from_libx264_into_the_lane_opens_in_the_background() { + for (current, wanted, behind) in [ + ("libx264", "h264_nvenc", true), + ("libx264", "hevc_amf", true), + ("libx264", "h264_qsv", true), + ("libx264", "libx265", true), + ("libx264", "libx264", false), + ("h264_nvenc", "hevc_nvenc", false), + ("h264_qsv", "h264_qsv", false), + ("hevc_amf", "libx264", false), + ("h264_videotoolbox", "hevc_videotoolbox", false), + ] { + assert_eq!( + opens_behind(current, wanted), + behind, + "{current} -> {wanted}" + ); + } + } + + #[test] + fn a_hardware_encoder_that_failed_is_remembered_at_its_size_for_the_launch() { + use GpuVendor::{Amd, Intel, Nvidia}; + let hd = (1920, 1080); + let mut failures = LaunchFailures::default(); + failures.failed("h264_nvenc", hd, || panic!("NVENC never reads the cards")); + assert!(failures.earlier("h264_nvenc", hd).is_some()); + // Another size, another encoder: tried. + assert!(failures.earlier("h264_nvenc", (2732, 2048)).is_none()); + assert!(failures.earlier("hevc_nvenc", hd).is_none()); + assert!(failures.any("_nvenc")); + assert!(!failures.any("_amf")); + // Software encoders are never left out. + failures.failed("libx264", hd, || panic!("read")); + failures.failed("libx265", hd, || panic!("read")); + assert!(failures.earlier("libx264", hd).is_none()); + assert!(failures.earlier("libx265", hd).is_none()); + // AMF on a PC with an AMD card: its driver gets the note. + failures.failed("h264_amf", hd, || Some(vec![Nvidia, Amd])); + assert_eq!(failures.vendors, [Amd]); + // Quick Sync on a PC without an Intel card: no note, as that + // failure is expected there. Still remembered. + failures.failed("h264_qsv", hd, || Some(vec![Nvidia, Amd])); + assert_eq!(failures.vendors, [Amd]); + assert!(failures.earlier("h264_qsv", hd).is_some()); + // Unknown cards (DXGI listed some, not all): no note. + failures.failed("hevc_qsv", hd, || None); + assert_eq!(failures.vendors, [Amd]); + failures.failed("h264_qsv", (1280, 720), || Some(vec![Intel])); + assert_eq!(failures.vendors, [Amd, Intel]); + // Each failure is kept once, and the cards are read once a vendor. + failures.failed("hevc_amf", hd, || panic!("read again")); + failures.failed("h264_nvenc", hd, || panic!("read")); + assert_eq!( + failures + .opens + .iter() + .filter(|(name, size)| name == "h264_nvenc" && *size == hd) + .count(), + 1 + ); + let why = failures.earlier("h264_amf", hd).unwrap(); + assert!( + why.contains("h264_amf failed to open at 1920x1080 earlier in this launch"), + "{why}" + ); + assert!(why.contains("Restart"), "{why}"); + } + + #[test] + fn an_encoder_that_failed_to_open_is_not_opened_again_or_logged_again() { + ffmpeg_next::init().unwrap(); + crate::diagnostics::install_ffmpeg_logging(); + let failures = parking_lot::Mutex::new(LaunchFailures::default()); + // A real open that fails: Quick Sync on a PC without an Intel card + // ("Error creating a MFX session"), or not in this FFmpeg build + // (macOS). Both are the encoder's own failure. + let open = |failures: &parking_lot::Mutex| { + logged(|| { + open_session( + failures, + "h264_qsv", + 1280, + 720, + 8_000_000, + 60, + &mut EncoderInput::Yuv420, + Opening::First, + &mut || {}, + ) + .err() + .map(|error| error.to_string()) + }) + }; + let (Some(first), first_lines) = open(&failures) else { + eprintln!("Quick Sync opens on this PC; nothing to remember"); + return; + }; + eprintln!("first open: {first}"); + assert!(failures.lock().earlier("h264_qsv", (1280, 720)).is_some()); + // Every pipeline after it: no open, no FFmpeg line, and the reason. + for _ in 0..3 { + let (again, lines) = open(&failures); + let again = again.expect("opened"); + assert!(again.contains("earlier in this launch"), "{again}"); + assert!( + !lines + .iter() + .any(|line| line.contains("ffmpeg") || line.contains("Opening encoder session")), + "{lines:?}" + ); + } + eprintln!( + "first open logged {} FFmpeg lines", + first_lines + .iter() + .filter(|line| line.contains("ffmpeg")) + .count() + ); + // libx264 is never held back, and a Restart tries Quick Sync again. + assert!(open_session( + &failures, + "libx264", + 1280, + 720, + 8_000_000, + 60, + &mut EncoderInput::Yuv420, + Opening::First, + &mut || {}, + ) + .is_ok()); + *failures.lock() = LaunchFailures::default(); + let (retried, _) = open(&failures); + let retried = retried.expect("opened"); + assert!(!retried.contains("earlier in this launch"), "{retried}"); + } + + /// A background open in `lanes` of `name` at `pixel_format`, parked + /// already, that hangs until the returned sender sends, then gives `open`. + fn parked_background_open( + lanes: &'static crate::stall::Lanes, + name: &str, + pixel_format: ffmpeg_next::format::Pixel, + open: impl FnOnce() -> Result + + Send + + 'static, + ) -> (std::sync::mpsc::Sender<()>, BackgroundOpen) { + let (release, hang) = stuck_open(); + let mut pending = crate::stall::start_in( + lanes, + crate::stall::Call::EncoderOpen, + crate::stall::Wait { + limits: open_limits(name, Opening::Behind), + slice: Duration::from_millis(1), + now: Instant::now as fn() -> Instant, + }, + move || { + hang(); + open() + }, + crate::stall::no_hooks(), + ) + .unwrap(); + assert!(pending.park().is_none()); + let args = SessionArgs { + encoder_name: name.to_string(), + size: (64, 64), + bitrate_bps: 8_000_000, + target_fps: 60, + pixel_format, + legacy_pts: false, + }; + ( + release, + BackgroundOpen { + pending, + args, + yuv_retry: false, + }, + ) + } + + /// A real libx264 session at 64x64 YUV420P, opened on the call's thread + /// as a hardware encoder would be. + fn real_yuv_encoder() -> Result + { + let codec = ffmpeg_next::encoder::find_by_name("libx264").expect("libx264"); + let mut video = ffmpeg_next::codec::Context::new_with_codec(codec) + .encoder() + .video()?; + video.set_width(64); + video.set_height(64); + video.set_format(ffmpeg_next::format::Pixel::YUV420P); + video.set_time_base(ffmpeg_next::Rational(1, 60)); + video.open_with(encoder_options("libx264")) + } + + #[test] + fn a_bgra_open_that_fails_in_the_background_is_tried_again_at_yuv420_there() { + use ffmpeg_next::format::Pixel; + ffmpeg_next::init().unwrap(); + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let refused = || { + Err(ffmpeg_next::Error::Other { + errno: libc::EINVAL, + }) + }; + // NVENC's Automatic BGRA open outlived the first picture's wait, and + // fails after it, as an NVENC that takes only YUV420 does. + let (release, mut open) = parked_background_open(lanes, "h264_nvenc", Pixel::BGRA, refused); + open.yuv_retry = true; + let mut input = EncoderInput::Automatic; + assert!(open + .poll(&mut input, |_| panic!("tried again before it failed")) + .is_none()); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + // YUV420 is tried at once, in the background too, and the pipeline + // keeps it; the BGRA failure is not handed back to be remembered. + let mut retried_at = None; + let mut yuv_release = None; + let polled = open.poll(&mut input, |args| { + retried_at = Some((args.encoder_name.clone(), args.size, args.bitrate_bps)); + let (release, yuv) = + parked_background_open(lanes, &args.encoder_name, Pixel::YUV420P, real_yuv_encoder); + yuv_release = Some(release); + Err(Box::new(StillOpening( + args.encoder_name.clone(), + parking_lot::Mutex::new(Some(yuv)), + ))) + }); + assert!(polled.is_none(), "the BGRA failure was handed back"); + assert_eq!( + retried_at, + Some(("h264_nvenc".to_string(), (64, 64), 8_000_000)) + ); + assert_eq!(input, EncoderInput::Yuv420); + assert_eq!(open.args.pixel_format, Pixel::YUV420P); + assert!(open + .poll(&mut input, |_| panic!("tried again while YUV420 opens")) + .is_none()); + // It opens: the stream takes that session. + yuv_release.unwrap().send(()).unwrap(); + wait_for_clear_lane(lanes); + let state = open + .poll(&mut input, |_| panic!("YUV420 is tried once")) + .expect("returned") + .expect("the YUV420 open failed"); + assert_eq!(state.opened_encoder, "h264_nvenc"); + assert_eq!(state.frame.format(), Pixel::YUV420P); + + // When YUV420 fails too, that failure is handed back, to be + // remembered for the launch. + let (release, mut open) = parked_background_open(lanes, "h264_nvenc", Pixel::BGRA, refused); + open.yuv_retry = true; + let mut input = EncoderInput::Automatic; + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + let failed = open.poll(&mut input, |_| { + Err("h264_nvenc open failed at YUV420".into()) + }); + let error = failed.expect("returned").err().expect("it opened"); + assert!(error.to_string().contains("at YUV420"), "{error}"); + assert!(!is_open_stall(error.as_ref())); + assert_eq!(input, EncoderInput::Yuv420); + + // A YUV420 open, or a picked BGRA, is not tried again; nor is a + // stall, which says nothing about BGRA. + let (release, mut open) = parked_background_open(lanes, "h264_nvenc", Pixel::BGRA, refused); + let mut input = EncoderInput::Bgra; + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + let failed = open.poll(&mut input, |_| panic!("a picked BGRA was tried again")); + assert!(failed.expect("returned").is_err()); + assert_eq!(input, EncoderInput::Bgra); + } + + #[test] + fn an_hevc_open_in_the_background_is_dropped_once_the_frame_wants_h264() { + use ffmpeg_next::format::Pixel; + ffmpeg_next::init().unwrap(); + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + // An H.264 encoder whose H.265 sibling this FFmpeg build has. + let Some(h264) = ["libx264", "h264_nvenc", "h264_videotoolbox"] + .into_iter() + .find(|name| { + ffmpeg_next::encoder::find_by_name(hevc_variant_of(name).unwrap()).is_some() + }) + else { + eprintln!("no H.265 encoder in this FFmpeg build"); + return; + }; + // The encode loop's negotiation, from the user's preference and the + // client's decoding. + let preferred = |hevc_enabled: bool, client_hevc: bool| { + preferred_encoder(h264, hevc_enabled && client_hevc, false, &mut true) + }; + let hevc = preferred(true, true); + assert!(is_hevc_encoder(&hevc), "{hevc}"); + for change in ["an H.264-only client connected", "HEVC was turned off"] { + let (release, open) = + parked_background_open(lanes, &hevc, Pixel::YUV420P, real_yuv_encoder); + let mut background = Some(Behind::first(open, 8_000_000)); + assert!(!drop_obsolete_background( + &mut background, + &preferred(true, true) + )); + assert!(background.is_some(), "{change}: dropped while wanted"); + let now = match change { + "an H.264-only client connected" => preferred(true, false), + _ => preferred(false, true), + }; + assert_eq!(now, h264, "{change}"); + // The HEVC session opens meanwhile, ready to be taken. + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + assert!( + drop_obsolete_background(&mut background, &now), + "{change}: the HEVC session would replace one the client decodes" + ); + assert!(background.is_none(), "{change}"); + } + // Still wanted, it is taken once it opens. + let (release, open) = + parked_background_open(lanes, &hevc, Pixel::YUV420P, real_yuv_encoder); + let mut background = Some(Behind::first(open, 8_000_000)); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + assert!(!drop_obsolete_background(&mut background, &hevc)); + let state = background + .as_mut() + .unwrap() + .open + .poll(&mut EncoderInput::Yuv420, |_| panic!("tried again")) + .expect("returned") + .expect("opened"); + assert_eq!(state.opened_encoder, hevc); + // An H.264 open is kept when HEVC is turned on: the client decodes + // it, and the stream reopens to HEVC from there. + let (release, open) = parked_background_open(lanes, h264, Pixel::YUV420P, real_yuv_encoder); + let mut background = Some(Behind::first(open, 8_000_000)); + assert!(!drop_obsolete_background(&mut background, &hevc)); + assert!(background.is_some()); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + } + + #[test] + fn the_reopen_line_says_why() { + let nvenc = ("h264_nvenc", 60); + let x264 = ("libx264", 60); + assert_eq!(reopen_reason(nvenc, nvenc, false, false, false), "bitrate"); + assert_eq!( + reopen_reason(nvenc, ("h264_nvenc", 30), false, false, false), + "fps" + ); + assert_eq!( + reopen_reason(nvenc, ("hevc_nvenc", 60), false, false, false), + "codec" + ); + assert_eq!(reopen_reason(x264, nvenc, false, false, true), "recovery"); + assert_eq!( + reopen_reason(x264, ("hevc_nvenc", 60), false, false, true), + "recovery" + ); + let amf = ("h264_amf", 60); + assert_eq!(reopen_reason(amf, amf, false, true, false), "keyframe"); + assert_eq!( + reopen_reason(nvenc, ("hevc_nvenc", 30), true, true, false), + "size" + ); + } + + #[test] + fn a_qsv_open_as_slow_as_the_slowest_seen_opens_on_qsv() { + use crate::stall::{encoder_open_limits, no_hooks, run_in, Call, Lanes, Wait}; + use std::sync::atomic::AtomicU64; + use std::sync::Arc; + let lanes: &'static Lanes = Box::leak(Box::new(Lanes::new())); + let out = Arc::new(parking_lot::Mutex::new(Vec::::new())); + struct Sink(Arc>>); + impl std::io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let writer = Arc::clone(&out); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || Sink(Arc::clone(&writer))) + .finish(); + let mut recovery = StallRecovery::default(); + let now = Instant::now(); + let desired = recovery.encoder_for_frame("h264_qsv", now, lane_clear(lanes)); + // The open returns once the caller's clock reads 4.4 s, the slowest + // QSV open in the field logs. + let start = Instant::now(); + let reads = Arc::new(AtomicU64::new(0)); + let clock_reads = Arc::clone(&reads); + crate::logging::prepare_log_capture(); + let opened = tracing::subscriber::with_default(subscriber, || { + run_in( + lanes, + Call::EncoderOpen, + Wait { + limits: encoder_open_limits(&desired), + slice: Duration::from_millis(1), + now: move || { + start + + Duration::from_millis( + 100 * clock_reads.fetch_add(1, Ordering::SeqCst), + ) + }, + }, + move || { + while reads.load(Ordering::SeqCst) < 44 { + std::thread::sleep(Duration::from_millis(1)); + } + "h264_qsv" + }, + || {}, + no_hooks(), + ) + }); + assert_eq!(opened, Ok("h264_qsv")); + assert!(!recovery.opened(&desired, "h264_qsv")); + assert!(recovery.owed.is_none()); + assert!(lane_clear(lanes)); + let lines = String::from_utf8(out.lock().clone()).unwrap(); + assert!(!lines.contains("stall=encoder_open"), "{lines}"); + // Not even slow: the warning starts above the slowest open seen. + assert!(!lines.contains("slow=encoder_open"), "{lines}"); + } + + #[test] + fn hevc_software_does_not_block_behind_a_stuck_hardware_open() { + use crate::stall::{no_hooks, run_in, Call, Lanes, Stall, Wait}; + use std::time::Duration; + for name in ["h264_nvenc", "hevc_amf", "h264_qsv", "libx265"] { + assert!(takes_codec_lock(name), "{name}"); + } + for name in ["libx264", "h264_videotoolbox", "hevc_videotoolbox"] { + assert!(!takes_codec_lock(name), "{name}"); + } + let lanes: &'static Lanes = Box::leak(Box::new(Lanes::new())); + // An NVENC open is stuck holding FFmpeg's lock. + let start = Instant::now(); + let reads = std::sync::atomic::AtomicU64::new(0); + let (release, released) = std::sync::mpsc::channel::<()>(); + let stuck = run_in( + lanes, + Call::EncoderOpen, + Wait { + limits: Call::EncoderOpen.limits(), + slice: Duration::from_millis(1), + now: move || { + start + Duration::from_millis(100 * reads.fetch_add(1, Ordering::SeqCst)) + }, + }, + move || { + let _ = released.recv(); + }, + || {}, + no_hooks(), + ); + assert!(matches!(stuck, Err(Stall::TimedOut { .. }))); + let context = |name: &str| { + let codec = ffmpeg_next::encoder::find_by_name(name)?; + let mut video = ffmpeg_next::codec::Context::new_with_codec(codec) + .encoder() + .video() + .ok()?; + video.set_width(64); + video.set_height(64); + video.set_format(ffmpeg_next::format::Pixel::YUV420P); + video.set_time_base(ffmpeg_next::Rational(1, 60)); + Some(video) + }; + // A reconnect asks for HEVC again: libx265 is refused at once + // instead of waiting on the lock with no limit, and the encode loop + // stays on H.264. + if let Some(video) = context("libx265") { + let started = Instant::now(); + let opened = now(open_encoder_on( + lanes, + video, + "libx265", + Opening::First, + &mut || {}, + )); + assert!( + matches!( + opened, + Err(Stall::Refused { + call: Call::EncoderOpen + }) + ), + "{:?}", + opened.map(|r| r.map(|_| ())) + ); + assert!(started.elapsed() < Duration::from_secs(1)); + } + // libx264 opens meanwhile. + let video = context("libx264").expect("libx264 is in every build"); + assert!(matches!( + now(open_encoder_on( + lanes, + video, + "libx264", + Opening::First, + &mut || {} + )), + Ok(Ok(_)) + )); + let _ = release.send(()); + } + + #[test] + fn startup_uses_software_only_after_all_other_hardware_fails() { + for failed in ["h264_nvenc", "h264_amf", "h264_qsv"] { + let mut attempts = Vec::new(); + let (selected, ()) = open_remaining_encoders((failed, "test"), no_list, |name| { + attempts.push(name.to_string()); + if name == "libx264" { + Ok(()) + } else { Err("device unavailable".into()) } }) @@ -4097,16 +6109,286 @@ mod tests { } #[test] - fn non_windows_backends_keep_their_software_fallback() { - let mut attempts = Vec::new(); - let (selected, ()) = - open_remaining_encoders(("h264_videotoolbox", "test"), no_list, |name| { - attempts.push(name.to_string()); - Ok(()) - }) + fn non_windows_backends_keep_their_software_fallback() { + let mut attempts = Vec::new(); + let (selected, ()) = + open_remaining_encoders(("h264_videotoolbox", "test"), no_list, |name| { + attempts.push(name.to_string()); + Ok(()) + }) + .unwrap(); + assert_eq!(selected, "libx264"); + assert_eq!(attempts, ["libx264"]); + } + + #[test] + fn nvenc_and_quick_sync_take_a_bitrate_in_place_and_amf_and_x265_reopen() { + use LiveBitrate::{NvencReconfig, QsvReset, X264}; + for (name, live) in [ + ("libx264", Some(X264)), + ("h264_qsv", Some(QsvReset)), + ("hevc_qsv", Some(QsvReset)), + ("h264_nvenc", Some(NvencReconfig)), + ("hevc_nvenc", Some(NvencReconfig)), + ("h264_amf", None), + ("hevc_amf", None), + ("libx265", None), + ("h264_videotoolbox", None), + ] { + assert_eq!(LiveBitrate::of(name), live, "{name}"); + assert_eq!( + takes_bitrate_in_place(name, |_| false), + live.is_some(), + "{name}" + ); + } + // A refusal stops one kind only, and libx264 never refuses. + let nvenc_refused = |live| live == NvencReconfig; + assert!(!takes_bitrate_in_place("h264_nvenc", nvenc_refused)); + assert!(takes_bitrate_in_place("h264_qsv", nvenc_refused)); + assert!(takes_bitrate_in_place("libx264", |_| true)); + // The harness's NVENC: in place on a card that changes it live. + let t0 = Instant::now(); + let mut p = Pipeline::new("h264_nvenc", "h264_nvenc", t0); + p.nvenc_refused = false; + assert_eq!( + play( + &mut p, + &[ + (0, Some(12_000_000)), + (100, Some(20_000_000)), + (200, Some(8_000_000)) + ], + 5_000 + ), + [] + ); + assert_eq!((p.in_place, p.opened.1), (3, 8_000_000)); + } + + #[test] + fn an_nvenc_change_is_settled_by_its_reconfigure_and_never_by_an_idr() { + let lines = |lines: &[&str]| { + lines + .iter() + .map(|line| line.to_string()) + .collect::>() + }; + // What FFmpeg 7.1.1's nvenc.c prints while it takes the change's + // frame. A reconfigure that ran: + assert!(nvenc_reconfigured(&lines(&[ + "[h264_nvenc @ 0x1] avg bitrate change: 8000000 -> 6000000", + "[h264_nvenc @ 0x1] vbv buffer size change: 16000000 -> 12000000", + ]))); + // One that failed: FFmpeg returns no error, so the line decides. + assert!(!nvenc_reconfigured(&lines(&[ + "[h264_nvenc @ 0x1] avg bitrate change: 8000000 -> 6000000", + "[h264_nvenc @ 0x1] failed to reconfigure nvenc: invalid param (8): ", + ]))); + // A card that cannot change its bitrate live: FFmpeg skips it and + // prints nothing, though the frame may come out as an IDR anyway, + // forced or the GOP's own. Before, that IDR confirmed it. + assert!(!nvenc_reconfigured(&[])); + assert!(!nvenc_reconfigured(&lines(&[ + "[h264_nvenc @ 0x1] aspect ratio change (DAR): 16:9 -> 16:9" + ]))); + } + + #[test] + fn ffmpeg_verbose_lines_reach_only_the_watching_thread_and_are_not_logged() { + ffmpeg_next::init().unwrap(); + crate::diagnostics::install_ffmpeg_logging(); + let say = |level: i32, text: &str| { + let message = std::ffi::CString::new(format!("{text}\n")).unwrap(); + unsafe { ffmpeg_next::ffi::av_log(std::ptr::null_mut(), level, message.as_ptr()) }; + }; + let (kept, logged_lines) = logged(|| { + let watch = crate::diagnostics::FfmpegWatch::start(); + say( + ffmpeg_next::ffi::AV_LOG_VERBOSE, + "avg bitrate change: 1 -> 2", + ); + say(ffmpeg_next::ffi::AV_LOG_WARNING, "a warning while watching"); + // Another thread's verbose line meanwhile is not kept here. + let other = std::thread::spawn(move || { + say( + ffmpeg_next::ffi::AV_LOG_VERBOSE, + "another thread's verbose line", + ); + }); + other.join().unwrap(); + let kept = watch.finish(); + // After the watch, verbose lines are dropped again. + say(ffmpeg_next::ffi::AV_LOG_VERBOSE, "verbose after the watch"); + kept + }); + assert_eq!(kept.len(), 2, "{kept:?}"); + assert!(kept[0].contains("avg bitrate change: 1 -> 2"), "{kept:?}"); + assert!(kept[1].contains("a warning while watching"), "{kept:?}"); + // The warning is logged as always; no verbose line ever is. + assert!( + logged_lines + .iter() + .any(|line| line.contains("a warning while watching")), + "{logged_lines:?}" + ); + assert!( + !logged_lines + .iter() + .any(|line| line.contains("verbose") || line.contains("avg bitrate change")), + "{logged_lines:?}" + ); + // A watch never touches FFmpeg's own level, which other threads + // read as they log. + assert_eq!( + unsafe { ffmpeg_next::ffi::av_log_get_level() }, + ffmpeg_next::ffi::AV_LOG_VERBOSE + ); + } + + #[test] + fn a_refused_nvenc_change_reopens_at_the_new_bitrate_even_with_abr_off() { + // ETERNAL_ABR=0: the target is the user's Max bitrate, and nothing + // changes it again for as long as the user leaves it. A change NVENC + // did not take must not leave the session on the old bitrate. + let mut abr = crate::transport::abr::AbrController::new(8_000_000, false); + let mut p = Pipeline::new("h264_nvenc", "h264_nvenc", Instant::now()); + p.nvenc_refused = false; + p.nvenc_reconfigures = false; + p.opened.1 = abr.current_bps(); + p.bitrate = abr.current_bps(); + let raised = abr.set_ceiling(20_000_000); + assert!(raised.changed); + p.bitrate = raised.target_bps; + // The frame it went in with: refused, the session is dropped. + assert_eq!(p.frame(Reopen::Opens), None); + assert!(p.nvenc_refused && p.dropped); + // The next frame opens one at the new bitrate. + p.later(16); + assert_eq!(p.frame(Reopen::Opens).as_deref(), Some("h264_nvenc")); + assert_eq!(p.opened.1, 20_000_000); + assert_eq!(p.in_place, 0); + // From then on its changes reopen, as after a Quick Sync refusal. + let lowered = abr.set_ceiling(10_000_000); + p.later(2_000); + p.bitrate = lowered.target_bps; + assert_eq!(p.frame(Reopen::Opens).as_deref(), Some("h264_nvenc")); + assert_eq!(p.opened.1, 10_000_000); + // A card that reconfigures takes the same changes in place. + let mut p = Pipeline::new("h264_nvenc", "h264_nvenc", Instant::now()); + p.nvenc_refused = false; + for bitrate in [20_000_000, 10_000_000] { + p.bitrate = bitrate; + assert_eq!(p.frame(Reopen::Opens), None); + assert_eq!(p.opened.1, bitrate); + } + assert_eq!(p.in_place, 2); + } + + /// Native, where NVENC opens: a bitrate change on the open session takes + /// effect with no new session, FFmpeg's reconfigure confirms each one, + /// including on a GOP's own IDR (frames 120 and 240) and on a frame that + /// forces an IDR (250), and the stream decodes throughout. Skips + /// elsewhere. + #[test] + fn nvenc_changes_its_bitrate_on_the_open_session() { + ffmpeg_next::init().unwrap(); + let Ok(mut state) = EncoderState::new( + "h264_nvenc", + 1280, + 720, + 8_000_000, + 60, + &mut EncoderInput::Yuv420, + Opening::First, + &mut || {}, + ) else { + eprintln!("skipping: h264_nvenc does not open here"); + return; + }; + let context = unsafe { state.encoder.as_ptr() }; + let codec = ffmpeg_next::decoder::find(ffmpeg_next::codec::Id::H264).unwrap(); + let mut decoder = ffmpeg_next::codec::Context::new_with_codec(codec) + .decoder() + .video() .unwrap(); - assert_eq!(selected, "libx264"); - assert_eq!(attempts, ["libx264"]); + let mut decoded = ffmpeg_next::frame::Video::empty(); + let mut decoded_frames = 0; + let mut sizes = [0usize; 3]; + let mut keyframes = Vec::new(); + let mut random = 1u32; + let mut confirmed = Vec::new(); + for number in 0..360u64 { + let change = match number { + 120 => Some(1_000_000), + 240 => Some(8_000_000), + 250 => Some(6_000_000), + _ => None, + }; + if let Some(bitrate) = change { + assert!(state.update_bitrate_without_reopening("h264_nvenc", bitrate, 60)); + assert_eq!( + state.bitrate_change.take(), + Some(Unconfirmed { + live: LiveBitrate::NvencReconfig + }) + ); + } + for value in state.frame.data_mut(0) { + random = random.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + *value = (random >> 24) as u8; + } + state.frame.data_mut(1).fill(128); + state.frame.data_mut(2).fill(128); + state.frame.set_pts(Some(number as i64 * 1_000_000 / 60)); + state.frame.set_kind(if number == 250 { + ffmpeg_next::picture::Type::I + } else { + ffmpeg_next::picture::Type::None + }); + let watch = change.map(|_| crate::diagnostics::FfmpegWatch::start()); + state.encoder.send_frame(&state.frame).unwrap(); + loop { + match state.encoder.receive_packet(&mut state.packet) { + Ok(()) => { + let out = state.packet.pts().unwrap(); + if state.packet.is_key() { + keyframes.push(out * 60 / 1_000_000); + } + if number % 120 >= 60 { + sizes[(number / 120) as usize] += state.packet.size(); + } + decoder.send_packet(&state.packet).unwrap(); + loop { + match decoder.receive_frame(&mut decoded) { + Ok(()) => decoded_frames += 1, + Err(error) if is_eagain(&error) => break, + Err(error) => panic!("decode failed: {error}"), + } + } + } + Err(error) if is_eagain(&error) => break, + Err(error) => panic!("encode failed: {error}"), + } + } + if let Some(watch) = watch { + let lines = watch.finish(); + assert!(nvenc_reconfigured(&lines), "frame {number}: {lines:?}"); + confirmed.push(number); + } + } + eprintln!("NVENC sizes {sizes:?}, keyframes at {keyframes:?}, confirmed {confirmed:?}"); + assert_eq!(confirmed, [120, 240, 250]); + assert_eq!(decoded_frames, 360); + assert_eq!(unsafe { state.encoder.as_ptr() }, context); + assert!( + sizes[1] * 2 < sizes[0], + "lower bitrate was not applied: {sizes:?}" + ); + assert!( + sizes[2] > sizes[1] * 2, + "higher bitrate was not applied: {sizes:?}" + ); } #[test] @@ -4187,10 +6469,38 @@ mod tests { // FPS and codec changes must still use the existing reopen path. assert!(!state.update_bitrate_without_reopening("libx264", 4_000_000, 120)); assert!(!state.update_bitrate_without_reopening("libx265", 4_000_000, 60)); - for name in ["h264_nvenc", "h264_amf", "h264_qsv", "libx265"] { + for name in ["h264_amf", "hevc_amf", "libx265"] { state.opened_encoder = name.to_string(); assert!(!state.update_bitrate_without_reopening(name, 4_000_000, 60)); } + // NVENC takes it in place, with the two-second VBV a session opened + // at it gets, to be confirmed by the IDR its reconfigure forces. + state.opened_encoder = "h264_nvenc".to_string(); + assert!(state.update_bitrate_without_reopening("h264_nvenc", 5_000_000, 60)); + assert_eq!(state.opened_bitrate, 5_000_000); + assert_eq!( + unsafe { (*state.encoder.as_ptr()).rc_buffer_size }, + 10_000_000 + ); + assert_eq!( + state.bitrate_change, + Some(Unconfirmed { + live: LiveBitrate::NvencReconfig + }) + ); + state.bitrate_change = None; + // Quick Sync takes it in place, to be confirmed by its next frame. + state.opened_encoder = "h264_qsv".to_string(); + assert!(state.bitrate_change.is_none()); + assert!(state.update_bitrate_without_reopening("h264_qsv", 4_000_000, 60)); + assert_eq!(state.opened_bitrate, 4_000_000); + assert_eq!( + state.bitrate_change, + Some(Unconfirmed { + live: LiveBitrate::QsvReset + }) + ); + assert!(!state.update_bitrate_without_reopening("h264_qsv", 4_000_000, 60)); } #[test] @@ -4209,6 +6519,26 @@ mod tests { /// Automatic's BGRA open failing on NVENC opens YUV420P at once and keeps /// it for later opens; a picked input, a stall, or a YUV420P open that /// fails is not tried again. + /// NVENC failing over to Quick Sync streams from the Intel card: the + /// NVIDIA card detection chose is unused and is not woken for iPads. + #[test] + fn an_encoder_open_publishes_the_card_to_wake() { + let shared = SharedControl::new(0, 8_000_000); + let gpu = GpuInfo { + encoder_name: "h264_nvenc".to_string(), + adapter_index: 1, + ..GpuInfo::software_fallback() + }; + note_encoder_opened(&shared, "h264_nvenc", &gpu); + assert_eq!(*shared.wake_adapter.lock(), Some(1)); + note_encoder_opened(&shared, "h264_qsv", &gpu); + assert_eq!(*shared.wake_adapter.lock(), None, "a cross-vendor fallback"); + note_encoder_opened(&shared, "libx264", &gpu); + assert_eq!(*shared.wake_adapter.lock(), None, "the CPU"); + note_encoder_opened(&shared, "hevc_nvenc", &gpu); + assert_eq!(*shared.wake_adapter.lock(), Some(1), "back on the card"); + } + #[test] fn a_failed_automatic_bgra_open_falls_back_to_yuv420_and_stays_there() { use ffmpeg_next::format::Pixel; @@ -4389,6 +6719,287 @@ mod tests { assert!(!pending); } + fn raw(frame_number: u64, data: &std::sync::Arc>) -> RawFrame { + RawFrame { + frame_number, + timestamp: Instant::now(), + data: std::sync::Arc::clone(data), + width: 2, + height: 2, + cached: false, + } + } + + /// A picture given again to bring out a keyframe keeps its capture time + /// and number, but is a separate frame to the encoder: it must not share + /// the PTS of the frame before it, under ETERNAL_LEGACY_PTS too. + #[test] + fn a_picture_given_again_gets_its_own_pts_in_both_pts_modes() { + ffmpeg_next::init().unwrap(); + for legacy in [false, true] { + let mut state = libx264_at(640, 360); + state.legacy_pts = legacy; + let captured = Instant::now(); + // Frame 7, given again twice, then capture's next frame. + let frames = [(captured, 7), (captured, 7), (captured, 7)] + .into_iter() + .chain([(captured + Duration::from_millis(16), 8)]); + let mut pts = Vec::new(); + for (at, n) in frames { + let next = state.next_pts(at, n); + state.frame.set_pts(Some(next)); + state.send_frame(next, at, n).unwrap(); + while state.encoder.receive_packet(&mut state.packet).is_ok() {} + pts.push(next); + } + assert!( + pts.windows(2).all(|pair| pair[0] < pair[1]), + "legacy={legacy}: PTS {pts:?}" + ); + } + } + + /// Capture recycles three buffers, and the mailbox may hold one while the + /// encoder works on another. A kept picture that outlived a newer capture + /// took the third: the synthetic source found no free buffer and panicked. + #[test] + fn a_newer_capture_releases_the_kept_picture_before_it_is_encoded() { + let (producer, consumer) = crate::capture::frame_slot(); + let kept = std::sync::Arc::new(vec![0u8; 16]); + let newer = std::sync::Arc::new(vec![0u8; 16]); + let mut owed = OwedKeyframe::until(Instant::now() + Duration::from_secs(5)); + owed.picture = Some(raw(1, &kept)); + producer.publish(raw(2, &newer)); + let Next::Frame(frame) = owed.next(&consumer) else { + panic!("no frame"); + }; + assert_eq!(frame.frame_number, 2); + assert_eq!( + std::sync::Arc::strong_count(&kept), + 1, + "the kept picture still pins a capture buffer while the newer one is encoded" + ); + } + + #[test] + fn the_deadline_counts_from_the_first_keyframe_submission() { + let kept = std::sync::Arc::new(vec![0u8; 16]); + let submitted = Instant::now() + .checked_sub(Duration::from_millis(200)) + .unwrap(); + // The keyframe went in 200 ms ago; sending and receiving took that long. + let owed = OwedKeyframe::after_encode( + None, + Some(raw(1, &kept)), + false, + submitted, + KEYFRAME_FLUSH_WAIT, + ) + .expect("the keyframe is owed"); + assert_eq!(owed.until, submitted + KEYFRAME_FLUSH_FOR); + // A retry submitted later keeps the first deadline. + let owed = OwedKeyframe::after_encode( + Some(owed), + Some(raw(1, &kept)), + false, + Instant::now(), + KEYFRAME_FLUSH_WAIT, + ) + .expect("still owed"); + assert_eq!(owed.until, submitted + KEYFRAME_FLUSH_FOR); + assert!( + OwedKeyframe::after_encode( + Some(owed), + Some(raw(1, &kept)), + true, + Instant::now(), + KEYFRAME_FLUSH_WAIT + ) + .is_none(), + "the keyframe came out" + ); + assert!( + OwedKeyframe::after_encode(None, None, false, submitted, KEYFRAME_FLUSH_WAIT).is_none() + ); + } + + const ENCODE: Duration = Duration::from_millis(2); + + /// Replays capture on a fake clock through the rule the encode loop + /// uses, as if a keyframe were owed after every picture: encoding ends + /// [`ENCODE`] after a picture is captured, and the picture is given + /// again if capture's next one has not come by then plus the wait. + /// `pictures` are capture times, each marked cached or real. Returns the + /// indices of the pictures that would have been given again. + fn given_again(target_fps: u32, pictures: &[(Instant, bool)]) -> Vec { + let mut cadence = CaptureCadence::default(); + let mut again = Vec::new(); + for (n, pair) in pictures.windows(2).enumerate() { + let (at, cached) = pair[0]; + cadence.captured(at, cached); + let wait = keyframe_flush_wait(target_fps, cached, cadence.pace()); + if pair[1].0 > at + ENCODE + wait { + again.push(n); + } + } + again + } + + /// Real pictures every `frame`, off by up to a quarter frame (offsets in + /// eighths of a frame). Gaps run from 0.75 to 1.5 frames. The 1.25- and + /// 1.5-frame gaps come once per 12 pictures, rarer than the 8 gaps the + /// pace remembers, so the 1.5-frame gap is longer than the pace when it + /// comes: the wait needs its half frame of margin (1.0x fails here). + fn moving(start: Instant, frame: Duration, count: u32) -> Vec<(Instant, bool)> { + let eighth = frame / 8; + let offsets = [0i32, 0, 2, 0, 0, -1, 0, 0, 0, 0, -2, 2]; + (0..count) + .map(|n| { + let at = start + frame * n; + let at = match offsets[n as usize % offsets.len()] { + k if k >= 0 => at + eighth * k as u32, + k => at - eighth * (-k) as u32, + }; + (at, false) + }) + .collect() + } + + #[test] + fn a_moving_desktop_never_has_its_picture_given_again_at_60_50_or_30_fps() { + let start = Instant::now() + Duration::from_secs(1); + let us = Duration::from_micros; + for (target_fps, frame, what) in [ + (60, us(16_667), "60 fps"), + (60, us(20_000), "a 50 Hz display streamed at 60 fps"), + (50, us(20_000), "50 fps"), + (30, us(33_333), "30 fps"), + (60, us(33_333), "30 fps content streamed at 60 fps"), + ] { + let pictures = moving(start, frame, 240); + // The pipeline's first picture has no pace to go by. It is + // encoded before any viewer arrives (see capture's IdleGate). + let again: Vec = given_again(target_fps, &pictures) + .into_iter() + .filter(|&n| n > 0) + .collect(); + assert!(again.is_empty(), "{what}: pictures {again:?} given again"); + } + } + + #[test] + fn motion_that_resumes_after_an_idle_spell_keeps_its_pace() { + let start = Instant::now() + Duration::from_secs(1); + for (target_fps, frame) in [ + (60, Duration::from_micros(16_667)), + (60, Duration::from_micros(20_000)), + (60, Duration::from_micros(33_333)), + (30, Duration::from_micros(33_333)), + ] { + let mut pictures = moving(start, frame, 20); + let last = pictures.last().unwrap().0; + // Still for 750 ms, with the keepalive's cached resend, then + // moving again at the same pace. + pictures.push((last + Duration::from_millis(750), true)); + let resumed = last + Duration::from_millis(1500); + pictures.extend(moving(resumed, frame, 20)); + // Judged from the first picture after the resend: the last one + // before the idle spell is given again, as the desktop went + // still after it. + let resent = 20; + let again: Vec = given_again(target_fps, &pictures) + .into_iter() + .filter(|&n| n > resent) + .collect(); + assert!( + again.is_empty(), + "{target_fps} fps, {frame:?} frames: real pictures {again:?} given again" + ); + } + // Past a long idle spell the pace is forgotten: the target's again. + let mut cadence = CaptureCadence::default(); + for (at, cached) in moving(start, Duration::from_micros(33_333), 10) { + cadence.captured(at, cached); + } + cadence.captured(start + PACE_FORGOTTEN * 2, false); + assert_eq!(cadence.pace(), None); + assert_eq!( + keyframe_flush_wait(60, false, cadence.pace()), + crate::capture::frame_budget_for(60) * 3 / 2 + ); + } + + #[test] + fn a_cached_picture_is_given_again_after_the_shortest_wait() { + let start = Instant::now() + Duration::from_secs(1); + for target_fps in [60, 50, 30] { + let frame = crate::capture::frame_budget_for(target_fps); + let mut pictures = moving(start, frame, 10); + let last = pictures.last().unwrap().0; + // A pointer redraw resends the cached picture after only 50 ms, + // then the desktop stays still. + pictures.push((last + Duration::from_millis(50), true)); + pictures.push((last + Duration::from_millis(800), true)); + let mut cadence = CaptureCadence::default(); + for &(at, cached) in &pictures[..pictures.len() - 1] { + cadence.captured(at, cached); + } + assert_eq!( + keyframe_flush_wait(target_fps, true, cadence.pace()), + KEYFRAME_FLUSH_WAIT, + "{target_fps} fps: a cached picture" + ); + assert!( + given_again(target_fps, &pictures).contains(&10), + "{target_fps} fps: the redrawn picture was not given again" + ); + // A real picture waits a frame and a half of the pace. + assert_eq!( + keyframe_flush_wait(target_fps, false, cadence.pace()), + frame.max(cadence.pace().unwrap()) * 3 / 2, + "{target_fps} fps: a real picture" + ); + } + // Past 75 fps a frame and a half is shorter than AMF's encode. + assert_eq!(keyframe_flush_wait(120, false, None), KEYFRAME_FLUSH_WAIT); + } + + #[test] + fn the_kept_picture_is_given_again_before_the_deadline() { + let (_producer, consumer) = crate::capture::frame_slot(); + let kept = std::sync::Arc::new(vec![0u8; 16]); + let mut owed = OwedKeyframe::until(Instant::now() + Duration::from_secs(60)); + owed.picture = Some(raw(1, &kept)); + let started = Instant::now(); + let Next::Again(frame) = owed.next(&consumer) else { + panic!("the picture was not given again"); + }; + assert_eq!(frame.frame_number, 1); + assert_eq!(owed.given, 1); + assert!( + started.elapsed() >= KEYFRAME_FLUSH_WAIT, + "it did not wait for capture first" + ); + } + + #[test] + fn the_kept_picture_is_not_given_again_after_the_deadline() { + let (_producer, consumer) = crate::capture::frame_slot(); + let kept = std::sync::Arc::new(vec![0u8; 16]); + let passed = Instant::now() + .checked_sub(Duration::from_millis(1)) + .unwrap(); + let mut owed = OwedKeyframe::until(passed); + owed.picture = Some(raw(1, &kept)); + assert!(matches!(owed.next(&consumer), Next::GaveUp)); + assert_eq!(owed.given, 0); + assert_eq!( + std::sync::Arc::strong_count(&kept), + 1, + "the picture is let go" + ); + } + #[test] fn amf_options_keep_the_settings_the_ipad_decoder_needs() { let h264 = encoder_options("h264_amf"); @@ -4434,6 +7045,8 @@ mod tests { collector: std::thread::JoinHandle<()>, /// When each packet arrived, from `started`. packets: std::sync::Arc>>, + /// When each keyframe arrived, from `started`. + keyframes: std::sync::Arc>>, started: Instant, number: u64, } @@ -4441,6 +7054,10 @@ mod tests { #[cfg(windows)] impl NativeStage { fn start() -> Self { + Self::start_with("h264_nvenc") + } + + fn start_with(encoder: &str) -> Self { use std::sync::Arc; let lines = Arc::new(parking_lot::Mutex::new(Vec::::new())); struct Sink(Arc>>); @@ -4469,8 +7086,8 @@ mod tests { let shared = SharedControl::new(0, 8_000_000); shared.running.store(true, Ordering::SeqCst); let gpu = GpuInfo { - encoder_name: "h264_nvenc".to_string(), - codec_display_name: "H.264 (NVENC)".to_string(), + encoder_name: encoder.to_string(), + codec_display_name: codec_display_name_for(encoder).to_string(), ..GpuInfo::software_fallback() }; let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); @@ -4481,11 +7098,17 @@ mod tests { }); let started = Instant::now(); let packets = Arc::new(parking_lot::Mutex::new(Vec::new())); + let keyframes = Arc::new(parking_lot::Mutex::new(Vec::new())); let collector = { let packets = Arc::clone(&packets); + let keyframes = Arc::clone(&keyframes); std::thread::spawn(move || { - while rx.blocking_recv().is_some() { - packets.lock().push(started.elapsed()); + while let Some(nal) = rx.blocking_recv() { + let at = started.elapsed(); + packets.lock().push(at); + if nal.is_keyframe { + keyframes.lock().push(at); + } } }) }; @@ -4496,6 +7119,7 @@ mod tests { stage, collector, packets, + keyframes, started, number: 0, } @@ -4514,6 +7138,7 @@ mod tests { data: std::sync::Arc::new(data), width: 1280, height: 720, + cached: false, }); self.number += 1; std::thread::sleep(std::time::Duration::from_millis(33)); @@ -4558,10 +7183,124 @@ mod tests { } } + /// Native, for measurements: the real encode stage on + /// `ETERNAL_MEASURE_ENCODER` (h264_nvenc unless set), steered by the real + /// ABR controller for `ETERNAL_MEASURE_MINUTES` (4) on a link that loses + /// whatever the encoder sends above 11 Mbps: 10 Mbps holds, and each + /// probe to 12 is lost. One receiver report a second. + /// Prints a MEASURE line: steps, reopens and in-place changes a minute, + /// and for each change the longest gap between packets in the next 2 s + /// (the hitch) and the time to the next keyframe. + /// `cargo test -p eternal-host --lib bitrate_steps_on_a_lossy_link -- --ignored --nocapture` + #[cfg(windows)] + #[test] + #[ignore = "native: measures a real encoder for minutes"] + fn bitrate_steps_on_a_lossy_link() { + use std::time::Duration; + let encoder = + std::env::var("ETERNAL_MEASURE_ENCODER").unwrap_or_else(|_| "h264_nvenc".to_string()); + let minutes: u64 = std::env::var("ETERNAL_MEASURE_MINUTES") + .ok() + .and_then(|m| m.parse().ok()) + .unwrap_or(4); + const CAPACITY: u32 = 11_000_000; + let mut stage = NativeStage::start_with(&encoder); + let mut abr = crate::transport::abr::AbrController::new(50_000_000, true); + stage + .shared + .abr_current_bps + .store(abr.current_bps(), Ordering::SeqCst); + let (mut received, mut lost) = (0u32, 0u32); + let mut changes = Vec::new(); + let end = stage.started + Duration::from_secs(minutes * 60); + let mut next_report = stage.started + Duration::from_secs(1); + while Instant::now() < end { + stage.publish(); + if Instant::now() < next_report { + continue; + } + next_report += Duration::from_secs(1); + // What the open session sends, as the encode loop records it. + let rate = PIPELINE_STATS.lock().bitrate_bps.max(1); + let over = rate.saturating_sub(CAPACITY) / 1_000; + received += rate / 1_000 - over; + lost += over; + let report = eternal_wire::v2::control::ReceiverReport { + frags_received: received, + frags_lost: lost, + ..Default::default() + }; + // AFTER-ONLY begin: the encoder's cost of a step, which ABR reads. + abr.encoder_reopens( + stage + .shared + .encoder_reopens_for_bitrate + .load(Ordering::Relaxed), + ); + // AFTER-ONLY end + let decision = abr.on_report(&report, None, Instant::now()); + if decision.changed { + stage + .shared + .abr_current_bps + .store(decision.target_bps, Ordering::SeqCst); + changes.push(stage.started.elapsed()); + } + } + for _ in 0..30 { + stage.publish(); + } + let packets = stage.packets.lock().clone(); + let keyframes = stage.keyframes.lock().clone(); + let text = stage.finish(); + let reopens = text.matches("Reopened encoder session").count(); + let in_place = text.matches("bitrate without reopening").count() + + text.matches("Updated software encoder bitrate").count(); + let mut hitches: Vec = Vec::new(); + let mut to_keyframe: Vec = Vec::new(); + for &change in &changes { + let window: Vec = packets + .iter() + .copied() + .filter(|&at| { + at + Duration::from_millis(100) >= change + && at <= change + Duration::from_secs(2) + }) + .collect(); + if let Some(gap) = window.windows(2).map(|pair| pair[1] - pair[0]).max() { + hitches.push(gap.as_millis()); + } + if let Some(key) = keyframes.iter().find(|&&at| at >= change) { + to_keyframe.push((*key - change).as_millis()); + } + } + let summary = |values: &mut Vec| { + values.sort(); + let at = |q: f64| { + values + .get(((values.len() as f64 - 1.0) * q).round() as usize) + .copied() + }; + (at(0.5), at(0.9), values.last().copied()) + }; + let per_minute = |count: usize| count as f64 / minutes as f64; + eprintln!( + "MEASURE encoder={encoder} minutes={minutes} steps={} steps_per_min={:.2} reopens={reopens} reopens_per_min={:.2} in_place={in_place} hitch_ms_p50_p90_max={:?} keyframe_after_change_ms_p50_p90_max={:?} packets={}", + changes.len(), + per_minute(changes.len()), + per_minute(reopens), + summary(&mut hitches), + summary(&mut to_keyframe), + packets.len(), + ); + assert!(!changes.is_empty(), "ABR never stepped"); + } + /// Native, on a PC with NVENC: the encode stage's first NVENC open is - /// held for twice its limit (`ETERNAL_SIMULATE_STALL`). The stage falls - /// through to software at once and streams, the late NVENC session is - /// closed on its own thread, and the stage then goes back to NVENC. + /// held for twice its limit (`ETERNAL_SIMULATE_STALL`). The stage streams + /// on software once the first picture is due, gives the open up at its + /// limit, closes the late NVENC session on its own thread, and then goes + /// back to NVENC through an open in the background. /// `cargo test -p eternal-host --lib a_real_nvenc_open -- --ignored --nocapture` #[cfg(windows)] #[test] @@ -4573,8 +7312,8 @@ mod tests { let started = stage.started; // When each step's line first shows, to within a frame. let steps = [ + "streaming on libx264 while it opens in the background", "stall=encoder_open", - "streaming on the fallback", "stall_ended=encoder_open", "Back on the preferred encoder", ]; @@ -4596,9 +7335,10 @@ mod tests { for _ in 0..30 { stage.publish(); } - let [stalled, _, ended, back] = seen.map(Option::unwrap); + let [outwaited, stalled, ended, back] = seen.map(Option::unwrap); + let first_packet = stage.packets.lock().first().copied().expect("no packet"); // A picture throughout: x264 while the open was stuck, NVENC after. - let on_x264 = stage.packets_between(stalled + Duration::from_secs(1), ended); + let on_x264 = stage.packets_between(outwaited + Duration::from_secs(1), ended); let on_nvenc = stage.packets_between(back, Duration::MAX); let text = stage.finish(); let at = |needle: &str| { @@ -4609,7 +7349,21 @@ mod tests { assert!(order.windows(2).all(|pair| pair[0] < pair[1]), "{order:?}"); let back_line = text[order[3]..].lines().next().unwrap(); assert!(back_line.contains("h264_nvenc"), "{back_line}"); - eprintln!("stalled at {stalled:?}, stall ended at {ended:?}, back on NVENC at {back:?}"); + assert!( + text.contains("Opening the encoder in the background"), + "the try froze the stream:\n{text}" + ); + eprintln!( + "first picture at {first_packet:?}, x264 from {outwaited:?}, stalled at {stalled:?}, \ + stall ended at {ended:?}, back on NVENC at {back:?}" + ); + // The first picture well before the iPad's 10 s timeout: 3 s, then + // libx264's open and first frame. + assert!( + outwaited >= Duration::from_secs(3) && outwaited < Duration::from_secs(5), + "{outwaited:?}" + ); + assert!(first_packet < Duration::from_secs(5), "{first_packet:?}"); // Given up on at NVENC's 15 s limit; the held open returns at 30 s. assert!( stalled >= Duration::from_secs(15) && stalled < Duration::from_secs(18), @@ -4625,6 +7379,62 @@ mod tests { assert!(!stats.using_software_fallback); } + /// Native, on a PC with NVENC: the first NVENC open takes 6 s, past the + /// first picture's 3 s but within its 15 s limit. The stage streams on + /// software from 3 s and switches to that same NVENC session when it + /// opens, with no stall and no second open. + /// `cargo test -p eternal-host --lib a_slow_real_nvenc_open -- --ignored --nocapture` + #[cfg(windows)] + #[test] + #[ignore = "native: needs an NVIDIA GPU"] + fn a_slow_real_nvenc_open_is_taken_once_it_opens() { + use std::time::Duration; + std::env::set_var("ETERNAL_SIMULATE_STALL", "encoder_open"); + std::env::set_var("ETERNAL_SIMULATE_STALL_MS", "6000"); + let mut stage = NativeStage::start(); + let (outwaited, from) = stage.publish_until( + "streaming on libx264 while it opens in the background", + 0, + Duration::from_secs(20), + ); + let (switched, _) = + stage.publish_until("in_background=true", from, Duration::from_secs(20)); + for _ in 0..30 { + stage.publish(); + } + let first_packet = stage.packets.lock().first().copied().expect("no packet"); + let on_x264 = stage.packets_between(outwaited + Duration::from_millis(500), switched); + let after = stage.packets_between(switched, Duration::MAX); + let text = stage.finish(); + eprintln!( + "first picture at {first_packet:?}, x264 from {outwaited:?}, NVENC from {switched:?}; \ + {on_x264} packets on x264, {after} after" + ); + assert!(first_packet < Duration::from_secs(5), "{first_packet:?}"); + assert!( + switched >= Duration::from_secs(6) && switched < Duration::from_secs(8), + "{switched:?}" + ); + assert!(on_x264 >= 30, "{on_x264} packets on x264"); + assert!(after >= 20, "{after} packets on NVENC"); + let line = text + .lines() + .find(|line| line.contains("in_background=true")) + .unwrap(); + assert!(line.contains("Reopened encoder session"), "{line}"); + assert!(line.contains("h264_nvenc"), "{line}"); + assert!(!text.contains("stall=encoder_open"), "{text}"); + // NVENC opened once: the session that went on in the background. + assert_eq!( + text.matches("Encoder opened").count(), + 2, + "x264 and NVENC, once each:\n{text}" + ); + let stats = PIPELINE_STATS.lock(); + assert_eq!(stats.codec_name, "H.264 (NVENC)"); + assert!(!stats.using_software_fallback); + } + /// Native, on a PC with NVENC: an ABR step's NVENC reopen /// is held for 10 s, so it is given up on at the 9 s reopen limit and /// the old session streams on. The lane clears 1 s later, before the @@ -4636,8 +7446,28 @@ mod tests { #[ignore = "native: needs an NVIDIA GPU"] fn a_bitrate_change_after_a_stalled_nvenc_reopen_clears_is_prompt() { use std::time::Duration; + // As on an NVIDIA card that cannot change its bitrate live: there an + // ABR step reopens NVENC, the reopen this test holds. + LiveBitrate::NvencReconfig + .refused() + .store(true, Ordering::Relaxed); let mut stage = NativeStage::start(); - let (_, opened) = stage.publish_until("Encoder opened", 0, Duration::from_secs(20)); + // NVENC streams before the ABR step. A first open slower than the + // first picture's wait streams libx264 until NVENC's session, opened + // in the background, takes over. + let mut from = 0; + let opened = loop { + let (_, at) = stage.publish_until("Encoder opened", from, Duration::from_secs(20)); + if stage.text()[at..] + .lines() + .next() + .unwrap() + .contains("h264_nvenc") + { + break at; + } + from = at + 1; + }; for _ in 0..30 { stage.publish(); } diff --git a/host/src/ffmpeg_log.c b/host/src/ffmpeg_log.c index 75f7dabc..b4b34a1a 100644 --- a/host/src/ffmpeg_log.c +++ b/host/src/ffmpeg_log.c @@ -3,8 +3,26 @@ /* Keep va_list in C: its ABI differs between MSVC, Intel and Apple ARM. */ static void (*log_sink)(int, const char *); +/* Set while a thread watches FFmpeg's verbose lines (see + `diagnostics::FfmpegWatch`); the sink gives them only to that thread. Read + from every thread that logs, so it is atomic. Relaxed is enough: a watching + thread sets it before its own lines, and other threads' lines only decide + what the sink drops. */ +#ifdef _MSC_VER +#include +static volatile long verbose_wanted; +static int verbose_is_wanted(void) { return _InterlockedCompareExchange(&verbose_wanted, 0, 0) != 0; } +static void set_verbose_wanted(int wanted) { _InterlockedExchange(&verbose_wanted, wanted ? 1 : 0); } +#else +#include +static atomic_int verbose_wanted; +static int verbose_is_wanted(void) { return atomic_load_explicit(&verbose_wanted, memory_order_relaxed) != 0; } +static void set_verbose_wanted(int wanted) { atomic_store_explicit(&verbose_wanted, wanted ? 1 : 0, memory_order_relaxed); } +#endif + static void forward_log(void *context, int level, const char *format, va_list args) { - if (level > AV_LOG_WARNING || !log_sink) return; + if (!log_sink) return; + if (level > AV_LOG_WARNING && !(level <= AV_LOG_VERBOSE && verbose_is_wanted())) return; char line[4096]; int prefix = 1; int length = av_log_format_line2(context, level, format, args, line, sizeof(line), &prefix); @@ -13,6 +31,13 @@ static void forward_log(void *context, int level, const char *format, va_list ar void eternal_install_ffmpeg_logger(void (*sink)(int, const char *)) { log_sink = sink; - av_log_set_level(AV_LOG_WARNING); + /* Set once, before FFmpeg is used: FFmpeg's level is a plain int that + every logging thread reads, so it is never changed afterwards. The + callback drops what is not wanted. */ + av_log_set_level(AV_LOG_VERBOSE); av_log_set_callback(forward_log); } + +void eternal_ffmpeg_verbose(int wanted) { + set_verbose_wanted(wanted); +} diff --git a/host/src/firewall.rs b/host/src/firewall.rs new file mode 100644 index 00000000..7ec55b41 --- /dev/null +++ b/host/src/firewall.rs @@ -0,0 +1,1921 @@ +//! Whether Windows Firewall lets iPads reach this copy of the host. +//! +//! An iPad whose HELLO never arrives says "Your PC didn't answer" while the +//! PC says it is ready, and neither side could tell why. The usual blocks are +//! on the PC: an inbound Block rule for this exe, which Windows adds when its +//! "allow access" prompt is dismissed and which beats every Allow rule, or +//! "Blocks all incoming connections" on the network's profile. A background +//! thread reads the firewall policy at startup and every [`RECHECK`] and logs +//! the result when it changes. +//! +//! The Stream page warns only when the block is certain for the network an +//! iPad would use, and only for what Setup or one switch removes: this +//! exe's own unscoped Block rule, or "Blocks all incoming connections", on +//! every profile in use, with a home network address and no iPad connected. +//! Everything else is only logged: a Block rule for every program or one +//! scoped to some addresses, interfaces, a service, a package, users or +//! machines, a block on one of two networks, and any block an Allow rule +//! asking for IPsec might let traffic past. +//! +//! The log line names profiles and counts rules. It holds no path, rule name +//! or address. + +use std::path::Path; +use std::sync::Mutex; +use std::time::Duration; + +#[cfg(any(windows, test))] +use tracing::{info, warn}; + +/// How often the policy is read again: a network can turn Public, and a rule +/// can be added, while the app runs. +pub const RECHECK: Duration = Duration::from_secs(60); + +/// The Windows Firewall profiles, as `NET_FW_PROFILE_TYPE2` bits. +pub const DOMAIN: u32 = 1; +pub const PRIVATE: u32 = 2; +pub const PUBLIC: u32 = 4; + +/// One profile in use on a connected network. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Profile { + pub bit: u32, + pub on: bool, + /// "Blocks all incoming connections, including those in the list of + /// allowed apps". + pub block_all_inbound: bool, +} + +/// An inbound rule, as far as the verdict needs it. Text fields hold what +/// Windows stores (`*` or empty means any); `None` is a field that could not +/// be read, which the verdict treats as unknown, never as any. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Rule { + /// The program path as stored, which may use `%ProgramFiles%` and the + /// like. Empty: the rule is for every program. + pub program: Option, + pub allow: bool, + pub enabled: bool, + /// UDP or any protocol. + pub udp: bool, + pub profiles: u32, + pub local_ports: Option, + pub remote_ports: Option, + pub local_addresses: Option, + pub remote_addresses: Option, + /// `All`, or some of `Wireless`, `Lan`, `RemoteAccess`. + pub interface_types: Option, + /// The rule names particular interfaces. + pub interfaces: Option, + /// The rule is for a service, not every process of the program. + pub service: Option, + /// INetFwRule3's restrictions: an app package, a user owning the + /// process, and users or machines allowed to connect. Any of them limits + /// whose traffic the rule is for. + pub package: Option, + pub local_user_owner: Option, + pub local_users: Option, + pub remote_users: Option, + pub remote_machines: Option, + /// `NET_FW_AUTHENTICATE_TYPE`: 0 is none; anything else asks for IPsec. + pub secure_flags: Option, +} + +impl Rule { + /// An inbound rule for `program` (empty: every program) with no scope + /// beyond its protocol. + pub fn unscoped(program: &str, allow: bool, profiles: u32) -> Self { + Self { + program: Some(program.into()), + allow, + enabled: true, + udp: true, + profiles, + local_ports: Some("*".into()), + remote_ports: Some("*".into()), + local_addresses: Some("*".into()), + remote_addresses: Some("*".into()), + interface_types: Some("All".into()), + interfaces: Some(false), + service: Some(String::new()), + package: Some(String::new()), + local_user_owner: Some(String::new()), + local_users: Some(String::new()), + remote_users: Some(String::new()), + remote_machines: Some(String::new()), + secure_flags: Some(0), + } + } +} + +/// The text fields of a firewall rule, read one by one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Field { + Program, + LocalPorts, + RemotePorts, + LocalAddresses, + RemoteAddresses, + InterfaceTypes, + Service, + /// INetFwRule3 from here on. + Package, + LocalUserOwner, + LocalUsers, + RemoteUsers, + RemoteMachines, +} + +/// A rule as Windows hands it out. Each getter can fail on its own. +pub trait RuleSource { + fn inbound(&self) -> Result; + fn allow(&self) -> Result; + fn enabled(&self) -> Result; + /// `NET_FW_IP_PROTOCOL`: 17 is UDP, 256 any. + fn protocol(&self) -> Result; + fn profiles(&self) -> Result; + fn text(&self, field: Field) -> Result; + /// Whether the rule names particular interfaces. + fn interfaces_listed(&self) -> Result; + /// INetFwRule3's `SecureFlags`. + fn secure_flags(&self) -> Result; +} + +/// The inbound rule `source` describes, or `None` for an outbound one. A +/// failed direction, action, state, protocol or profile read fails the +/// whole reading; a text field that can't be read stays unknown. +pub fn rule_from(source: &impl RuleSource) -> Result, String> { + if !source.inbound()? { + return Ok(None); + } + let text = |field| source.text(field).ok(); + let protocol = source.protocol()?; + Ok(Some(Rule { + program: text(Field::Program), + allow: source.allow()?, + enabled: source.enabled()?, + udp: protocol == 17 || protocol == 256, + profiles: source.profiles()? as u32, + local_ports: text(Field::LocalPorts), + remote_ports: text(Field::RemotePorts), + local_addresses: text(Field::LocalAddresses), + remote_addresses: text(Field::RemoteAddresses), + interface_types: text(Field::InterfaceTypes), + interfaces: source.interfaces_listed().ok(), + service: text(Field::Service), + package: text(Field::Package), + local_user_owner: text(Field::LocalUserOwner), + local_users: text(Field::LocalUsers), + remote_users: text(Field::RemoteUsers), + remote_machines: text(Field::RemoteMachines), + secure_flags: source.secure_flags().ok(), + })) +} + +/// How far a rule reaches the stream's traffic. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Reach { + /// Every datagram an iPad sends to the stream port. + All, + /// Only some, or unknown: it names addresses, ports, interfaces, a + /// service, a package, users or machines, asks for IPsec, or a field of + /// it could not be read. + Some, + /// None of it: another port. + None, +} + +/// Whether `ports` (`*`, or numbers and ranges such as `80,9000-9999`) +/// covers `port`. Keywords such as `RPC` cover no stream port. +pub fn covers(ports: &str, port: u16) -> bool { + let ports = ports.trim(); + if ports.is_empty() || ports == "*" { + return true; + } + ports.split(',').map(str::trim).any(|part| { + let (low, high) = part.split_once('-').unwrap_or((part, part)); + match (low.trim().parse::(), high.trim().parse::()) { + (Ok(low), Ok(high)) => (low..=high).contains(&port), + _ => false, + } + }) +} + +/// Read, and any: an unknown field is not. +fn any(value: &Option) -> bool { + value + .as_deref() + .map(str::trim) + .is_some_and(|value| value.is_empty() || value == "*") +} + +/// How far `rule` reaches datagrams to `port`. +pub fn reach(rule: &Rule, port: u16) -> Reach { + if rule + .local_ports + .as_deref() + .is_some_and(|ports| !covers(ports, port)) + { + return Reach::None; + } + let all_types = rule + .interface_types + .as_deref() + .map(str::trim) + .is_some_and(|types| types.is_empty() || types.eq_ignore_ascii_case("all")); + // Ports that cover the stream's are no limit; ports never read are. + let unscoped = rule.local_ports.is_some() + && any(&rule.remote_ports) + && any(&rule.local_addresses) + && any(&rule.remote_addresses) + && all_types + && rule.interfaces == Some(false) + && [ + &rule.service, + &rule.package, + &rule.local_user_owner, + &rule.local_users, + &rule.remote_users, + &rule.remote_machines, + ] + .iter() + .all(|value| value.as_deref().is_some_and(|v| v.trim().is_empty())) + && rule.secure_flags == Some(0); + if unscoped { + Reach::All + } else { + Reach::Some + } +} + +/// Whose traffic a rule is for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Whose { + /// This exe. + Ours, + /// Every program: Setup's program-specific delete can't remove it. + AllPrograms, + /// The program could not be read. + Unknown, + /// A path with this exe's file name that names a variable the machine + /// has no value for: it may be this exe's. + Unresolved, + Other, +} + +pub(crate) fn ours(rule: &Rule, exe: &Path, lookup: impl Fn(&str) -> Option) -> bool { + whose(rule, exe, lookup) == Whose::Ours +} + +fn whose(rule: &Rule, exe: &Path, lookup: impl Fn(&str) -> Option) -> Whose { + match rule.program.as_deref() { + None => Whose::Unknown, + Some(program) if program.trim().is_empty() => Whose::AllPrograms, + Some(program) => match expand_known(program, lookup) { + Some(path) if same_path(&path, exe) => Whose::Ours, + None if file_name(program) == file_name(&exe.to_string_lossy()) => Whose::Unresolved, + _ => Whose::Other, + }, + } +} + +/// What the firewall does on one profile. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Verdict { + /// An Allow rule for this exe or every program, and nothing certain + /// that beats it. + Allowed, + /// An unscoped Block rule for this exe. + Blocked, + /// An unscoped Block rule for every program on the stream's port. + BlockedForAllApps, + /// The profile blocks all incoming connections. + BlocksAll, + /// Neither rule exists: Windows asks, or blocks quietly if it may not. + NoRule, + /// Windows Firewall is off on this profile: another firewall may be on. + Off, + /// An unscoped Block rule that may be this exe's: its path has this + /// exe's file name but names a variable the machine has no value for. + Uncertain, +} + +impl Verdict { + fn label(self) -> &'static str { + match self { + Verdict::Allowed => "allowed", + Verdict::Blocked => "blocked", + Verdict::BlockedForAllApps => "blocked_all_programs", + Verdict::BlocksAll => "blocks_all", + Verdict::NoRule => "no_rule", + Verdict::Off => "off", + Verdict::Uncertain => "uncertain", + } + } + + fn blocks(self) -> bool { + matches!( + self, + Verdict::Blocked | Verdict::BlockedForAllApps | Verdict::BlocksAll + ) + } +} + +/// What the firewall does to this exe's inbound UDP on each network in use. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Report { + /// Each profile in use, with its verdict. + pub profiles: Vec<(u32, Verdict)>, + /// The stream's port. + pub port: u16, + pub allow_rules: usize, + /// Unscoped Block rules for this exe. + pub block_rules: usize, + /// Unscoped Block rules for every program that cover the stream's port. + pub all_programs_block_rules: usize, + /// Block rules that reach only some of the stream's traffic, or that + /// could not be read whole. + pub scoped_block_rules: usize, + /// Allow rules for this exe or every program that may let IPsec + /// traffic past Block rules (see [`judge`]). + pub bypass_rules: usize, + /// The PC has a home network address (see [`crate::network::Reach::Lan`]). + pub lan: bool, +} + +/// What the Stream page may say: a block certain on every network in use, +/// and one Setup or a single switch removes. A rule for every program is +/// only logged. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CertainBlock { + /// Block rules for this exe, and on `all_blocked` networks "Blocks all + /// incoming connections" as well. + Rule { all_blocked: Vec<&'static str> }, + /// Only "Blocks all incoming connections", on these networks. + AllIncoming(Vec<&'static str>), +} + +impl Report { + /// The whole picture in one word for the log. + pub fn verdict(&self) -> &'static str { + let blocking = self.profiles.iter().filter(|(_, v)| v.blocks()).count(); + let unsure = self.profiles.iter().any(|(_, v)| *v == Verdict::Uncertain); + if unsure || (blocking > 0 && self.bypass_rules > 0) { + "uncertain" + } else if !self.profiles.is_empty() && blocking == self.profiles.len() { + "blocked" + } else if blocking > 0 { + "partly_blocked" + } else if self.profiles.iter().all(|(_, v)| *v == Verdict::Off) { + "off" + } else if self.profiles.iter().any(|(_, v)| *v == Verdict::NoRule) { + "no_rule" + } else { + "allowed" + } + } + + /// A block certain for the network an iPad would use, and one Setup or + /// a single switch removes: every profile in use is blocked by this + /// exe's own rule or blocks all incoming connections, the PC has a home + /// network address, and no Allow rule may let traffic past. Otherwise + /// `None`, and the log line says what is known. + pub fn certain_block(&self) -> Option { + let fixable = |v: &Verdict| matches!(v, Verdict::Blocked | Verdict::BlocksAll); + if !self.lan + || self.bypass_rules > 0 + || self.profiles.is_empty() + || !self.profiles.iter().all(|(_, v)| fixable(v)) + { + return None; + } + let all_blocked: Vec<_> = self + .profiles + .iter() + .filter(|(_, v)| *v == Verdict::BlocksAll) + .map(|(bit, _)| profile_name(*bit)) + .collect(); + Some(if all_blocked.len() == self.profiles.len() { + CertainBlock::AllIncoming(all_blocked) + } else { + CertainBlock::Rule { all_blocked } + }) + } + + /// The log line: profile names, verdicts and counts only. + pub fn summary(&self) -> String { + let profiles: Vec = self + .profiles + .iter() + .map(|(bit, verdict)| format!("{}:{}", profile_name(*bit), verdict.label())) + .collect(); + format!( + "Windows Firewall: profiles=[{}] allow_rules={} block_rules={} all_programs_block_rules={} scoped_block_rules={} bypass_rules={} lan={} verdict={}", + profiles.join(" "), + self.allow_rules, + self.block_rules, + self.all_programs_block_rules, + self.scoped_block_rules, + self.bypass_rules, + self.lan, + self.verdict(), + ) + } +} + +pub fn profile_name(bit: u32) -> &'static str { + match bit { + DOMAIN => "domain", + PRIVATE => "private", + PUBLIC => "public", + _ => "other", + } +} + +/// What `%name%` in a rule's program path stands for, from the machine's +/// own settings only: the known folders for Program Files, the Windows +/// folder and ProgramData, then the system environment in +/// `HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment`. +/// Never this process's environment, which any user can change for their +/// own programs: a rule for another program must not look like this exe's, +/// to the verdict or to the elevated fix. A system value that names other +/// variables is expanded from these values too ([`resolve`]); one only a +/// user has is unknown. +#[cfg(windows)] +pub fn machine_env(name: &str) -> Option { + resolve(name, &windows_policy::machine_value) +} + +/// Off Windows no rule has a path to expand. +#[cfg(not(windows))] +pub fn machine_env(_name: &str) -> Option { + None +} + +/// A machine value as stored. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Stored { + /// A known folder, or a REG_SZ value: as written, `%` and all. + Literal(String), + /// A REG_EXPAND_SZ value: its `%name%`s are machine values too. + Expand(String), +} + +/// How deep REG_EXPAND_SZ values may name one another. +pub const EXPAND_DEPTH: usize = 8; + +/// How many `%name%`s one [`resolve`] may follow, counting repeats. +pub const MAX_REFERENCES: usize = 256; + +/// The most text one expansion may make, in bytes: more than any path +/// Windows takes. +pub const MAX_EXPANDED: usize = 32 * 1024; + +/// `name`'s value among the machine values `stored`, each REG_EXPAND_SZ +/// value's own `%name%`s resolved the same way, each name once. `None` when +/// `name`, or a value it names, has no machine value, when values name each +/// other in a cycle, when more than [`EXPAND_DEPTH`] of them nest, or when +/// it takes more than [`MAX_REFERENCES`] or makes more than +/// [`MAX_EXPANDED`]. +pub fn resolve(name: &str, stored: &impl Fn(&str) -> Option) -> Option { + Resolving { + stored, + known: std::collections::HashMap::new(), + expanding: Vec::new(), + references: 0, + } + .value(name) +} + +struct Resolving<'a, F> { + stored: &'a F, + /// Each name resolved so far. Any `None` fails the whole resolve, so + /// remembering one is never wrong. + known: std::collections::HashMap>, + expanding: Vec, + references: usize, +} + +impl Option> Resolving<'_, F> { + fn value(&mut self, name: &str) -> Option { + self.references += 1; + if self.references > MAX_REFERENCES { + return None; + } + let key = name.to_lowercase(); + if let Some(known) = self.known.get(&key) { + return known.clone(); + } + let value = match (self.stored)(name) { + None => None, + Some(Stored::Literal(value)) => Some(value), + Some(Stored::Expand(value)) => { + if self.expanding.len() >= EXPAND_DEPTH || self.expanding.contains(&key) { + return None; + } + self.expanding.push(key.clone()); + let expanded = expand_known(&value, |inner| self.value(inner)); + self.expanding.pop(); + expanded + } + }; + self.known.insert(key, value.clone()); + value + } +} + +/// `path` with every `%NAME%` replaced with `lookup(NAME)`, as Windows +/// stores a rule made in its own window (`%ProgramFiles%\...`), or `None` if +/// one has no value or it would make more than [`MAX_EXPANDED`]. `%%` and a +/// lone `%` stay as written. +fn expand_known(path: &str, mut lookup: impl FnMut(&str) -> Option) -> Option { + let mut out = String::new(); + let mut rest = path; + while let Some(start) = rest.find('%') { + let Some(len) = rest[start + 1..].find('%') else { + break; + }; + let name = &rest[start + 1..start + 1 + len]; + out.push_str(&rest[..start]); + if name.is_empty() { + out.push_str("%%"); + } else { + out.push_str(&lookup(name)?); + } + if out.len() > MAX_EXPANDED { + return None; + } + rest = &rest[start + len + 2..]; + } + out.push_str(rest); + Some(out).filter(|out| out.len() <= MAX_EXPANDED) +} + +/// A path as Windows compares it: no quotes or `\\?\`, backslashes, no case. +fn plain(path: &str) -> String { + let path = path + .trim() + .trim_matches('"') + .replace('/', "\\") + .to_lowercase(); + path.strip_prefix(r"\\?\") + .map(str::to_string) + .unwrap_or(path) +} + +fn same_path(path: &str, exe: &Path) -> bool { + let exe = plain(&exe.to_string_lossy()); + !exe.is_empty() && plain(path) == exe +} + +/// The last part of a path, as Windows compares it. +fn file_name(path: &str) -> String { + let path = plain(path); + path.rsplit('\\').next().unwrap_or_default().to_string() +} + +/// Whether a rule's program is `exe`. Windows compares paths without case, +/// and its prompt stores them in lower case. +pub fn same_program(program: &str, exe: &Path, lookup: impl Fn(&str) -> Option) -> bool { + expand_known(program, lookup).is_some_and(|path| same_path(&path, exe)) +} + +/// The verdict on each profile for inbound UDP to `exe` on `port`. Windows +/// lets "block all" beat every rule and a Block rule beat every Allow rule. +/// Only a Block rule, for this exe or every program, that reaches all of +/// the stream's traffic and was read whole is a block. +pub fn judge( + profiles: &[Profile], + rules: &[Rule], + exe: &Path, + port: u16, + lan: bool, + lookup: impl Fn(&str) -> Option, +) -> Report { + let on: u32 = profiles.iter().filter(|p| p.on).map(|p| p.bit).sum(); + let relevant: Vec<(&Rule, Whose, Reach)> = rules + .iter() + .filter(|r| r.enabled && r.udp) + .map(|r| (r, whose(r, exe, &lookup), reach(r, port))) + .filter(|(_, whose, reach)| *whose != Whose::Other && *reach != Reach::None) + .collect(); + let count = |mask: u32, wanted: &dyn Fn(&Rule, Whose, Reach) -> bool| { + relevant + .iter() + .filter(|(r, whose, reach)| r.profiles & mask != 0 && wanted(r, *whose, *reach)) + .count() + }; + // Windows has many Allow rules for every program, each limited to a + // service or some addresses: only an unscoped one lets any app in. + let allows = |r: &Rule, whose, reach| { + r.allow && (whose == Whose::Ours || (whose == Whose::AllPrograms && reach == Reach::All)) + }; + let ours = |r: &Rule, whose, reach| !r.allow && whose == Whose::Ours && reach == Reach::All; + let everyone = + |r: &Rule, whose, reach| !r.allow && whose == Whose::AllPrograms && reach == Reach::All; + let unsure = |r: &Rule, whose, reach| { + !r.allow && (matches!(whose, Whose::Unknown | Whose::Unresolved) || reach == Reach::Some) + }; + let maybe_ours = + |r: &Rule, whose, reach| !r.allow && whose == Whose::Unresolved && reach == Reach::All; + // An Allow rule that asks for IPsec (SecureFlags) can come with + // "Override block rules": IPsec-authenticated traffic then gets past + // Block rules for the exe. iPads never authenticate, so it changes + // nothing for them, but the COM API doesn't say whether the override is + // set: any such rule, or one whose flags couldn't be read, makes a block + // uncertain. Protocol and port aren't looked at: fewer warnings, never + // a wrong one. + let bypass_rules = rules + .iter() + .filter(|r| { + r.enabled + && r.allow + && r.secure_flags != Some(0) + && matches!(whose(r, exe, &lookup), Whose::Ours | Whose::AllPrograms) + }) + .count(); + let verdicts = profiles + .iter() + .map(|p| { + let verdict = if !p.on { + Verdict::Off + } else if p.block_all_inbound { + Verdict::BlocksAll + } else if count(p.bit, &everyone) > 0 { + Verdict::BlockedForAllApps + } else if count(p.bit, &ours) > 0 { + Verdict::Blocked + } else if count(p.bit, &maybe_ours) > 0 { + Verdict::Uncertain + } else if count(p.bit, &allows) > 0 { + Verdict::Allowed + } else { + Verdict::NoRule + }; + (p.bit, verdict) + }) + .collect(); + Report { + profiles: verdicts, + port, + allow_rules: count(on, &allows), + block_rules: count(on, &ours), + all_programs_block_rules: count(on, &everyone), + scoped_block_rules: count(on, &unsure), + bypass_rules, + lan, + } +} + +static LATEST: Mutex> = Mutex::new(None); + +/// Set to read the policy again now rather than at the next [`RECHECK`]. +static RECHECK_NOW: (Mutex, std::sync::Condvar) = + (Mutex::new(false), std::sync::Condvar::new()); + +/// Reads the policy again at once (after "Fix it" ran), on the thread +/// [`watch`] started. +pub fn recheck_now() { + let (wanted, wake) = &RECHECK_NOW; + if let Ok(mut wanted) = wanted.lock() { + *wanted = true; + wake.notify_all(); + } +} + +/// Waits up to `timeout`, or until [`recheck_now`]. +#[cfg(any(windows, test))] +fn wait_for_recheck(timeout: Duration) { + let (wanted, wake) = &RECHECK_NOW; + let Ok(guard) = wanted.lock() else { + return; + }; + if let Ok((mut guard, _)) = wake.wait_timeout_while(guard, timeout, |wanted| !*wanted) { + *guard = false; + } +} + +/// The last reading, if the last read worked. `None` off Windows. +pub fn latest() -> Option { + LATEST.lock().ok()?.clone() +} + +/// Takes one read's result: stores and logs a report when it changes, and +/// clears it when the read failed, so an old block never stays on screen. +/// The first failure in a row is a warning. +#[cfg(any(windows, test))] +fn observe(read: Result, failed: &mut bool) { + let Ok(mut latest) = LATEST.lock() else { + return; + }; + match read { + Ok(report) => { + *failed = false; + if latest.as_ref() == Some(&report) { + return; + } + let line = report.summary(); + if report.verdict() == "allowed" || report.verdict() == "no_rule" { + info!("{line}"); + } else { + warn!("{line}"); + } + *latest = Some(report); + } + Err(error) => { + *latest = None; + if !std::mem::replace(failed, true) { + warn!(%error, "Could not read the Windows Firewall policy"); + } + } + } +} + +/// Reads the policy for the stream's `port` now and every [`RECHECK`] on +/// its own thread. +pub fn watch(port: u16) { + #[cfg(windows)] + { + let Ok(exe) = std::env::current_exe() else { + return; + }; + let _ = std::thread::Builder::new() + .name("firewall".into()) + .spawn(move || { + let mut failed = false; + loop { + let lan = crate::network::adapters().is_some_and(|adapters| { + crate::network::rank(&adapters) + .iter() + .any(|address| address.reach == crate::network::Reach::Lan) + }); + let read = windows_policy::read(&exe, port, lan).map_err(|e| e.to_string()); + observe(read, &mut failed); + wait_for_recheck(RECHECK); + } + }); + } + #[cfg(not(windows))] + let _ = port; +} + +#[cfg(windows)] +pub(crate) mod windows_policy { + use std::path::Path; + + use windows::core::{IUnknown, Interface, VARIANT}; + use windows::Win32::Foundation::E_FAIL; + use windows::Win32::NetworkManagement::WindowsFirewall::{ + INetFwPolicy2, INetFwRule, INetFwRule3, NetFwPolicy2, NET_FW_ACTION_ALLOW, + NET_FW_PROFILE_TYPE2, NET_FW_RULE_DIR_IN, + }; + use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_MULTITHREADED, + }; + use windows::Win32::System::Ole::IEnumVARIANT; + + use super::{ + judge, rule_from, Field, Profile, Report, Rule, RuleSource, Stored, DOMAIN, PRIVATE, PUBLIC, + }; + + /// VT_DISPATCH: what the rule enumerator hands out. + const VT_DISPATCH: u16 = 9; + + pub(crate) struct Apartment; + + impl Apartment { + pub(crate) fn new() -> windows::core::Result { + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED).ok()? }; + Ok(Self) + } + } + + impl Drop for Apartment { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } + } + + pub fn read(exe: &Path, port: u16, lan: bool) -> windows::core::Result { + let _apartment = Apartment::new()?; + let policy: INetFwPolicy2 = + unsafe { CoCreateInstance(&NetFwPolicy2, None, CLSCTX_INPROC_SERVER)? }; + let profiles = current_profiles(&policy)?; + let rules = inbound_rules(&policy)?; + Ok(judge(&profiles, &rules, exe, port, lan, super::machine_env)) + } + + /// One machine value as stored, for [`super::resolve`]. + pub(crate) fn machine_value(name: &str) -> Option { + use windows::Win32::UI::Shell::{ + FOLDERID_ProgramData, FOLDERID_ProgramFiles, FOLDERID_ProgramFilesCommon, + FOLDERID_ProgramFilesCommonX64, FOLDERID_ProgramFilesCommonX86, + FOLDERID_ProgramFilesX64, FOLDERID_ProgramFilesX86, FOLDERID_Windows, + }; + let folder = match name.to_ascii_lowercase().as_str() { + "programfiles" => FOLDERID_ProgramFiles, + "programw6432" => FOLDERID_ProgramFilesX64, + "programfiles(x86)" => FOLDERID_ProgramFilesX86, + "commonprogramfiles" => FOLDERID_ProgramFilesCommon, + "commonprogramw6432" => FOLDERID_ProgramFilesCommonX64, + "commonprogramfiles(x86)" => FOLDERID_ProgramFilesCommonX86, + "systemroot" | "windir" => FOLDERID_Windows, + "programdata" | "allusersprofile" => FOLDERID_ProgramData, + "systemdrive" => { + return known_folder(&FOLDERID_Windows) + .and_then(|windows| windows.get(..2).map(str::to_string)) + .filter(|drive| drive.ends_with(':')) + .map(Stored::Literal) + } + _ => return system_variable(name), + }; + known_folder(&folder).map(Stored::Literal) + } + + fn known_folder(id: &windows::core::GUID) -> Option { + use windows::Win32::Foundation::HANDLE; + use windows::Win32::System::Com::CoTaskMemFree; + use windows::Win32::UI::Shell::{SHGetKnownFolderPath, KF_FLAG_DEFAULT}; + let path = unsafe { SHGetKnownFolderPath(id, KF_FLAG_DEFAULT, HANDLE::default()) }.ok()?; + let text = unsafe { path.to_string() }.ok(); + unsafe { CoTaskMemFree(Some(path.0 as *const _)) }; + text + } + + /// A value of the system environment as stored, never expanded by + /// Windows: that would use this process's environment. + fn system_variable(name: &str) -> Option { + use windows::core::{w, PCWSTR}; + use windows::Win32::Foundation::ERROR_SUCCESS; + use windows::Win32::System::Registry::{ + RegGetValueW, HKEY_LOCAL_MACHINE, REG_EXPAND_SZ, REG_SZ, REG_VALUE_TYPE, RRF_NOEXPAND, + RRF_RT_REG_EXPAND_SZ, RRF_RT_REG_SZ, + }; + let name: Vec = name.encode_utf16().chain(Some(0)).collect(); + let mut value = [0u16; 1024]; + let mut bytes = std::mem::size_of_val(&value) as u32; + let mut kind = REG_VALUE_TYPE::default(); + let result = unsafe { + RegGetValueW( + HKEY_LOCAL_MACHINE, + w!(r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment"), + PCWSTR(name.as_ptr()), + RRF_RT_REG_SZ | RRF_RT_REG_EXPAND_SZ | RRF_NOEXPAND, + Some(&mut kind), + Some(value.as_mut_ptr().cast()), + Some(&mut bytes), + ) + }; + if result != ERROR_SUCCESS { + return None; + } + let length = value.iter().position(|&c| c == 0).unwrap_or(value.len()); + let value = String::from_utf16_lossy(&value[..length]); + match kind { + REG_SZ => Some(Stored::Literal(value)), + REG_EXPAND_SZ => Some(Stored::Expand(value)), + _ => None, + } + } + + /// The profiles of the networks in use now, as the verdict sees them. + pub(crate) fn current_profiles(policy: &INetFwPolicy2) -> windows::core::Result> { + let current = unsafe { policy.CurrentProfileTypes()? } as u32; + let mut profiles = Vec::new(); + for bit in [DOMAIN, PRIVATE, PUBLIC] { + if current & bit == 0 { + continue; + } + let profile = NET_FW_PROFILE_TYPE2(bit as i32); + profiles.push(Profile { + bit, + on: unsafe { policy.get_FirewallEnabled(profile)? }.as_bool(), + block_all_inbound: unsafe { policy.get_BlockAllInboundTraffic(profile)? }.as_bool(), + }); + } + Ok(profiles) + } + + impl RuleSource for INetFwRule { + fn inbound(&self) -> Result { + Ok(unsafe { self.Direction() }.map_err(|e| e.to_string())? == NET_FW_RULE_DIR_IN) + } + + fn allow(&self) -> Result { + Ok(unsafe { self.Action() }.map_err(|e| e.to_string())? == NET_FW_ACTION_ALLOW) + } + + fn enabled(&self) -> Result { + Ok(unsafe { self.Enabled() } + .map_err(|e| e.to_string())? + .as_bool()) + } + + fn protocol(&self) -> Result { + unsafe { self.Protocol() }.map_err(|e| e.to_string()) + } + + fn profiles(&self) -> Result { + unsafe { self.Profiles() }.map_err(|e| e.to_string()) + } + + fn text(&self, field: Field) -> Result { + let value = unsafe { + match field { + Field::Program => self.ApplicationName(), + Field::LocalPorts => self.LocalPorts(), + Field::RemotePorts => self.RemotePorts(), + Field::LocalAddresses => self.LocalAddresses(), + Field::RemoteAddresses => self.RemoteAddresses(), + Field::InterfaceTypes => self.InterfaceTypes(), + Field::Service => self.ServiceName(), + restriction => { + let rule3: INetFwRule3 = self.cast().map_err(|e| e.to_string())?; + match restriction { + Field::Package => rule3.LocalAppPackageId(), + Field::LocalUserOwner => rule3.LocalUserOwner(), + Field::LocalUsers => rule3.LocalUserAuthorizedList(), + Field::RemoteUsers => rule3.RemoteUserAuthorizedList(), + _ => rule3.RemoteMachineAuthorizedList(), + } + } + } + }; + value.map(|v| v.to_string()).map_err(|e| e.to_string()) + } + + fn interfaces_listed(&self) -> Result { + Ok(!unsafe { self.Interfaces() } + .map_err(|e| e.to_string())? + .is_empty()) + } + + fn secure_flags(&self) -> Result { + let rule3: INetFwRule3 = self.cast().map_err(|e| e.to_string())?; + unsafe { rule3.SecureFlags() }.map_err(|e| e.to_string()) + } + } + + /// Every inbound rule, each with what could be read of it. + fn inbound_rules(policy: &INetFwPolicy2) -> windows::core::Result> { + let mut rules = Vec::new(); + for_each_rule(policy, |rule| { + if let Some(rule) = rule_from(rule).map_err(|e| windows::core::Error::new(E_FAIL, e))? { + rules.push(rule); + } + Ok(()) + })?; + Ok(rules) + } + + /// Hands every rule of the policy, in any direction, to `each`. + pub(crate) fn for_each_rule( + policy: &INetFwPolicy2, + mut each: impl FnMut(&INetFwRule) -> windows::core::Result<()>, + ) -> windows::core::Result<()> { + let all = unsafe { policy.Rules()? }; + let rules: IEnumVARIANT = unsafe { all._NewEnum()? }.cast()?; + loop { + let mut item = [VARIANT::default()]; + let mut fetched = 0; + unsafe { rules.Next(&mut item, &mut fetched).ok()? }; + if fetched == 0 { + break; + } + let raw = item[0].as_raw(); + let (vt, pointer) = unsafe { + ( + raw.Anonymous.Anonymous.vt, + raw.Anonymous.Anonymous.Anonymous.pdispVal, + ) + }; + if vt != VT_DISPATCH { + continue; + } + // IDispatch is an IUnknown; the VARIANT keeps its reference. + let Some(unknown) = (unsafe { IUnknown::from_raw_borrowed(&pointer) }) else { + continue; + }; + let Ok(rule) = unknown.cast::() else { + continue; + }; + each(&rule)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + const EXE: &str = r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"; + const PORT: u16 = 9876; + + /// The machine's values come from the machine: a variable only this + /// process has, or only a user has, is unknown. + #[cfg(windows)] + #[test] + fn machine_values_never_come_from_this_process() { + const OWN: &str = "EM_FIREWALL_MACHINE_TEST"; + std::env::set_var(OWN, r"C:\Program Files"); + assert_eq!(machine_env(OWN), None); + std::env::remove_var(OWN); + for name in ["USERPROFILE", "LOCALAPPDATA", "APPDATA"] { + assert_eq!(machine_env(name), None, "{name}"); + } + for name in ["ProgramFiles", "SystemRoot", "windir", "ProgramData"] { + let value = machine_env(name).unwrap_or_else(|| panic!("{name}")); + assert!(Path::new(&value).is_dir(), "{name}"); + } + let drive = machine_env("SystemDrive").unwrap(); + assert!(machine_env("SystemRoot").unwrap().starts_with(&drive)); + } + + fn env(name: &str) -> Option { + match name.to_lowercase().as_str() { + "programfiles" => Some(r"C:\Program Files".into()), + _ => None, + } + } + + fn on(bit: u32) -> Profile { + Profile { + bit, + on: true, + block_all_inbound: false, + } + } + + /// The installer's rule: all profiles, UDP. + fn installer_rule() -> Rule { + Rule::unscoped(EXE, true, 0x7fff_ffff) + } + + /// What Windows adds when its prompt is dismissed: lower case, the + /// profiles in use then, no other scope. + fn prompt_block(profiles: u32) -> Rule { + Rule::unscoped(&EXE.to_lowercase(), false, profiles) + } + + fn report(profiles: &[Profile], rules: &[Rule]) -> Report { + judge(profiles, rules, &PathBuf::from(EXE), PORT, true, env) + } + + fn verdicts(profiles: &[Profile], rules: &[Rule]) -> Vec { + report(profiles, rules) + .profiles + .iter() + .map(|(_, v)| *v) + .collect() + } + + #[test] + fn the_installer_rule_allows_on_every_network() { + for bit in [DOMAIN, PRIVATE, PUBLIC] { + let report = report(&[on(bit)], &[installer_rule()]); + assert_eq!(report.profiles, [(bit, Verdict::Allowed)]); + assert_eq!(report.verdict(), "allowed"); + assert_eq!(report.certain_block(), None); + } + } + + #[test] + fn a_dismissed_prompt_is_certain_on_the_only_network() { + let report = report(&[on(PUBLIC)], &[installer_rule(), prompt_block(PUBLIC)]); + assert_eq!(report.profiles, [(PUBLIC, Verdict::Blocked)]); + assert_eq!(report.verdict(), "blocked"); + assert_eq!( + report.certain_block(), + Some(CertainBlock::Rule { + all_blocked: vec![] + }) + ); + // Another network, the same rules: nothing blocks there. + let private = + super::tests::report(&[on(PRIVATE)], &[installer_rule(), prompt_block(PUBLIC)]); + assert_eq!(private.profiles, [(PRIVATE, Verdict::Allowed)]); + } + + /// Ethernet on a private network, Wi-Fi on a public one: which an iPad + /// uses isn't known, so the block is logged per network, not shown. + #[test] + fn a_block_on_one_of_two_networks_is_logged_not_certain() { + let report = report( + &[on(PRIVATE), on(PUBLIC)], + &[installer_rule(), prompt_block(PUBLIC)], + ); + assert_eq!( + report.profiles, + [(PRIVATE, Verdict::Allowed), (PUBLIC, Verdict::Blocked)] + ); + assert_eq!(report.verdict(), "partly_blocked"); + assert_eq!(report.certain_block(), None); + // Blocked on both: certain again. + let both = super::tests::report( + &[on(PRIVATE), on(PUBLIC)], + &[installer_rule(), prompt_block(PRIVATE | PUBLIC)], + ); + assert!(both.certain_block().is_some()); + } + + #[test] + fn blocking_all_incoming_connections_beats_every_rule() { + let shields = Profile { + block_all_inbound: true, + ..on(PUBLIC) + }; + let alone = report(&[shields], &[installer_rule()]); + assert_eq!(alone.profiles, [(PUBLIC, Verdict::BlocksAll)]); + assert_eq!( + alone.certain_block(), + Some(CertainBlock::AllIncoming(vec!["public"])) + ); + // A private network that lets iPads in: not certain. + let mixed = report(&[on(PRIVATE), shields], &[installer_rule()]); + assert_eq!(mixed.certain_block(), None); + // Both networks blocked, one by a rule and one by blocking all. + let both = report(&[on(PRIVATE), shields], &[prompt_block(PRIVATE)]); + assert_eq!( + both.certain_block(), + Some(CertainBlock::Rule { + all_blocked: vec!["public"] + }) + ); + } + + #[test] + fn a_block_for_another_port_does_not_count() { + for ports in ["443", "1-1024,9877", "RPC", "9000-9875"] { + let rule = Rule { + local_ports: Some(ports.into()), + ..prompt_block(PUBLIC) + }; + let report = report(&[on(PUBLIC)], &[installer_rule(), rule]); + assert_eq!(report.profiles, [(PUBLIC, Verdict::Allowed)], "{ports}"); + assert_eq!((report.block_rules, report.scoped_block_rules), (0, 0)); + } + for ports in ["*", "", "9876", "53, 9000-9999"] { + let rule = Rule { + local_ports: Some(ports.into()), + ..prompt_block(PUBLIC) + }; + assert_eq!( + verdicts(&[on(PUBLIC)], &[rule]), + [Verdict::Blocked], + "{ports}" + ); + } + // The port the stream really listens on decides. + let rule = Rule { + local_ports: Some("19876".into()), + ..prompt_block(PUBLIC) + }; + let moved = judge( + &[on(PUBLIC)], + &[rule], + &PathBuf::from(EXE), + 19876, + true, + env, + ); + assert_eq!(moved.profiles, [(PUBLIC, Verdict::Blocked)]); + } + + #[test] + fn a_scoped_block_is_logged_not_certain() { + let scoped = [ + Rule { + remote_addresses: Some("203.0.113.0/24".into()), + ..prompt_block(PUBLIC) + }, + Rule { + remote_addresses: Some("LocalSubnet".into()), + ..prompt_block(PUBLIC) + }, + Rule { + local_addresses: Some("10.0.0.5".into()), + ..prompt_block(PUBLIC) + }, + Rule { + remote_ports: Some("5000".into()), + ..prompt_block(PUBLIC) + }, + Rule { + interface_types: Some("RemoteAccess".into()), + ..prompt_block(PUBLIC) + }, + Rule { + interfaces: Some(true), + ..prompt_block(PUBLIC) + }, + Rule { + service: Some("SomeService".into()), + ..prompt_block(PUBLIC) + }, + ]; + for rule in scoped { + let report = report(&[on(PUBLIC)], &[installer_rule(), rule.clone()]); + assert_eq!(report.profiles, [(PUBLIC, Verdict::Allowed)], "{rule:?}"); + assert_eq!((report.block_rules, report.scoped_block_rules), (0, 1)); + assert_eq!(report.certain_block(), None); + assert!(report.summary().contains("scoped_block_rules=1")); + } + let all_types = Rule { + interface_types: Some("All".into()), + ..prompt_block(PUBLIC) + }; + assert_eq!(verdicts(&[on(PUBLIC)], &[all_types]), [Verdict::Blocked]); + } + + /// A rule for every program blocks the stream as surely as one for this + /// exe, but Setup's delete by program can't remove it: it is logged and + /// never shown. + #[test] + fn a_block_for_every_program_on_the_port_is_logged_not_shown() { + let everyone = report( + &[on(PUBLIC)], + &[installer_rule(), Rule::unscoped("", false, PUBLIC)], + ); + assert_eq!(everyone.profiles, [(PUBLIC, Verdict::BlockedForAllApps)]); + assert_eq!( + (everyone.block_rules, everyone.all_programs_block_rules), + (0, 1) + ); + assert_eq!(everyone.verdict(), "blocked"); + assert_eq!(everyone.certain_block(), None); + assert!(everyone.summary().contains("public:blocked_all_programs")); + // With this exe's own block as well: Setup would leave the other. + let both = report( + &[on(PUBLIC)], + &[prompt_block(PUBLIC), Rule::unscoped(" ", false, PUBLIC)], + ); + assert_eq!(both.profiles, [(PUBLIC, Verdict::BlockedForAllApps)]); + assert_eq!(both.certain_block(), None); + // Another port, or scoped: not a block, or only logged. + let other_port = Rule { + local_ports: Some("443".into()), + ..Rule::unscoped("", false, PUBLIC) + }; + let scoped = Rule { + remote_addresses: Some("LocalSubnet".into()), + ..Rule::unscoped("", false, PUBLIC) + }; + let unsure = report(&[on(PUBLIC)], &[installer_rule(), other_port, scoped]); + assert_eq!(unsure.profiles, [(PUBLIC, Verdict::Allowed)]); + assert_eq!( + (unsure.all_programs_block_rules, unsure.scoped_block_rules), + (0, 1) + ); + assert_eq!(unsure.certain_block(), None); + // An Allow rule for every program lets the stream in too, unless it + // is one of Windows' own, limited to a service or some addresses. + let service_only = Rule { + service: Some("Dnscache".into()), + ..Rule::unscoped("", true, PRIVATE) + }; + let windows_own = report(&[on(PRIVATE)], &[service_only]); + assert_eq!(windows_own.profiles, [(PRIVATE, Verdict::NoRule)]); + assert_eq!(windows_own.allow_rules, 0); + let open = report(&[on(PRIVATE)], &[Rule::unscoped("", true, PRIVATE)]); + assert_eq!(open.profiles, [(PRIVATE, Verdict::Allowed)]); + } + + /// A rule as a fake COM object: every getter answers, except `broken`. + struct Fake { + rule: Rule, + protocol: i32, + broken: Option<&'static str>, + } + + impl Fake { + fn get(&self, name: &str, value: T) -> Result { + if self.broken == Some(name) { + Err(format!("{name} failed (0x80004005)")) + } else { + Ok(value) + } + } + } + + impl RuleSource for Fake { + fn inbound(&self) -> Result { + self.get("direction", true) + } + fn allow(&self) -> Result { + self.get("action", self.rule.allow) + } + fn enabled(&self) -> Result { + self.get("enabled", self.rule.enabled) + } + fn protocol(&self) -> Result { + self.get("protocol", self.protocol) + } + fn profiles(&self) -> Result { + self.get("profiles", self.rule.profiles as i32) + } + fn text(&self, field: Field) -> Result { + let (name, value) = match field { + Field::Program => ("program", &self.rule.program), + Field::LocalPorts => ("local_ports", &self.rule.local_ports), + Field::RemotePorts => ("remote_ports", &self.rule.remote_ports), + Field::LocalAddresses => ("local_addresses", &self.rule.local_addresses), + Field::RemoteAddresses => ("remote_addresses", &self.rule.remote_addresses), + Field::InterfaceTypes => ("interface_types", &self.rule.interface_types), + Field::Service => ("service", &self.rule.service), + Field::Package => ("package", &self.rule.package), + Field::LocalUserOwner => ("local_user_owner", &self.rule.local_user_owner), + Field::LocalUsers => ("local_users", &self.rule.local_users), + Field::RemoteUsers => ("remote_users", &self.rule.remote_users), + Field::RemoteMachines => ("remote_machines", &self.rule.remote_machines), + }; + self.get(name, value.clone().unwrap_or_default()) + } + fn interfaces_listed(&self) -> Result { + self.get("interfaces", self.rule.interfaces.unwrap_or(false)) + } + fn secure_flags(&self) -> Result { + self.get("secure_flags", self.rule.secure_flags.unwrap_or(0)) + } + } + + fn fake(rule: Rule, broken: Option<&'static str>) -> Fake { + Fake { + rule, + protocol: 17, + broken, + } + } + + /// A scope field that can't be read is unknown, never "any": the rule + /// is logged and can't make a block certain. + #[test] + fn a_scope_that_could_not_be_read_never_makes_a_block_certain() { + for block in [prompt_block(PUBLIC), Rule::unscoped("", false, PUBLIC)] { + let read = rule_from(&fake(block.clone(), None)).unwrap().unwrap(); + assert_eq!(read, block); + // Read whole, it blocks (this exe's rule or every program's). + assert!(report(&[on(PUBLIC)], &[read]).profiles[0].1.blocks()); + for broken in [ + "program", + "local_ports", + "remote_ports", + "local_addresses", + "remote_addresses", + "interface_types", + "interfaces", + "service", + "package", + "local_user_owner", + "local_users", + "remote_users", + "remote_machines", + "secure_flags", + ] { + let read = rule_from(&fake(block.clone(), Some(broken))) + .unwrap() + .unwrap(); + let judged = report(&[on(PUBLIC)], &[installer_rule(), read]); + assert_eq!(judged.profiles, [(PUBLIC, Verdict::Allowed)], "{broken}"); + assert_eq!( + ( + judged.block_rules, + judged.all_programs_block_rules, + judged.scoped_block_rules + ), + (0, 0, 1), + "{broken}" + ); + assert_eq!(judged.certain_block(), None, "{broken}"); + } + } + // Unknown ports on another program's rule: not ours, not counted. + let other = rule_from(&fake( + Rule::unscoped(r"C:\Tools\Other.exe", false, PUBLIC), + Some("local_ports"), + )) + .unwrap() + .unwrap(); + assert_eq!(report(&[on(PUBLIC)], &[other]).scoped_block_rules, 0); + } + + /// INetFwRule3's restrictions limit whose traffic a rule is for: a + /// package-only rule for every program is not a block on this exe. Each + /// one, set or unread, makes a rule scoped. + #[test] + fn a_rule_limited_to_a_package_user_or_machine_is_never_certain() { + type Set = fn(&mut Rule); + let limits: [(&str, Set); 6] = [ + ("package", |r| r.package = Some("S-1-15-2-1".into())), + ("local_user_owner", |r| { + r.local_user_owner = Some("S-1-5-21-1".into()) + }), + ("local_users", |r| { + r.local_users = Some("O:LSD:(A;;CC;;;S-1-5-21-1)".into()) + }), + ("remote_users", |r| { + r.remote_users = Some("O:LSD:(A;;CC;;;S-1-5-21-2)".into()) + }), + ("remote_machines", |r| { + r.remote_machines = Some("O:LSD:(A;;CC;;;S-1-5-21-3)".into()) + }), + ("secure_flags", |r| r.secure_flags = Some(1)), + ]; + for program in [EXE.to_lowercase(), String::new()] { + for (name, set) in limits { + let mut limited = Rule::unscoped(&program, false, PUBLIC); + set(&mut limited); + let read = rule_from(&fake(limited, None)).unwrap().unwrap(); + let judged = report(&[on(PUBLIC)], &[installer_rule(), read]); + assert_eq!( + judged.profiles, + [(PUBLIC, Verdict::Allowed)], + "{name} {program:?}" + ); + assert_eq!(judged.scoped_block_rules, 1, "{name}"); + assert_eq!(judged.certain_block(), None, "{name}"); + } + } + // Empty restrictions and no IPsec: the prompt's rule as Windows + // stores it stays a certain block. + let plain = rule_from(&fake(prompt_block(PUBLIC), None)) + .unwrap() + .unwrap(); + assert!(report(&[on(PUBLIC)], &[plain]).certain_block().is_some()); + } + + /// An Allow rule asking for IPsec may override Block rules for + /// authenticated traffic. Whatever blocks, the verdict is uncertain and + /// nothing is shown; it is logged as bypass_rules. + #[test] + fn an_allow_rule_that_may_bypass_blocks_makes_any_block_uncertain() { + let secure = |program: &str, flags: Option| Rule { + secure_flags: flags, + ..Rule::unscoped(program, true, 0x7fff_ffff) + }; + let shields = Profile { + block_all_inbound: true, + ..on(PUBLIC) + }; + for bypass in [ + secure(EXE, Some(2)), + secure("", Some(4)), + secure(&EXE.to_lowercase(), None), + ] { + for (profiles, rules) in [ + (vec![on(PUBLIC)], vec![prompt_block(PUBLIC), bypass.clone()]), + (vec![shields], vec![bypass.clone()]), + ] { + let judged = report(&profiles, &rules); + assert_eq!(judged.bypass_rules, 1, "{bypass:?}"); + assert_eq!(judged.verdict(), "uncertain"); + assert_eq!(judged.certain_block(), None); + assert!(judged.summary().contains("bypass_rules=1 ")); + } + } + // Not ours, disabled, or a Block rule: no bypass. + let others = [ + secure(r"C:\Tools\Other.exe", Some(2)), + Rule { + enabled: false, + ..secure(EXE, Some(2)) + }, + Rule { + allow: false, + remote_addresses: Some("10.0.0.0/8".into()), + ..secure(EXE, Some(2)) + }, + ]; + for other in others { + let judged = report(&[on(PUBLIC)], &[prompt_block(PUBLIC), other.clone()]); + assert_eq!(judged.bypass_rules, 0, "{other:?}"); + assert!(judged.certain_block().is_some(), "{other:?}"); + } + // Nothing blocks: a bypass rule alone changes nothing. + let open = report(&[on(PUBLIC)], &[secure(EXE, Some(2))]); + assert_eq!(open.verdict(), "allowed"); + } + + /// The fields the verdict can't do without fail the whole reading, + /// which clears what the Stream page shows. + #[test] + fn a_failed_action_or_profile_read_fails_the_reading() { + for broken in ["direction", "action", "enabled", "protocol", "profiles"] { + let failed = rule_from(&fake(prompt_block(PUBLIC), Some(broken))); + assert!(failed.is_err(), "{broken}"); + } + let any_protocol = Fake { + protocol: 256, + ..fake(prompt_block(PUBLIC), None) + }; + assert!(rule_from(&any_protocol).unwrap().unwrap().udp); + let tcp = Fake { + protocol: 6, + ..fake(prompt_block(PUBLIC), None) + }; + assert!(!rule_from(&tcp).unwrap().unwrap().udp); + } + + #[test] + fn without_a_home_network_address_no_block_is_certain() { + let report = judge( + &[on(PUBLIC)], + &[prompt_block(PUBLIC)], + &PathBuf::from(EXE), + PORT, + false, + env, + ); + assert_eq!(report.verdict(), "blocked"); + assert_eq!(report.certain_block(), None); + } + + #[test] + fn rules_that_do_not_apply_are_not_counted() { + let tcp = Rule { + udp: false, + ..prompt_block(PUBLIC) + }; + let disabled = Rule { + enabled: false, + ..prompt_block(PUBLIC) + }; + let other_exe = Rule::unscoped(r"C:\Tools\EternalMonitor-host.exe", false, PUBLIC); + let report = report( + &[on(PUBLIC)], + &[ + installer_rule(), + tcp, + disabled, + other_exe, + prompt_block(PRIVATE), + ], + ); + assert_eq!((report.allow_rules, report.block_rules), (1, 0)); + assert_eq!(report.profiles, [(PUBLIC, Verdict::Allowed)]); + } + + #[test] + fn no_rule_and_firewall_off_are_told_apart() { + assert_eq!(report(&[on(PRIVATE)], &[]).verdict(), "no_rule"); + let off = Profile { + on: false, + ..on(PRIVATE) + }; + let report_off = report(&[off], &[prompt_block(PRIVATE)]); + assert_eq!(report_off.verdict(), "off"); + assert_eq!(report_off.certain_block(), None); + // A network whose firewall is off lets iPads in: not certain. + let half = report(&[off, on(PUBLIC)], &[prompt_block(PUBLIC)]); + assert_eq!( + half.profiles, + [(PRIVATE, Verdict::Off), (PUBLIC, Verdict::Blocked)] + ); + assert_eq!(half.certain_block(), None); + } + + #[test] + fn a_rule_made_in_windows_security_matches_through_its_variables() { + let ui_rule = Rule::unscoped( + r"%ProgramFiles%\EternalMonitor\EternalMonitor-host.exe", + true, + 0x7fff_ffff, + ); + assert_eq!(verdicts(&[on(PRIVATE)], &[ui_rule]), [Verdict::Allowed]); + assert!(same_program( + "\"c:/program files/eternalmonitor/eternalmonitor-host.exe\"", + Path::new(EXE), + env + )); + assert!(same_program(EXE, Path::new(&format!(r"\\?\{EXE}")), env)); + assert!(!same_program("", Path::new(EXE), env)); + } + + #[test] + fn expansion_refuses_unknown_names_and_keeps_unpaired_ones() { + assert_eq!(expand_known(r"%nope%\a", env), None); + assert_eq!( + expand_known(r"%%\%ProgramFiles%\a%", env).as_deref(), + Some(r"%%\C:\Program Files\a%") + ); + } + + /// Machine values as Windows stores them: known folders and REG_SZ + /// values as written, REG_EXPAND_SZ values naming others. + fn stored(name: &str) -> Option { + let expand = |value: &str| Some(Stored::Expand(value.into())); + match name.to_lowercase().as_str() { + "systemdrive" => Some(Stored::Literal("C:".into())), + "programfiles" => Some(Stored::Literal(r"C:\Program Files".into())), + "emroot" => expand(r"%SystemDrive%\Apps"), + "literal" => Some(Stored::Literal(r"C:\100%\%SystemDrive%".into())), + "userish" => expand(r"%USERPROFILE%\Apps"), + "loop_a" => expand(r"%loop_b%\a"), + "loop_b" => expand(r"%LOOP_A%\b"), + "self" => expand(r"%self%"), + deep => { + // deep0 is a folder; each deepN names deepN-1. + let n: usize = deep.strip_prefix("deep")?.parse().ok()?; + if n == 0 { + Some(Stored::Literal(r"C:\Deep".into())) + } else { + expand(&format!(r"%deep{}%\{n}", n - 1)) + } + } + } + } + + #[test] + fn machine_values_naming_others_are_expanded_from_machine_values_only() { + let value = |name| resolve(name, &stored); + assert_eq!(value("EMROOT").as_deref(), Some(r"C:\Apps")); + assert_eq!(value("ProgramFiles").as_deref(), Some(r"C:\Program Files")); + // A REG_SZ value is never expanded, as Windows doesn't. + assert_eq!(value("literal").as_deref(), Some(r"C:\100%\%SystemDrive%")); + // A value only a user has, or no value at all. + assert_eq!(value("userish"), None); + assert_eq!(value("nope"), None); + // Values naming each other, or themselves. + assert_eq!(value("loop_a"), None); + assert_eq!(value("self"), None); + // Up to EXPAND_DEPTH nested values, and no deeper. + let deepest = format!("deep{EXPAND_DEPTH}"); + assert_eq!(value(&deepest).as_deref(), Some(r"C:\Deep\1\2\3\4\5\6\7\8")); + assert_eq!(value(&format!("deep{}", EXPAND_DEPTH + 1)), None); + } + + /// Values that each name the next several times would grow fourfold a + /// level: each name is looked up once, and the expansion stops at its + /// size or reference limit. + #[test] + fn a_branching_expansion_stops_at_its_limits() { + let branching = |name: &str| { + // b0 is a folder; each bN names bN-1 four times. + let n: usize = name.strip_prefix('b')?.parse().ok()?; + Some(if n == 0 { + Stored::Literal("x".repeat(20)) + } else { + Stored::Expand(format!("%b{0}%", n - 1).repeat(4)) + }) + }; + let lookups = std::cell::Cell::new(0); + let counted = |name: &str| { + lookups.set(lookups.get() + 1); + branching(name) + }; + // 4^8 × 20 bytes, 1.3 MB. + assert_eq!(resolve(&format!("b{EXPAND_DEPTH}"), &counted), None); + assert!(lookups.get() <= EXPAND_DEPTH + 1, "{}", lookups.get()); + // 4^4 × 20 bytes is within the limits. + assert_eq!( + resolve("b4", &branching).map(|value| value.len()), + Some(4usize.pow(4) * 20) + ); + // One value naming another too many times. + let wide = |name: &str| match name { + "wide" => Some(Stored::Expand("%leaf%".repeat(MAX_REFERENCES))), + "leaf" => Some(Stored::Literal("a".into())), + _ => None, + }; + assert_eq!(resolve("wide", &wide), None); + let within = |name: &str| match name { + "wide" => Some(Stored::Expand("%leaf%".repeat(MAX_REFERENCES - 1))), + _ => wide(name), + }; + assert_eq!( + resolve("wide", &within).map(|value| value.len()), + Some(MAX_REFERENCES - 1) + ); + // Past the size limit, nothing more is looked up. + let after = std::cell::Cell::new(false); + let big = |name: &str| match name { + "both" => Some(Stored::Expand("%big%%after%".into())), + "big" => Some(Stored::Literal("x".repeat(MAX_EXPANDED + 1))), + _ => { + after.set(true); + Some(Stored::Literal(String::new())) + } + }; + assert_eq!(resolve("both", &big), None); + assert!(!after.get()); + // Text after the last name counts too. + let tail = |name: &str| match name { + "tail" => Some(Stored::Expand("%full%x".into())), + "full" => Some(Stored::Literal("x".repeat(MAX_EXPANDED))), + _ => None, + }; + assert_eq!( + resolve("full", &tail).map(|value| value.len()), + Some(MAX_EXPANDED) + ); + assert_eq!(resolve("tail", &tail), None); + } + + /// A Block rule whose path names a machine value that names another + /// (`EMROOT=%SystemDrive%\Apps`) is this exe's block. One whose path + /// can't be resolved but has this exe's file name may be: never + /// "allowed", and never certain. + #[test] + fn a_block_through_nested_machine_values_counts_and_an_unresolved_one_is_uncertain() { + let exe = r"C:\Apps\EternalMonitor\EternalMonitor-host.exe"; + let allow = Rule::unscoped(exe, true, 0x7fff_ffff); + let nested = Rule::unscoped( + r"%EMROOT%\EternalMonitor\EternalMonitor-host.exe", + false, + PUBLIC, + ); + let judged = |rules: &[Rule]| { + judge(&[on(PUBLIC)], rules, Path::new(exe), PORT, true, |name| { + resolve(name, &stored) + }) + }; + let blocked = judged(&[allow.clone(), nested]); + assert_eq!(blocked.profiles, [(PUBLIC, Verdict::Blocked)]); + assert!(blocked.certain_block().is_some()); + + let unresolved = Rule::unscoped( + r"%USERISH%\EternalMonitor\EternalMonitor-host.exe", + false, + PUBLIC, + ); + let unsure = judged(&[allow.clone(), unresolved]); + assert_eq!(unsure.profiles, [(PUBLIC, Verdict::Uncertain)]); + assert_eq!(unsure.verdict(), "uncertain"); + assert_eq!(unsure.certain_block(), None); + assert_eq!(unsure.scoped_block_rules, 1); + + // Another file name is another program's rule. + let other = Rule::unscoped(r"%USERISH%\Other\other.exe", false, PUBLIC); + assert_eq!( + judged(&[allow, other]).profiles, + [(PUBLIC, Verdict::Allowed)] + ); + } + + /// A read that fails drops the last reading, so a block that is gone + /// can't stay on the Stream page. + #[test] + fn a_failed_read_clears_the_last_reading() { + let blocked = report(&[on(PUBLIC)], &[prompt_block(PUBLIC)]); + let mut failed = false; + observe(Ok(blocked.clone()), &mut failed); + assert_eq!(latest(), Some(blocked.clone())); + observe(Err("RPC server unavailable".into()), &mut failed); + assert!(failed); + assert_eq!(latest(), None); + observe(Err("again".into()), &mut failed); + assert_eq!(latest(), None); + observe(Ok(blocked.clone()), &mut failed); + assert!(!failed); + assert_eq!(latest(), Some(blocked)); + } + + /// `key=value` or `key="value with spaces"` from a netsh command line. + fn netsh_arg(params: &str, key: &str) -> Option { + let mut rest = params; + while let Some(start) = rest.find(&format!(" {key}=")) { + let value = &rest[start + key.len() + 2..]; + if let Some(quoted) = value.strip_prefix('"') { + return quoted.split('"').next().map(str::to_string); + } + let value = value.split(' ').next().unwrap_or_default(); + if !value.is_empty() { + return Some(value.to_string()); + } + rest = &rest[start + 1..]; + } + None + } + + /// What the firewall holds after Setup runs the netsh lines of its + /// [Run] section, in order, into `app`. Every rule here is inbound. + fn after_setup(mut rules: Vec<(String, Rule)>, app: &str) -> Vec { + let iss = include_str!("../../installer/EternalMonitor.iss"); + let run = iss + .split("\n[Run]") + .nth(1) + .and_then(|section| section.split("\n[").next()) + .expect("EternalMonitor.iss has a [Run] section"); + for line in run.lines().filter(|line| line.contains(r"\netsh.exe")) { + let params = line + .split("Parameters: \"") + .nth(1) + .and_then(|rest| rest.split("\"; ").next()) + .expect("a netsh line has Parameters") + .replace("\"\"", "\"") + .replace("{app}", app); + let arg = |key| netsh_arg(¶ms, key); + let name = arg("name").expect("a netsh rule line names its rule"); + if params.starts_with("advfirewall firewall delete rule ") { + match arg("program") { + Some(program) if name == "all" => { + assert_ne!(arg("dir").as_deref(), Some("out")); + rules.retain(|(_, rule)| { + !rule + .program + .as_deref() + .is_some_and(|ours| same_program(ours, Path::new(&program), env)) + }); + } + _ => rules.retain(|(named, _)| *named != name), + } + } else if params.starts_with("advfirewall firewall add rule ") { + assert_eq!(arg("dir").as_deref(), Some("in"), "{params}"); + let profiles = arg("profile").unwrap_or_default(); + let bits = [("domain", DOMAIN), ("private", PRIVATE), ("public", PUBLIC)]; + rules.push(( + name, + Rule { + enabled: arg("enable").as_deref() != Some("no"), + udp: arg("protocol").as_deref() == Some("UDP"), + ..Rule::unscoped( + &arg("program").unwrap_or_default(), + arg("action").as_deref() == Some("allow"), + bits.iter() + .filter(|(word, _)| profiles.split(',').any(|p| p == *word)) + .map(|(_, bit)| bit) + .sum(), + ) + }, + )); + } else { + panic!("unexpected netsh line in [Run]: {params}"); + } + } + rules.into_iter().map(|(_, rule)| rule).collect() + } + + #[test] + fn running_setup_again_removes_a_dismissed_prompts_block() { + let app = r"C:\Program Files\EternalMonitor"; + let other = Rule::unscoped(r"C:\Tools\Other.exe", false, PUBLIC); + let before = vec![ + ("EternalMonitor Host UDP".to_string(), installer_rule()), + ( + format!("UDP Query User{{0-1-2-3-4}}{}", EXE.to_lowercase()), + prompt_block(PRIVATE | PUBLIC), + ), + ("Other".to_string(), other.clone()), + ]; + let rules: Vec = before.iter().map(|(_, rule)| rule.clone()).collect(); + assert_eq!(verdicts(&[on(PUBLIC)], &rules), [Verdict::Blocked]); + + let after = after_setup(before, app); + assert_eq!( + verdicts(&[on(PUBLIC)], &after), + [Verdict::Allowed], + "{after:?}" + ); + assert_eq!(verdicts(&[on(PRIVATE)], &after), [Verdict::Allowed]); + assert!( + after.contains(&other), + "Setup removed another program's rule" + ); + // A first install: nothing to remove, the Allow rule still comes. + assert_eq!( + verdicts(&[on(DOMAIN)], &after_setup(Vec::new(), app)), + [Verdict::Allowed] + ); + } + + /// After "Fix it", the policy is read again at once, not at the next + /// minute's check. + #[test] + fn a_recheck_wakes_the_watch_at_once() { + let started = std::time::Instant::now(); + let waiting = std::thread::spawn(|| wait_for_recheck(Duration::from_secs(30))); + std::thread::sleep(Duration::from_millis(50)); + recheck_now(); + waiting.join().unwrap(); + assert!( + started.elapsed() < Duration::from_secs(10), + "{:?}", + started.elapsed() + ); + // Used up: the next wait waits its time. + let started = std::time::Instant::now(); + wait_for_recheck(Duration::from_millis(100)); + assert!(started.elapsed() >= Duration::from_millis(100)); + } + + #[test] + fn the_log_line_names_no_path_or_rule() { + let scoped = Rule { + remote_addresses: Some("192.168.1.0/24".into()), + ..prompt_block(PRIVATE) + }; + let report = report( + &[on(PRIVATE), on(PUBLIC)], + &[installer_rule(), prompt_block(PUBLIC), scoped], + ); + assert_eq!( + report.summary(), + "Windows Firewall: profiles=[private:allowed public:blocked] allow_rules=1 \ + block_rules=1 all_programs_block_rules=0 scoped_block_rules=1 bypass_rules=0 lan=true verdict=partly_blocked" + ); + let line = format!("2026-11-05T12:00:00.000000Z WARN {}", report.summary()); + assert_eq!(crate::telemetry::uploaded(&line), line); + } +} diff --git a/host/src/firewall_fix.rs b/host/src/firewall_fix.rs new file mode 100644 index 00000000..048681ae --- /dev/null +++ b/host/src/firewall_fix.rs @@ -0,0 +1,1223 @@ +//! The Stream page's "Fix it" for a Windows Firewall block on this exe. +//! +//! The block Windows adds when its "allow access" prompt is dismissed beats +//! the installer's Allow rule, and the only fix the PC offered was to run +//! Setup again. "Fix it" starts this exe as administrator, with one Windows +//! prompt, with [`FIX_ARG`] and the stream's port. That copy changes the +//! firewall through its COM interface, never a shell, and exits. +//! +//! It can't leave the PC worse off than before. Setup deletes every rule +//! for the exe, then adds its Allow rule; deletions with no add after them +//! could lose rules that worked. The fix is narrower ([`fix_rules`]), and +//! judges every step as the Stream page's verdict does +//! ([`crate::firewall::judge`]), on the networks in use now: +//! +//! 1. Unless the exe would be allowed on every network in use once its +//! blocks are gone, it adds the installer's Allow rule, first, so a +//! failure here changes nothing. An Allow rule the user made, however +//! narrow, that already covers those networks means no add. +//! 2. It removes only this exe's own enabled Block rules that reach the +//! stream port with no other scope, on a network in use: the blocks the +//! verdict counts. Each is renamed to a fresh name, then removed by that +//! name, so no other rule can go with it. Removing a Block rule never +//! stops anything that worked. Allow rules, scoped rules, blocks for +//! networks not in use and other programs' rules stay as they are. +//! 3. It reads everything again and fails unless the verdict now finds the +//! exe allowed, and not blocked by its own rule, on every network in use. +//! A Block rule that may be this exe's, with its file name and a path the +//! machine's values can't resolve, is never removed and keeps the fix +//! from claiming success. +//! +//! The exe's path comes from the elevated process itself, and the port is +//! only ever a number. A rule's path is this exe's only as written, or with +//! `%name%` read from the machine's own settings +//! ([`crate::firewall::machine_env`]), never from the environment the +//! elevated copy inherits from the user who clicked. A user who answers No to the Windows prompt changes +//! nothing: the notice stays as it was. After the copy ran, the firewall +//! is read again at once. + +use std::path::Path; + +use crate::firewall::{judge, reach, Profile, Reach, Rule, Verdict, DOMAIN, PRIVATE, PUBLIC}; + +/// The argument the elevated copy runs with, before the stream's port. +pub const FIX_ARG: &str = "--fix-firewall"; + +/// The installer's Allow rule, as its `[Run]` section adds it: this name, +/// inbound, UDP, every profile, enabled. +pub const ALLOW_NAME: &str = "EternalMonitor Host UDP"; +pub const ALLOW_PROFILES: u32 = DOMAIN | PRIVATE | PUBLIC; + +/// A firewall rule as the last read found it. `id` names it for +/// [`RuleStore::remove`] until the next read. `rule` is None for an +/// outbound rule. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Found { + pub id: usize, + pub rule: Option, +} + +/// The firewall, as the elevated copy changes it. The real one is the +/// Windows Firewall COM interface; tests fake it. +pub trait RuleStore { + /// The profiles of the networks in use now. + fn profiles(&mut self) -> Result, String>; + /// Every rule, in any direction. + fn rules(&mut self) -> Result, String>; + /// Adds the installer's Allow rule for `exe`. + fn add_allow(&mut self, exe: &str) -> Result<(), String>; + /// Removes the rule the last read found as `id`, and no other. + fn remove(&mut self, id: usize) -> Result<(), String>; +} + +/// Why the fix failed, as the elevated copy's exit code. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Failure { + /// The copy got no usable path or port. + Arguments = 3, + /// The rules or the networks in use couldn't be read. + Read = 10, + /// The Allow rule couldn't be added. Nothing was changed. + Add = 11, + /// A Block rule couldn't be removed. + Remove = 12, + /// Read again, the verdict still finds the exe blocked by its own rule, + /// or not allowed, on a network in use. + StillBlocked = 14, +} + +impl Failure { + pub fn code(self) -> u32 { + self as u32 + } +} + +/// What `%name%` in a rule's path stands for: [`crate::firewall::machine_env`] +/// in the elevated copy, never its environment, which the user who clicked +/// "Fix it" could have set. +type Lookup<'a> = &'a dyn Fn(&str) -> Option; + +/// The bits of the profiles in use with Windows Firewall on: where the +/// verdict counts rules. +fn in_use(profiles: &[Profile]) -> u32 { + profiles + .iter() + .filter(|profile| profile.on) + .map(|profile| profile.bit) + .sum() +} + +/// This exe's own Block rules the verdict counts: enabled, reaching every +/// datagram to `port`, on a network in use. All the fix removes. +fn is_target(rule: &Rule, exe: &Path, port: u16, in_use: u32, lookup: Lookup) -> bool { + !rule.allow + && rule.enabled + && rule.udp + && rule.profiles & in_use != 0 + && crate::firewall::ours(rule, exe, lookup) + && reach(rule, port) == Reach::All +} + +/// Whether the verdict on `rules` finds the exe allowed, and not blocked by +/// its own rule, on every network in use. A network that blocks all +/// incoming connections or has the firewall off is the user's switch, not +/// a rule the fix could change. +fn allowed_everywhere( + profiles: &[Profile], + rules: &[&Rule], + exe: &Path, + port: u16, + lookup: Lookup, +) -> bool { + let rules: Vec = rules.iter().map(|rule| (*rule).clone()).collect(); + let report = judge(profiles, &rules, exe, port, true, lookup); + report.profiles.iter().all(|(_, verdict)| { + matches!( + verdict, + Verdict::Allowed | Verdict::BlocksAll | Verdict::Off + ) + }) +} + +fn inbound(found: &[Found]) -> impl Iterator { + found + .iter() + .filter_map(|found| found.rule.as_ref().map(|rule| (found.id, rule))) +} + +/// The fix, on `store`, for this exe at `exe` and the stream's `port`. +pub fn fix_rules( + store: &mut impl RuleStore, + exe: &Path, + port: u16, + lookup: Lookup, +) -> Result<(), Failure> { + let program = exe + .to_str() + .filter(|path| !path.is_empty()) + .ok_or(Failure::Arguments)?; + let profiles = store.profiles().map_err(|_| Failure::Read)?; + let found = store.rules().map_err(|_| Failure::Read)?; + let bits = in_use(&profiles); + let targets: Vec = inbound(&found) + .filter(|(_, rule)| is_target(rule, exe, port, bits, lookup)) + .map(|(id, _)| id) + .collect(); + let without_targets: Vec<&Rule> = inbound(&found) + .filter(|(id, _)| !targets.contains(id)) + .map(|(_, rule)| rule) + .collect(); + if !allowed_everywhere(&profiles, &without_targets, exe, port, lookup) { + store.add_allow(program).map_err(|_| Failure::Add)?; + } + for id in targets { + store.remove(id).map_err(|_| Failure::Remove)?; + } + let profiles = store.profiles().map_err(|_| Failure::Read)?; + let after = store.rules().map_err(|_| Failure::Read)?; + let after: Vec<&Rule> = inbound(&after).map(|(_, rule)| rule).collect(); + if allowed_everywhere(&profiles, &after, exe, port, lookup) { + Ok(()) + } else { + Err(Failure::StillBlocked) + } +} + +/// [`fix_rules`] as the elevated copy runs it, with the machine's own +/// values for `%name%` in rule paths. +#[cfg(any(windows, test))] +pub(crate) fn fix_on_this_machine( + store: &mut impl RuleStore, + exe: &Path, + port: u16, +) -> Result<(), Failure> { + fix_rules(store, exe, port, &crate::firewall::machine_env) +} + +/// Runs `fix` on a store opened inside an apartment, and drops the store +/// before the apartment: COM objects released after `CoUninitialize` may +/// call into a library Windows has already unloaded. +#[cfg(any(windows, test))] +pub(crate) fn in_apartment( + apartment: impl FnOnce() -> Result, + store: impl FnOnce() -> Result, + fix: impl FnOnce(&mut S) -> Result, +) -> Result { + let apartment = apartment()?; + let mut store = store()?; + let result = fix(&mut store); + drop(store); + drop(apartment); + result +} + +/// How starting the elevated copy went. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Elevated { + /// It ran and exited with this code. + Exited(u32), + /// The user answered No to the Windows prompt. + Declined, + /// Windows couldn't start it. + Failed(String), +} + +/// Starts this exe again as administrator with `args`, and waits for it to +/// exit, however long it takes: a second copy never starts meanwhile. The +/// real one asks Windows; tests fake it. +pub trait Elevate { + fn run_elevated(&self, exe: &Path, args: &str) -> Elevated; +} + +/// What the Stream page shows after "Fix it". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Outcome { + /// Read again, the verdict finds the exe allowed on every network in + /// use; the Stream page reads it again too, and the notice follows. + Fixed, + /// No to the Windows prompt: nothing changed. + Declined, + /// The fix didn't run or didn't finish. English, as other errors are. + Failed(&'static str), +} + +/// The arguments the elevated copy gets: [`FIX_ARG`] and the port. +pub fn arguments(port: u16) -> String { + format!("{FIX_ARG} {port}") +} + +/// Runs the fix: the elevated copy, then a fresh read of the firewall +/// (`recheck`), unless the user said No, so the notice shows what is true. +pub fn fix(elevate: &impl Elevate, exe: &Path, port: u16, recheck: impl FnOnce()) -> Outcome { + if exe.to_str().is_none_or(str::is_empty) { + return Outcome::Failed(FAILED); + } + let outcome = match elevate.run_elevated(exe, &arguments(port)) { + Elevated::Declined => return Outcome::Declined, + Elevated::Exited(0) => Outcome::Fixed, + Elevated::Exited(code) if code == Failure::Add.code() => Outcome::Failed(NOT_CHANGED), + Elevated::Exited(_) | Elevated::Failed(_) => Outcome::Failed(FAILED), + }; + recheck(); + outcome +} + +/// The errors the Stream page shows when the fix didn't work. +pub const FAILED: &str = + "Windows didn't apply the whole fix. Run the EternalMonitor installer again."; +pub const NOT_CHANGED: &str = "Windows didn't let EternalMonitor add its firewall rule, so nothing was changed. Run the EternalMonitor installer again."; + +/// Where the Stream page's "Fix it" is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Progress { + #[default] + Idle, + /// The Windows prompt or the fix is running. + Running, + Done(Outcome), +} + +static PROGRESS: std::sync::Mutex = std::sync::Mutex::new(Progress::Idle); + +pub fn progress() -> Progress { + PROGRESS.lock().map_or(Progress::Idle, |progress| *progress) +} + +/// Forgets how the last fix went: the user dismissed its message. +pub fn dismiss() { + if let Ok(mut progress) = PROGRESS.lock() { + if matches!(*progress, Progress::Done(_)) { + *progress = Progress::Idle; + } + } +} + +/// Runs `fix` off the window's thread, once at a time: the Windows prompt +/// waits for the user, and the fix for Windows. +pub fn start(fix: impl FnOnce() -> Outcome + Send + 'static) { + { + let Ok(mut progress) = PROGRESS.lock() else { + return; + }; + if *progress == Progress::Running { + return; + } + *progress = Progress::Running; + } + let spawned = std::thread::Builder::new() + .name("firewall-fix".into()) + .spawn(move || { + let outcome = fix(); + tracing::info!(?outcome, "Firewall fix finished"); + if let Ok(mut progress) = PROGRESS.lock() { + *progress = Progress::Done(outcome); + } + }); + if spawned.is_err() { + if let Ok(mut progress) = PROGRESS.lock() { + *progress = Progress::Done(Outcome::Failed(FAILED)); + } + } +} + +/// The stream's port the last fix was for, for Try again. +static LAST_PORT: std::sync::atomic::AtomicU16 = std::sync::atomic::AtomicU16::new(0); + +/// Starts the fix for the stream on `port` ([`fix_now`]). +pub fn start_for(port: u16) { + LAST_PORT.store(port, std::sync::atomic::Ordering::Relaxed); + start(move || fix_now(port)); +} + +/// The last fix's port, if one was started. +pub fn last_port() -> Option { + Some(LAST_PORT.load(std::sync::atomic::Ordering::Relaxed)).filter(|port| *port != 0) +} + +/// The elevated copy's arguments after [`FIX_ARG`]: the port, a number. +pub fn parse_port(args: &[String]) -> Option { + match args { + [port] => port.parse().ok().filter(|port| *port != 0), + _ => None, + } +} + +/// If this process is the elevated copy ([`FIX_ARG`]), runs the fix and +/// exits with its code. Called first thing in `main`. +pub fn run_if_requested() { + let args: Vec = std::env::args().skip(1).collect(); + if args.first().map(String::as_str) != Some(FIX_ARG) { + return; + } + let code = match (std::env::current_exe(), parse_port(&args[1..])) { + (Ok(exe), Some(port)) => windows_firewall::fix(&exe, port), + _ => Failure::Arguments.code(), + }; + std::process::exit(code as i32); +} + +/// The fix as the Stream page runs it, on Windows. +#[cfg(windows)] +pub fn fix_now(port: u16) -> Outcome { + let Ok(exe) = std::env::current_exe() else { + return Outcome::Failed(FAILED); + }; + fix( + &windows_elevate::Shell, + &exe, + port, + crate::firewall::recheck_now, + ) +} + +/// Off Windows there is no firewall notice, so no fix. +#[cfg(not(windows))] +pub fn fix_now(_port: u16) -> Outcome { + Outcome::Failed(FAILED) +} + +#[cfg(windows)] +mod windows_firewall { + use std::path::Path; + + use windows::core::BSTR; + use windows::Win32::Foundation::{E_FAIL, VARIANT_TRUE}; + use windows::Win32::NetworkManagement::WindowsFirewall::{ + INetFwPolicy2, INetFwRule, NetFwPolicy2, NetFwRule, NET_FW_ACTION_ALLOW, NET_FW_RULE_DIR_IN, + }; + use windows::Win32::System::Com::{CoCreateGuid, CoCreateInstance, CLSCTX_INPROC_SERVER}; + + use super::{ + fix_on_this_machine, in_apartment, Failure, Found, RuleStore, ALLOW_NAME, ALLOW_PROFILES, + }; + use crate::firewall::windows_policy::{current_profiles, for_each_rule, Apartment}; + use crate::firewall::{rule_from, Profile}; + + /// UDP, as `NET_FW_IP_PROTOCOL`. + const UDP: i32 = 17; + + /// The policy, and the rules its last read found, by id. + struct Com { + policy: INetFwPolicy2, + found: Vec, + } + + impl RuleStore for Com { + fn profiles(&mut self) -> Result, String> { + current_profiles(&self.policy).map_err(|e| e.to_string()) + } + + fn rules(&mut self) -> Result, String> { + let mut found = Vec::new(); + let mut rules = Vec::new(); + for_each_rule(&self.policy, |rule| { + let read = rule_from(rule).map_err(|e| windows::core::Error::new(E_FAIL, e))?; + found.push(Found { + id: rules.len(), + rule: read, + }); + rules.push(rule.clone()); + Ok(()) + }) + .map_err(|e| e.to_string())?; + self.found = rules; + Ok(found) + } + + fn add_allow(&mut self, exe: &str) -> Result<(), String> { + let add = || -> windows::core::Result<()> { + let rule: INetFwRule = + unsafe { CoCreateInstance(&NetFwRule, None, CLSCTX_INPROC_SERVER)? }; + unsafe { + rule.SetName(&BSTR::from(ALLOW_NAME))?; + rule.SetApplicationName(&BSTR::from(exe))?; + rule.SetProtocol(UDP)?; + rule.SetDirection(NET_FW_RULE_DIR_IN)?; + rule.SetAction(NET_FW_ACTION_ALLOW)?; + rule.SetProfiles(ALLOW_PROFILES as i32)?; + rule.SetEnabled(VARIANT_TRUE)?; + self.policy.Rules()?.Add(&rule) + } + }; + add().map_err(|e| e.to_string()) + } + + /// Renames the rule to a fresh name first, so removing by name takes + /// it alone: not a rule that shares its name, whatever the case, nor + /// one added since the read. A failed remove leaves a renamed Block + /// rule, no worse than before. + fn remove(&mut self, id: usize) -> Result<(), String> { + let rule = self.found.get(id).ok_or("no such rule")?; + let remove = || -> windows::core::Result<()> { + let guid = unsafe { CoCreateGuid()? }; + let name = BSTR::from(format!("EternalMonitor fix {guid:?}")); + unsafe { + rule.SetName(&name)?; + self.policy.Rules()?.Remove(&name) + } + }; + remove().map_err(|e| e.to_string()) + } + } + + /// The elevated copy's work: the exit code. + pub fn fix(exe: &Path, port: u16) -> u32 { + in_apartment( + || Apartment::new().map_err(|_| Failure::Read), + || { + let policy: INetFwPolicy2 = + unsafe { CoCreateInstance(&NetFwPolicy2, None, CLSCTX_INPROC_SERVER) } + .map_err(|_| Failure::Read)?; + Ok(Com { + policy, + found: Vec::new(), + }) + }, + |store| fix_on_this_machine(store, exe, port), + ) + .map_or_else(Failure::code, |()| 0) + } +} + +#[cfg(not(windows))] +mod windows_firewall { + pub fn fix(_exe: &std::path::Path, _port: u16) -> u32 { + super::Failure::Read.code() + } +} + +#[cfg(windows)] +mod windows_elevate { + use std::os::windows::ffi::OsStrExt; + use std::path::Path; + + use windows::core::PCWSTR; + use windows::Win32::Foundation::{CloseHandle, ERROR_CANCELLED, WAIT_OBJECT_0, WAIT_TIMEOUT}; + use windows::Win32::System::Com::{ + CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED, COINIT_DISABLE_OLE1DDE, + }; + use windows::Win32::System::Threading::{GetExitCodeProcess, WaitForSingleObject}; + use windows::Win32::UI::Shell::{ + ShellExecuteExW, SEE_MASK_NOASYNC, SEE_MASK_NOCLOSEPROCESS, SHELLEXECUTEINFOW, + }; + use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, SW_HIDE}; + + use super::{Elevate, Elevated}; + + /// How long to wait between looks at a copy that hasn't finished. + const WAIT_MS: u32 = 60_000; + + pub struct Shell; + + fn wide(text: &std::ffi::OsStr) -> Vec { + text.encode_wide().chain(Some(0)).collect() + } + + /// The single-threaded apartment ShellExecuteEx expects, for this call. + struct Sta(bool); + + impl Drop for Sta { + fn drop(&mut self) { + if self.0 { + unsafe { CoUninitialize() }; + } + } + } + + impl Elevate for Shell { + fn run_elevated(&self, exe: &Path, args: &str) -> Elevated { + let _sta = Sta(unsafe { + CoInitializeEx(None, COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE).is_ok() + }); + let verb = wide("runas".as_ref()); + let file = wide(exe.as_os_str()); + let params = wide(args.as_ref()); + let mut info = SHELLEXECUTEINFOW { + cbSize: std::mem::size_of::() as u32, + // No message loop here, and the thread blocks below: the + // call must finish before it returns. + fMask: SEE_MASK_NOCLOSEPROCESS | SEE_MASK_NOASYNC, + // The window just clicked: the Windows prompt belongs to it, + // not to whatever is behind. + hwnd: unsafe { GetForegroundWindow() }, + lpVerb: PCWSTR(verb.as_ptr()), + lpFile: PCWSTR(file.as_ptr()), + lpParameters: PCWSTR(params.as_ptr()), + nShow: SW_HIDE.0, + ..Default::default() + }; + if let Err(error) = unsafe { ShellExecuteExW(&mut info) } { + return if error.code() == ERROR_CANCELLED.to_hresult() { + Elevated::Declined + } else { + Elevated::Failed(error.to_string()) + }; + } + if info.hProcess.is_invalid() { + return Elevated::Failed("no process".into()); + } + // Until it exits, however long: the page stays on "running", so + // Try again can't start a second copy. + let elevated = loop { + match unsafe { WaitForSingleObject(info.hProcess, WAIT_MS) } { + WAIT_OBJECT_0 => { + let mut code = 0u32; + break match unsafe { GetExitCodeProcess(info.hProcess, &mut code) } { + Ok(()) => Elevated::Exited(code), + Err(error) => Elevated::Failed(error.to_string()), + }; + } + WAIT_TIMEOUT => { + tracing::warn!("The firewall fix is taking a while; still waiting for it"); + } + _ => break Elevated::Failed("lost track of the fix".into()), + } + }; + unsafe { + let _ = CloseHandle(info.hProcess); + } + elevated + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::{Cell, RefCell}; + + const EXE: &str = r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"; + const PORT: u16 = 9876; + + fn exe() -> &'static Path { + Path::new(EXE) + } + + fn on(bit: u32) -> Profile { + Profile { + bit, + on: true, + block_all_inbound: false, + } + } + + /// What Windows adds when its prompt is dismissed. + fn prompt_block(profiles: u32) -> Rule { + Rule::unscoped(&EXE.to_lowercase(), false, profiles) + } + + fn installer_allow() -> Rule { + Rule::unscoped(EXE, true, ALLOW_PROFILES) + } + + /// The machine's own values. + fn machine(name: &str) -> Option { + name.eq_ignore_ascii_case("ProgramFiles") + .then(|| r"C:\Program Files".to_string()) + } + + /// A firewall in memory. Removes take the rule they name and no other. + struct Fake { + profiles: Vec, + rules: Vec<(String, Option)>, + add_fails: bool, + remove_fails: bool, + read_fails: bool, + calls: Vec, + } + + impl Fake { + fn new(profiles: &[Profile], rules: Vec<(&str, Rule)>) -> Self { + Self { + profiles: profiles.to_vec(), + rules: rules + .into_iter() + .map(|(name, rule)| (name.to_string(), Some(rule))) + .chain([("Outbound".to_string(), None)]) + .collect(), + add_fails: false, + remove_fails: false, + read_fails: false, + calls: Vec::new(), + } + } + + fn names(&self) -> Vec<&str> { + self.rules.iter().map(|(name, _)| name.as_str()).collect() + } + } + + impl RuleStore for Fake { + fn profiles(&mut self) -> Result, String> { + if self.read_fails { + return Err("read".into()); + } + Ok(self.profiles.clone()) + } + + fn rules(&mut self) -> Result, String> { + if self.read_fails { + return Err("read".into()); + } + Ok(self + .rules + .iter() + .enumerate() + .map(|(id, (_, rule))| Found { + id, + rule: rule.clone(), + }) + .collect()) + } + + fn add_allow(&mut self, exe: &str) -> Result<(), String> { + self.calls.push(format!("add {exe}")); + if self.add_fails { + return Err("add".into()); + } + self.rules.push(( + ALLOW_NAME.into(), + Some(Rule::unscoped(exe, true, ALLOW_PROFILES)), + )); + Ok(()) + } + + fn remove(&mut self, id: usize) -> Result<(), String> { + self.calls.push(format!("remove {}", self.rules[id].0)); + if self.remove_fails { + return Err("remove".into()); + } + self.rules.remove(id); + Ok(()) + } + } + + const PROMPT_UDP: &str = "UDP Query User{0-1-2-3-4}eternalmonitor-host.exe"; + + fn dismissed_prompt() -> Vec<(&'static str, Rule)> { + vec![ + (ALLOW_NAME, installer_allow()), + (PROMPT_UDP, prompt_block(PRIVATE | PUBLIC)), + ( + "TCP Query User{0-1-2-3-4}eternalmonitor-host.exe", + Rule { + udp: false, + ..prompt_block(PRIVATE) + }, + ), + ( + "Other", + Rule::unscoped(r"C:\Tools\Other.exe", false, PUBLIC), + ), + ] + } + + /// Windows' Block rule for the exe goes; its Allow rule, the TCP block + /// that doesn't reach the stream, other programs' rules and outbound + /// rules stay. + #[test] + fn the_fix_removes_only_this_exes_block_and_keeps_the_rest() { + let mut store = Fake::new(&[on(PUBLIC)], dismissed_prompt()); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); + assert_eq!( + store.names(), + [ + ALLOW_NAME, + "TCP Query User{0-1-2-3-4}eternalmonitor-host.exe", + "Other", + "Outbound" + ] + ); + assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")]); + } + + /// The user's own block for a network not in use was never counted, + /// and stays. + #[test] + fn a_block_on_a_network_not_in_use_is_left() { + let mut rules = dismissed_prompt(); + rules.push(("Block EternalMonitor on Public", prompt_block(PUBLIC))); + let mut store = Fake::new(&[on(PRIVATE)], rules); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); + assert!(store.names().contains(&"Block EternalMonitor on Public")); + assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")]); + } + + /// A network with Windows Firewall off, or one that blocks all incoming + /// connections, is the user's switch: a block there is left, and the + /// fix still works on the other networks. + #[test] + fn the_users_switches_are_left_as_they_are() { + let mut rules = dismissed_prompt(); + rules.push(("Block EternalMonitor on Domain", prompt_block(DOMAIN))); + let switches = [ + Profile { + bit: DOMAIN, + on: false, + block_all_inbound: false, + }, + Profile { + bit: PRIVATE, + on: true, + block_all_inbound: true, + }, + ]; + let mut store = Fake::new(&[switches[0], switches[1], on(PUBLIC)], rules); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); + assert!(store.names().contains(&"Block EternalMonitor on Domain")); + assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")]); + } + + /// No Allow rule at all (deleted, or never added): it comes first, as + /// the installer adds it, then the block goes. + #[test] + fn a_missing_allow_rule_is_added_before_anything_is_removed() { + let mut rules = dismissed_prompt(); + rules.remove(0); + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); + assert_eq!( + store.calls, + [format!("add {EXE}"), format!("remove {PROMPT_UDP}")] + ); + } + + /// The user's own Allow rule that already lets the stream in on the + /// networks in use, however narrow, means no add: the fix doesn't + /// widen what the user chose. + #[test] + fn the_users_own_allow_rule_means_no_add() { + let private_only = Rule::unscoped(EXE, true, PRIVATE); + let lan_only = Rule { + remote_addresses: Some("LocalSubnet".into()), + ..Rule::unscoped(EXE, true, PRIVATE) + }; + for allow in [private_only, lan_only] { + let mut store = Fake::new( + &[on(PRIVATE)], + vec![("Mine", allow.clone()), (PROMPT_UDP, prompt_block(PRIVATE))], + ); + assert_eq!( + fix_rules(&mut store, exe(), PORT, &machine), + Ok(()), + "{allow:?}" + ); + assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")], "{allow:?}"); + } + } + + /// The fix's Allow rule is the one the installer's [Run] section adds. + #[test] + fn the_allow_rule_is_the_installers() { + let iss = include_str!("../../installer/EternalMonitor.iss"); + let run = iss + .split("\n[Run]") + .nth(1) + .and_then(|section| section.split("\n[").next()) + .unwrap(); + let add = run + .lines() + .find(|line| line.contains("advfirewall firewall add rule")) + .expect("the installer adds its Allow rule"); + assert!(add.contains(&format!(r#"name=""{ALLOW_NAME}"""#)), "{add}"); + assert!( + add.contains(r#"program=""{app}\EternalMonitor-host.exe"""#), + "{add}" + ); + for part in [ + "dir=in", + "action=allow", + "protocol=UDP", + "profile=domain,private,public", + "enable=yes", + ] { + assert!(add.contains(part), "{part} in {add}"); + } + } + + /// Never worse than before: when the Allow rule can't be added, + /// nothing is removed, and the rules are exactly as they were. + #[test] + fn a_failed_add_leaves_the_rules_exactly_as_they_were() { + let mut rules = dismissed_prompt(); + rules.remove(0); + let mut store = Fake::new(&[on(PUBLIC)], rules); + let before = store.rules.clone(); + store.add_fails = true; + assert_eq!( + fix_rules(&mut store, exe(), PORT, &machine), + Err(Failure::Add) + ); + assert_eq!(store.rules, before); + assert_eq!(store.calls, [format!("add {EXE}")]); + } + + /// A Block rule that can't be removed is a failure, not a success, and + /// nothing that let traffic through went. + #[test] + fn a_failed_remove_is_a_failure() { + let mut store = Fake::new(&[on(PUBLIC)], dismissed_prompt()); + let before = store.rules.clone(); + store.remove_fails = true; + assert_eq!( + fix_rules(&mut store, exe(), PORT, &machine), + Err(Failure::Remove) + ); + assert_eq!(store.rules, before); + } + + /// A remove that reports success but leaves the block, or an add that + /// leaves no Allow rule, is caught when everything is read again. + #[test] + fn a_fix_that_did_not_take_is_a_failure() { + struct Stubborn(Fake); + impl RuleStore for Stubborn { + fn profiles(&mut self) -> Result, String> { + self.0.profiles() + } + fn rules(&mut self) -> Result, String> { + self.0.rules() + } + fn add_allow(&mut self, exe: &str) -> Result<(), String> { + self.0.add_allow(exe) + } + fn remove(&mut self, _id: usize) -> Result<(), String> { + Ok(()) + } + } + let mut store = Stubborn(Fake::new(&[on(PUBLIC)], dismissed_prompt())); + assert_eq!( + fix_rules(&mut store, exe(), PORT, &machine), + Err(Failure::StillBlocked) + ); + + struct NoAdd(Fake); + impl RuleStore for NoAdd { + fn profiles(&mut self) -> Result, String> { + self.0.profiles() + } + fn rules(&mut self) -> Result, String> { + self.0.rules() + } + fn add_allow(&mut self, _exe: &str) -> Result<(), String> { + Ok(()) + } + fn remove(&mut self, id: usize) -> Result<(), String> { + self.0.remove(id) + } + } + let mut rules = dismissed_prompt(); + rules.remove(0); + let mut store = NoAdd(Fake::new(&[on(PUBLIC)], rules)); + assert_eq!( + fix_rules(&mut store, exe(), PORT, &machine), + Err(Failure::StillBlocked) + ); + } + + /// A Windows prompt answered for one network type can leave an Allow + /// and a Block under one name: only the block goes. + #[test] + fn a_block_sharing_its_name_goes_alone() { + let mut rules = dismissed_prompt(); + rules.push(( + PROMPT_UDP, + Rule::unscoped(&EXE.to_lowercase(), true, PRIVATE), + )); + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); + let left: Vec<_> = store + .rules + .iter() + .filter(|(name, _)| name == PROMPT_UDP) + .collect(); + assert_eq!(left.len(), 1); + assert!(left[0].1.as_ref().unwrap().allow, "the Allow rule stays"); + } + + /// A scoped Block rule, a disabled one, or one for another port is the + /// user's own: left, as the verdict leaves it. + #[test] + fn blocks_the_verdict_does_not_count_are_left() { + let rules = vec![ + (ALLOW_NAME, installer_allow()), + ( + "Scoped", + Rule { + remote_addresses: Some("10.0.0.0/8".into()), + ..prompt_block(PUBLIC) + }, + ), + ( + "Off", + Rule { + enabled: false, + ..prompt_block(PUBLIC) + }, + ), + ( + "Port", + Rule { + local_ports: Some("80".into()), + ..prompt_block(PUBLIC) + }, + ), + ]; + let mut store = Fake::new(&[on(PUBLIC)], rules); + let before = store.rules.clone(); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); + assert_eq!(store.rules, before); + assert!(store.calls.is_empty()); + } + + /// A rule's `%name%` means what the machine says, not what this + /// process's environment says: a user can set their own value, and the + /// elevated copy inherits it. Another program's rule never goes because + /// of it. + #[test] + fn only_the_machines_values_make_a_rule_this_exes() { + const USERS_OWN: &str = "EM_FIREWALL_FIX_TEST_ROOT"; + // A value only this process has, set as a user would. + std::env::set_var(USERS_OWN, r"C:\Program Files"); + let installed = r"D:\Apps\EternalMonitor\EternalMonitor-host.exe"; + let rules = vec![ + (ALLOW_NAME, Rule::unscoped(installed, true, ALLOW_PROFILES)), + ( + "Program Files copy", + Rule::unscoped( + r"%ProgramFiles%\EternalMonitor\EternalMonitor-host.exe", + false, + PUBLIC, + ), + ), + ]; + // The process would say D:\Apps for ProgramFiles; the machine says + // C:\Program Files. + let mut store = Fake::new(&[on(PUBLIC)], rules.clone()); + assert_eq!( + fix_rules(&mut store, Path::new(installed), PORT, &machine), + Ok(()) + ); + assert!(store.calls.is_empty(), "{:?}", store.calls); + + // The same rules, for the copy the machine's value names: its + // block goes, and it gets its own Allow rule. + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); + assert_eq!( + store.calls, + [format!("add {EXE}"), "remove Program Files copy".into()] + ); + + // A variable only this process has, as the elevated copy runs it: + // the rule is never removed, and as it may be this exe's, the fix + // doesn't claim success. + let rules = vec![ + (ALLOW_NAME, installer_allow()), + ( + "Users own", + Rule::unscoped( + &format!(r"%{USERS_OWN}%\EternalMonitor\EternalMonitor-host.exe"), + false, + PUBLIC, + ), + ), + ]; + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!( + fix_on_this_machine(&mut store, exe(), PORT), + Err(Failure::StillBlocked) + ); + assert!(store.names().contains(&"Users own")); + assert!(!store.calls.iter().any(|call| call.starts_with("remove"))); + std::env::remove_var(USERS_OWN); + } + + /// A machine value naming another (`EMROOT=%SystemDrive%\Apps`) makes + /// the rule this exe's: it goes. One that can't be resolved, with this + /// exe's file name, stays, and the fix fails rather than claim success. + #[test] + fn a_block_through_nested_machine_values_goes_and_an_unresolved_one_fails_the_fix() { + use crate::firewall::{resolve, Stored}; + let stored = |name: &str| match name.to_lowercase().as_str() { + "systemdrive" => Some(Stored::Literal("C:".into())), + "emroot" => Some(Stored::Expand(r"%SystemDrive%\Apps".into())), + "userish" => Some(Stored::Expand(r"%USERPROFILE%\Apps".into())), + _ => None, + }; + let lookup = |name: &str| resolve(name, &stored); + let exe = Path::new(r"C:\Apps\EternalMonitor\EternalMonitor-host.exe"); + let allow = Rule::unscoped(&exe.to_string_lossy(), true, ALLOW_PROFILES); + let block = |root: &str| { + Rule::unscoped( + &format!(r"%{root}%\EternalMonitor\EternalMonitor-host.exe"), + false, + PUBLIC, + ) + }; + + let rules = vec![(ALLOW_NAME, allow.clone()), ("Nested", block("EMROOT"))]; + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!(fix_rules(&mut store, exe, PORT, &lookup), Ok(())); + assert_eq!(store.calls, ["remove Nested"]); + + let rules = vec![(ALLOW_NAME, allow), ("Unresolved", block("USERISH"))]; + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!( + fix_rules(&mut store, exe, PORT, &lookup), + Err(Failure::StillBlocked) + ); + assert!(store.names().contains(&"Unresolved")); + assert!(!store.calls.iter().any(|call| call.starts_with("remove"))); + } + + #[test] + fn rules_that_cant_be_read_are_a_failure() { + let mut store = Fake::new(&[on(PUBLIC)], dismissed_prompt()); + store.read_fails = true; + assert_eq!( + fix_rules(&mut store, exe(), PORT, &machine), + Err(Failure::Read) + ); + assert!(store.calls.is_empty()); + assert_eq!( + fix_rules(&mut store, Path::new(""), PORT, &machine), + Err(Failure::Arguments) + ); + } + + /// The store goes before the apartment, whatever the fix returns. + #[test] + fn the_store_is_released_before_the_apartment() { + struct Noted(&'static str, std::rc::Rc>>); + impl Drop for Noted { + fn drop(&mut self) { + self.1.borrow_mut().push(self.0); + } + } + for result in [Ok(()), Err(Failure::Remove)] { + let order = std::rc::Rc::new(RefCell::new(Vec::new())); + let got = in_apartment( + || Ok(Noted("apartment", order.clone())), + || Ok(Noted("store", order.clone())), + |_| result, + ); + assert_eq!(got, result); + assert_eq!(*order.borrow(), ["store", "apartment"]); + } + } + + /// The elevated copy gets the fixed argument and a port, nothing else. + #[test] + fn the_elevated_copy_takes_only_a_port() { + assert_eq!(arguments(PORT), "--fix-firewall 9876"); + assert_eq!(parse_port(&["9876".into()]), Some(9876)); + let bad: [&[&str]; 5] = [&[], &["0"], &["98765"], &["9876 x"], &["1", "2"]]; + for args in bad { + let args: Vec = args.iter().map(|arg| arg.to_string()).collect(); + assert_eq!(parse_port(&args), None, "{args:?}"); + } + } + + struct FakeElevate { + answer: Elevated, + asked: RefCell>, + } + + impl Elevate for FakeElevate { + fn run_elevated(&self, exe: &Path, args: &str) -> Elevated { + self.asked + .borrow_mut() + .push((exe.display().to_string(), args.to_string())); + self.answer.clone() + } + } + + fn elevate(answer: Elevated) -> FakeElevate { + FakeElevate { + answer, + asked: RefCell::new(Vec::new()), + } + } + + /// The elevated copy is this exe with the fixed arguments, and the + /// firewall is read again once it ran. + #[test] + fn the_fix_starts_this_exe_elevated_then_reads_the_firewall_again() { + let elevate = elevate(Elevated::Exited(0)); + let rechecked = Cell::new(false); + assert_eq!( + fix(&elevate, exe(), PORT, || rechecked.set(true)), + Outcome::Fixed + ); + assert!(rechecked.get()); + assert_eq!( + *elevate.asked.borrow(), + [(EXE.to_string(), "--fix-firewall 9876".to_string())] + ); + } + + /// No to the Windows prompt: nothing ran, nothing is read again, and no + /// error shows. + #[test] + fn declining_the_windows_prompt_changes_nothing() { + let rechecked = Cell::new(false); + assert_eq!( + fix(&elevate(Elevated::Declined), exe(), PORT, || rechecked + .set(true)), + Outcome::Declined + ); + assert!(!rechecked.get()); + } + + /// Each failure says what happened, and the firewall is read again in + /// case some of it ran. + #[test] + fn a_fix_that_fails_says_so() { + for (answer, said) in [ + (Elevated::Exited(Failure::Add.code()), NOT_CHANGED), + (Elevated::Exited(Failure::Remove.code()), FAILED), + (Elevated::Exited(Failure::StillBlocked.code()), FAILED), + (Elevated::Exited(1), FAILED), + (Elevated::Failed("x".into()), FAILED), + ] { + let rechecked = Cell::new(false); + assert_eq!( + fix(&elevate(answer.clone()), exe(), PORT, || rechecked + .set(true)), + Outcome::Failed(said), + "{answer:?}" + ); + assert!(rechecked.get(), "{answer:?}"); + } + let nobody = elevate(Elevated::Exited(0)); + assert_eq!( + fix(&nobody, Path::new(""), PORT, || {}), + Outcome::Failed(FAILED) + ); + assert!(nobody.asked.borrow().is_empty()); + } + + /// One fix at a time; the window hears how it went until dismissed. + #[test] + fn the_page_sees_the_fix_run_and_finish() { + let (go, wait) = std::sync::mpsc::channel::<()>(); + start(move || { + let _ = wait.recv(); + Outcome::Failed(FAILED) + }); + assert_eq!(progress(), Progress::Running); + start(|| panic!("a second fix started")); + dismiss(); + assert_eq!(progress(), Progress::Running, "a running fix stays"); + go.send(()).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while progress() == Progress::Running { + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert_eq!(progress(), Progress::Done(Outcome::Failed(FAILED))); + dismiss(); + assert_eq!(progress(), Progress::Idle); + } +} diff --git a/host/src/gpu.rs b/host/src/gpu.rs index bcc382d8..7558e3d3 100644 --- a/host/src/gpu.rs +++ b/host/src/gpu.rs @@ -4,6 +4,8 @@ use windows::Win32::Graphics::Dxgi::{ CreateDXGIFactory1, IDXGIAdapter1, IDXGIFactory1, DXGI_ERROR_NOT_FOUND, }; +use crate::stall::Call; + const VENDOR_NVIDIA: u32 = 0x10DE; const VENDOR_AMD: u32 = 0x1002; const VENDOR_INTEL: u32 = 0x8086; @@ -49,19 +51,24 @@ pub struct GpuInfo { pub dedicated_vram_mb: u64, pub encoder_name: String, pub codec_display_name: String, + /// Detection did not return within its limit: these are the software + /// facts, and the processor encodes only because of that. A later + /// pipeline takes [`take_late_detection`]'s facts if it has returned. + pub detection_stalled: bool, } impl GpuInfo { /// Detect the best GPU via DXGI and resolve the preferred encoder. - /// Falls back to software encoding if detection fails. + /// Falls back to software encoding if detection fails, or does not + /// return within its limit ([`Call::GpuDetect`]), so a stuck driver + /// cannot hold startup and the window. pub fn detect() -> Self { - match detect_from_dxgi() { - Ok(info) => info, - Err(e) => { - warn!(error = %e, "DXGI GPU detection failed, falling back to software encoding"); - Self::software_fallback() - } - } + detect_within( + crate::stall::lanes(), + Call::GpuDetect.limits(), + &FOUND, + || detect_from_dxgi().map_err(|e| e.to_string()), + ) } pub fn software_fallback() -> Self { @@ -72,7 +79,46 @@ impl GpuInfo { dedicated_vram_mb: 0, encoder_name: "libx264".to_string(), codec_display_name: "H.264 (x264)".to_string(), + detection_stalled: false, + } + } +} + +/// The graphics card the last detection found, kept when it returns, also +/// after startup gave up on it. +static FOUND: parking_lot::Mutex> = parking_lot::const_mutex(None); + +/// What a startup detection that was given up on found once it returned: +/// None until then, or if it found no graphics card. Taken once. +pub fn take_late_detection() -> Option { + FOUND.lock().take() +} + +/// `detect`'s result, run on a thread of its own in `lanes` under `limits`. +/// What it finds is also kept in `found`, so a detection given up on still +/// leaves its facts for later pipelines when it returns. The stall guard +/// logs the one line for a timeout. +fn detect_within( + lanes: &'static crate::stall::Lanes, + limits: crate::stall::Limits, + found: &'static parking_lot::Mutex>, + detect: impl FnOnce() -> Result + Send + 'static, +) -> GpuInfo { + let detect = move || { + let result = detect(); + *found.lock() = result.as_ref().ok().cloned(); + result + }; + match crate::stall::run_on(lanes, Call::GpuDetect, limits, detect, || {}) { + Ok(Ok(info)) => info, + Ok(Err(e)) => { + warn!(error = %e, "DXGI GPU detection failed, falling back to software encoding"); + GpuInfo::software_fallback() } + Err(_) => GpuInfo { + detection_stalled: true, + ..GpuInfo::software_fallback() + }, } } @@ -230,6 +276,7 @@ fn detect_from_dxgi() -> Result> { dedicated_vram_mb: vram_mb, encoder_name, codec_display_name, + detection_stalled: false, }) } @@ -324,6 +371,138 @@ mod tests { }) } + /// A 300 ms limit, for a detection that hangs. + const HELD: crate::stall::Limits = crate::stall::Limits { + warn: std::time::Duration::from_millis(100), + limit: std::time::Duration::from_millis(300), + }; + + /// Limits a detection that returns at once never comes near, however + /// slowly the machine runs. + const ROOMY: crate::stall::Limits = crate::stall::Limits { + warn: std::time::Duration::from_secs(1), + limit: std::time::Duration::from_secs(30), + }; + + /// `detect_within` over `detect` in lanes and a slot of its own, under + /// `limits`: what it found, how long it took, what it logged, the lanes + /// and the slot. + fn detect_held( + limits: crate::stall::Limits, + detect: impl FnOnce() -> Result + Send + 'static, + ) -> ( + GpuInfo, + std::time::Duration, + String, + &'static crate::stall::Lanes, + &'static parking_lot::Mutex>, + ) { + use std::sync::Arc; + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let found: &'static parking_lot::Mutex> = + Box::leak(Box::new(parking_lot::Mutex::new(None))); + let out = Arc::new(parking_lot::Mutex::new(Vec::::new())); + struct Sink(Arc>>); + impl std::io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let writer = Arc::clone(&out); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || Sink(Arc::clone(&writer))) + .finish(); + crate::logging::prepare_log_capture(); + let started = std::time::Instant::now(); + let info = tracing::subscriber::with_default(subscriber, || { + detect_within(lanes, limits, found, detect) + }); + let took = started.elapsed(); + let text = String::from_utf8(out.lock().clone()).unwrap(); + (info, took, text, lanes, found) + } + + fn found(name: &str) -> GpuInfo { + GpuInfo { + vendor: GpuVendor::Nvidia, + name: name.to_string(), + encoder_name: "h264_nvenc".to_string(), + codec_display_name: "H.264 (NVENC)".to_string(), + ..GpuInfo::software_fallback() + } + } + + #[test] + fn a_stuck_detection_starts_at_its_limit_and_leaves_its_late_facts() { + // The detector waits on a driver that does not return until the + // test lets it. + let (release, held) = std::sync::mpsc::channel::<()>(); + let (started, detecting) = std::sync::mpsc::channel::<()>(); + let (info, took, text, lanes, slot) = detect_held(HELD, move || { + let _ = started.send(()); + let _ = held.recv(); + Ok(found("late")) + }); + // The software facts at the limit, marked as a stall so the + // processor's encoding is explained. + assert_eq!(info.name, "Software"); + assert_eq!(info.encoder_name, "libx264"); + assert_eq!(info.vendor, GpuVendor::Unknown); + assert!(info.detection_stalled); + // Never before the limit; that it returned at all while the detector + // still blocks shows it did not wait for it. + assert!(took >= HELD.limit, "{took:?}"); + // One WARN, saying detection timed out and after how long. + let warns: Vec<_> = text.lines().filter(|l| l.contains("WARN")).collect(); + assert_eq!(warns.len(), 1, "{text}"); + assert!(warns[0].contains("stall=gpu_detect after_ms="), "{text}"); + assert!(warns[0].contains("limit_ms=300"), "{text}"); + // Until it returns, the lane holds it and nothing was found. + assert_eq!(lanes.abandoned(Call::GpuDetect), 1); + assert!(slot.lock().is_none()); + detecting.recv().unwrap(); + release.send(()).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while lanes.abandoned(Call::GpuDetect) != 0 { + assert!( + std::time::Instant::now() < deadline, + "the lane did not free" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + // What it found once it returned is there for later pipelines. + let late = slot.lock().take().expect("the late facts were not kept"); + assert_eq!( + (late.name.as_str(), late.encoder_name.as_str()), + ("late", "h264_nvenc") + ); + assert!(!late.detection_stalled); + } + + #[test] + fn a_detection_that_returns_in_time_keeps_its_facts() { + let (info, _, text, _, _) = detect_held(ROOMY, || Ok(found("RTX"))); + assert_eq!( + (info.name.as_str(), info.encoder_name.as_str()), + ("RTX", "h264_nvenc") + ); + assert!(!info.detection_stalled); + assert!(!text.contains("WARN"), "{text}"); + // A failed one falls back with its own line, as before, and is not + // a stall: no notice blames a driver that answered. + let (info, _, text, _, slot) = detect_held(ROOMY, || Err("no adapters".to_string())); + assert_eq!(info.encoder_name, "libx264"); + assert!(!info.detection_stalled); + assert!(slot.lock().is_none()); + assert!(text.contains("DXGI GPU detection failed"), "{text}"); + assert!(!text.contains("stall="), "{text}"); + } + #[test] fn every_adapter_is_listed_or_there_is_no_list() { use GpuVendor::{Amd, Intel, Nvidia, Unknown}; diff --git a/host/src/gpu_other.rs b/host/src/gpu_other.rs index 82d009d2..a11f1c91 100644 --- a/host/src/gpu_other.rs +++ b/host/src/gpu_other.rs @@ -33,6 +33,8 @@ pub struct GpuInfo { pub dedicated_vram_mb: u64, pub encoder_name: String, pub codec_display_name: String, + /// Always false: detection here only asks FFmpeg. + pub detection_stalled: bool, } impl GpuInfo { @@ -53,6 +55,7 @@ impl GpuInfo { dedicated_vram_mb: 0, encoder_name: name.to_string(), codec_display_name: display_name.to_string(), + detection_stalled: false, }; } info!(encoder = name, "Encoder not available, trying next"); @@ -70,10 +73,16 @@ impl GpuInfo { dedicated_vram_mb: 0, encoder_name: "libx264".to_string(), codec_display_name: "H.264 (x264)".to_string(), + detection_stalled: false, } } } +/// Detection here never stalls, so there is never a late one. +pub fn take_late_detection() -> Option { + None +} + /// No NVIDIA drivers to read outside Windows. pub fn nvidia_driver_versions() -> Vec { Vec::new() diff --git a/host/src/gpu_preference.rs b/host/src/gpu_preference.rs index 26c5b950..43f12221 100644 --- a/host/src/gpu_preference.rs +++ b/host/src/gpu_preference.rs @@ -1439,6 +1439,7 @@ mod tests { .with_writer(move || Sink(Arc::clone(&writer))) .finish(); let mut store = key(None); + crate::logging::prepare_log_capture(); let error = tracing::subscriber::with_default(subscriber, || { switch( &mut store, diff --git a/host/src/gpu_wake.rs b/host/src/gpu_wake.rs new file mode 100644 index 00000000..5828f715 --- /dev/null +++ b/host/src/gpu_wake.rs @@ -0,0 +1,285 @@ +//! Wakes the encoder's graphics card when an iPad is probably about to +//! connect. A laptop's discrete card powers off entirely (D3cold) about 20 s +//! after its last use: its PCIe link and the ports above it go down too. On +//! an AMD hybrid laptop the next D3D11 device or AMF open on it then took +//! about 800 ms instead of 160, all of it inside the first session's first +//! picture. An iPad looks for PCs over mDNS on its connect screen, seconds +//! before it sends HELLO, so its query (see `discovery::watch_browsers`) +//! starts the card waking: one tiny D3D11 device on it, dropped at once. +//! Nothing is kept open, and nothing wakes it on a timer. The first AMF open +//! then took about 280 ms instead of 830; holding the device for 10 s +//! changed nothing measurable, so it isn't held. + +use std::time::{Duration, Instant}; + +/// The shortest time between two wakes. A card woken this recently is still +/// in D3hot at worst, which wakes in about 160 ms. +pub const WAKE_EVERY: Duration = Duration::from_secs(15); +/// The longest time between two wakes after many wasted ones. +const WAKE_EVERY_AT_MOST: Duration = Duration::from_secs(600); + +/// When a browse may wake the card. Devices browse without connecting too +/// (an iPad left on its connect screen, another app's scan), so each wake no +/// session followed doubles the time to the next, up to +/// [`WAKE_EVERY_AT_MOST`]: a few wakes an hour at most, never a keep-alive. +/// A session resets it. +#[derive(Debug)] +pub struct WakeGate { + last: Option, + every: Duration, + /// `Session::sessions_started` at the last wake. + sessions_at_wake: Option, +} + +impl Default for WakeGate { + fn default() -> Self { + Self { + last: None, + every: WAKE_EVERY, + sessions_at_wake: None, + } + } +} + +impl WakeGate { + /// A device browsed for the service at `now`. Whether to wake the card: + /// never while a viewer is connected (`in_session`, the card is in use). + /// `sessions` is `Session::sessions_started`: a session since the last + /// wake means the wake paid off. + pub fn browse(&mut self, now: Instant, in_session: bool, sessions: u32) -> bool { + if in_session { + return false; + } + // A session since the last wake restores the short wait first: after + // a long backoff, the next viewer's browse mustn't wait it out. + let paid = self.sessions_at_wake.is_none_or(|at| at != sessions); + if paid { + self.every = WAKE_EVERY; + } + if let Some(last) = self.last { + if now.saturating_duration_since(last) < self.every { + return false; + } + } + if !paid { + self.every = (self.every * 2).min(WAKE_EVERY_AT_MOST); + } + self.sessions_at_wake = Some(sessions); + self.last = Some(now); + true + } +} + +/// The adapter to wake for `opened`, the encoder that opened in a pipeline +/// using `gpu`: the card detection chose, when `opened` is a hardware encoder +/// of its family (its HEVC sibling too). None for a CPU encoder, or for a +/// fallback to another card's encoder (NVENC or AMF failing over to Quick +/// Sync): the card detection chose is then unused, and waking it costs power +/// for nothing. +pub fn wake_adapter_for(opened: &str, gpu: &crate::gpu::GpuInfo) -> Option { + let family = |name: &str| { + name.strip_prefix("h264_") + .or_else(|| name.strip_prefix("hevc_")) + .map(str::to_owned) + }; + (family(opened).is_some() && family(opened) == family(&gpu.encoder_name)) + .then_some(gpu.adapter_index) +} + +/// Runs `touch` unless `still_wanted`, checked just before, says a session +/// has started since the wake was decided: the card is then the session's, +/// and a device of the wake's during encoder or capture startup would only +/// compete with it. Returns whether it ran. +#[cfg(any(windows, test))] +fn run_wake(still_wanted: impl FnOnce() -> bool, touch: impl FnOnce()) -> bool { + if !still_wanted() { + return false; + } + touch(); + true +} + +/// Starts waking the graphics card at DXGI adapter `adapter_index`, on a +/// thread of its own, unless a wake is still running. The thread drops it +/// if `still_wanted` returns false when it starts (see [`run_wake`]). +#[cfg(windows)] +pub fn wake(adapter_index: u32, still_wanted: impl FnOnce() -> bool + Send + 'static) { + use std::sync::atomic::{AtomicBool, Ordering}; + static RUNNING: AtomicBool = AtomicBool::new(false); + if RUNNING.swap(true, Ordering::SeqCst) { + return; + } + let spawned = std::thread::Builder::new() + .name("gpu-wake".into()) + .spawn(move || { + let started = Instant::now(); + let ran = run_wake(still_wanted, || match touch(adapter_index) { + Ok(()) => tracing::debug!( + adapter_index, + took_ms = started.elapsed().as_millis() as u64, + "Woke the encoder's graphics card for a viewer that may connect" + ), + Err(error) => { + tracing::debug!(adapter_index, %error, "Could not wake the graphics card") + } + }); + if !ran { + tracing::debug!( + adapter_index, + "A session started first: not waking the graphics card" + ); + } + RUNNING.store(false, Ordering::SeqCst); + }); + if spawned.is_err() { + RUNNING.store(false, Ordering::SeqCst); + } +} + +#[cfg(not(windows))] +pub fn wake(_adapter_index: u32, _still_wanted: impl FnOnce() -> bool + Send + 'static) {} + +/// A D3D11 device on the adapter, dropped at once: enough to power it up. +#[cfg(windows)] +fn touch(adapter_index: u32) -> windows::core::Result<()> { + use windows::core::Interface; + use windows::Win32::Graphics::Direct3D::D3D_DRIVER_TYPE_UNKNOWN; + use windows::Win32::Graphics::Direct3D11::{D3D11CreateDevice, D3D11_SDK_VERSION}; + use windows::Win32::Graphics::Dxgi::{CreateDXGIFactory1, IDXGIAdapter, IDXGIFactory1}; + let factory: IDXGIFactory1 = unsafe { CreateDXGIFactory1() }?; + let adapter: IDXGIAdapter = unsafe { factory.EnumAdapters1(adapter_index) }?.cast()?; + let mut device = None; + unsafe { + D3D11CreateDevice( + &adapter, + D3D_DRIVER_TYPE_UNKNOWN, + None, + Default::default(), + None, + D3D11_SDK_VERSION, + Some(&mut device), + None, + None, + ) + }?; + drop(device); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at(start: Instant, s: u64) -> Instant { + start + Duration::from_secs(s) + } + + #[test] + fn a_browse_wakes_the_card_at_most_once_per_interval_and_never_in_a_session() { + let start = Instant::now(); + let mut gate = WakeGate::default(); + assert!(gate.browse(at(start, 0), false, 7)); + assert!( + !gate.browse(at(start, 5), false, 7), + "a scan's burst of queries" + ); + assert!(!gate.browse(at(start, 14), false, 7)); + assert!(!gate.browse(at(start, 40), true, 8), "the card is in use"); + // The iPad connected and left: the wake paid off. + assert!(gate.browse(at(start, 60), false, 8)); + assert!(gate.browse(at(start, 75), false, 9)); + } + + fn gpu(encoder: &str, adapter_index: u32) -> crate::gpu::GpuInfo { + crate::gpu::GpuInfo { + encoder_name: encoder.to_string(), + adapter_index, + ..crate::gpu::GpuInfo::software_fallback() + } + } + + #[test] + fn only_the_encoder_that_streams_on_the_detected_card_wakes_it() { + let nvenc = gpu("h264_nvenc", 1); + assert_eq!(wake_adapter_for("h264_nvenc", &nvenc), Some(1)); + assert_eq!( + wake_adapter_for("hevc_nvenc", &nvenc), + Some(1), + "its HEVC sibling" + ); + assert_eq!( + wake_adapter_for("h264_qsv", &nvenc), + None, + "a Quick Sync fallback" + ); + assert_eq!(wake_adapter_for("h264_amf", &nvenc), None, "another card"); + assert_eq!(wake_adapter_for("libx264", &nvenc), None, "the CPU"); + assert_eq!(wake_adapter_for("libx264", &gpu("libx264", 0)), None); + } + + #[test] + fn a_wake_is_dropped_when_a_session_started_first() { + let touched = std::cell::Cell::new(false); + assert!(!run_wake(|| false, || touched.set(true))); + assert!( + !touched.get(), + "the card was touched during the session's startup" + ); + assert!(run_wake(|| true, || touched.set(true))); + assert!(touched.get()); + } + + #[test] + fn a_session_after_a_long_backoff_restores_the_short_interval_at_once() { + let start = Instant::now(); + let mut gate = WakeGate::default(); + // Browses with nobody connecting, until the wait is at its longest. + let mut last = 0; + for s in (0..7200).step_by(20) { + if gate.browse(at(start, s), false, 7) { + last = s; + } + } + assert_eq!(gate.every, WAKE_EVERY_AT_MOST); + // An iPad connects, leaves, and browses again 20 s after that wake. + assert!( + gate.browse(at(start, last + 20), false, 8), + "the session's browse waited out the old ten-minute wait" + ); + assert!( + !gate.browse(at(start, last + 30), false, 8), + "the 15 s minimum" + ); + } + + #[test] + fn wakes_no_session_followed_come_ever_less_often() { + let start = Instant::now(); + let mut gate = WakeGate::default(); + // A device that browses every 20 s and never connects. + let mut wakes = Vec::new(); + for s in (0..3600).step_by(20) { + if gate.browse(at(start, s), false, 7) { + wakes.push(s); + } + } + let gaps: Vec = wakes.windows(2).map(|pair| pair[1] - pair[0]).collect(); + assert!(gaps.windows(2).all(|pair| pair[0] <= pair[1]), "{gaps:?}"); + // Then one every ten minutes: six an hour. + let settled = &gaps[gaps.len() - 3..]; + assert!( + settled + .iter() + .all(|&gap| gap >= WAKE_EVERY_AT_MOST.as_secs()), + "{gaps:?}" + ); + assert!( + wakes.len() <= 12, + "{} wakes in the first hour: {wakes:?}", + wakes.len() + ); + // An iPad connects: back to the short interval. + assert!(gate.browse(at(start, 4000), false, 8)); + assert!(gate.browse(at(start, 4015), false, 9)); + } +} diff --git a/host/src/gui/driver_wait.rs b/host/src/gui/driver_wait.rs new file mode 100644 index 00000000..9ae0900b --- /dev/null +++ b/host/src/gui/driver_wait.rs @@ -0,0 +1,1043 @@ +//! The window waits for a graphics driver that answers, and says so. +//! +//! The window's thread lists adapters, creates its device and draws through +//! the graphics driver, and Windows cannot stop a thread stuck inside it. +//! So before that thread makes its first graphics call ([`before_window`]): +//! - the graphics cards are listed through DXGI on a thread of their own, +//! in the startup graphics lane ([`Call::GpuDetect`]), once nothing else +//! is stuck there. The renderer's crash avoidance keys its history by that +//! list, so it is configured only with a list that returned; +//! - then the same wgpu calls the window's thread makes first are made once +//! on a thread of their own ([`Call::GpuProbe`]). +//! +//! Each is one call, waited for as long as it runs: past its limit the +//! notice goes up, and a driver that answers late still opens the window. +//! +//! Past that, the window's thread can still stall in calls that cannot be +//! moved off it (its own device and swap chain, a surface it recreates), so +//! [`watch_first_paint`] watches from another thread until the window is +//! confirmed drawing. +//! +//! Whenever the window waits on the driver, a notice drawn by Windows +//! without the graphics card says why ([`Prompt`]). OK hides it for a +//! while; Cancel quits. It goes for good when the wait is over. +// Only Windows waits; the logic is tested everywhere. +#![cfg_attr(not(windows), allow(dead_code))] + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use parking_lot::Mutex; +use tracing::{info, warn}; + +use crate::stall::{Call, Lanes, Limits}; + +/// How often a waiting launch looks whether the stuck call has returned. +pub(super) const POLL: Duration = Duration::from_millis(250); + +/// How long OK hides the notice while the wait goes on. +pub(super) const REMIND_AFTER: Duration = Duration::from_secs(15); + +/// How long the window may run UI passes without being confirmed drawing. +/// A working one is confirmed about 3.5 s after its first pass (3 s and 10 +/// frames of drawing, then the screenshot that proves a present); one whose +/// paints return without a surface keeps passing and never is. +pub(super) const PROOF_WITHIN: Duration = Duration::from_secs(12); + +/// How the user closed the notice. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Answer { + /// OK: keep waiting. + Wait, + /// Cancel: quit EternalMonitor. + Quit, +} + +/// What the user sees while the window waits. +pub(super) trait Notice: Send { + fn show(&mut self); + /// How the user closed the shown notice, once. + fn answer(&mut self) -> Option; + /// Takes the notice down, if it is up. + fn close(&mut self); +} + +/// Where the wait stands. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Phase { + Waiting, + /// The wait is over and the user did not quit. + Over, + /// The user chose Cancel, also one that came as the wait ended. + Quit, +} + +/// The notice while startup waits on the driver, shared by the steps on +/// the window's thread, the first-paint watcher and the renderer: up from +/// the first wait, back [`REMIND_AFTER`] after OK, down for good once the +/// wait is over. Every change happens under one lock, so the end of the +/// wait, the user's answer and showing the notice never cross. +#[derive(Clone)] +pub(super) struct Prompt(Arc>); + +struct PromptState { + notice: Box, + up: bool, + raised: bool, + hidden_at: Option, + phase: Phase, +} + +impl Prompt { + pub(super) fn new(notice: impl Notice + 'static) -> Self { + Self(Arc::new(Mutex::new(PromptState { + notice: Box::new(notice), + up: false, + raised: false, + hidden_at: None, + phase: Phase::Waiting, + }))) + } + + /// Show the notice at `now`, unless it is up, OK hid it less than + /// [`REMIND_AFTER`] ago, or the wait is over. + pub(super) fn raise(&self, now: Instant) { + let mut state = self.0.lock(); + state.read_answer(now); + if state.up || state.phase != Phase::Waiting { + return; + } + if state + .hidden_at + .is_some_and(|at| now.saturating_duration_since(at) < REMIND_AFTER) + { + return; + } + if !state.raised { + warn!("The graphics driver is not answering; the window opens once it does"); + } + state.notice.show(); + state.up = true; + state.raised = true; + state.hidden_at = None; + } + + /// Where the wait stands at `now`, with the user's answer read. + pub(super) fn phase(&self, now: Instant) -> Phase { + let mut state = self.0.lock(); + state.read_answer(now); + state.phase + } + + /// The notice was ever shown. + pub(super) fn raised(&self) -> bool { + self.0.lock().raised + } + + /// The wait is over: the answer given by now is kept, and the notice + /// goes down and stays down. Where it stands after. + pub(super) fn finish(&self) -> Phase { + let mut state = self.0.lock(); + state.read_answer(Instant::now()); + if state.phase == Phase::Waiting { + state.phase = Phase::Over; + } + state.up = false; + state.notice.close(); + state.phase + } +} + +impl PromptState { + fn read_answer(&mut self, now: Instant) { + if !self.up { + return; + } + match self.notice.answer() { + Some(Answer::Wait) => { + self.up = false; + self.hidden_at = Some(now); + } + Some(Answer::Quit) => { + self.up = false; + if self.phase == Phase::Waiting { + self.phase = Phase::Quit; + } + } + None => {} + } + } +} + +/// A limit no startup call reaches: it is waited for as long as it runs. +const NEVER: Duration = Duration::from_secs(365 * 24 * 60 * 60); + +/// Starts a call from `make` in `call`'s lane once none of `free` is stuck, +/// and waits for it as long as it runs, with the notice up while a lane is +/// stuck or once the call has run for `limits`' limit. Its result, also a +/// late one; None when the user quit, also by a Cancel that came as the +/// driver answered. +fn await_call( + lanes: &'static Lanes, + call: Call, + limits: Limits, + free: &[Call], + mut make: impl FnMut() -> F, + prompt: &Prompt, + poll: Duration, +) -> Option +where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, +{ + let started = Instant::now(); + let unlimited = Limits { + warn: limits.warn, + limit: NEVER, + }; + let quit = |prompt: &Prompt| { + let quit = prompt.phase(Instant::now()) == Phase::Quit; + if quit { + info!( + waited_ms = started.elapsed().as_millis() as u64, + "Quit while the window waited for the graphics driver" + ); + } + quit + }; + let mut pending = loop { + if free.iter().all(|&stuck| lanes.abandoned(stuck) == 0) { + if let Ok(pending) = crate::stall::start_on(lanes, call, unlimited, make()) { + break pending; + } + } + prompt.raise(Instant::now()); + if quit(prompt) { + return None; + } + std::thread::sleep(poll); + }; + let mut up_to = limits.limit; + loop { + match pending.wait_up_to(up_to, &mut || {}) { + Some(Ok(value)) => { + if quit(prompt) { + return None; + } + if prompt.raised() { + info!( + call = call.name(), + waited_ms = started.elapsed().as_millis() as u64, + "The graphics driver answered" + ); + } + return Some(value); + } + // Only past a limit no call reaches. + Some(Err(_)) => return None, + None => { + prompt.raise(Instant::now()); + // Dropped, a call still in the driver keeps its lane taken + // until it returns. + if quit(prompt) { + return None; + } + up_to += poll; + } + } + } +} + +/// The steps before the window's thread makes its first graphics call: +/// a list from `list`, in [`Call::GpuDetect`] under `list_limits`, which +/// `configure` gets; then a probe from `probe`, over what `configure` +/// returned, in [`Call::GpuProbe`] under `probe_limits`. Each is waited for +/// as long as it runs. False when the user quit from the notice. +#[allow(clippy::too_many_arguments)] +pub(super) fn before_window( + lanes: &'static Lanes, + list_limits: Limits, + probe_limits: Limits, + prompt: &Prompt, + poll: Duration, + list: impl FnMut() -> L, + configure: impl FnOnce(T) -> S, + probe: impl Fn(&S) -> F, +) -> bool +where + T: Send + 'static, + L: FnOnce() -> T + Send + 'static, + F: FnOnce() + Send + 'static, +{ + let Some(adapters) = await_call( + lanes, + Call::GpuDetect, + list_limits, + &[Call::GpuDetect], + list, + prompt, + poll, + ) else { + return false; + }; + let spec = configure(adapters); + await_call( + lanes, + Call::GpuProbe, + probe_limits, + &[Call::GpuDetect, Call::GpuProbe], + || probe(&spec), + prompt, + poll, + ) + .is_some() +} + +/// What the first-paint watcher has seen, and when it says the window is +/// stuck. Every look is made from the watcher's thread, in order. +pub(super) struct Watch { + started: Instant, + limit: Duration, + proof_within: Duration, + last_held: Option, + first_pass: Option, +} + +impl Watch { + pub(super) fn new(started: Instant, limit: Duration, proof_within: Duration) -> Self { + Self { + started, + limit, + proof_within, + last_held: None, + first_pass: None, + } + } + + /// Whether the notice should be up at `at`, given the window thread's + /// last UI pass and whether the window is held now (minimized, or its + /// title bar or system menu held, when a working window runs no UI pass + /// either). Counted from the latest of the start and the last time it + /// was held: `limit` without a UI pass, or `proof_within` of UI passes + /// without the window confirmed drawing. + pub(super) fn look(&mut self, at: Instant, last_pass: Option, held: bool) -> bool { + if held { + self.last_held = Some(at); + return false; + } + if let Some(pass) = last_pass { + self.first_pass.get_or_insert(pass); + } + let since = self + .last_held + .map_or(self.started, |held| held.max(self.started)); + let quiet = last_pass.map_or(since, |pass| pass.max(since)); + let silent = at.saturating_duration_since(quiet) >= self.limit; + let unproven = self.first_pass.is_some_and(|first| { + at.saturating_duration_since(first.max(since)) >= self.proof_within + }); + silent || unproven + } +} + +/// Watches the window from another thread until the wait ends +/// ([`FirstPaintWatch::finisher`], called when the window is confirmed +/// drawing, or [`FirstPaintWatch::stop`]), and raises the notice when +/// [`Watch::look`] says so. Cancel runs `quit`, which ends the process: the +/// window's thread may never return. +#[allow(clippy::too_many_arguments)] +pub(super) fn watch_first_paint( + prompt: Prompt, + limit: Duration, + proof_within: Duration, + poll: Duration, + now: impl Fn() -> Instant + Send + 'static, + last_pass: impl Fn() -> Option + Send + 'static, + held: impl Fn() -> bool + Send + 'static, + quit: impl FnOnce() + Send + 'static, +) -> FirstPaintWatch { + let mut watch = Watch::new(now(), limit, proof_within); + let watched = prompt.clone(); + let run = move || loop { + match watched.phase(now()) { + Phase::Quit => { + info!("Quit while the window waited for the graphics driver"); + quit(); + return; + } + Phase::Over => return, + Phase::Waiting => {} + } + let at = now(); + if watch.look(at, last_pass(), held()) { + watched.raise(at); + } + std::thread::sleep(poll); + }; + let thread = std::thread::Builder::new() + .name("first-paint-watch".into()) + .spawn(run) + .map_err(|error| warn!(%error, "Could not watch the window's first frames")) + .ok(); + FirstPaintWatch { prompt, thread } +} + +/// A running [`watch_first_paint`]. +pub(super) struct FirstPaintWatch { + prompt: Prompt, + thread: Option>, +} + +impl FirstPaintWatch { + /// Ends the wait, for the renderer to call when the window is + /// confirmed drawing. + pub(super) fn finisher(&self) -> Box { + let prompt = self.prompt.clone(); + Box::new(move || { + prompt.finish(); + }) + } + + /// The window ended: the wait is over, and the watcher is done. A + /// Cancel given by then still quits. Err if the watcher panicked. + pub(super) fn stop(self) -> std::thread::Result<()> { + self.prompt.finish(); + self.thread.map_or(Ok(()), std::thread::JoinHandle::join) + } +} + +/// A message box: Windows draws it without the graphics card's driver. +#[cfg(windows)] +#[derive(Default)] +pub(super) struct MessageNotice { + shown: Option, +} + +#[cfg(windows)] +struct Shown { + /// The box's thread, to find its window by. + thread: Arc, + answer: std::thread::JoinHandle, +} + +#[cfg(windows)] +const NOTICE: &str = "The graphics driver is not responding, so the EternalMonitor window can't open yet. It opens on its own as soon as the driver answers.\n\nIf this stays, restart the PC or update the graphics driver from NVIDIA, AMD or Intel.\n\nClick OK to keep waiting, or Cancel to quit EternalMonitor."; + +#[cfg(windows)] +impl Notice for MessageNotice { + fn show(&mut self) { + use std::sync::atomic::{AtomicU32, Ordering}; + use windows::core::HSTRING; + use windows::Win32::UI::WindowsAndMessaging::{ + MessageBoxW, MB_ICONWARNING, MB_OKCANCEL, MB_SETFOREGROUND, + }; + let thread = Arc::new(AtomicU32::new(0)); + let id = Arc::clone(&thread); + let spawned = std::thread::Builder::new() + .name("driver-wait-notice".into()) + .spawn(move || unsafe { + id.store( + windows::Win32::System::Threading::GetCurrentThreadId(), + Ordering::SeqCst, + ); + MessageBoxW( + None, + &HSTRING::from(NOTICE), + &HSTRING::from("EternalMonitor"), + MB_OKCANCEL | MB_ICONWARNING | MB_SETFOREGROUND, + ) + }); + match spawned { + Ok(answer) => self.shown = Some(Shown { thread, answer }), + Err(error) => warn!(%error, "Could not show the graphics driver notice"), + } + } + + fn answer(&mut self) -> Option { + use windows::Win32::UI::WindowsAndMessaging::IDCANCEL; + if !self.shown.as_ref()?.answer.is_finished() { + return None; + } + let shown = self.shown.take()?; + // A box that could not be shown keeps waiting, as OK does. + Some(match shown.answer.join() { + Ok(answer) if answer == IDCANCEL => Answer::Quit, + _ => Answer::Wait, + }) + } + + fn close(&mut self) { + use std::sync::atomic::Ordering; + use windows::Win32::Foundation::{BOOL, HWND, LPARAM, WPARAM}; + use windows::Win32::UI::WindowsAndMessaging::{EnumThreadWindows, PostMessageW, WM_CLOSE}; + unsafe extern "system" fn close_window(hwnd: HWND, _: LPARAM) -> BOOL { + let _ = PostMessageW(hwnd, WM_CLOSE, WPARAM(0), LPARAM(0)); + true.into() + } + let Some(shown) = self.shown.take() else { + return; + }; + // On a thread of its own: this may run on the window's thread, under + // the prompt's lock. The box may not be up yet when the driver + // answers at once, so ask again until its thread ends, for 2 s. + let closing = move || { + let deadline = Instant::now() + Duration::from_secs(2); + while !shown.answer.is_finished() && Instant::now() < deadline { + let thread = shown.thread.load(Ordering::SeqCst); + if thread != 0 { + unsafe { + let _ = EnumThreadWindows(thread, Some(close_window), LPARAM(0)); + } + } + std::thread::sleep(Duration::from_millis(50)); + } + }; + if let Err(error) = std::thread::Builder::new() + .name("driver-wait-notice-close".into()) + .spawn(closing) + { + warn!(%error, "Could not close the graphics driver notice"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst}; + use std::sync::mpsc; + + /// For a call that hangs until the test releases it. + const SHORT: Limits = Limits { + warn: Duration::from_millis(100), + limit: Duration::from_millis(300), + }; + /// For a call that answers: a slow machine never reaches it, so the + /// events stay exactly what the test lists. + const LONG: Limits = Limits { + warn: Duration::from_secs(30), + limit: Duration::from_secs(60), + }; + const SLICE: Duration = Duration::from_millis(1); + + fn lanes() -> &'static Lanes { + Box::leak(Box::new(Lanes::new())) + } + + /// What happened, in order. + #[derive(Clone, Default)] + struct Events(Arc>>); + + impl Events { + fn push(&self, event: impl Into) { + self.0.lock().push(event.into()); + } + fn all(&self) -> Vec { + self.0.lock().clone() + } + fn count(&self, event: &str) -> usize { + self.0.lock().iter().filter(|e| *e == event).count() + } + /// Backstop only; each wait ends as soon as the event happens. + fn wait_for(&self, event: &str, times: usize) { + let deadline = Instant::now() + Duration::from_secs(60); + while self.count(event) < times { + assert!(Instant::now() < deadline, "no {event}: {:?}", self.all()); + std::thread::sleep(Duration::from_millis(1)); + } + } + } + + /// A notice the test answers by setting `answer` while it is up. + #[derive(Clone, Default)] + struct Fake { + events: Events, + answer: Arc>>, + up: Arc, + } + + impl Notice for Fake { + fn show(&mut self) { + self.up.store(true, SeqCst); + self.events.push("shown"); + } + fn answer(&mut self) -> Option { + if !self.up.load(SeqCst) { + return None; + } + let answer = self.answer.lock().take()?; + self.up.store(false, SeqCst); + self.events.push(match answer { + Answer::Wait => "ok", + Answer::Quit => "cancel", + }); + Some(answer) + } + fn close(&mut self) { + if self.up.swap(false, SeqCst) { + self.events.push("closed"); + } + } + } + + /// Something that blocks in `lanes`' startup lane until the returned + /// sender sends, already given up on: a stuck GPU detection. + fn stuck_detection(lanes: &'static Lanes) -> mpsc::Sender<()> { + let (release, held) = mpsc::channel::<()>(); + let result = crate::stall::run_on( + lanes, + Call::GpuDetect, + SHORT, + move || { + let _ = held.recv(); + }, + || {}, + ); + assert!(result.is_err()); + release + } + + type ListStep = Box String + Send>; + type ProbeStep = Box; + + /// [`before_window`] on its own thread: each list's result and each + /// probe come from the test, and `configure` records what it got. + fn start_before_window( + lanes: &'static Lanes, + fake: &Fake, + (list_limits, probe_limits): (Limits, Limits), + list: impl FnMut() -> ListStep + Send + 'static, + probe: impl Fn() -> ProbeStep + Send + 'static, + ) -> std::thread::JoinHandle { + let prompt = Prompt::new(fake.clone()); + let events = fake.events.clone(); + std::thread::spawn(move || { + let configured = events.clone(); + before_window( + lanes, + list_limits, + probe_limits, + &prompt, + SLICE, + list, + move |gpus: String| { + // Crash avoidance is configured only with a list that + // returned, and nothing is stuck in the startup lane. + assert_eq!(lanes.abandoned(Call::GpuDetect), 0); + configured.push(format!("configure {gpus}")); + }, + |_| probe(), + ) + }) + } + + fn listing(events: &Events, gpus: &'static str) -> ListStep { + let events = events.clone(); + Box::new(move || { + events.push("list"); + gpus.to_string() + }) + } + + fn probing(events: &Events) -> ProbeStep { + let events = events.clone(); + Box::new(move || events.push("probe")) + } + + /// A step that blocks until the test sends on the returned sender, and + /// says when it started. + fn held_step( + events: &Events, + name: &'static str, + ) -> ( + mpsc::Sender<()>, + mpsc::Receiver<()>, + Box, + ) { + let (release, held) = mpsc::channel::<()>(); + let (started, running) = mpsc::channel::<()>(); + let events = events.clone(); + let step = Box::new(move || { + events.push(name); + let _ = started.send(()); + let _ = held.recv(); + events.push(format!("{name} returned")); + }); + (release, running, step) + } + + #[test] + fn a_driver_that_answers_opens_the_window_without_a_notice() { + let fake = Fake::default(); + let (list_events, probe_events) = (fake.events.clone(), fake.events.clone()); + let opened = start_before_window( + lanes(), + &fake, + (LONG, LONG), + move || listing(&list_events, "10de:2c02 32.0"), + move || probing(&probe_events), + ); + assert!(opened.join().unwrap()); + assert_eq!( + fake.events.all(), + ["list", "configure 10de:2c02 32.0", "probe"] + ); + } + + #[test] + fn while_detection_is_stuck_nothing_lists_and_the_notice_is_up() { + let lanes = lanes(); + let release = stuck_detection(lanes); + let fake = Fake::default(); + let (list_events, probe_events) = (fake.events.clone(), fake.events.clone()); + let opened = start_before_window( + lanes, + &fake, + (LONG, LONG), + move || listing(&list_events, "gpus"), + move || probing(&probe_events), + ); + fake.events.wait_for("shown", 1); + // Nothing went near the stuck driver. + assert_eq!(fake.events.all(), ["shown"]); + release.send(()).unwrap(); + assert!(opened.join().unwrap()); + // The notice stays up for the first-paint watcher to take down. + assert_eq!( + fake.events.all(), + ["shown", "list", "configure gpus", "probe"] + ); + } + + #[test] + fn a_list_that_answers_after_its_limit_is_waited_for_and_configures_once() { + let lanes = lanes(); + let fake = Fake::default(); + let (release, list_running, step) = held_step(&fake.events, "list"); + let step = Arc::new(Mutex::new(Some(step))); + let probe_events = fake.events.clone(); + let opened = start_before_window( + lanes, + &fake, + (SHORT, LONG), + move || { + let step = step.lock().take().expect("listed once"); + Box::new(move || { + step(); + "late".to_string() + }) + }, + move || probing(&probe_events), + ); + list_running.recv().unwrap(); + fake.events.wait_for("shown", 1); + release.send(()).unwrap(); + assert!(opened.join().unwrap()); + // One list, past its limit; its late result configured the renderer. + assert_eq!( + fake.events.all(), + ["list", "shown", "list returned", "configure late", "probe"] + ); + assert_eq!(lanes.abandoned(Call::GpuDetect), 0); + } + + #[test] + fn a_probe_that_answers_after_its_limit_opens_the_window() { + let lanes = lanes(); + let fake = Fake::default(); + let (release, probe_running, step) = held_step(&fake.events, "probe"); + let step = Arc::new(Mutex::new(Some(step))); + let list_events = fake.events.clone(); + let opened = start_before_window( + lanes, + &fake, + (LONG, SHORT), + move || listing(&list_events, "gpus"), + move || step.lock().take().expect("probed once"), + ); + probe_running.recv().unwrap(); + fake.events.wait_for("shown", 1); + release.send(()).unwrap(); + assert!(opened.join().unwrap()); + assert_eq!( + fake.events.all(), + ["list", "configure gpus", "probe", "shown", "probe returned"] + ); + assert_eq!(lanes.abandoned(Call::GpuProbe), 0); + } + + #[test] + fn cancel_while_a_call_hangs_opens_no_window_and_leaves_its_lane_taken() { + let lanes = lanes(); + let fake = Fake::default(); + let (release, probe_running, step) = held_step(&fake.events, "probe"); + let step = Arc::new(Mutex::new(Some(step))); + let list_events = fake.events.clone(); + let opened = start_before_window( + lanes, + &fake, + (LONG, SHORT), + move || listing(&list_events, "gpus"), + move || step.lock().take().expect("probed once"), + ); + probe_running.recv().unwrap(); + fake.events.wait_for("shown", 1); + *fake.answer.lock() = Some(Answer::Quit); + assert!(!opened.join().unwrap()); + assert_eq!( + fake.events.all(), + ["list", "configure gpus", "probe", "shown", "cancel"] + ); + // The probe still runs in the driver; its lane says so until it + // returns. + assert_eq!(lanes.abandoned(Call::GpuProbe), 1); + release.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(60); + while lanes.abandoned(Call::GpuProbe) != 0 { + assert!(Instant::now() < deadline, "the lane did not free"); + std::thread::sleep(Duration::from_millis(1)); + } + } + + #[test] + fn cancel_while_detection_is_stuck_opens_no_window() { + let lanes = lanes(); + let _release = stuck_detection(lanes); + let fake = Fake::default(); + *fake.answer.lock() = Some(Answer::Quit); + let (list_events, probe_events) = (fake.events.clone(), fake.events.clone()); + let opened = start_before_window( + lanes, + &fake, + (LONG, LONG), + move || listing(&list_events, "gpus"), + move || probing(&probe_events), + ); + assert!(!opened.join().unwrap()); + assert_eq!(fake.events.all(), ["shown", "cancel"]); + } + + #[test] + fn a_cancel_as_the_driver_answers_still_quits() { + let lanes = lanes(); + let release = stuck_detection(lanes); + let fake = Fake::default(); + let answer = Arc::clone(&fake.answer); + let list_events = fake.events.clone(); + // The user clicks Cancel while the probe runs, the moment before it + // returns. + let opened = start_before_window( + lanes, + &fake, + (LONG, LONG), + move || listing(&list_events, "gpus"), + move || { + let answer = Arc::clone(&answer); + Box::new(move || *answer.lock() = Some(Answer::Quit)) + }, + ); + fake.events.wait_for("shown", 1); + release.send(()).unwrap(); + assert!(!opened.join().unwrap()); + assert_eq!( + fake.events.all(), + ["shown", "list", "configure gpus", "cancel"] + ); + } + + #[test] + fn ok_hides_the_notice_until_the_reminder_and_the_end_of_the_wait_ends_it() { + let fake = Fake::default(); + let prompt = Prompt::new(fake.clone()); + let start = Instant::now(); + prompt.raise(start); + prompt.raise(start + Duration::from_secs(1)); + assert_eq!(fake.events.all(), ["shown"]); + *fake.answer.lock() = Some(Answer::Wait); + let ok_at = start + Duration::from_secs(2); + assert_eq!(prompt.phase(ok_at), Phase::Waiting); + prompt.raise(ok_at + REMIND_AFTER - Duration::from_millis(1)); + assert_eq!(fake.events.all(), ["shown", "ok"]); + prompt.raise(ok_at + REMIND_AFTER); + assert_eq!(fake.events.all(), ["shown", "ok", "shown"]); + // Cancel on the second showing quits. + *fake.answer.lock() = Some(Answer::Quit); + assert_eq!(prompt.phase(ok_at + REMIND_AFTER), Phase::Quit); + // Once the wait is over, nothing shows it again. + let fake = Fake::default(); + let prompt = Prompt::new(fake.clone()); + prompt.raise(start); + assert_eq!(prompt.finish(), Phase::Over); + prompt.raise(start + REMIND_AFTER * 2); + assert_eq!(fake.events.all(), ["shown", "closed"]); + } + + const LIMIT: Duration = Duration::from_secs(4); + const PROOF: Duration = Duration::from_secs(12); + + /// Looks at the window as the watcher does, each at a time in ms from + /// the start: (ms, last UI pass in ms, held). The times the notice + /// would be raised at. + fn looks(script: &[(u64, Option, bool)]) -> Vec { + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let mut watch = Watch::new(start, LIMIT, PROOF); + script + .iter() + .filter(|(ms, pass, held)| watch.look(at(*ms), pass.map(at), *held)) + .map(|(ms, _, _)| *ms) + .collect() + } + + /// UI passes every 200 ms from `from` to `to` ms, looked at each 250 ms. + fn passing(from: u64, to: u64) -> Vec<(u64, Option, bool)> { + (from..=to) + .step_by(250) + .map(|ms| (ms, Some(ms / 200 * 200), false)) + .collect() + } + + #[test] + fn a_window_that_passes_and_draws_in_time_shows_no_notice() { + // Confirmed after about 3.5 s, well before either bound. + assert!(looks(&passing(0, 3_500)).is_empty()); + } + + #[test] + fn a_failed_first_acquire_then_a_stuck_recreation_raises_the_notice_4_s_after_the_last_pass() { + // One pass at 200 ms, whose frame's surface acquire failed; the + // recreation at the start of the next paint never returns. + let script: Vec<_> = (250..=4_500) + .step_by(50) + .map(|ms| (ms, Some(200), false)) + .collect(); + assert_eq!(looks(&script)[0], 4_200); + } + + #[test] + fn passes_without_a_proof_of_a_presented_frame_raise_the_notice_at_the_bound() { + // The surface is gone and every paint returns before drawing: UI + // passes go on, the screenshots are never answered. + let raised = looks(&passing(0, 12_500)); + assert_eq!(raised.first(), Some(&12_000), "{raised:?}"); + } + + #[test] + fn held_windows_suspend_both_bounds_and_count_again_from_when_they_are_let_go() { + // One pass, then minimized: eframe runs no pass for a minimized + // window, so only the live check sees it. Held for a minute: nothing. + let mut script = vec![(100, Some(100), false)]; + script.extend((250..=60_000).step_by(250).map(|ms| (ms, Some(100), true))); + assert!(looks(&script).is_empty()); + // Restored at 60 s, and no pass comes: the notice 4 s later. + let mut stuck = script.clone(); + stuck.extend( + (60_250..=64_500) + .step_by(250) + .map(|ms| (ms, Some(100), false)), + ); + assert_eq!(looks(&stuck).first(), Some(&64_000)); + // Restored and passing, but never confirmed: the proof bound counts + // from the release, not from the first pass. + let mut unproven = script.clone(); + unproven.extend(passing(60_250, 72_500)); + assert_eq!(looks(&unproven).first(), Some(&72_000)); + // The title bar held while passes stop counts the same way. + let mut dragged = passing(0, 2_000); + dragged.extend( + (2_250..=20_000) + .step_by(250) + .map(|ms| (ms, Some(2_000), true)), + ); + assert!(looks(&dragged).is_empty()); + } + + #[test] + fn cancel_while_the_window_is_stuck_quits_from_the_watcher() { + let fake = Fake::default(); + let quits = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&quits); + *fake.answer.lock() = Some(Answer::Quit); + // Nothing passes: the notice goes up at once with a zero limit. + let watch = watch_first_paint( + Prompt::new(fake.clone()), + Duration::ZERO, + PROOF, + SLICE, + Instant::now, + || None, + || false, + move || { + count.fetch_add(1, SeqCst); + }, + ); + watch.thread.unwrap().join().unwrap(); + assert_eq!(quits.load(SeqCst), 1); + assert_eq!(fake.events.all(), ["shown", "cancel"]); + } + + #[test] + fn a_cancel_that_comes_with_the_window_drawing_still_quits() { + let fake = Fake::default(); + let quits = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&quits); + let prompt = Prompt::new(fake.clone()); + prompt.raise(Instant::now()); + // The user clicks Cancel; before the watcher reads it, the renderer + // confirms the window and ends the wait. + *fake.answer.lock() = Some(Answer::Quit); + assert_eq!(prompt.finish(), Phase::Quit); + let watch = watch_first_paint( + prompt, + REMIND_AFTER, + PROOF, + SLICE, + Instant::now, + || None, + || false, + move || { + count.fetch_add(1, SeqCst); + }, + ); + watch.thread.unwrap().join().unwrap(); + assert_eq!(quits.load(SeqCst), 1); + assert_eq!(fake.events.all(), ["shown", "cancel"]); + } + + #[test] + fn a_window_that_draws_while_the_watcher_decides_to_raise_shows_no_notice() { + let fake = Fake::default(); + let start = Instant::now(); + // The watcher reads the clock just before it raises the notice. The + // test holds that read until the renderer has ended the wait. + let (reading, clock_read) = mpsc::channel::<()>(); + let (go, proceed) = mpsc::channel::<()>(); + let proceed = std::sync::Mutex::new(proceed); + let reads = AtomicUsize::new(0); + let now = move || { + // Read 0 is the watcher's start, 1 its phase check; 2 is the + // time it raises at. + if reads.fetch_add(1, SeqCst) == 2 { + let _ = reading.send(()); + let _ = proceed.lock().unwrap().recv(); + } + start + SHORT.limit * 2 + }; + let never_quit = || panic!("quit"); + let watch = watch_first_paint( + Prompt::new(fake.clone()), + SHORT.limit, + PROOF, + SLICE, + now, + || None, + || false, + never_quit, + ); + clock_read.recv().unwrap(); + (watch.finisher())(); + go.send(()).unwrap(); + watch.stop().unwrap(); + assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); + } +} diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 53d7cf8e..670017eb 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -19,6 +19,7 @@ //! `author_id`, see `widgets::automation_id`): scripts and tests find them by //! that id first, then by name, so the same name works in every language. +mod driver_wait; mod feedback_panel; mod fonts; mod icons; @@ -44,7 +45,7 @@ use tracing::{info, warn}; use crate::autostart::{first_lan_ipv4, read_startup_registry, set_startup_registry}; use crate::capture::recovery::CaptureBlock; -use crate::capture::{enumerate_outputs, OutputInfo}; +use crate::capture::{listing, OutputInfo}; use crate::control::{CaptureTarget, GuiControl}; use crate::gpu_preference::{Change, Preference, SwitchRecord}; use crate::handoff::Handoff; @@ -81,6 +82,8 @@ struct StatsSnapshot { codec_name: String, using_software_fallback: bool, nvidia_driver_outdated: bool, + gpu_detection_stalled: bool, + driver_update_vendors: Vec, gpu_name: String, capture_display: String, transport_fps: f64, @@ -119,6 +122,8 @@ impl StatsSnapshot { codec_name: s.codec_name.clone(), using_software_fallback: s.using_software_fallback, nvidia_driver_outdated: s.nvidia_driver_outdated, + gpu_detection_stalled: s.gpu_detection_stalled, + driver_update_vendors: s.driver_update_vendors.clone(), gpu_name: s.gpu_name.clone(), capture_display: s.capture_display.clone(), transport_fps: s.transport_fps, @@ -303,6 +308,7 @@ pub struct AnalyzerApp { settings_fps_target: u32, settings_encoder_input: crate::encoder::input::EncoderInput, settings_check_updates: bool, + settings_beta_updates: bool, updates: crate::updates::Updater, /// The update popup is on screen; it stays until a button is chosen. update_popup_open: bool, @@ -363,7 +369,9 @@ pub struct AnalyzerApp { update_refused: bool, history: History, quality: QualityMeter, - logo: Option<(f32, egui::TextureHandle)>, + /// The logo for a scale and the window renderer's device it was made + /// on: textures go with a lost device. + logo: Option<(f32, u64, egui::TextureHandle)>, /// The saved language choice (see `i18n::resolve`). settings_language: Option, /// A language whose fonts were just installed; it shows from the next @@ -375,6 +383,7 @@ pub struct AnalyzerApp { impl AnalyzerApp { pub fn new(cc: &eframe::CreationContext<'_>, control: GuiControl) -> Self { + renderer::remember_window(cc); // Load persisted settings first; values fall back to the live runtime state when the // file is missing or unreadable. let persisted = SettingsFile::load(); @@ -441,7 +450,8 @@ impl AnalyzerApp { String::new() } }; - let available_outputs = enumerate_outputs(); + // Before the first paint, so a stuck driver must not hold it. + let available_outputs = listing::outputs_at_startup(&control.shared); // Same model as encoder_override: push the persisted preference into // SharedControl so the encoder honours it without a restart. @@ -473,6 +483,7 @@ impl AnalyzerApp { let ctx = cc.egui_ctx.clone(); crate::updates::Updater::start( persisted.check_for_updates, + persisted.beta_updates, control.shared.session.clone(), control.shared.handoff.clone(), std::sync::Arc::new(move |version: Option<&str>| { @@ -489,6 +500,7 @@ impl AnalyzerApp { Self { settings_encoder_input: persisted.encoder_input, settings_check_updates: persisted.check_for_updates, + settings_beta_updates: persisted.beta_updates, updates, update_popup_open: false, ipad_last_seen: Some(popup_now()), @@ -565,6 +577,7 @@ impl AnalyzerApp { encoder_override, encoder_input: self.settings_encoder_input, check_for_updates: self.settings_check_updates, + beta_updates: self.settings_beta_updates, capture_display: if self.settings_capture_display.is_empty() { None } else { @@ -704,7 +717,7 @@ impl AnalyzerApp { } fn refresh_outputs(&mut self) { - self.available_outputs = enumerate_outputs(); + self.available_outputs = listing::outputs(&self.control.shared); info!( count = self.available_outputs.len(), "Re-enumerated display outputs" @@ -787,11 +800,17 @@ impl AnalyzerApp { self.persist_settings(); } + fn set_beta_updates(&mut self, enabled: bool) { + self.settings_beta_updates = enabled; + self.updates.set_beta(enabled); + self.persist_settings(); + } + fn set_require_pairing(&mut self, required: bool) { { let mut pairing = self.control.shared.pairing.lock(); pairing.required = required; - pairing.warn_if_off(); + pairing.log_if_off(); } self.persist_settings(); } @@ -820,6 +839,8 @@ impl AnalyzerApp { fn restart_stream(&mut self) { self.restart_pending = false; + // The user may have updated a driver, as the fallback notice asks. + crate::encoder::forget_launch_failures(); self.control.request_restart(); } @@ -1074,8 +1095,9 @@ impl eframe::App for AnalyzerApp { CANVAS.to_normalized_gamma_f32() } - fn ui(&mut self, ui: &mut egui::Ui, _frame: &mut eframe::Frame) { + fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { renderer::frame_started(ui.ctx()); + renderer::release_render_state(frame); crate::startup::simulate_window_failure(); let ctx = ui.ctx().clone(); if let Some(language) = self.pending_language.take() { @@ -1270,14 +1292,19 @@ impl AnalyzerApp { fn draw_brand(&mut self, ui: &mut egui::Ui) { const LOGO: f32 = 30.0; let ppp = ui.ctx().pixels_per_point(); - if self.logo.as_ref().is_none_or(|(scale, _)| *scale != ppp) { + let device = renderer::devices(); + if self + .logo + .as_ref() + .is_none_or(|(scale, made_on, _)| (*scale, *made_on) != (ppp, device)) + { let pixels = (LOGO * ppp).round().max(1.0) as usize; - self.logo = logo_texture(ui.ctx(), pixels).map(|texture| (ppp, texture)); + self.logo = logo_texture(ui.ctx(), pixels).map(|texture| (ppp, device, texture)); } ui.horizontal(|ui| { ui.spacing_mut().item_spacing.x = 10.0; let (rect, _) = ui.allocate_exact_size(vec2(LOGO, LOGO), Sense::hover()); - if let Some((_, texture)) = &self.logo { + if let Some((_, _, texture)) = &self.logo { ui.painter().image( texture.id(), rect, @@ -1760,8 +1787,23 @@ pub fn shown_address(ip: std::net::Ipv4Addr, listen_port: u16) -> String { } } +/// The app quits from outside its window, as when Windows closes it for an +/// installer, perhaps before the window is confirmed drawing. As for a quit +/// from the graphics driver notice, this launch's renderer marker goes +/// without counting, so the next launch avoids no backend, and nothing still +/// in the driver saves it again. +pub fn quitting_from_outside() { + renderer::abandon(); +} + /// Launch the GUI window. Blocks the calling thread. -pub fn run_gui(control: GuiControl) -> eframe::Result<()> { +/// Runs the window until it closes. `quit_while_stuck` is the app's quit +/// path, for a Cancel in the graphics driver notice while the window's +/// thread is stuck before its first frame; it ends the process. +pub fn run_gui( + control: GuiControl, + quit_while_stuck: Box, +) -> eframe::Result<()> { let icon = eframe::icon_data::from_png_bytes(include_bytes!("../../assets/icon_256.png")) .expect("embedded icon PNG is valid"); @@ -1773,13 +1815,61 @@ pub fn run_gui(control: GuiControl) -> eframe::Result<()> { .with_icon(std::sync::Arc::new(icon)), ..Default::default() }; - renderer::configure(&mut options.wgpu_options); + // The window's thread makes its first graphics call only once the + // driver has answered the same calls on another thread, and is watched + // until it is confirmed drawing. + let prepared = renderer::prepare(); + #[cfg(windows)] + let watch = { + let prompt = driver_wait::Prompt::new(driver_wait::MessageNotice::default()); + let skip_listing = prepared.listing_avoided(); + let probe_limits = crate::stall::Call::GpuProbe.limits(); + let opened = driver_wait::before_window( + crate::stall::lanes(), + crate::stall::Call::GpuDetect.limits(), + probe_limits, + &prompt, + driver_wait::POLL, + || renderer::adapter_list(skip_listing), + |adapters| { + renderer::configure(&mut options.wgpu_options, prepared, adapters); + renderer::probe_spec(&options.wgpu_options) + }, + |spec| renderer::adapter_probe(spec.clone()), + ); + if !opened { + renderer::abandon(); + return Ok(()); + } + let watch = driver_wait::watch_first_paint( + prompt, + probe_limits.limit, + driver_wait::PROOF_WITHIN, + driver_wait::POLL, + std::time::Instant::now, + renderer::last_pass, + renderer::window_held, + move || { + renderer::abandon(); + quit_while_stuck(); + }, + ); + renderer::when_drawing(watch.finisher()); + watch + }; + #[cfg(not(windows))] + { + renderer::configure(&mut options.wgpu_options, prepared, Vec::new()); + drop(quit_while_stuck); + } let result = eframe::run_native( "EternalMonitor", options, Box::new(|cc| Ok(Box::new(AnalyzerApp::new(cc, control)))), ); + #[cfg(windows)] + let _ = watch.stop(); if result.is_ok() { renderer::window_closed(); } diff --git a/host/src/gui/page_settings.rs b/host/src/gui/page_settings.rs index 58bfba3d..dacc6164 100644 --- a/host/src/gui/page_settings.rs +++ b/host/src/gui/page_settings.rs @@ -24,10 +24,11 @@ impl AnalyzerApp { widgets::list_card().show(ui, |ui| { ui.set_width(ui.available_width()); let mut refresh = false; + let windows_10 = crate::capture::second_screen::windows_unsupported(&self.control.shared); widgets::setting_row( ui, tr("How the iPad shows your PC"), - "Mirror my screen shows your main screen. Second screen adds the iPad as an extra screen while it is connected. You can also mirror one particular monitor.", + display_description(windows_10), 300.0, |ui| { refresh = widgets::icon_button(ui, super::icons::Icon::Refresh, tr("Refresh displays")).clicked(); @@ -229,7 +230,7 @@ impl AnalyzerApp { ui.horizontal(|ui| { ui.spacing_mut().item_spacing.x = 14.0; let (rect, _) = ui.allocate_exact_size(vec2(44.0, 44.0), Sense::hover()); - if let Some((_, texture)) = &self.logo { + if let Some((_, _, texture)) = &self.logo { ui.painter().image( texture.id(), rect, @@ -328,6 +329,16 @@ pub(super) fn env_or(env: Option<&str>, description: &str) -> String { /// What "Match the iPad's resolution" does, and when a change applies: the /// driver reads the modes when the second screen starts, and restarting /// the stream keeps the second screen that is already there. +/// What the display choice does, and on Windows 10 that Second screen +/// can't run there. +pub(super) fn display_description(windows_10: bool) -> &'static str { + if windows_10 { + "Mirror my screen shows your main screen. You can also mirror one particular monitor. Second screen needs Windows 11, so on this PC the iPad shows your main screen instead." + } else { + "Mirror my screen shows your main screen. Second screen adds the iPad as an extra screen while it is connected. You can also mirror one particular monitor." + } +} + pub(super) fn vdd_match_description(extending: bool) -> &'static str { if extending { "Makes the second screen exactly as sharp and smooth as the iPad's display. A change applies the next time the second screen starts, when the iPad connects again." diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index 679e95e1..6e963513 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -56,6 +56,41 @@ impl AnalyzerApp { /// Update, restart and fault notices, most urgent first. fn notices(&mut self, ui: &mut Ui, snap: &StatsSnapshot, client: &ClientSnapshot) { self.port_notice(ui); + let firewall = crate::firewall::latest(); + let progress = crate::firewall_fix::progress(); + // Its own banner, whatever the firewall reads now: a fix that went + // part of the way can leave no notice to show the failure in. + if let Some(error) = firewall_fix_failure(progress) { + widgets::banner( + ui, + WARNING, + Icon::Warning, + FIREWALL_FIX_FAILED_TITLE, + error, + 220.0, + |ui| { + if let Some(port) = crate::firewall_fix::last_port() { + if widgets::button(ui, Tone::Primary, None, tr("Try again")).clicked() { + crate::firewall_fix::start_for(port); + } + } + if widgets::button(ui, Tone::Secondary, None, tr("Dismiss")).clicked() { + crate::firewall_fix::dismiss(); + } + }, + ); + ui.add_space(GAP - 10.0); + } + if let Some((title, body)) = firewall_notice(firewall.as_ref(), client.peer.is_some()) { + let fixable = firewall_fix_offered(firewall.as_ref()); + let width = if fixable { 110.0 } else { 0.0 }; + widgets::banner(ui, WARNING, Icon::Warning, &title, &body, width, |ui| { + if let (true, Some(report)) = (fixable, firewall.as_ref()) { + firewall_fix_action(ui, progress, report.port); + } + }); + ui.add_space(GAP - 10.0); + } if let Some((title, body)) = usb_cable_notice( snap.usb_service_reachable, snap.usb_cabled_devices, @@ -69,9 +104,12 @@ impl AnalyzerApp { } if snap.using_software_fallback { // Choices made while troubleshooting are the usual reason a - // working graphics card is skipped, so offer to undo them first. - let customized = self.settings_encoder_choice != ENCODER_AUTO_LABEL - || self.settings_encoder_input != crate::encoder::input::EncoderInput::default(); + // working graphics card is skipped, so offer to undo them first, + // unless the driver did not answer at startup: then Restart. + let customized = !snap.gpu_detection_stalled + && (self.settings_encoder_choice != ENCODER_AUTO_LABEL + || self.settings_encoder_input + != crate::encoder::input::EncoderInput::default()); let mut reset = false; let mut restart = false; widgets::banner( @@ -79,7 +117,12 @@ impl AnalyzerApp { WARNING, Icon::Warning, SOFTWARE_FALLBACK_TITLE, - &software_fallback_body(customized, snap.nvidia_driver_outdated), + &software_fallback_body( + customized, + snap.nvidia_driver_outdated, + snap.gpu_detection_stalled, + &snap.driver_update_vendors, + ), if customized { 230.0 } else { 110.0 }, |ui| { if customized { @@ -127,7 +170,7 @@ impl AnalyzerApp { } // The user asked for the extended (virtual) display but it could not be brought up, // so capture silently fell back to the primary monitor. Say so. - let vdd_failure = crate::capture::second_screen::remembered_failure(&self.control.shared); + let vdd_failure = crate::capture::second_screen::notice_failure(&self.control.shared); if let Some(mut notice) = second_screen_notice(vdd_status, vdd_health, vdd_problem, vdd_failure) { @@ -560,12 +603,14 @@ impl AnalyzerApp { } }); ui.add_space(2.0); + let windows_10 = crate::capture::second_screen::windows_unsupported(&self.control.shared); ui.add( egui::Label::new(muted(screen_mode_text( ¤t, output.as_deref(), vdd, running, + windows_10, ))) .wrap(), ); @@ -824,21 +869,44 @@ enum HeaderAction { pub(super) const SOFTWARE_FALLBACK_TITLE: &str = "Video is using the processor instead of the graphics card"; -/// What the CPU fallback notice says: undo custom settings first, say the -/// NVIDIA driver is too old when that is what the graphics card refused, or +/// What the CPU fallback notice says: that the graphics driver did not +/// answer at startup, undo custom settings first, say the NVIDIA driver is +/// too old when that is what the graphics card refused, name the AMD or +/// Intel driver whose card could not start encoding (`driver_vendors`), or /// suggest a driver update. -pub(super) fn software_fallback_body(customized: bool, nvidia_driver_outdated: bool) -> String { - if customized { +pub(super) fn software_fallback_body( + customized: bool, + nvidia_driver_outdated: bool, + detection_stalled: bool, + driver_vendors: &[crate::gpu::GpuVendor], +) -> String { + use crate::gpu::GpuVendor::{Amd, Intel}; + let amd = driver_vendors.contains(&Amd); + let intel = driver_vendors.contains(&Intel); + if detection_stalled { + "The graphics driver did not respond when EternalMonitor started, so the processor encodes the video for now. The picture may lag and the PC may run hot. Click Restart to try the graphics card again.".to_string() + } else if customized { "The graphics card could not encode video with the options chosen in Settings, so the picture may lag and the PC may run hot. Go back to the recommended options first.".to_string() } else if nvidia_driver_outdated { format!( "The NVIDIA graphics driver on this PC is too old for the graphics card to encode video, so the picture may lag and the PC may run hot. Update the NVIDIA driver to version {} or newer, if your graphics card supports it, then click Restart. Older cards keep using the processor.", crate::encoder::NVENC_MIN_DRIVER ) + } else if amd && intel { + "The AMD and Intel graphics drivers on this PC could not start video encoding on the graphics cards, so the picture may lag and the PC may run hot. Update the AMD and Intel graphics drivers, then click Restart.".to_string() + } else if amd || intel { + let vendor = if amd { "AMD" } else { "Intel" }; + format!( + "The {vendor} graphics driver on this PC could not start video encoding on the graphics card, so the picture may lag and the PC may run hot. Update the {vendor} graphics driver, then click Restart." + ) } else { "The picture may lag and the PC may run hot. Update your graphics driver from NVIDIA, AMD or Intel, then click Restart.".to_string() } } +/// Why Second screen mirrors on Windows 10: the notice, and the note beside +/// the choice. +pub(super) const WINDOWS_10_NOTE: &str = + "Second screen needs Windows 11, so the iPad shows your main screen instead."; pub(super) const SECOND_SCREEN_FAILED_TITLE: &str = "The second screen couldn't start, so your main screen is mirrored"; const INSTALLER_URL: &str = "https://eternalmonitor.dev/download.html"; @@ -917,6 +985,26 @@ pub(super) fn second_screen_notice( }) }; match (status, health, problem) { + // Nothing was tried, and trying again can't help. + (Failed | TaskFailed, _, _) if failure == Some(crate::capture::second_screen::WINDOWS_10) => { + notice( + &format!("{WINDOWS_10_NOTE} Mirror my screen works as usual."), + false, + false, + ) + } + // Windows left the device as not extended too, but Extend can't help. + (Failed, _, _) if failure == Some("panel_lost") => notice( + "Your laptop's built-in screen turned off when the second screen started, and Windows only turns it back on after a restart. Until then your desktop is on the iPad: save your work there, then restart the PC. On this PC the second screen can turn the built-in screen off again, so the iPad shows your main screen until you click Try again or choose Second screen again.", + true, + false, + ), + // The same, after the restart that brought the panel back. + (Failed, _, _) if failure == Some("panel_lost_before") => notice( + "Second screen is paused on this laptop because it turned off the built-in screen last time. Click Try again to use it.", + true, + false, + ), (Failed | TaskFailed, VddTaskHealth::NotResponding, _) => notice( "Windows' display driver stopped responding. Restart the PC to use the second screen.", false, @@ -1150,18 +1238,23 @@ pub(super) fn capture_block_notice( } /// What the Stream page says under the mirror / second screen choice. +/// `windows_10`: Second screen can't run on this Windows +/// ([`WINDOWS_10_NOTE`]). pub(super) fn screen_mode_text( capture: &str, output: Option<&str>, vdd: VddStatus, running: bool, + windows_10: bool, ) -> String { let restart = if running { " Switching restarts the stream, and the iPad reconnects by itself." } else { "" }; - let what = if capture == CAPTURE_VIRTUAL_SENTINEL { + let what = if capture == CAPTURE_VIRTUAL_SENTINEL && windows_10 { + WINDOWS_10_NOTE.to_string() + } else if capture == CAPTURE_VIRTUAL_SENTINEL { match vdd { VddStatus::Active => "The iPad is an extra screen. Drag windows past the edge of your main screen to move them onto it.", VddStatus::Failed | VddStatus::TaskFailed => { @@ -1177,7 +1270,12 @@ pub(super) fn screen_mode_text( } else { "The iPad shows the same picture as your main screen.".to_string() }; - what + restart + let windows = if windows_10 && capture != CAPTURE_VIRTUAL_SENTINEL { + " Second screen needs Windows 11." + } else { + "" + }; + what + windows + restart } /// The home network's address, without the port (the iPad has its own Port @@ -1352,6 +1450,85 @@ pub(super) fn usb_cable_notice( }) } +/// "Fix it" on the firewall notice: for this exe's own Block rule, which +/// the fix removes, not for "Blocks all incoming connections", which only +/// the user's switch in Windows Security turns off. +pub(super) fn firewall_fix_offered(report: Option<&crate::firewall::Report>) -> bool { + matches!( + report.and_then(crate::firewall::Report::certain_block), + Some(crate::firewall::CertainBlock::Rule { .. }) + ) +} + +/// Runs the fix; it asks Windows for administrator rights once. +fn firewall_fix_action(ui: &mut Ui, progress: crate::firewall_fix::Progress, port: u16) { + let running = progress == crate::firewall_fix::Progress::Running; + let clicked = ui + .add_enabled_ui(!running, |ui| { + widgets::button(ui, Tone::Primary, None, tr("Fix it")) + }) + .inner + .clicked(); + if clicked { + crate::firewall_fix::start_for(port); + } +} + +/// The title of the banner for a firewall fix that didn't work. +pub(super) const FIREWALL_FIX_FAILED_TITLE: &str = "EternalMonitor couldn't fix Windows Firewall"; + +/// The error to show for the last fix, until dismissed: a failure only. +/// A fix that worked, or a No to the Windows prompt, shows nothing. +pub(super) fn firewall_fix_failure( + progress: crate::firewall_fix::Progress, +) -> Option<&'static str> { + match progress { + crate::firewall_fix::Progress::Done(crate::firewall_fix::Outcome::Failed(error)) => { + Some(error) + } + _ => None, + } +} + +/// What the Stream page says while Windows Firewall keeps iPads out: only +/// for a block certain on the network an iPad would use that Setup or one +/// switch removes (see [`crate::firewall::Report::certain_block`]), and +/// never while an iPad is connected, over Wi-Fi or the cable. +pub(super) fn firewall_notice( + report: Option<&crate::firewall::Report>, + session_active: bool, +) -> Option<(String, String)> { + use crate::firewall::CertainBlock; + if session_active { + return None; + } + let lead = "iPads on Wi-Fi can't reach this PC, so they say it didn't answer."; + let usb = "A USB cable still works."; + let turn_off = |networks: &[&str]| { + format!( + "In Windows Security, open Firewall & network protection, choose the {} network and turn off Blocks all incoming connections.", + networks.join(" and ") + ) + }; + Some(match report?.certain_block()? { + CertainBlock::Rule { all_blocked } => { + let mut body = + format!("{lead} Run the EternalMonitor installer again: it removes the block."); + if !all_blocked.is_empty() { + body = format!("{body} {}", turn_off(&all_blocked)); + } + ( + "Windows Firewall is blocking EternalMonitor".into(), + format!("{body} {usb}"), + ) + } + CertainBlock::AllIncoming(networks) => ( + "Windows Firewall blocks every incoming connection".into(), + format!("{lead} {} {usb}", turn_off(&networks)), + ), + }) +} + /// What the USB row says. The Apple Devices hint appears only when Apple's /// device service cannot be reached. pub(super) fn usb_summary( @@ -1439,6 +1616,13 @@ pub(super) fn audio_summary( ACCENT, ); } + if audio.status == crate::audio::DEVICE_IN_USE_STATUS { + return ( + "Another app is using your speakers on its own. PC audio comes back when it lets go." + .into(), + WARNING, + ); + } // The audio stage's idle status names the protocol, not the device. if audio.status == "Waiting for client" { return ("Starts when an iPad connects.".into(), TEXT_MUTED); diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 322c1d74..b2403584 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -66,6 +66,9 @@ const RETRY_AFTER: u32 = 5; /// What ended a launch that crashed in no module the reporter could name, or /// that no reporter described. pub(crate) const UNKNOWN: &str = "unknown"; +/// What ended a launch whose graphics device was lost before the window +/// worked. +pub(crate) const DEVICE_LOST: &str = "device_lost"; /// What ended a launch that was ended without a crash, as by Task Manager. #[cfg(any(windows, test))] pub(crate) const TERMINATED: &str = "terminated"; @@ -105,6 +108,10 @@ const DRIVER_MODULES: [&str; 18] = [ /// PCI vendor id of Intel GPUs. const INTEL: u32 = 0x8086; const HISTORY_FILE: &str = "history.json"; +/// The marker backend of a launch listing the graphics cards through DXGI, +/// before it knows their keys. A launch that dies there makes the next ones +/// key the GPUs by wgpu's driver text instead, until clean launches retry. +const LISTING: &str = "list dxgi"; /// Held while one instance reads, changes and saves the history. const HISTORY_LOCK: &str = "history.lock"; /// The window counts as working after this long and this many frames with no @@ -112,13 +119,34 @@ const HISTORY_LOCK: &str = "history.lock"; /// at the first present, within milliseconds. const CONFIRM_AFTER: Duration = Duration::from_secs(3); const CONFIRM_FRAMES: u64 = 10; +/// Frames after asking for the proof of a presented frame before asking +/// again: the ask is dropped with a frame whose surface was not acquired. +const PROOF_AGAIN_AFTER: u64 = 3 * CONFIRM_FRAMES; static STATE_DIR: OnceLock = OnceLock::new(); -static STARTED: Mutex> = Mutex::new(None); +static LAUNCH: Mutex = Mutex::new(Launch::new()); static PRESENT: Mutex = Mutex::new(PresentTracker::new()); /// Frames whose surface texture could not be acquired, so nothing was presented. static FAILED_ACQUIRES: AtomicU64 = AtomicU64::new(0); static STATUS: Mutex> = Mutex::new(None); +/// The DXGI adapter list taken before the window, which the window's GPU +/// keys come from until it is confirmed drawing: no more DXGI calls on its +/// thread before then. +static ADAPTERS: Mutex>> = Mutex::new(None); +/// When the window's thread last began a UI pass. +static LAST_PASS: Mutex> = Mutex::new(None); +/// The window, once it exists (an HWND on Windows), for checks made from +/// other threads while its own may be stuck. +#[cfg_attr(not(windows), allow(dead_code))] +static WINDOW: std::sync::atomic::AtomicIsize = std::sync::atomic::AtomicIsize::new(0); +/// Run once, when the window is confirmed drawing. +type OnDrawing = Box; +static ON_DRAWING: Mutex> = Mutex::new(None); +/// Adapters chosen for the window this launch. After the first, each +/// replaces a lost device (`vendor/egui-wgpu`). +static SELECTIONS: AtomicU64 = AtomicU64::new(0); +/// The painter's count of devices dropped as lost, when last looked at. +static LOSSES_SEEN: AtomicU64 = AtomicU64::new(0); /// How this launch draws its window, for Settings > Troubleshooting. #[derive(Clone, Debug, Default, PartialEq)] @@ -272,9 +300,13 @@ fn driver_module(module: &str) -> bool { DRIVER_MODULES.iter().any(|name| module.starts_with(name)) } -/// Choose the instance backends and install the adapter selector, after -/// recording failed earlier starts. -pub(super) fn configure(config: &mut WgpuConfiguration) { +/// The failed starts this launch knows of: earlier launches' markers are +/// recorded first. Makes no graphics call. +pub(super) struct Prepared { + history: Vec, +} + +pub(super) fn prepare() -> Prepared { let history = match crate::settings::app_data_dir() { Some(base) => { let directory = base.join("renderer"); @@ -285,6 +317,57 @@ pub(super) fn configure(config: &mut WgpuConfiguration) { } None => Vec::new(), }; + Prepared { history } +} + +impl Prepared { + /// A launch died listing the graphics cards through DXGI, and that is + /// not retried yet. + #[cfg_attr(not(windows), allow(dead_code))] + pub(super) fn listing_avoided(&self) -> bool { + avoided(&self.history, "", LISTING) + } +} + +/// The DXGI adapter list, for another thread to take before the window +/// (`skip`: a launch died there; the GPUs are keyed by wgpu's driver text). +/// This launch's marker names the listing before it starts, and the thread +/// it runs on once it does, until a later step replaces it: a crash there, +/// even after the caller gave up on a list that did not return, is the +/// window's. +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn adapter_list(skip: bool) -> impl FnOnce() -> Vec<(u32, u32, String)> + Send { + if skip { + info!( + "A launch died listing the graphics cards; keying them by the renderer's driver text" + ); + } else { + let _ = start(Marker { + gpu: String::new(), + backend: LISTING.to_string(), + thread: None, + }); + } + move || { + if skip { + return Vec::new(); + } + mark_thread(current_thread()); + list_dxgi_adapters() + } +} + +/// Choose the instance backends and install the adapter selector, with the +/// graphics cards `adapters` listed before the window. +pub(super) fn configure( + config: &mut WgpuConfiguration, + prepared: Prepared, + adapters: Vec<(u32, u32, String)>, +) { + let Prepared { history } = prepared; + if let Ok(mut listed) = ADAPTERS.lock() { + *listed = Some(adapters); + } let choice = crate::settings::SettingsFile::load().window_drawing; // Saved by the last update check; this launch's choice only. let rules = if cfg!(windows) { @@ -305,6 +388,138 @@ pub(super) fn configure(config: &mut WgpuConfiguration) { }); } +/// The instance the window's renderer will create, for [`adapter_probe`]. +/// None when it is given an existing device. +#[derive(Clone)] +pub(super) struct ProbeSpec( + Option<( + Backends, + wgpu::InstanceFlags, + wgpu::BackendOptions, + wgpu::MemoryBudgetThresholds, + )>, +); + +/// What `config`, as [`configure`] left it, will create. +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn probe_spec(config: &WgpuConfiguration) -> ProbeSpec { + ProbeSpec(match &config.wgpu_setup { + WgpuSetup::CreateNew(setup) => Some(( + setup.instance_descriptor.backends, + setup.instance_descriptor.flags, + setup.instance_descriptor.backend_options.clone(), + setup.instance_descriptor.memory_budget_thresholds, + )), + WgpuSetup::Existing(_) => None, + }) +} + +/// The wgpu calls the window's thread makes first, for another thread to +/// make before it: an instance with this launch's backends lists its +/// adapters (DX12 creates a device on each to describe it). Built on the +/// window's thread, run on another by [`super::driver_wait::before_window`]. +/// While it lists, the marker names that thread, so a driver crash there is +/// still the window's. +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn adapter_probe(spec: ProbeSpec) -> impl FnOnce() + Send + 'static { + let window = current_thread(); + move || { + let Some((backends, flags, backend_options, memory_budget_thresholds)) = spec.0 else { + return; + }; + mark_thread(current_thread()); + let started = Instant::now(); + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { + backends, + flags, + backend_options, + memory_budget_thresholds, + display: None, + }); + let listed = tokio::runtime::Builder::new_current_thread() + .build() + .map(|runtime| { + runtime + .block_on(instance.enumerate_adapters(backends)) + .len() + }); + drop(instance); + info!( + adapters = listed.unwrap_or(0), + took_ms = started.elapsed().as_millis() as u64, + "Window renderer: the graphics driver answered before the window" + ); + mark_thread(window); + } +} + +/// Name `thread` in this launch's marker and save it again. +fn mark_thread(thread: Option) { + if let Ok(mut launch) = LAUNCH.lock() { + launch.mark_thread( + STATE_DIR.get().map(PathBuf::as_path), + ProcessId::current(), + thread, + ); + } +} + +/// This launch's marker, and whether the user quit before the window drew. +/// One lock holds both, so a worker still in the driver cannot save the +/// marker again after the quit cleared it. +struct Launch { + started: Option, + abandoned: bool, +} + +impl Launch { + const fn new() -> Self { + Self { + started: None, + abandoned: false, + } + } + + /// Leave `marker` in `directory` until the window is confirmed working. + /// False when it could not be saved, or the launch was abandoned. + fn start(&mut self, directory: Option<&Path>, owner: ProcessId, marker: Marker) -> bool { + if self.abandoned { + return false; + } + let saved = directory.is_some_and(|directory| save_marker(directory, owner, &marker)); + self.started = Some(marker); + saved + } + + fn mark_thread(&mut self, directory: Option<&Path>, owner: ProcessId, thread: Option) { + if self.abandoned { + return; + } + let Some(marker) = self.started.as_mut() else { + return; + }; + marker.thread = thread; + if let Some(directory) = directory { + let _ = save_marker(directory, owner, marker); + } + } + + /// Clear the marker, after recording that its backend drew when `drew`. + fn clear(&mut self, directory: &Path, owner: ProcessId, drew: bool) { + if let Some(marker) = self.started.as_ref() { + if finish_launch(directory, marker, owner, drew) { + self.started = None; + } + } + } + + /// The user quit: the marker goes without counting, for good. + fn abandon(&mut self, directory: &Path, owner: ProcessId) { + self.abandoned = true; + self.clear(directory, owner, false); + } +} + /// `WGPU_BACKEND` decides when set, as in wgpu, and nothing is narrowed or /// recorded then. Otherwise the saved choice, unless it failed before. Also /// returns why a remote rule can't apply, if one of them decides. @@ -329,6 +544,9 @@ fn choose_backends( backend: plan.stage().to_string(), thread: current_thread(), }); + } else { + // The listing's marker: nothing is recorded under WGPU_BACKEND. + clear_marker(false); } descriptor.backends = instance_backends(forced, plan); } @@ -414,13 +632,13 @@ fn current_thread() -> Option { /// Leave this launch's marker until the window is confirmed working. False /// when it could not be saved. fn start(marker: Marker) -> bool { - let saved = STATE_DIR - .get() - .is_some_and(|directory| save_marker(directory, ProcessId::current(), &marker)); - if let Ok(mut started) = STARTED.lock() { - *started = Some(marker); - } - saved + LAUNCH.lock().is_ok_and(|mut launch| { + launch.start( + STATE_DIR.get().map(PathBuf::as_path), + ProcessId::current(), + marker, + ) + }) } fn save_marker(directory: &Path, owner: ProcessId, marker: &Marker) -> bool { @@ -428,31 +646,250 @@ fn save_marker(directory: &Path, owner: ProcessId, marker: &Marker) -> bool { .is_ok_and(|json| fs::write(marker_path(directory, owner), json).is_ok()) } +/// What a screenshot that proves a frame was presented carries: the span it +/// was asked in (a new one after each surface error, skipped frame or new +/// device) and which ask it answers. Only an answer to the current span's +/// asks counts. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct PresentProof { + span: u64, + request: u64, +} + /// Called at the start of each UI pass; clears this launch's marker once the /// window has kept drawing without a surface error. pub(super) fn frame_started(ctx: &egui::Context) { crate::startup::simulate_surface_error(&FAILED_ACQUIRES); + // Whether or not a replacement is ever made. + let lost = eframe::egui_wgpu::winit::DEVICES_LOST.load(Ordering::SeqCst); + if LOSSES_SEEN.swap(lost, Ordering::SeqCst) != lost { + lost_before_drawing(); + } let visible = ctx.input(|input| input.viewport().minimized != Some(true)); + if let Ok(mut last) = LAST_PASS.lock() { + *last = Some(Instant::now()); + } + let proofs: Vec = ctx.input(|input| { + input + .events + .iter() + .filter_map(|event| match event { + egui::Event::Screenshot { user_data, .. } => user_data + .data + .as_ref()? + .downcast_ref::() + .copied(), + _ => None, + }) + .collect() + }); let Ok(mut tracker) = PRESENT.lock() else { return; }; - let now_drawing = tracker.observe( + let step = tracker.observe( ctx.cumulative_frame_nr(), - FAILED_ACQUIRES.load(Ordering::SeqCst), + failures(), visible, Instant::now(), + &proofs, ); let drawing = tracker.confirmed; drop(tracker); - if now_drawing { + if let Step::Prove(proof) = step { + // egui-wgpu answers only after it acquired the surface, drew the + // frame into it and submitted it, right before the present. + ctx.send_viewport_cmd(egui::ViewportCommand::Screenshot(egui::UserData::new( + proof, + ))); + } + if step == Step::Confirmed { crate::startup::mark_window_shown(); info!("Window renderer is drawing"); + let on_drawing = ON_DRAWING.lock().ok().and_then(|mut hook| hook.take()); + if let Some(on_drawing) = on_drawing { + on_drawing(); + } } if drawing { clear_marker(true); } } +/// When the window's thread last began a UI pass. None before the first. +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn last_pass() -> Option { + LAST_PASS.lock().ok().and_then(|last| *last) +} + +/// Remember the window, for [`window_held`]. +pub(super) fn remember_window(cc: &eframe::CreationContext<'_>) { + #[cfg(windows)] + { + use eframe::wgpu::rwh::{HasWindowHandle, RawWindowHandle}; + if let Ok(handle) = cc.window_handle() { + if let RawWindowHandle::Win32(window) = handle.as_raw() { + WINDOW.store(window.hwnd.get(), Ordering::SeqCst); + } + } + } + #[cfg(not(windows))] + let _ = cc; +} + +/// Whether the window is minimized, or its thread is in the move/size or +/// menu loop Windows runs while the title bar is held: no UI pass runs then +/// in a working window either. Read from another thread, live. +#[cfg(windows)] +pub(super) fn window_held() -> bool { + use windows::Win32::Foundation::HWND; + use windows::Win32::UI::WindowsAndMessaging::{ + GetGUIThreadInfo, GetWindowThreadProcessId, IsIconic, GUITHREADINFO, GUI_INMENUMODE, + GUI_INMOVESIZE, + }; + let hwnd = WINDOW.load(Ordering::SeqCst); + if hwnd == 0 { + return false; + } + let hwnd = HWND(hwnd as _); + if unsafe { IsIconic(hwnd) }.as_bool() { + return true; + } + let thread = unsafe { GetWindowThreadProcessId(hwnd, None) }; + let mut info = GUITHREADINFO { + cbSize: std::mem::size_of::() as u32, + ..Default::default() + }; + thread != 0 + && unsafe { GetGUIThreadInfo(thread, &mut info) }.is_ok() + && (info.flags & (GUI_INMOVESIZE | GUI_INMENUMODE)).0 != 0 +} + +#[cfg(not(windows))] +#[allow(dead_code)] +pub(super) fn window_held() -> bool { + false +} + +/// Run `hook` once, when the window is confirmed drawing: frames presented +/// for a while with no surface error, visible ([`PresentTracker`]). +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn when_drawing(hook: OnDrawing) { + if let Ok(mut on_drawing) = ON_DRAWING.lock() { + *on_drawing = Some(hook); + } +} + +/// The user quit before the window drew. A deliberate quit is no failed +/// start: the marker goes without counting, and no worker still in the +/// driver writes it again. +pub(super) fn abandon() { + if let (Some(directory), Ok(mut launch)) = (STATE_DIR.get(), LAUNCH.lock()) { + launch.abandon(directory, ProcessId::current()); + } +} + +/// Surface errors, devices and frames the painter skipped while its device +/// was lost or being replaced: any of them starts the wait for a working +/// window again. egui-wgpu reports no surface error for a frame it skips. +fn failures() -> u64 { + FAILED_ACQUIRES.load(Ordering::SeqCst) + + devices() + + eframe::egui_wgpu::winit::SKIPPED_FRAMES.load(Ordering::SeqCst) +} + +/// The painter dropped a lost device. Before the window was confirmed +/// working, that means the backend it started on failed to draw. +fn lost_before_drawing() { + let confirmed = PRESENT.lock().is_ok_and(|tracker| tracker.confirmed); + if let (Some(directory), Ok(mut launch)) = (STATE_DIR.get(), LAUNCH.lock()) { + lose_started( + directory, + &mut launch.started, + confirmed, + ProcessId::current(), + ); + } +} + +/// Record the failure of this launch's `started` backend, once, unless the +/// window was `confirmed`. The marker ends with it, so neither a replacement +/// that draws later nor a normal quit counts that backend as drawing. +fn lose_started(directory: &Path, started: &mut Option, confirmed: bool, owner: ProcessId) { + if confirmed { + return; + } + if let Some(marker) = started.take() { + warn!(gpu = %marker.gpu, backend = %marker.backend, "Window device lost before the window worked"); + lose_launch(directory, &marker, owner); + } +} + +/// Record `marker`'s backend as failed and remove the marker. While another +/// instance holds the history the marker stays, and the next launch counts it. +fn lose_launch(directory: &Path, marker: &Marker, owner: ProcessId) { + if update_history(directory, |history| { + record_failure(history, marker, DEVICE_LOST) + }) + .is_some() + { + let _ = fs::remove_file(marker_path(directory, owner)); + } +} + +/// Drop eframe's copy of the window's first device, which is kept for paint +/// callbacks the window has none of. A lost device is only replaced once +/// nothing holds it: D3D12 keeps one device per GPU in a process, and the +/// window's old swapchain stays with it. +pub(super) fn release_render_state(frame: &mut eframe::Frame) { + #[cfg(any(debug_assertions, feature = "fault-injection"))] + if crate::startup::device_loss_requested() { + // The copy is kept until the simulated loss. + if crate::startup::simulate_device_loss_now() { + if let Some(render_state) = frame.wgpu_render_state.take() { + lose_device(&render_state.device); + } + } + return; + } + frame.wgpu_render_state = None; +} + +/// Lose the window's device on purpose, when [`crate::startup::SIMULATE_ENV`] +/// asks: with the `fault-injection` feature DX12 removes it, as a driver reset +/// does; otherwise it is destroyed. +#[cfg(any(debug_assertions, feature = "fault-injection"))] +fn lose_device(device: &wgpu::Device) { + warn!("Simulated loss of the window's graphics device"); + #[cfg(all(windows, feature = "fault-injection"))] + if remove_dx12_device(device) { + return; + } + device.destroy(); +} + +/// `ID3D12Device5::RemoveDevice` on a DX12 device; false on other backends. +#[cfg(all(windows, feature = "fault-injection"))] +fn remove_dx12_device(device: &wgpu::Device) -> bool { + use windows::core::Interface; + use windows::Win32::Graphics::Direct3D12::{ID3D12Device, ID3D12Device5}; + let Some(hal) = (unsafe { device.as_hal::() }) else { + return false; + }; + // wgpu-hal uses a newer windows crate, so the device crosses as its COM + // pointer, which is all either version's interface type holds. + let raw: *mut std::ffi::c_void = unsafe { std::mem::transmute_copy(hal.raw_device()) }; + let Some(device) = (unsafe { ID3D12Device::from_raw_borrowed(&raw) }) else { + return false; + }; + match device.cast::() { + Ok(device) => { + unsafe { device.RemoveDevice() }; + true + } + Err(_) => false, + } +} + /// The window closed normally. A launch that quits before its window is /// confirmed did not fail, so its marker goes without counting either way. pub(super) fn window_closed() { @@ -463,13 +900,8 @@ pub(super) fn window_closed() { /// `drew`. The history may be busy with another instance; the next frame /// tries again, so the GUI never waits for it. fn clear_marker(drew: bool) { - let (Some(directory), Ok(mut started)) = (STATE_DIR.get(), STARTED.lock()) else { - return; - }; - if let Some(marker) = started.as_ref() { - if finish_launch(directory, marker, ProcessId::current(), drew) { - started.take(); - } + if let (Some(directory), Ok(mut launch)) = (STATE_DIR.get(), LAUNCH.lock()) { + launch.clear(directory, ProcessId::current(), drew); } } @@ -481,23 +913,42 @@ fn finish_launch(directory: &Path, marker: &Marker, owner: ProcessId, drew: bool true } -/// Confirms a working window from the frame counter, the time and surface -/// errors. egui-wgpu presents inside its painter and reports nothing after a -/// present; a surface it fails to recreate is not reported at all. So the -/// window must keep drawing for a while after the last reported error, which -/// takes it past device creation and the first presents, where the drivers of -/// issue #59 fail. +/// Confirms a working window from the frame counter, the time, surface +/// errors and a proof of a presented frame. egui-wgpu presents inside its +/// painter and reports nothing after a present; a surface it fails to +/// recreate is not reported at all, and its later paints return before +/// drawing. So the window must keep drawing for a while after the last +/// reported error, which takes it past device creation and the first +/// presents, where the drivers of issue #59 fail, and then answer a +/// screenshot, which egui-wgpu takes only from a frame it acquired and drew. struct PresentTracker { span: Option, confirmed: bool, + /// The id the next span gets, and the next ask. + next_span: u64, + next_request: u64, } /// Visible frames since the last surface error. -#[derive(Clone, Copy)] +#[derive(Clone)] struct Span { + id: u64, since: Instant, first_frame: u64, failures: u64, + /// The frame that last asked for the proof in this span, and the asks + /// made in it. + asked: Option<(u64, std::ops::RangeInclusive)>, +} + +/// What a UI pass does for the confirmation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Step { + Wait, + /// Ask for the proof: a screenshot of this frame, carrying this. + Prove(PresentProof), + /// Confirmed, once. + Confirmed, } impl PresentTracker { @@ -505,37 +956,75 @@ impl PresentTracker { Self { span: None, confirmed: false, + next_span: 0, + next_request: 0, } } - /// True once: after [`CONFIRM_AFTER`] and [`CONFIRM_FRAMES`] of visible - /// frames with no new surface error. A surface error or a minimized - /// window starts the span again. - fn observe(&mut self, frame: u64, failures: u64, visible: bool, now: Instant) -> bool { + /// After [`CONFIRM_AFTER`] and [`CONFIRM_FRAMES`] of visible frames with + /// no new surface error, asks for the proof (again after + /// [`PROOF_AGAIN_AFTER`] frames without it), and confirms once an + /// answer to one of this span's asks came (`proofs`). A surface error, + /// a skipped frame, a new device or a minimized window starts the span + /// again, and answers to an earlier span's asks are ignored. + fn observe( + &mut self, + frame: u64, + failures: u64, + visible: bool, + now: Instant, + proofs: &[PresentProof], + ) -> Step { if self.confirmed { - return false; + return Step::Wait; } - match self.span { + let request = self.next_request; + match self.span.as_mut() { Some(span) if visible && span.failures == failures => { - if frame >= span.first_frame + CONFIRM_FRAMES - && now.duration_since(span.since) >= CONFIRM_AFTER - { - self.confirmed = true; - return true; + let ask = if let Some((asked, requests)) = &span.asked { + if proofs + .iter() + .any(|proof| proof.span == span.id && requests.contains(&proof.request)) + { + self.confirmed = true; + return Step::Confirmed; + } + (frame >= asked + PROOF_AGAIN_AFTER).then_some(*requests.start()..=request) + } else { + (frame >= span.first_frame + CONFIRM_FRAMES + && now.duration_since(span.since) >= CONFIRM_AFTER) + .then_some(request..=request) + }; + if let Some(requests) = ask { + span.asked = Some((frame, requests)); + self.next_request += 1; + return Step::Prove(PresentProof { + span: span.id, + request, + }); } } _ => { self.span = visible.then_some(Span { + id: self.next_span, since: now, first_frame: frame, failures, + asked: None, }); + self.next_span += 1; } } - false + Step::Wait } } +/// How many devices the window has drawn with, or tried to: one more after +/// each loss. Its textures must be made again on each. +pub(super) fn devices() -> u64 { + SELECTIONS.load(Ordering::SeqCst) +} + fn select( adapters: &[wgpu::Adapter], surface: Option<&wgpu::Surface<'_>>, @@ -544,6 +1033,14 @@ fn select( rules: &[RendererRule], overruled: Option<&'static str>, ) -> Result { + // A replacement for a lost device (on egui-wgpu's replacement thread) + // chooses as the start did, and starts no marker: the loss itself was + // recorded when the device was dropped (`frame_started`). + let replacing = SELECTIONS.fetch_add(1, Ordering::SeqCst) > 0; + #[cfg(any(debug_assertions, feature = "fault-injection"))] + if replacing { + crate::startup::simulate_replacement_failure()?; + } let usable: Vec<&wgpu::Adapter> = adapters .iter() .filter(|adapter| surface.is_none_or(|surface| adapter.is_surface_supported(surface))) @@ -561,7 +1058,7 @@ fn select( apply_rule(&candidates, &gpus, picked, rules, overruled), picked, STATE_DIR.get().map(PathBuf::as_path), - |index| start(marker(index)), + |index| replacing || start(marker(index)), ); if let Some((rule, outcome)) = ruled { info!("{}", rule_line(rule, outcome, &gpus[picked])); @@ -579,6 +1076,7 @@ fn select( "remote rule" }, avoided = ?history.iter().filter(|f| gpus.contains(&f.gpu) && avoided(history, &f.gpu, &f.backend)).map(|f| (&f.gpu, &f.backend, &f.fault)).collect::>(), + replacing_lost_device = replacing, "Window renderer selected" ); let drawing = match info.backend { @@ -814,8 +1312,34 @@ fn system_gpus() -> String { } /// Vendor, device and driver version of each DXGI adapter. Creates no device. +/// Until the window is confirmed drawing, the list taken before it. After +/// that (a renderer rebuilt after a lost device), a fresh one under a limit +/// on a thread of its own: empty if it does not return in time. #[cfg(windows)] fn dxgi_adapters() -> Vec<(u32, u32, String)> { + let shown = crate::startup::window_shown(); + if !shown { + if let Some(listed) = ADAPTERS.lock().ok().and_then(|listed| listed.clone()) { + return listed; + } + } + crate::stall::run(adapters_lane(shown), list_dxgi_adapters, || {}).unwrap_or_default() +} + +/// The lane of a DXGI adapter list the window takes itself: startup's until +/// it is confirmed drawing, then the one the second screen's graphics reads +/// use, so a later renderer neither waits behind nor holds startup's. +#[cfg_attr(not(windows), allow(dead_code))] +fn adapters_lane(painted: bool) -> crate::stall::Call { + if painted { + crate::stall::Call::GpuFacts + } else { + crate::stall::Call::GpuDetect + } +} + +#[cfg(windows)] +fn list_dxgi_adapters() -> Vec<(u32, u32, String)> { use windows::core::Interface; use windows::Win32::Graphics::Dxgi::{CreateDXGIFactory1, IDXGIDevice, IDXGIFactory1}; let Ok(factory) = (unsafe { CreateDXGIFactory1::() }) else { @@ -843,6 +1367,11 @@ fn dxgi_adapters() -> Vec<(u32, u32, String)> { Vec::new() } +#[cfg(not(windows))] +fn list_dxgi_adapters() -> Vec<(u32, u32, String)> { + Vec::new() +} + /// A crash inside a graphics driver stays avoided until the driver changes, /// which changes the GPU key. Every other failure, and one on a GPU whose /// driver version is unknown, is retried after clean launches. @@ -1617,6 +2146,88 @@ mod tests { fs::remove_dir_all(directory).unwrap(); } + #[test] + fn a_quit_from_the_driver_notice_leaves_no_marker_and_no_worker_writes_it_again() { + let directory = temp_dir("abandon"); + let owner = exited_process(); + let mut launch = Launch::new(); + assert!(launch.start( + Some(&directory), + owner, + marker("10de:2c02 32.0", "enumerate") + )); + assert!(marker_path(&directory, owner).exists()); + // Cancel while the window's thread is stuck in the driver. + launch.abandon(&directory, owner); + assert!(!marker_path(&directory, owner).exists()); + // A probe or list still in the driver returns later, and the + // window's thread may still reach its adapter selector. + launch.mark_thread(Some(&directory), owner, Some(WINDOW)); + assert!(!launch.start(Some(&directory), owner, marker("10de:2c02 32.0", "dx12"))); + assert!(!marker_path(&directory, owner).exists()); + // The next launch counts no failed start. + assert!(sweep(&directory, ProcessId::current()).is_empty()); + let _ = fs::remove_dir_all(&directory); + } + + /// The app quits from outside its window (Windows closing it for an + /// installer) before the window is confirmed drawing, through this + /// launch's own marker: none is left for the next launch to count, and a + /// probe still in the driver doesn't write it again. The only test of + /// the launch-wide marker; it puts it back as it found it. + #[test] + fn a_quit_from_outside_the_window_before_it_draws_leaves_no_marker() { + let directory = STATE_DIR.get_or_init(|| temp_dir("quit-outside")).clone(); + let owner = ProcessId::current(); + assert!(start(marker("10de:2c02 32.0", "dx12"))); + assert!(marker_path(&directory, owner).exists()); + crate::gui::quitting_from_outside(); + assert!(!marker_path(&directory, owner).exists()); + mark_thread(Some(WINDOW)); + assert!(!start(marker("10de:2c02 32.0", "vulkan"))); + assert!(!marker_path(&directory, owner).exists()); + assert!(sweep(&directory, ProcessId::current()).is_empty()); + *LAUNCH.lock().unwrap() = Launch::new(); + let _ = fs::remove_dir_all(&directory); + } + + #[test] + fn a_launch_that_dies_listing_the_graphics_cards_skips_the_list_next_time() { + let directory = temp_dir("listing"); + let owner = exited_process(); + let mut launch = Launch::new(); + // The marker is there before the list starts, and names the thread + // that lists once it runs. A crash there is the window's. + launch.start(Some(&directory), owner, marker("", LISTING)); + launch.mark_thread(Some(&directory), owner, Some(0x5150)); + let saved: Marker = + serde_json::from_slice(&fs::read(marker_path(&directory, owner)).unwrap()).unwrap(); + assert_eq!(saved.thread, Some(0x5150)); + // That launch died in the driver while listing. + let history = sweep(&directory, ProcessId::current()); + assert_eq!(history.len(), 1, "{history:?}"); + assert_eq!( + (history[0].gpu.as_str(), history[0].backend.as_str()), + ("", LISTING) + ); + assert!(Prepared { history }.listing_avoided()); + assert!(!Prepared { + history: Vec::new() + } + .listing_avoided()); + let _ = fs::remove_dir_all(&directory); + } + + #[test] + fn after_the_first_frame_the_adapter_list_leaves_the_startup_lane() { + use crate::stall::Call; + // Before the window is confirmed, a stuck startup call refuses it. + assert_eq!(adapters_lane(false), Call::GpuDetect); + // A renderer started later (after a lost device) neither waits behind + // nor holds startup's lane, nor logs its stalls as startup's. + assert_eq!(adapters_lane(true), Call::GpuFacts); + } + #[test] fn the_windows_instance_leaves_out_opengl_unless_wgpu_backend_asks() { let full = instance_backends(None, Plan::Full); @@ -1953,63 +2564,290 @@ mod tests { } /// Visible frames every 200 ms, as the GUI repaints, counted from `start`. + const NO_PROOF: &[PresentProof] = &[]; + + /// The proof a step asks for; panics on any other step. + fn ask(step: Step) -> PresentProof { + match step { + Step::Prove(proof) => proof, + other => panic!("{other:?}"), + } + } + + /// Frames `frames` a 200 ms apart, the proof answering the frame after + /// each ask: whether the window was confirmed. fn run( tracker: &mut PresentTracker, start: Instant, frames: std::ops::Range, failures: u64, ) -> bool { + let mut answer = None; frames.into_iter().any(|frame| { - tracker.observe( + let proofs: Vec = answer.take().into_iter().collect(); + let step = tracker.observe( frame, failures, true, start + Duration::from_millis(200 * frame), - ) + &proofs, + ); + if let Step::Prove(proof) = step { + answer = Some(proof); + } + step == Step::Confirmed }) } #[test] - fn the_window_works_after_frames_and_time_without_surface_errors() { + fn the_window_works_after_frames_and_time_without_surface_errors_and_a_proof() { let start = Instant::now(); let mut tracker = PresentTracker::new(); // Frames alone are not enough: egui may lay out one frame many times. for frame in 0..100 { - assert!(!tracker.observe(frame, 0, true, start)); + assert_eq!(tracker.observe(frame, 0, true, start, NO_PROOF), Step::Wait); } - assert!(tracker.observe(100, 0, true, start + CONFIRM_AFTER)); + let proof = ask(tracker.observe(100, 0, true, start + CONFIRM_AFTER, NO_PROOF)); + assert_eq!( + tracker.observe(101, 0, true, start + CONFIRM_AFTER, &[proof]), + Step::Confirmed + ); let mut tracker = PresentTracker::new(); // Nor is time alone, with one frame laid out repeatedly. - assert!(!tracker.observe(0, 0, true, start)); - assert!(!tracker.observe(0, 0, true, start + 10 * CONFIRM_AFTER)); - assert!(tracker.observe(CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER)); - assert!(!tracker.observe(CONFIRM_FRAMES + 1, 0, true, start + CONFIRM_AFTER)); + assert_eq!(tracker.observe(0, 0, true, start, NO_PROOF), Step::Wait); + assert_eq!( + tracker.observe(0, 0, true, start + 10 * CONFIRM_AFTER, NO_PROOF), + Step::Wait + ); + let proof = ask(tracker.observe(CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER, NO_PROOF)); + assert_eq!( + tracker.observe(CONFIRM_FRAMES + 1, 0, true, start + CONFIRM_AFTER, &[proof]), + Step::Confirmed + ); + // Once. + assert_eq!( + tracker.observe(CONFIRM_FRAMES + 2, 0, true, start + CONFIRM_AFTER, &[proof]), + Step::Wait + ); + } + + #[test] + fn frames_that_present_nothing_are_never_confirmed() { + let start = Instant::now(); + let later = start + CONFIRM_AFTER; + let mut tracker = PresentTracker::new(); + assert_eq!(tracker.observe(0, 0, true, start, NO_PROOF), Step::Wait); + let first = ask(tracker.observe(CONFIRM_FRAMES, 0, true, later, NO_PROOF)); + // The surface was removed and its recreation failed: egui-wgpu's + // paints return before drawing, with no surface error, and the UI + // passes go on. No screenshot answers, so nothing is confirmed; the + // ask is repeated now and then. + for frame in CONFIRM_FRAMES + 1..CONFIRM_FRAMES + PROOF_AGAIN_AFTER { + assert_eq!(tracker.observe(frame, 0, true, later, NO_PROOF), Step::Wait); + } + let again = + ask(tracker.observe(CONFIRM_FRAMES + PROOF_AGAIN_AFTER, 0, true, later, NO_PROOF)); + assert_eq!(again.span, first.span); + assert_ne!(again.request, first.request); + // A late answer to the first ask of this span still proves a frame. + assert_eq!( + tracker.observe( + CONFIRM_FRAMES + PROOF_AGAIN_AFTER + 1, + 0, + true, + later, + &[first] + ), + Step::Confirmed + ); + } + + #[test] + fn a_late_answer_from_before_a_surface_error_or_a_new_device_proves_nothing() { + let start = Instant::now(); + let later = start + CONFIRM_AFTER; + // `failures` counts surface errors, frames the painter skipped and + // devices: a lost device skips frames and its replacement is one more. + for (what, after) in [ + ("a surface error", 1), + ("skipped frames and a new device", 4), + ] { + let mut tracker = PresentTracker::new(); + tracker.observe(0, 0, true, start, NO_PROOF); + let old = ask(tracker.observe(CONFIRM_FRAMES, 0, true, later, NO_PROOF)); + // The old frame's answer arrives only after the change. + let frame = CONFIRM_FRAMES + 1; + assert_eq!( + tracker.observe(frame, after, true, later, &[old]), + Step::Wait, + "{what}" + ); + let new_start = later; + let new_later = new_start + CONFIRM_AFTER; + assert_eq!( + tracker.observe(frame + 1, after, true, new_start, &[old]), + Step::Wait, + "{what}" + ); + let new = + ask(tracker.observe(frame + 1 + CONFIRM_FRAMES, after, true, new_later, &[old])); + assert_ne!(new.span, old.span, "{what}"); + // Still only the old answer: nothing. + assert_eq!( + tracker.observe(frame + 2 + CONFIRM_FRAMES, after, true, new_later, &[old]), + Step::Wait, + "{what}" + ); + // The new span's own answer confirms. + assert_eq!( + tracker.observe( + frame + 3 + CONFIRM_FRAMES, + after, + true, + new_later, + &[old, new] + ), + Step::Confirmed, + "{what}" + ); + } + // An answer naming this span but an ask it never made proves nothing. + let mut tracker = PresentTracker::new(); + tracker.observe(0, 0, true, start, NO_PROOF); + let asked = ask(tracker.observe(CONFIRM_FRAMES, 0, true, later, NO_PROOF)); + let forged = PresentProof { + request: asked.request + 7, + ..asked + }; + assert_eq!( + tracker.observe(CONFIRM_FRAMES + 1, 0, true, later, &[forged]), + Step::Wait + ); } #[test] fn a_lost_surface_that_is_recreated_late_is_not_a_working_window() { let start = Instant::now(); let mut tracker = PresentTracker::new(); - assert!(!tracker.observe(0, 0, true, start)); + assert_eq!(tracker.observe(0, 0, true, start, NO_PROOF), Step::Wait); // Lost: egui reports it and recreates the surface on the next frame, // which can fail without a report and without a present. - assert!(!tracker.observe(1, 1, true, start + Duration::from_millis(16))); - assert!(!tracker.observe(2, 1, true, start + Duration::from_millis(32))); - assert!(!tracker.observe(3, 1, true, start + Duration::from_millis(48))); + for (frame, ms) in [(1, 16), (2, 32), (3, 48)] { + assert_eq!( + tracker.observe(frame, 1, true, start + Duration::from_millis(ms), NO_PROOF), + Step::Wait + ); + } // Confirmed only once the window kept drawing after the error. assert!(!run(&mut tracker, start, 4..15, 1)); let mut tracker = PresentTracker::new(); assert!(run(&mut tracker, start, 0..20, 0)); } + #[test] + fn frames_skipped_for_a_lost_device_or_a_new_device_are_not_a_working_window() { + use eframe::egui_wgpu::winit::SKIPPED_FRAMES; + let start = Instant::now(); + let mut tracker = PresentTracker::new(); + assert!(!run(&mut tracker, start, 0..10, failures())); + // The device is lost: egui-wgpu skips frames without a surface error. + SKIPPED_FRAMES.fetch_add(3, Ordering::SeqCst); + assert!(!run(&mut tracker, start, 10..24, failures())); + // A new device is made, and it must draw for a while too. + SELECTIONS.fetch_add(1, Ordering::SeqCst); + assert!(!run(&mut tracker, start, 24..38, failures())); + assert!(run(&mut tracker, start, 38..60, failures())); + } + + #[test] + fn a_device_lost_before_the_window_works_fails_its_backend() { + let directory = temp_dir("lost-start"); + let owner = ProcessId::current(); + let gpu = "10de:2c02 32.0.15.9186"; + let started = marker(gpu, "dx12"); + assert!(save_marker(&directory, owner, &started)); + lose_launch(&directory, &started, owner); + let history = read_history(&directory); + assert_eq!(history.len(), 1); + assert_eq!( + (history[0].gpu.as_str(), history[0].backend.as_str()), + (gpu, "dx12") + ); + assert_eq!(history[0].failures, 1); + assert_eq!(history[0].fault.as_deref(), Some(DEVICE_LOST)); + assert!(!marker_path(&directory, owner).exists()); + // Not a driver crash: retried after clean launches, like any other end. + assert!(!history[0].in_driver()); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn a_device_lost_before_the_window_works_is_recorded_once_even_if_never_replaced() { + let directory = temp_dir("lost-unreplaced"); + let owner = ProcessId::current(); + let gpu = "10de:2c02 32.0.15.9186"; + let mut started = Some(marker(gpu, "vulkan")); + assert!(save_marker(&directory, owner, started.as_ref().unwrap())); + // The painter drops the device, then every replacement fails making + // its surface, so no adapter is ever chosen, until it gives up. The + // host sees the drop on the next frames. + for _ in 0..3 { + lose_started(&directory, &mut started, false, owner); + } + // Then a normal quit (`window_closed`). + if let Some(marker) = started.as_ref() { + assert!(finish_launch(&directory, marker, owner, false)); + } + let history = read_history(&directory); + assert_eq!(history.len(), 1); + assert_eq!(history[0].backend, "vulkan"); + assert_eq!(history[0].failures, 1); + assert_eq!(history[0].fault.as_deref(), Some(DEVICE_LOST)); + assert!(!marker_path(&directory, owner).exists()); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn a_device_lost_after_the_window_works_is_not_a_failed_start() { + let directory = temp_dir("lost-confirmed"); + let owner = ProcessId::current(); + let mut started = Some(marker("10de:2c02 32.0.15.9186", "dx12")); + lose_started(&directory, &mut started, true, owner); + assert!(started.is_some()); + assert!(read_history(&directory).is_empty()); + fs::remove_dir_all(directory).unwrap(); + } + #[test] fn a_minimized_window_is_not_confirmed() { let start = Instant::now(); let mut tracker = PresentTracker::new(); - assert!(!tracker.observe(0, 0, true, start)); - assert!(!tracker.observe(CONFIRM_FRAMES, 0, false, start + CONFIRM_AFTER)); - assert!(!tracker.observe(2 * CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER)); - assert!(tracker.observe(3 * CONFIRM_FRAMES, 0, true, start + 2 * CONFIRM_AFTER)); + assert_eq!(tracker.observe(0, 0, true, start, NO_PROOF), Step::Wait); + assert_eq!( + tracker.observe(CONFIRM_FRAMES, 0, false, start + CONFIRM_AFTER, NO_PROOF), + Step::Wait + ); + assert_eq!( + tracker.observe(2 * CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER, NO_PROOF), + Step::Wait + ); + let proof = ask(tracker.observe( + 3 * CONFIRM_FRAMES, + 0, + true, + start + 2 * CONFIRM_AFTER, + NO_PROOF, + )); + assert_eq!( + tracker.observe( + 3 * CONFIRM_FRAMES + 1, + 0, + true, + start + 2 * CONFIRM_AFTER, + &[proof] + ), + Step::Confirmed + ); } // ── Remote renderer rules ─────────────────────────────────────────────── diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index d7f2b148..389e777f 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -1,9 +1,138 @@ use super::page_stream::{ - audio_summary, capture_block_notice, second_screen_notice, software_fallback_body, - usb_cable_notice, usb_summary, + audio_summary, capture_block_notice, firewall_fix_failure, firewall_fix_offered, + firewall_notice, second_screen_notice, software_fallback_body, usb_cable_notice, usb_summary, }; use super::*; +/// A fix that failed says so in its own banner, whatever the firewall +/// reads now. One that worked, or a No to the Windows prompt, says nothing. +#[test] +fn a_failed_firewall_fix_shows_its_error_on_its_own() { + use crate::firewall_fix::{Outcome, Progress, FAILED}; + assert_eq!( + firewall_fix_failure(Progress::Done(Outcome::Failed(FAILED))), + Some(FAILED) + ); + for quiet in [ + Progress::Idle, + Progress::Running, + Progress::Done(Outcome::Fixed), + Progress::Done(Outcome::Declined), + ] { + assert_eq!(firewall_fix_failure(quiet), None, "{quiet:?}"); + } +} + +#[test] +fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { + use crate::firewall::{judge, Profile, Rule, PRIVATE, PUBLIC}; + let exe = std::path::Path::new(r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"); + let profile = |bit, block_all_inbound| Profile { + bit, + on: true, + block_all_inbound, + }; + let block = |profiles| Rule::unscoped(&exe.to_string_lossy().to_lowercase(), false, profiles); + let allow = Rule::unscoped(&exe.to_string_lossy(), true, 0x7fff_ffff); + let report = |profiles: &[Profile], rules: &[Rule], lan| { + judge(profiles, rules, exe, 9876, lan, |_| None) + }; + + let blocked = report( + &[profile(PUBLIC, false)], + &[allow.clone(), block(PUBLIC)], + true, + ); + let (title, body) = firewall_notice(Some(&blocked), false).unwrap(); + assert_eq!(title, "Windows Firewall is blocking EternalMonitor"); + // This exe's own Block rule: "Fix it" removes it. + assert!(firewall_fix_offered(Some(&blocked))); + assert!( + body.contains("Run the EternalMonitor installer again"), + "{body}" + ); + assert!(body.contains("A USB cable still works."), "{body}"); + // Any connected iPad, over Wi-Fi or the cable, hides it. + assert_eq!(firewall_notice(Some(&blocked), true), None); + // No reading, or the last read failed. + assert_eq!(firewall_notice(None, false), None); + + let shields = report(&[profile(PUBLIC, true)], std::slice::from_ref(&allow), true); + let (title, body) = firewall_notice(Some(&shields), false).unwrap(); + assert_eq!(title, "Windows Firewall blocks every incoming connection"); + // Only the user's switch turns that off: no button. + assert!(!firewall_fix_offered(Some(&shields))); + assert!(!firewall_fix_offered(None)); + assert!(body.contains("choose the public network"), "{body}"); + let both = report( + &[profile(PRIVATE, false), profile(PUBLIC, true)], + &[block(PRIVATE)], + true, + ); + let (title, body) = firewall_notice(Some(&both), false).unwrap(); + assert_eq!(title, "Windows Firewall is blocking EternalMonitor"); + assert!( + body.contains("installer again") && body.contains("choose the public network"), + "{body}" + ); + // The rule part has the button; the switch keeps its instructions. + assert!(firewall_fix_offered(Some(&both))); + + // A rule for every program: Setup can't remove it, so it is only + // logged, alone or beside this exe's own block. + let everyone = report( + &[profile(PUBLIC, false)], + &[allow.clone(), Rule::unscoped("", false, PUBLIC)], + true, + ); + assert_eq!(firewall_notice(Some(&everyone), false), None); + let mine_too = report( + &[profile(PUBLIC, false)], + &[block(PUBLIC), Rule::unscoped("", false, PUBLIC)], + true, + ); + assert_eq!(firewall_notice(Some(&mine_too), false), None); + // An Allow rule that asks for IPsec may let traffic past: not shown. + let bypass = Rule { + secure_flags: Some(2), + ..allow.clone() + }; + let maybe = report(&[profile(PUBLIC, false)], &[bypass, block(PUBLIC)], true); + assert_eq!(firewall_notice(Some(&maybe), false), None); + + // Not certain for the network an iPad uses: logged, not shown. + let one_of_two = report( + &[profile(PRIVATE, false), profile(PUBLIC, false)], + &[allow.clone(), block(PUBLIC)], + true, + ); + assert_eq!(firewall_notice(Some(&one_of_two), false), None); + let scoped = Rule { + remote_addresses: Some("LocalSubnet".into()), + ..block(PUBLIC) + }; + assert_eq!( + firewall_notice( + Some(&report(&[profile(PUBLIC, false)], &[scoped], true)), + false + ), + None + ); + let no_lan = report(&[profile(PUBLIC, false)], &[block(PUBLIC)], false); + assert_eq!(firewall_notice(Some(&no_lan), false), None); + assert_eq!( + firewall_notice( + Some(&report(&[profile(PUBLIC, false)], &[allow], true)), + false + ), + None + ); + assert_eq!( + firewall_notice(Some(&report(&[profile(PUBLIC, false)], &[], true)), false), + None + ); +} + #[test] fn audio_row_reports_rate_and_keeps_failure_separate_from_video() { let mut audio = crate::audio::AudioStats::started("USB Audio Device"); @@ -37,6 +166,25 @@ fn audio_row_reports_rate_and_keeps_failure_separate_from_video() { assert_eq!(color, TEXT_MUTED); } +#[test] +fn audio_row_says_why_while_another_app_has_the_output() { + let mut audio = crate::audio::AudioStats::started("Speakers"); + audio.stopped(crate::audio::DEVICE_IN_USE_STATUS); + let (text, color) = audio_summary(&audio, true); + assert_eq!( + text, + "Another app is using your speakers on its own. PC audio comes back when it lets go." + ); + assert_eq!(color, WARNING); + + // Gone once audio is back. + let mut audio = crate::audio::AudioStats::started("Speakers"); + audio.kbps = 128.0; + let (text, color) = audio_summary(&audio, true); + assert!(text.starts_with("Streaming Speakers"), "{text}"); + assert_eq!(color, ACCENT); +} + #[test] fn usb_row_shows_install_hint_only_when_the_service_is_unavailable() { let missing = usb_summary(false, 2, "Connected", 0, false); @@ -210,12 +358,12 @@ fn screen_choice_says_what_the_ipad_shows_and_that_it_reconnects() { use super::page_stream::screen_mode_text; use crate::control::VddStatus; - let mirror = screen_mode_text("", None, VddStatus::Inactive, false); + let mirror = screen_mode_text("", None, VddStatus::Inactive, false, false); assert_eq!( mirror, "The iPad shows the same picture as your main screen." ); - let running = screen_mode_text("", None, VddStatus::Inactive, true); + let running = screen_mode_text("", None, VddStatus::Inactive, true, false); assert!(running.contains("reconnects by itself"), "{running}"); let monitor = screen_mode_text( @@ -223,6 +371,7 @@ fn screen_choice_says_what_the_ipad_shows_and_that_it_reconnects() { Some("DISPLAY2"), VddStatus::Inactive, false, + false, ); assert!( monitor.contains("DISPLAY2") && monitor.contains("Settings"), @@ -234,12 +383,19 @@ fn screen_choice_says_what_the_ipad_shows_and_that_it_reconnects() { None, VddStatus::WaitingForClient, true, + false, ); assert!(waiting.contains("as soon as it connects"), "{waiting}"); - let active = screen_mode_text(CAPTURE_VIRTUAL_SENTINEL, None, VddStatus::Active, true); + let active = screen_mode_text( + CAPTURE_VIRTUAL_SENTINEL, + None, + VddStatus::Active, + true, + false, + ); assert!(active.contains("Drag windows"), "{active}"); for failed in [VddStatus::Failed, VddStatus::TaskFailed] { - let text = screen_mode_text(CAPTURE_VIRTUAL_SENTINEL, None, failed, true); + let text = screen_mode_text(CAPTURE_VIRTUAL_SENTINEL, None, failed, true, false); assert!(text.contains("mirrored"), "{text}"); } // No driver or protocol words on the main screen. @@ -321,7 +477,7 @@ fn display_choices_use_the_names_shared_with_the_ipad() { #[test] fn an_outdated_nvidia_driver_is_named_with_the_version_to_install() { - let body = software_fallback_body(false, true); + let body = software_fallback_body(false, true, false, &[]); assert!( body.contains("NVIDIA driver to version 570 or newer"), "{body}" @@ -338,10 +494,82 @@ fn an_outdated_nvidia_driver_is_named_with_the_version_to_install() { } // Custom settings are still the first thing to undo. assert_eq!( - software_fallback_body(true, true), - software_fallback_body(true, false) + software_fallback_body(true, true, false, &[]), + software_fallback_body(true, false, false, &[]) + ); + assert!(!software_fallback_body(false, false, false, &[]).contains("570")); +} + +#[test] +fn an_amd_or_intel_card_that_could_not_encode_is_named_for_its_driver_update() { + use crate::gpu::GpuVendor::{Amd, Intel, Nvidia}; + let amd = software_fallback_body(false, false, false, &[Amd]); + assert!( + amd.contains("The AMD graphics driver on this PC could not start video encoding"), + "{amd}" + ); + assert!( + amd.contains("Update the AMD graphics driver, then click Restart"), + "{amd}" + ); + let intel = software_fallback_body(false, false, false, &[Intel]); + assert!( + intel.contains("Update the Intel graphics driver, then click Restart"), + "{intel}" + ); + assert!(!intel.contains("AMD"), "{intel}"); + let both = software_fallback_body(false, false, false, &[Intel, Amd]); + assert!( + both.contains("Update the AMD and Intel graphics drivers, then click Restart"), + "{both}" + ); + for body in [&amd, &intel, &both] { + for jargon in [ + "AMF", + "Quick Sync", + "QSV", + "MFX", + "libx264", + "encoder", + "http", + ] { + assert!(!body.contains(jargon), "{body}"); + } + } + // An outdated NVIDIA driver names the version to install, and custom + // settings are still the first thing to undo. + assert_eq!( + software_fallback_body(false, true, false, &[Amd]), + software_fallback_body(false, true, false, &[]) + ); + assert_eq!( + software_fallback_body(true, false, false, &[Amd]), + software_fallback_body(true, false, false, &[]) + ); + // No vendor named: the general advice, as before. + assert_eq!( + software_fallback_body(false, false, false, &[Nvidia]), + software_fallback_body(false, false, false, &[]) + ); + assert!(software_fallback_body(false, false, false, &[]).contains("NVIDIA, AMD or Intel")); +} + +#[test] +fn a_driver_that_did_not_answer_at_startup_is_named_with_a_restart() { + let body = software_fallback_body(false, false, true, &[]); + assert!( + body.contains("graphics driver did not respond when EternalMonitor started"), + "{body}" + ); + assert!(body.contains("Click Restart"), "{body}"); + for jargon in ["DXGI", "libx264", "encoder", "timed out"] { + assert!(!body.contains(jargon), "{body}"); + } + // It is the reason, whatever else is true. + assert_eq!( + body, + software_fallback_body(true, true, true, &[crate::gpu::GpuVendor::Amd]) ); - assert!(!software_fallback_body(false, false).contains("570")); } #[test] @@ -680,6 +908,116 @@ fn the_update_popup_waits_until_no_ipad_has_been_connected_for_a_while() { assert!(!ipad_nearby(false, &mut last, gone + quiet, quiet)); } +/// The hybrid laptop's lost panel: Windows also leaves the display "not +/// extended", but Extend can't bring the panel back, so the notice says to +/// restart, and that Try again can take the panel again. +#[test] +fn a_lost_built_in_screen_says_to_restart_not_to_extend() { + use crate::control::VddStatus; + use crate::vdd::VddTaskHealth; + use crate::vdd_device::VddProblem; + for problem in [None, Some(VddProblem::NotExtended)] { + let lost = second_screen_notice( + VddStatus::Failed, + VddTaskHealth::Ok, + problem, + Some("panel_lost"), + ) + .unwrap(); + assert!( + lost.body.starts_with( + "Your laptop's built-in screen turned off when the second screen started" + ), + "{}", + lost.body + ); + assert!(lost.body.contains("restart the PC"), "{}", lost.body); + assert!(!lost.body.contains("Extend"), "{}", lost.body); + assert!(lost.retry && !lost.installer); + } +} + +/// After the restart that brought the panel back, the notice is calm: no +/// restart, only that Second screen is paused until it is chosen again. +#[test] +fn a_lost_built_in_screen_from_before_the_restart_only_says_it_is_paused() { + use crate::control::VddStatus; + use crate::vdd::VddTaskHealth; + use crate::vdd_device::VddProblem; + for problem in [None, Some(VddProblem::NotExtended)] { + let paused = second_screen_notice( + VddStatus::Failed, + VddTaskHealth::Ok, + problem, + Some("panel_lost_before"), + ) + .unwrap(); + assert_eq!( + paused.body, + "Second screen is paused on this laptop because it turned off the built-in screen last time. Click Try again to use it." + ); + assert!(paused.retry && !paused.installer); + } +} + +/// On Windows 10 Second screen isn't tried: the notice, the note beside the +/// choice and Settings say it needs Windows 11, with no Try again, and the +/// note goes from Windows 11's text only. +#[test] +fn windows_10_says_second_screen_needs_windows_11() { + use super::page_settings::display_description; + use super::page_stream::{add_later_retry, screen_mode_text, WINDOWS_10_NOTE}; + use crate::control::VddStatus; + use crate::vdd::VddTaskHealth; + let windows_10 = Some(crate::capture::second_screen::WINDOWS_10); + for status in [VddStatus::Failed, VddStatus::TaskFailed] { + for health in [VddTaskHealth::Ok, VddTaskHealth::TasksMissing] { + let mut notice = second_screen_notice(status, health, None, windows_10).unwrap(); + assert!(notice.body.starts_with(WINDOWS_10_NOTE), "{}", notice.body); + assert!(!notice.retry && !notice.installer); + // A failure saved by an earlier version doesn't offer a retry. + let before = notice.body.clone(); + add_later_retry(&mut notice, true); + assert_eq!(notice.body, before); + } + } + assert!(WINDOWS_10_NOTE.contains("needs Windows 11")); + + for status in [ + VddStatus::WaitingForClient, + VddStatus::Failed, + VddStatus::Active, + ] { + let second = screen_mode_text(CAPTURE_VIRTUAL_SENTINEL, None, status, false, true); + assert_eq!(second, WINDOWS_10_NOTE); + } + let mirror = screen_mode_text("", None, VddStatus::Inactive, true, true); + assert_eq!( + mirror, + "The iPad shows the same picture as your main screen. Second screen needs Windows 11. \ + Switching restarts the stream, and the iPad reconnects by itself." + ); + assert!(display_description(true).contains("Second screen needs Windows 11")); + + // Windows 11 reads as before. + assert!(!display_description(false).contains("Windows")); + for (capture, status) in [ + ("", VddStatus::Inactive), + (CAPTURE_VIRTUAL_SENTINEL, VddStatus::WaitingForClient), + (CAPTURE_VIRTUAL_SENTINEL, VddStatus::Failed), + ] { + assert!(!screen_mode_text(capture, None, status, true, false).contains("Windows")); + } + let attach = second_screen_notice( + VddStatus::Failed, + VddTaskHealth::Ok, + None, + Some("attach_timeout"), + ) + .unwrap(); + assert!(!attach.body.contains("Windows 11") && attach.retry); +} + /// A second screen failure remembered for this PC is tried again later by /// itself; the notice says so beside Try again. #[test] diff --git a/host/src/gui/update_banner.rs b/host/src/gui/update_banner.rs index 84443e47..7d13c20a 100644 --- a/host/src/gui/update_banner.rs +++ b/host/src/gui/update_banner.rs @@ -457,6 +457,22 @@ impl AnalyzerApp { self.set_check_updates(automatic); } widgets::divider(ui); + if crate::updates::is_beta(env!("CARGO_PKG_VERSION")) { + let mut beta = self.settings_beta_updates; + widgets::setting_row( + ui, + "Get Beta updates", + "This is a Beta build. Turn this off to be offered only stable releases.", + 64.0, + |ui| { + widgets::toggle(ui, &mut beta, "Get Beta updates"); + }, + ); + if beta != self.settings_beta_updates { + self.set_beta_updates(beta); + } + widgets::divider(ui); + } let status = self.updates.check_status(); let available = self.updates.available_version(); let description = match (status, &available) { diff --git a/host/src/i18n/strings/en.json b/host/src/i18n/strings/en.json index 4e691899..9c2b59dd 100644 --- a/host/src/i18n/strings/en.json +++ b/host/src/i18n/strings/en.json @@ -23,11 +23,13 @@ "Copied": "Copied", "Copy address": "Copy address", "Delay": "Delay", + "Dismiss": "Dismiss", "Display": "Display", "Done": "Done", "EternalMonitor runs in the background so your iPad can find this PC.": "EternalMonitor runs in the background so your iPad can find this PC.", "Excellent": "Excellent", "Fair": "Fair", + "Fix it": "Fix it", "Forget all iPads": "Forget all iPads", "Frame rate": "Frame rate", "General": "General", @@ -89,6 +91,7 @@ "The iPad is an extra screen next to your main one.": "The iPad is an extra screen next to your main one.", "The iPad shows the same picture as your main screen.": "The iPad shows the same picture as your main screen.", "Then tap this PC, or tap Scan QR code and point the camera at this code.": "Then tap this PC, or tap Scan QR code and point the camera at this code.", + "Try again": "Try again", "USB connection": "USB connection", "USB needs Apple's free Apple Devices app.": "USB needs Apple's free Apple Devices app.", "Unavailable": "Unavailable", diff --git a/host/src/i18n/tests.rs b/host/src/i18n/tests.rs index 94a3db96..892301d7 100644 --- a/host/src/i18n/tests.rs +++ b/host/src/i18n/tests.rs @@ -22,17 +22,25 @@ fn placeholders(text: &str) -> Vec<&str> { found } +/// English keys whose translations come in a separate pass. Until then +/// every language shows them in English. A language that has one already +/// is checked like any other key. +const AWAITING_TRANSLATION: &[&str] = &["Dismiss", "Fix it", "Try again"]; + #[test] fn every_language_has_exactly_the_english_keys_and_placeholders() { let english = strings(Lang::En); for (key, value) in &english { assert_eq!(key, value, "en.json maps each key to itself"); } + for key in AWAITING_TRANSLATION { + assert!(english.contains_key(*key), "{key:?} is not an English key"); + } for lang in Lang::ALL { let translated = strings(lang); let missing: Vec<_> = english .keys() - .filter(|k| !translated.contains_key(*k)) + .filter(|k| !translated.contains_key(*k) && !AWAITING_TRANSLATION.contains(&k.as_str())) .collect(); let extra: Vec<_> = translated .keys() diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs new file mode 100644 index 00000000..64e7782d --- /dev/null +++ b/host/src/input/injector.rs @@ -0,0 +1,1308 @@ +//! Input injection on its own thread. +//! +//! Windows' input calls (`SendInput`, `InjectSyntheticPointerInput`, +//! `GetSystemMetrics`, creating the pen device) can wait while Windows +//! reconfigures its displays. They ran on the transport loop, which then +//! answered nobody: no heartbeats, no HELLO answers, and a stopped stream +//! kept its port. Now the transport only queues input, and one thread for +//! the whole app hands it to Windows in order. +//! +//! Each stream generation queues on the same thread, behind the one before +//! it. A generation's end releases what its input still holds before the +//! next one's input lands, however long Windows kept the thread waiting, +//! and a restart never starts a second thread that could inject alongside +//! a stuck one. +//! +//! The queue is bounded. While Windows takes nothing, a pointer or Pencil +//! move replaces the newest queued move of its kind that no press or +//! release follows, so a backlog never replays a path the iPad already +//! left. Input that waited longer than +//! [`INPUT_LATE`] goes to the relay as late, which lands only a release of +//! what the PC holds, as for input that arrived late from the iPad. Ends, +//! key releases and session resets are never merged, and never lost: at +//! the bound, one becomes a release of everything after the last command +//! queued. + +use std::collections::VecDeque; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant}; + +use eternal_wire::v2::control::InputEvent; +use parking_lot::{Condvar, Mutex}; +use tracing::{info, warn}; + +use super::{ + CaptureGeometry, InputRelay, KIND_HOVER, KIND_MOUSE_ABS, KIND_PENCIL, KIND_TOUCH, PHASE_MOVED, +}; +use crate::transport::session::INPUT_LATE; + +/// Commands kept at most, besides one start and one end per stream +/// generation. A second of a Pencil at 240 Hz with every other kind of +/// input. While Windows is stuck, each kind of move keeps only its newest +/// after the last press or release, so the queue grows by presses and +/// releases. +const CAPACITY: usize = 1024; + +/// From this many waiting commands, a move replaces the newest move of its +/// kind. Below it every sample goes to Windows, as the Pencil's ink needs. +const MERGE_FROM: usize = 32; + +/// How often a native pen contact is refreshed; Windows expires a still one. +const PEN_TICK: Duration = Duration::from_millis(50); + +/// A Windows call this slow is logged: it would have stalled the stream. +const SLOW_CALL: Duration = Duration::from_millis(250); + +/// A Windows call held this long is logged while it is still held, from +/// the transport's next input: one that never returns would log nothing. +const HELD_LOGGED: Duration = Duration::from_secs(5); + +/// Waits this short are timed precisely, for pen frames held back to keep +/// them `FRAME_SPACING` apart; a condition variable wakes in whole +/// milliseconds or worse on Windows. +const PRECISE_WAIT: Duration = Duration::from_millis(2); + +/// How long the first stream waits for the thread to say whether Windows +/// gave it a pen device. Later streams know at once. +const START_WAIT: Duration = Duration::from_secs(1); + +/// What the injection thread drives: the input relay, or a test's fake. +pub(crate) trait Sink { + fn relay(&mut self, session_id: u32, event: &InputEvent, output: CaptureGeometry, late: bool); + fn reset(&mut self); + fn keep_alive(&mut self, output: Option); + fn pen_available(&self) -> bool; + fn spaced_until(&self) -> Option; + fn tick_needed(&self) -> bool; +} + +impl Sink for InputRelay { + fn relay(&mut self, session_id: u32, event: &InputEvent, output: CaptureGeometry, late: bool) { + if late { + self.relay_late(session_id, event, output); + } else { + InputRelay::relay(self, session_id, event, output); + } + } + fn reset(&mut self) { + InputRelay::reset(self); + } + fn keep_alive(&mut self, output: Option) { + self.keep_pen_alive(output); + } + fn pen_available(&self) -> bool { + InputRelay::pen_available(self) + } + fn spaced_until(&self) -> Option { + self.pen_spaced_until() + } + fn tick_needed(&self) -> bool { + self.pen_tick_needed() + } +} + +type Geometry = Arc>>; + +#[derive(Debug, Clone)] +enum Command { + /// A stream generation starts: what the one before it holds is + /// released first. `geometry` is the capture rect a still Pencil + /// follows. + Begin { geometry: Geometry }, + Relay { + session_id: u32, + event: InputEvent, + output: CaptureGeometry, + late: bool, + }, + /// Release what the PC holds: the session ended, expired or is on hold. + Reset, + /// The generation's stream stopped: release what it holds. + End, + /// The app quits: release what the PC holds, whichever stream it is + /// from, and take no more input. + ReleaseAll, +} + +impl Command { + /// A pointer or Pencil move or hover, which a later one can stand in for. + fn merges_with(&self, earlier: &Command) -> bool { + let ( + Command::Relay { + session_id, + event, + late, + .. + }, + Command::Relay { + session_id: earlier_session, + event: earlier_event, + late: earlier_late, + .. + }, + ) = (self, earlier) + else { + return false; + }; + let moves = |event: &InputEvent| { + event.phase == PHASE_MOVED + && matches!( + event.kind, + KIND_TOUCH | KIND_MOUSE_ABS | KIND_HOVER | KIND_PENCIL + ) + }; + session_id == earlier_session + && late == earlier_late + && moves(event) + && moves(earlier_event) + && event.kind == earlier_event.kind + && event.input_ver == earlier_event.input_ver + && event.buttons == earlier_event.buttons + } + + /// A press, release, reset or generation change: what a move must not + /// pass. + fn is_edge(&self) -> bool { + match self { + Command::Relay { event, .. } => event.phase != PHASE_MOVED, + Command::Begin { .. } | Command::Reset | Command::End | Command::ReleaseAll => true, + } + } + + /// An end or reset: what keeps the PC from holding a key or button. + fn releases(&self) -> bool { + match self { + Command::Reset | Command::End | Command::ReleaseAll => true, + Command::Begin { .. } => false, + Command::Relay { event, .. } => event.phase > PHASE_MOVED, + } + } +} + +#[derive(Debug)] +struct Queued { + generation: u64, + command: Command, + at: Instant, + /// A release came while the queue was full: once this command has + /// landed, release everything its generation holds. + then_reset: bool, +} + +#[derive(Debug, Default)] +struct State { + commands: VecDeque, + /// Since when the thread has been in Windows, with a command or a pen + /// refresh. + in_windows_since: Option, + /// That call was logged as held. + held_logged: bool, + #[cfg(test)] + held_logs: u64, + merged: u64, + dropped: u64, + /// Releases that came while the queue was full, each now part of a + /// release of everything. + folded: u64, + /// Input events queued since the thread started, for tests that must + /// know one is behind another. + #[cfg(any(test, feature = "test-hooks"))] + relays_queued: u64, +} + +impl State { + fn leave_windows(&mut self) { + self.in_windows_since = None; + self.held_logged = false; + } +} + +/// What the transport's handles and the thread share. +#[derive(Debug)] +struct Shared { + state: Mutex, + wake: Condvar, + /// The queue is empty and the thread waits: for `InputThread::finish`. + idle: Condvar, + limits: Limits, + next_generation: AtomicU64, + /// Whether the relay has a native pen device: 0 not known yet, 1 no, + /// 2 yes. + pen: AtomicU8, +} + +impl Shared { + fn push(&self, generation: u64, command: Command, now: Instant) { + let mut state = self.state.lock(); + let waiting = state.commands.len(); + if let Some(since) = state.in_windows_since { + let held = now.saturating_duration_since(since); + if held >= HELD_LOGGED && !state.held_logged { + state.held_logged = true; + #[cfg(test)] + { + state.held_logs += 1; + } + warn!( + held_s = held.as_secs(), + waiting, "Windows has held input for seconds and has not returned" + ); + } + } + if waiting >= self.limits.merge_from { + // The newest move of this kind, unless a press or release, or + // another generation, came after it. + let merges = state + .commands + .iter() + .enumerate() + .rev() + .take_while(|(_, queued)| { + queued.generation == generation + && !queued.then_reset + && !queued.command.is_edge() + }) + .find(|(_, queued)| command.merges_with(&queued.command)) + .map(|(index, _)| index); + if let Some(index) = merges { + // It goes last, where it came: the other kinds' newest + // moves keep their order before it. + let earlier = state.commands.remove(index).expect("found above"); + state.commands.push_back(Queued { command, ..earlier }); + state.merged += 1; + return; + } + } + // A generation's start and end always go in: there is one of each + // per stream. + let boundary = matches!( + command, + Command::Begin { .. } | Command::End | Command::ReleaseAll + ); + if waiting >= self.limits.capacity && !boundary { + let barrier = state + .commands + .back_mut() + .filter(|last| command.releases() && last.generation == generation); + match barrier { + Some(last) => { + last.then_reset = true; + state.folded += 1; + } + None => state.dropped += 1, + } + return; + } + #[cfg(any(test, feature = "test-hooks"))] + { + state.relays_queued += u64::from(matches!(command, Command::Relay { .. })); + } + state.commands.push_back(Queued { + generation, + command, + at: now, + then_reset: false, + }); + drop(state); + self.wake.notify_one(); + } +} + +/// The app's one input thread, started with the first stream that takes +/// input. `SharedControl` keeps it across stream generations. +#[derive(Default)] +pub struct InputThread { + shared: OnceLock>, +} + +impl std::fmt::Debug for InputThread { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InputThread") + .field("started", &self.shared.get().is_some()) + .finish() + } +} + +impl InputThread { + /// A new stream generation's handle; starts the thread with the relay + /// the first time. + pub fn begin(&self, geometry: Geometry) -> Injector { + let shared = self.shared.get_or_init(|| { + #[cfg(all(test, not(windows)))] + let recorder = super::recorder::current(); + spawn(Limits::default(), move || { + #[cfg(all(test, not(windows)))] + super::recorder::adopt(recorder); + InputRelay::default() + }) + }); + Injector::begin(Arc::clone(shared), geometry) + } + + /// A thread driving a test's fake instead of Windows. + #[cfg(test)] + pub(crate) fn with_fake(sink: impl FnOnce() -> S + Send + 'static) -> Self { + Self::with_sink(Limits::default(), sink) + } + + #[cfg(test)] + fn with_sink(limits: Limits, sink: impl FnOnce() -> S + Send + 'static) -> Self { + let thread = Self::default(); + let _ = thread.shared.set(spawn(limits, sink)); + thread + } + + /// Input events queued so far, merged and dropped ones aside. + #[cfg(any(test, feature = "test-hooks"))] + pub fn relays_queued(&self) -> u64 { + self.shared + .get() + .map_or(0, |shared| shared.state.lock().relays_queued) + } + + /// At quit: release what the PC holds, after everything queued, and + /// wait up to `limit` for it to reach Windows. Whether it did. A stream + /// still running need not have queued its end. + pub fn finish(&self, limit: Duration) -> bool { + let Some(shared) = self.shared.get() else { + return true; + }; + shared.push(0, Command::ReleaseAll, Instant::now()); + let done = self.idle(limit); + if !done { + warn!( + waiting = shared.state.lock().commands.len(), + "Quitting before Windows took the last input" + ); + } + done + } + + /// Wait up to `limit` for everything queued to reach Windows, without + /// releasing anything. + #[cfg(any(test, feature = "test-hooks"))] + pub fn wait_idle(&self, limit: Duration) -> bool { + self.idle(limit) + } + + fn idle(&self, limit: Duration) -> bool { + let Some(shared) = self.shared.get() else { + return true; + }; + let deadline = Instant::now() + limit; + let mut state = shared.state.lock(); + while state.in_windows_since.is_some() || !state.commands.is_empty() { + if shared.idle.wait_until(&mut state, deadline).timed_out() { + return false; + } + } + true + } +} + +fn spawn(limits: Limits, sink: impl FnOnce() -> S + Send + 'static) -> Arc { + let shared = Arc::new(Shared { + state: Mutex::new(State::default()), + wake: Condvar::new(), + idle: Condvar::new(), + limits, + next_generation: AtomicU64::new(1), + pen: AtomicU8::new(0), + }); + let worker = Arc::clone(&shared); + let spawned = std::thread::Builder::new() + .name("input-inject".into()) + .spawn(move || { + let mut sink = sink(); + note_pen(&worker, &sink); + run(&worker, &mut sink); + }); + if let Err(error) = spawned { + warn!(%error, "Could not start the input thread; input is ignored"); + shared.pen.store(1, Ordering::SeqCst); + } + shared +} + +fn note_pen(shared: &Shared, sink: &impl Sink) { + let pen = if sink.pen_available() { 2 } else { 1 }; + shared.pen.store(pen, Ordering::SeqCst); +} + +/// One stream generation's handle on the input thread. Dropping it +/// releases what the generation's input holds, in turn, without waiting +/// for Windows. +#[derive(Debug)] +pub struct Injector { + shared: Arc, + generation: u64, + pen_available: bool, +} + +impl Injector { + fn begin(shared: Arc, geometry: Geometry) -> Self { + let generation = shared.next_generation.fetch_add(1, Ordering::SeqCst); + shared.push(generation, Command::Begin { geometry }, Instant::now()); + let started = Instant::now(); + let pen_available = loop { + match shared.pen.load(Ordering::SeqCst) { + 0 if started.elapsed() < START_WAIT => std::thread::sleep(Duration::from_millis(1)), + 0 => { + warn!( + wait_ms = START_WAIT.as_millis() as u64, + "Windows did not set up input in time; the Pencil uses the mouse this stream" + ); + break false; + } + pen => break pen == 2, + } + }; + Self { + shared, + generation, + pen_available, + } + } + + /// Whether the relay has a native pen device, for the HELLO answer. + pub fn pen_available(&self) -> bool { + self.pen_available + } + + /// Queue one session-validated event. Never waits for Windows. + pub fn relay(&self, session_id: u32, event: InputEvent, output: CaptureGeometry, late: bool) { + self.push(Command::Relay { + session_id, + event, + output, + late, + }); + } + + /// Queue a release of every key, button and pen contact the PC holds. + pub fn reset(&self) { + self.push(Command::Reset); + } + + fn push(&self, command: Command) { + self.shared.push(self.generation, command, Instant::now()); + } +} + +impl Drop for Injector { + fn drop(&mut self) { + self.push(Command::End); + } +} + +#[derive(Debug, Clone, Copy)] +struct Limits { + capacity: usize, + merge_from: usize, + stale_after: Duration, +} + +impl Default for Limits { + fn default() -> Self { + Self { + capacity: CAPACITY, + merge_from: MERGE_FROM, + stale_after: INPUT_LATE, + } + } +} + +enum Next { + Command(Queued), + /// A pen frame's spacing or the refresh tick is due. + Due, +} + +/// Input that waited for Windows, counted until the queue empties. +#[derive(Debug, Default)] +struct Backlog { + stale: u64, + longest: Duration, +} + +/// The generation whose input the thread takes now. +struct Current { + generation: u64, + geometry: Geometry, + /// Its input reached the relay, so its end has something to release. + relayed: bool, +} + +fn run(shared: &Shared, sink: &mut S) { + let mut timer = crate::capture::timing::FrameTimer::new(); + let mut next_tick = Instant::now() + PEN_TICK; + let mut backlog = Backlog::default(); + let mut current: Option = None; + loop { + let next = { + let mut state = shared.state.lock(); + state.leave_windows(); + loop { + if let Some(queued) = state.commands.pop_front() { + state.in_windows_since = Some(Instant::now()); + break Next::Command(queued); + } + shared.idle.notify_all(); + let (merged, dropped, folded) = ( + std::mem::take(&mut state.merged), + std::mem::take(&mut state.dropped), + std::mem::take(&mut state.folded), + ); + if backlog.stale > 0 || merged > 0 || dropped > 0 || folded > 0 { + info!( + stale = backlog.stale, + merged, + dropped, + folded, + longest_ms = backlog.longest.as_millis() as u64, + "Input waited for Windows: moves merged, only releases sent of what waited too long, and everything released for releases past the bound" + ); + backlog = Backlog::default(); + } + let now = Instant::now(); + let spaced = current.as_ref().and(sink.spaced_until()); + let ticks = current.is_some() && sink.tick_needed(); + let due = spaced.or_else(|| ticks.then_some(next_tick)); + match due { + Some(at) if at <= now => break Next::Due, + Some(at) if spaced.is_some() && at - now <= PRECISE_WAIT => { + drop(state); + timer.wait_until(at); + state = shared.state.lock(); + if state.commands.is_empty() { + break Next::Due; + } + } + Some(at) => { + if shared.wake.wait_until(&mut state, at).timed_out() { + break Next::Due; + } + } + None => shared.wake.wait(&mut state), + } + } + }; + if let Next::Command(Queued { + generation, + command, + at, + then_reset, + }) = next + { + let started = Instant::now(); + let waited = started.saturating_duration_since(at); + backlog.longest = backlog.longest.max(waited); + let ours = current + .as_ref() + .is_some_and(|current| current.generation == generation); + match command { + Command::Begin { geometry } => { + // A generation that never said it ended releases what + // it holds before the new one's input. + if current.take().is_some_and(|current| current.relayed) { + sink.reset(); + } + current = Some(Current { + generation, + geometry, + relayed: false, + }); + } + Command::End if ours => { + if current.take().is_some_and(|current| current.relayed) { + sink.reset(); + } + } + Command::Reset if ours => sink.reset(), + Command::Relay { + session_id, + event, + output, + late, + } if ours => { + let stale = waited > shared.limits.stale_after; + backlog.stale += u64::from(stale && !late); + sink.relay(session_id, &event, output, late || stale); + if let Some(current) = current.as_mut() { + current.relayed = true; + } + } + Command::ReleaseAll => { + current = None; + sink.reset(); + } + // An ended generation's: its end released what it held. + Command::End | Command::Reset | Command::Relay { .. } => {} + } + if then_reset + && current + .as_ref() + .is_some_and(|current| current.generation == generation) + { + sink.reset(); + } + note_slow(started, "input"); + note_pen(shared, sink); + } + // Pen frames held back for their spacing, and the refresh a still + // contact needs, go between commands. + let Some(current) = current.as_ref() else { + continue; + }; + let now = Instant::now(); + let spaced_due = sink.spaced_until().is_some_and(|at| at <= now); + let tick_due = sink.spaced_until().is_none() && sink.tick_needed() && next_tick <= now; + if spaced_due || tick_due { + if tick_due { + next_tick = now + PEN_TICK; + } + // Copied out first: the transport takes this lock for every + // input datagram, and Windows can hold the refresh. + let output = *current.geometry.lock(); + let started = Instant::now(); + shared.state.lock().in_windows_since = Some(started); + sink.keep_alive(output); + shared.state.lock().leave_windows(); + note_slow(started, "pen_refresh"); + } + } +} + +fn note_slow(started: Instant, what: &'static str) { + let took = started.elapsed(); + if took >= SLOW_CALL { + warn!( + what, + took_ms = took.as_millis() as u64, + "Windows took a long time to take input" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::input::{KIND_KEY, PHASE_BEGAN, PHASE_ENDED}; + use std::sync::mpsc; + + const OUTPUT: CaptureGeometry = CaptureGeometry { + left: 0, + top: 0, + width: 1920, + height: 1080, + }; + + #[derive(Debug, Clone, PartialEq)] + enum Call { + Relay { + kind: u8, + phase: u8, + id: u32, + late: bool, + }, + Reset, + KeepAlive, + } + + /// Records calls, and can hold the first relay until told to go on, + /// as Windows does while it reconfigures displays. + struct Fake { + calls: Arc>>, + hold: Option>, + entered: Option>, + } + + impl Sink for Fake { + fn relay(&mut self, _: u32, event: &InputEvent, _: CaptureGeometry, late: bool) { + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(hold) = self.hold.take() { + let _ = hold.recv(); + } + self.calls.lock().push(Call::Relay { + kind: event.kind, + phase: event.phase, + id: event.event_id, + late, + }); + } + fn reset(&mut self) { + self.calls.lock().push(Call::Reset); + } + fn keep_alive(&mut self, _: Option) { + self.calls.lock().push(Call::KeepAlive); + } + fn pen_available(&self) -> bool { + true + } + fn spaced_until(&self) -> Option { + None + } + fn tick_needed(&self) -> bool { + false + } + } + + struct Harness { + thread: InputThread, + injector: Option, + calls: Arc>>, + go: mpsc::Sender<()>, + entered: mpsc::Receiver<()>, + } + + fn harness(limits: Limits) -> Harness { + let calls = Arc::new(Mutex::new(Vec::new())); + let (go, hold) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + let recorded = Arc::clone(&calls); + let thread = InputThread::with_sink(limits, move || Fake { + calls: recorded, + hold: Some(hold), + entered: Some(entered_tx), + }); + let injector = thread.begin(geometry()); + Harness { + thread, + injector: Some(injector), + calls, + go, + entered, + } + } + + fn geometry() -> Geometry { + Arc::new(Mutex::new(Some(OUTPUT))) + } + + impl Harness { + fn send(&self, kind: u8, phase: u8, id: u32) { + self.injector + .as_ref() + .unwrap() + .relay(1, event(kind, phase, id), OUTPUT, false); + } + + /// Queued as though at `at`. + fn send_at(&self, kind: u8, phase: u8, id: u32, at: Instant) { + let injector = self.injector.as_ref().unwrap(); + injector.shared.push( + injector.generation, + Command::Relay { + session_id: 1, + event: event(kind, phase, id), + output: OUTPUT, + late: false, + }, + at, + ); + } + + /// Holds Windows in the first event's call until `go`. + fn stuck(&self) { + self.send(KIND_KEY, PHASE_BEGAN, 0); + self.entered + .recv_timeout(Duration::from_secs(5)) + .expect("the thread takes the first event"); + } + + fn wait_for(&self, count: usize) -> Vec { + let started = Instant::now(); + while self.calls.lock().len() < count { + assert!( + started.elapsed() < Duration::from_secs(5), + "{:?}", + self.calls.lock() + ); + std::thread::sleep(Duration::from_millis(1)); + } + self.calls.lock().clone() + } + + /// Nothing more reaches Windows. + fn settled(&self, count: usize) { + assert!(self.thread.wait_idle(Duration::from_secs(5))); + assert_eq!(self.calls.lock().len(), count, "{:?}", self.calls.lock()); + } + } + + fn event(kind: u8, phase: u8, id: u32) -> InputEvent { + InputEvent { + input_ver: 1, + kind, + phase, + buttons: 0, + event_id: id, + x_norm: id as u16, + y_norm: 0, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: 0x04, + modifiers: 0, + client_time_us: 0, + tilt_x: 0, + tilt_y: 0, + } + } + + fn relayed(kind: u8, phase: u8, id: u32) -> Call { + Call::Relay { + kind, + phase, + id, + late: false, + } + } + + #[test] + fn queueing_never_waits_for_a_stuck_windows() { + let harness = harness(Limits::default()); + harness.stuck(); + let started = Instant::now(); + for id in 1..=5000 { + harness.send(KIND_MOUSE_ABS, PHASE_MOVED, id); + } + harness.injector.as_ref().unwrap().reset(); + assert!(started.elapsed() < Duration::from_millis(500)); + harness.go.send(()).unwrap(); + } + + #[test] + fn every_command_reaches_windows_in_the_order_it_came() { + let harness = harness(Limits::default()); + harness.stuck(); + let sent = [ + (KIND_MOUSE_ABS, PHASE_BEGAN, 1), + (KIND_MOUSE_ABS, PHASE_MOVED, 2), + (KIND_PENCIL, PHASE_MOVED, 3), + (KIND_MOUSE_ABS, PHASE_MOVED, 4), + (KIND_KEY, PHASE_ENDED, 5), + (KIND_MOUSE_ABS, PHASE_ENDED, 6), + ]; + for (kind, phase, id) in sent { + harness.send(kind, phase, id); + } + harness.injector.as_ref().unwrap().reset(); + harness.send(KIND_KEY, PHASE_BEGAN, 7); + harness.go.send(()).unwrap(); + let mut expected = vec![relayed(KIND_KEY, PHASE_BEGAN, 0)]; + expected.extend(sent.map(|(kind, phase, id)| relayed(kind, phase, id))); + expected.push(Call::Reset); + expected.push(relayed(KIND_KEY, PHASE_BEGAN, 7)); + assert_eq!(harness.wait_for(expected.len()), expected); + } + + #[test] + fn a_backed_up_queue_merges_moves_but_keeps_every_edge_in_place() { + let harness = harness(Limits { + merge_from: 4, + ..Limits::default() + }); + harness.stuck(); + // Four wait as they came; then each move stands in for the one + // before it, up to the next edge. + let mut id = 0; + for _ in 0..4 { + id += 1; + harness.send(KIND_HOVER, PHASE_MOVED, id); + } + for _ in 0..50 { + id += 1; + harness.send(KIND_HOVER, PHASE_MOVED, id); + } + harness.send(KIND_MOUSE_ABS, PHASE_BEGAN, 100); + for move_id in 101..150 { + harness.send(KIND_MOUSE_ABS, PHASE_MOVED, move_id); + } + // A Pencil move never stands in for a mouse move. + harness.send(KIND_PENCIL, PHASE_MOVED, 150); + harness.send(KIND_MOUSE_ABS, PHASE_ENDED, 151); + harness.injector.as_ref().unwrap().reset(); + harness.go.send(()).unwrap(); + let expected = vec![ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_HOVER, PHASE_MOVED, 1), + relayed(KIND_HOVER, PHASE_MOVED, 2), + relayed(KIND_HOVER, PHASE_MOVED, 3), + relayed(KIND_HOVER, PHASE_MOVED, 54), + relayed(KIND_MOUSE_ABS, PHASE_BEGAN, 100), + relayed(KIND_MOUSE_ABS, PHASE_MOVED, 149), + relayed(KIND_PENCIL, PHASE_MOVED, 150), + relayed(KIND_MOUSE_ABS, PHASE_ENDED, 151), + Call::Reset, + ]; + assert_eq!(harness.wait_for(expected.len()), expected); + harness.settled(expected.len()); + } + + /// A Pencil hovering while the pointer moves sends both kinds of move + /// interleaved, about 250 a second. While Windows is stuck each kind + /// keeps only its newest move after the last press or release, in the + /// order those newest moves came. + #[test] + fn interleaved_moves_of_different_kinds_each_merge_up_to_the_last_edge() { + let harness = harness(Limits { + merge_from: 4, + ..Limits::default() + }); + harness.stuck(); + for id in 1..=104 { + let kind = if id % 2 == 1 { KIND_PENCIL } else { KIND_HOVER }; + harness.send(kind, PHASE_MOVED, id); + } + harness.send(KIND_MOUSE_ABS, PHASE_BEGAN, 200); + for id in 201..=204 { + let kind = if id % 2 == 1 { KIND_PENCIL } else { KIND_HOVER }; + harness.send(kind, PHASE_MOVED, id); + } + harness.send(KIND_MOUSE_ABS, PHASE_ENDED, 300); + harness.injector.as_ref().unwrap().reset(); + harness.go.send(()).unwrap(); + let expected = vec![ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_PENCIL, PHASE_MOVED, 1), + relayed(KIND_HOVER, PHASE_MOVED, 2), + relayed(KIND_PENCIL, PHASE_MOVED, 103), + relayed(KIND_HOVER, PHASE_MOVED, 104), + // Moves never merge across a press or release. + relayed(KIND_MOUSE_ABS, PHASE_BEGAN, 200), + relayed(KIND_PENCIL, PHASE_MOVED, 203), + relayed(KIND_HOVER, PHASE_MOVED, 204), + relayed(KIND_MOUSE_ABS, PHASE_ENDED, 300), + Call::Reset, + ]; + assert!(harness.thread.wait_idle(Duration::from_secs(5))); + assert_eq!(*harness.calls.lock(), expected); + } + + #[test] + fn input_that_waited_too_long_goes_as_late_and_releases_still_land() { + // A limit no pick-up under load comes near, and input queued as if + // long ago: no sleep decides which is late. + let harness = harness(Limits { + stale_after: Duration::from_secs(10), + ..Limits::default() + }); + harness.stuck(); + let long_ago = Instant::now() - Duration::from_secs(20); + harness.send_at(KIND_KEY, PHASE_ENDED, 1, long_ago); + harness.send_at(KIND_MOUSE_ABS, PHASE_BEGAN, 2, long_ago); + harness.go.send(()).unwrap(); + // Fresh input after Windows came back is not late. + harness.wait_for(3); + harness.send(KIND_MOUSE_ABS, PHASE_ENDED, 3); + let late = |kind, phase, id| Call::Relay { + kind, + phase, + id, + late: true, + }; + assert_eq!( + harness.wait_for(4), + [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + late(KIND_KEY, PHASE_ENDED, 1), + late(KIND_MOUSE_ABS, PHASE_BEGAN, 2), + relayed(KIND_MOUSE_ABS, PHASE_ENDED, 3), + ] + ); + } + + #[test] + fn a_full_queue_drops_new_moves_and_presses_and_turns_a_release_into_a_reset() { + let harness = harness(Limits { + capacity: 3, + merge_from: 100, + stale_after: INPUT_LATE, + }); + harness.stuck(); + for id in 1..=3 { + harness.send(KIND_KEY, PHASE_BEGAN, id); + } + harness.send(KIND_KEY, PHASE_BEGAN, 4); + harness.send(KIND_MOUSE_ABS, PHASE_MOVED, 5); + harness.send(KIND_KEY, PHASE_ENDED, 6); + harness.injector.as_ref().unwrap().reset(); + harness.go.send(()).unwrap(); + // The key-up and the reset after it become one release of + // everything, after the last press queued. + let expected = [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_KEY, PHASE_BEGAN, 1), + relayed(KIND_KEY, PHASE_BEGAN, 2), + relayed(KIND_KEY, PHASE_BEGAN, 3), + Call::Reset, + ]; + assert_eq!(harness.wait_for(5), expected); + harness.settled(expected.len()); + } + + /// Windows stays stuck while the queue fills to its bound and then + /// releases keep coming: the key-ups, and the disconnect's reset after + /// them. However many pile up, what the PC holds is released once + /// Windows answers, and nothing pressed lands after that. + #[test] + fn releases_past_the_hard_limit_still_release_everything() { + let harness = harness(Limits::default()); + harness.stuck(); + for id in 1..=CAPACITY as u32 { + harness.send(KIND_KEY, PHASE_BEGAN, id); + } + for id in 1..=CAPACITY as u32 { + harness.send(KIND_KEY, PHASE_ENDED, 10_000 + id); + } + harness.send(KIND_KEY, PHASE_BEGAN, 20_000); + harness.injector.as_ref().unwrap().reset(); + harness.go.send(()).unwrap(); + assert!(harness.thread.wait_idle(Duration::from_secs(5))); + let calls = harness.calls.lock().clone(); + assert_eq!( + calls.last(), + Some(&Call::Reset), + "{:?}", + &calls[calls.len() - 3..] + ); + assert!(calls.len() <= 2 * CAPACITY + 1, "{}", calls.len()); + let pressed: Vec<_> = calls[1..=CAPACITY].to_vec(); + assert_eq!( + pressed, + (1..=CAPACITY as u32) + .map(|id| relayed(KIND_KEY, PHASE_BEGAN, id)) + .collect::>() + ); + } + + #[test] + fn a_dropped_handle_returns_at_once_and_its_input_still_lands_then_is_released() { + let mut harness = harness(Limits::default()); + harness.stuck(); + harness.send(KIND_KEY, PHASE_ENDED, 1); + let started = Instant::now(); + drop(harness.injector.take()); + assert!(started.elapsed() < Duration::from_millis(100)); + harness.go.send(()).unwrap(); + let expected = [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_KEY, PHASE_ENDED, 1), + Call::Reset, + ]; + assert_eq!(harness.wait_for(3), expected); + harness.settled(expected.len()); + } + + /// A stream restarts while Windows holds the old one's input. The new + /// generation's handle is ready at once, on the same thread, and its + /// input lands only after the old one's release: a key it presses + /// stays down. Restarts over and over start no thread of their own. + #[test] + fn a_restart_while_windows_is_stuck_releases_the_old_input_before_the_new() { + let mut harness = harness(Limits::default()); + harness.stuck(); + drop(harness.injector.take()); + let started = Instant::now(); + let mut generations: Vec<_> = (0..20).map(|_| harness.thread.begin(geometry())).collect(); + assert!(started.elapsed() < Duration::from_millis(100)); + let newest = generations.pop().unwrap(); + generations.clear(); + newest.relay(1, event(KIND_KEY, PHASE_BEGAN, 1), OUTPUT, false); + harness.go.send(()).unwrap(); + let expected = [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + Call::Reset, + relayed(KIND_KEY, PHASE_BEGAN, 1), + ]; + assert_eq!(harness.wait_for(3), expected); + // The old generations' ends, queued behind the new one's start, + // released nothing of it. + harness.settled(expected.len()); + drop(newest); + assert_eq!(harness.wait_for(4)[3], Call::Reset); + } + + /// A Windows call that never returns is logged once, from the input + /// that queues behind it, and again only for the next held call. + #[test] + fn a_call_windows_holds_for_seconds_is_logged_once() { + let harness = harness(Limits::default()); + harness.stuck(); + let logs = || { + harness + .injector + .as_ref() + .unwrap() + .shared + .state + .lock() + .held_logs + }; + let later = |seconds| Instant::now() + Duration::from_secs(seconds); + harness.send(KIND_HOVER, PHASE_MOVED, 1); + assert_eq!(logs(), 0); + harness.send_at(KIND_HOVER, PHASE_MOVED, 2, later(6)); + harness.send_at(KIND_HOVER, PHASE_MOVED, 3, later(60)); + assert_eq!(logs(), 1); + harness.go.send(()).unwrap(); + assert!(harness.thread.finish(Duration::from_secs(5))); + assert_eq!(logs(), 1); + } + + /// A quit while a stream still runs, its end not yet queued, still + /// releases what the PC holds. + #[test] + fn quitting_releases_what_a_running_stream_holds() { + let harness = harness(Limits::default()); + harness.stuck(); + harness.go.send(()).unwrap(); + assert!(harness.thread.finish(Duration::from_secs(5))); + assert_eq!( + *harness.calls.lock(), + [relayed(KIND_KEY, PHASE_BEGAN, 0), Call::Reset] + ); + } + + #[test] + fn quitting_waits_for_queued_releases_but_not_past_its_limit() { + let harness = harness(Limits::default()); + harness.stuck(); + harness.send(KIND_KEY, PHASE_ENDED, 1); + let started = Instant::now(); + assert!(!harness.thread.finish(Duration::from_millis(50))); + assert!(started.elapsed() < Duration::from_secs(1)); + harness.go.send(()).unwrap(); + assert!(harness.thread.wait_idle(Duration::from_secs(5))); + assert_eq!( + *harness.calls.lock(), + [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_KEY, PHASE_ENDED, 1), + Call::Reset, + ] + ); + } + + /// The hop the thread adds: from the transport queueing an event to the + /// call that hands it to Windows, at a steady 1 kHz. Prints p50, p99 and + /// the worst; the bound only catches a thread that waits for a timer. + #[test] + fn the_hop_to_the_input_thread_is_short() { + struct Timed(Arc>>); + impl Sink for Timed { + fn relay(&mut self, _: u32, event: &InputEvent, _: CaptureGeometry, _: bool) { + self.0.lock().push((event.event_id, Instant::now())); + } + fn reset(&mut self) {} + fn keep_alive(&mut self, _: Option) {} + fn pen_available(&self) -> bool { + false + } + fn spaced_until(&self) -> Option { + None + } + fn tick_needed(&self) -> bool { + false + } + } + const EVENTS: u32 = 2000; + let taken = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&taken); + // Every event is timed: a thread the machine starves for a moment + // gets a backlog, which would merge moves or drop them at the bound. + let limits = Limits { + capacity: EVENTS as usize, + merge_from: usize::MAX, + ..Limits::default() + }; + let thread = InputThread::with_sink(limits, move || Timed(recorded)); + let injector = thread.begin(geometry()); + let mut queued = Vec::new(); + let mut next = Instant::now(); + for id in 0..EVENTS { + next += Duration::from_millis(1); + std::thread::sleep(next.saturating_duration_since(Instant::now())); + queued.push(Instant::now()); + injector.relay(1, event(KIND_HOVER, PHASE_MOVED, id), OUTPUT, false); + } + assert!(thread.finish(Duration::from_secs(5))); + let mut hops: Vec<_> = taken + .lock() + .iter() + .map(|(id, at)| *at - queued[*id as usize]) + .collect(); + assert_eq!(hops.len(), EVENTS as usize); + hops.sort(); + let at = |q: f64| hops[((hops.len() - 1) as f64 * q) as usize]; + let (p50, p99, worst) = (at(0.5), at(0.99), *hops.last().unwrap()); + eprintln!( + "input hop to the input thread: p50 {} us, p99 {} us, max {} us", + p50.as_micros(), + p99.as_micros(), + worst.as_micros() + ); + assert!(p99 < Duration::from_millis(50), "{p99:?}"); + } + + /// The real relay, with a native pen device that takes every frame + /// (Windows' own off Windows). A stall delivers 64 samples at once; the + /// spacing holds back all but the first burst, and the thread sends the + /// rest within a few times the spacing, not at the 50 ms refresh. + #[cfg(not(windows))] + #[test] + fn the_thread_sends_a_held_pen_backlog_at_its_spacing() { + use crate::input::{native_pen, PenPhase, PenSample}; + const SAMPLES: usize = 64; + let (taken_tx, taken_rx) = mpsc::channel(); + let delivered = Arc::new(Mutex::new(None)); + let at = Arc::clone(&delivered); + let thread = InputThread::with_sink(Limits::default(), move || { + let taken = native_pen::install(); + *at.lock() = Some(Instant::now()); + native_pen::with(|device| { + for x in 0..SAMPLES as u16 { + let sample = PenSample { + phase: if x == 0 { + PenPhase::Down + } else { + PenPhase::Move + }, + x: i32::from(x), + y: 10, + x_norm: x, + y_norm: 10, + pressure: 512, + tilt_x: 0, + tilt_y: 0, + }; + device.deliver(sample, native_pen::SCREEN, Some(OUTPUT), Instant::now()); + } + }); + assert!( + taken.lock().len() < SAMPLES / 2, + "the spacing must hold the backlog back" + ); + taken_tx.send(taken).unwrap(); + InputRelay::default() + }); + let injector = thread.begin(geometry()); + let taken = taken_rx.recv().unwrap(); + let started = Instant::now(); + while taken.lock().len() < SAMPLES && started.elapsed() < Duration::from_secs(2) { + std::thread::sleep(Duration::from_millis(1)); + } + drop(injector); + let taken = taken.lock(); + let order: Vec<_> = taken.iter().map(|(frame, _)| frame.x_norm).collect(); + assert_eq!(order, (0..SAMPLES as u16).collect::>()); + let drained = taken.last().unwrap().1 - delivered.lock().unwrap(); + // 58 held frames 250 µs apart take 15 ms; the 50 ms tick alone + // would take 500 ms. + assert!(drained < Duration::from_millis(250), "{drained:?}"); + } +} diff --git a/host/src/input/mod.rs b/host/src/input/mod.rs index eb451ac3..1b8d790d 100644 --- a/host/src/input/mod.rs +++ b/host/src/input/mod.rs @@ -10,6 +10,11 @@ use eternal_wire::v2::control::InputEvent; mod hid; pub use hid::hid_to_scancode; +mod injector; +#[cfg(test)] +pub(crate) use injector::Sink; +pub use injector::{Injector, InputThread}; + #[cfg(windows)] pub mod windows_inject; @@ -136,6 +141,12 @@ pub struct PenSample { /// Map a wire-normalized point (0..65535 over the displayed video, which the /// client letterbox-corrects before sending) to a desktop pixel on the /// captured output. +/// +/// `output` is the whole capture, also when the video leaves out its odd +/// last column or row ([`crate::encoder::encoded_size`]): the video's `n` +/// pixels then span `n` steps of this mapping, so each one's centre lands +/// on its own desktop pixel, and the picture's far edge on the pixel left +/// out, which stays reachable. pub fn norm_to_output_pixel(x_norm: u16, y_norm: u16, output: CaptureGeometry) -> (i32, i32) { let x = output.left + ((u64::from(x_norm) * u64::from(output.width.saturating_sub(1))) / 65_535) as i32; @@ -361,8 +372,9 @@ pub fn resolve( pub mod recorder { use super::Injection; - /// The end-to-end tests record on the transport's thread and read on - /// their own. Unit tests run in parallel, so each keeps its own record. + /// The end-to-end tests record on the input thread and read on their + /// own. Unit tests run in parallel, so each keeps its own record, which + /// the input threads it starts write to. fn recorded(f: impl FnOnce(&mut Vec) -> R) -> R { #[cfg(not(test))] { @@ -371,14 +383,39 @@ pub mod recorder { } #[cfg(test)] { - thread_local! { - static RECORDED: std::cell::RefCell> = - const { std::cell::RefCell::new(Vec::new()) }; - } - RECORDED.with_borrow_mut(f) + current().with(f) + } + } + + /// A unit test's own record, which the threads it starts share. + #[cfg(test)] + #[derive(Debug, Clone, Default)] + pub struct Record(std::sync::Arc>>); + + #[cfg(test)] + impl Record { + fn with(&self, f: impl FnOnce(&mut Vec) -> R) -> R { + f(&mut self.0.lock().unwrap()) } } + #[cfg(test)] + thread_local! { + static RECORD: std::cell::RefCell = std::cell::RefCell::default(); + } + + /// This thread's record, for a thread it starts to [`adopt`]. + #[cfg(test)] + pub fn current() -> Record { + RECORD.with_borrow(Record::clone) + } + + /// Record on this thread into `record`. + #[cfg(test)] + pub fn adopt(record: Record) { + RECORD.set(record); + } + pub fn record(injections: &[Injection]) { recorded(|recorded| recorded.extend_from_slice(injections)); } @@ -400,8 +437,9 @@ pub(crate) mod native_pen { use super::pen_contact::{Outcome, PointerKind}; use super::pen_device::{PenApi, PenDevice}; use super::{CaptureGeometry, Injection, PenSample, VirtualScreen}; + use parking_lot::Mutex; use std::cell::RefCell; - use std::rc::Rc; + use std::sync::Arc; use std::time::Instant; pub(crate) const SCREEN: VirtualScreen = VirtualScreen { @@ -412,7 +450,7 @@ pub(crate) mod native_pen { }; /// Frames Windows took, with when. - pub(crate) type Taken = Rc>>; + pub(crate) type Taken = Arc>>; #[derive(Debug)] pub(crate) struct Windows(Taken); @@ -438,7 +476,7 @@ pub(crate) mod native_pen { _: VirtualScreen, _: Option, ) -> Outcome { - self.0.borrow_mut().push((frame, Instant::now())); + self.0.lock().push((frame, Instant::now())); Outcome::Injected } fn send_mouse(&mut self, inputs: &[Injection]) -> usize { @@ -457,7 +495,7 @@ pub(crate) mod native_pen { /// Give this thread's relays a device, and return what Windows takes. pub(crate) fn install() -> Taken { let taken = Taken::default(); - DEVICE.set(Some(PenDevice::new(Windows(Rc::clone(&taken))))); + DEVICE.set(Some(PenDevice::new(Windows(Arc::clone(&taken))))); taken } @@ -467,6 +505,68 @@ pub(crate) mod native_pen { } } +/// Lets an end-to-end test hold every injection where Windows' calls would +/// be, as Windows held them while it reconfigured displays. +#[cfg(feature = "test-hooks")] +pub mod hold { + use parking_lot::{Condvar, Mutex}; + + #[derive(Default)] + struct Held { + on: bool, + /// Only the first injection waits; later ones pass, as on a thread + /// Windows is not holding. + first_only: bool, + waiting: usize, + } + + static HELD: Mutex = Mutex::new(Held { + on: false, + first_only: false, + waiting: 0, + }); + static RELEASED: Condvar = Condvar::new(); + + /// Every injection waits until [`release`]. + pub fn hold() { + *HELD.lock() = Held { + on: true, + ..Held::default() + }; + } + + /// The next injection waits until [`release`]; the rest pass. + pub fn hold_next() { + *HELD.lock() = Held { + on: true, + first_only: true, + waiting: 0, + }; + } + + pub fn release() { + HELD.lock().on = false; + RELEASED.notify_all(); + } + + /// Injections waiting now. + pub fn waiting() -> usize { + HELD.lock().waiting + } + + pub(super) fn pass() { + let mut held = HELD.lock(); + if !held.on || (held.first_only && held.waiting > 0) { + return; + } + held.waiting += 1; + while held.on { + RELEASED.wait(&mut held); + } + held.waiting -= 1; + } +} + fn log_injections(injections: &[Injection]) { if std::env::var("ETERNAL_INPUT_RECORDER_LOG").is_ok_and(|value| value == "1") { for injection in injections { @@ -726,6 +826,8 @@ impl InputRelay { } fn inject(&mut self, injections: &[Injection]) { + #[cfg(feature = "test-hooks")] + hold::pass(); #[cfg(windows)] windows_inject::inject(injections, &mut self.pen_device, self.output); #[cfg(not(windows))] @@ -1019,6 +1121,42 @@ mod tests { assert!((cy - 540).abs() <= 1, "center y {cy}"); } + /// A 1365x945 screen streams 1364x944. A touch on the centre of any + /// pixel of that picture, normalized as the iPad does, lands on that + /// pixel, and the picture's far edge on the column and row left out. + #[test] + fn a_cropped_picture_maps_each_pixel_onto_itself() { + let output = CaptureGeometry { + left: -1365, + top: 0, + width: 1365, + height: 945, + }; + let (width, height) = crate::encoder::encoded_size(output.width, output.height); + assert_eq!((width, height), (1364, 944)); + // TouchRelay.swift: the touch's fraction of the picture, rounded. + let norm = |pixel: u32, pixels: u32| { + ((f64::from(pixel) + 0.5) / f64::from(pixels) * 65_535.0).round() as u16 + }; + for x in 0..width { + for y in [0, height / 2, height - 1] { + assert_eq!( + norm_to_output_pixel(norm(x, width), norm(y, height), output), + (output.left + x as i32, y as i32), + "pixel {x},{y}" + ); + } + } + for y in 0..height { + assert_eq!( + norm_to_output_pixel(norm(width - 1, width), norm(y, height), output), + (-2, y as i32), + "row {y}" + ); + } + assert_eq!(norm_to_output_pixel(65_535, 65_535, output), (-1, 944)); + } + #[test] fn extended_display_offset_is_applied() { // Virtual display sits LEFT of the primary at -1920. diff --git a/host/src/input/pen_contact.rs b/host/src/input/pen_contact.rs index 074d772d..56beb608 100644 --- a/host/src/input/pen_contact.rs +++ b/host/src/input/pen_contact.rs @@ -129,7 +129,7 @@ const NOT_READY_WINDOW: Duration = Duration::from_millis(2); /// Retries ERROR_NOT_READY briefly while Windows is accepting frames. A frame /// still refused after that means Windows' input queue is stuck; later /// frames get one attempt each until one is accepted, so a stuck queue costs -/// one call per sample or refresh tick rather than stalling the transport. +/// one call per sample or refresh tick rather than stalling the input thread. #[derive(Debug, Default)] pub struct NotReadyRetry { stalled: bool, @@ -173,8 +173,7 @@ pub(super) const FRAME_SPACING: Duration = Duration::from_micros(250); pub(super) const FRAME_BURST: u32 = 6; /// The most frames one `flush` sends before it yields. Injection runs on the -/// transport loop, so a backlog goes out in chunks with NACK answers, input -/// and video pacing in between. +/// input thread, so a backlog goes out in chunks with newer input in between. const PASS_FRAMES: u32 = FRAME_BURST; /// The most time one `flush` spends sending frames before it yields, should @@ -385,9 +384,8 @@ impl PenContact { self.abandoned.push((mapped(sample, output), now)); return; } - // Behind frames held back, the sample waits for the flush at the - // top of the next transport pass, so one pass never sends more than - // one budget. + // Behind frames held back, the sample waits for the input thread's + // next flush, so one pass never sends more than one budget. let held = self.spaced_until().is_some(); self.enqueue(sample, now); if !held { diff --git a/host/src/lib.rs b/host/src/lib.rs index 94cd3e26..c7ad0c90 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -12,7 +12,10 @@ pub mod discovery; pub mod display_guard; pub mod encoder; pub mod feedback; +pub mod firewall; +pub mod firewall_fix; pub mod gpu_preference; +pub mod gpu_wake; pub mod gui; pub mod handoff; pub mod i18n; @@ -22,6 +25,7 @@ pub mod logging; pub mod network; pub mod pairing; pub mod pipeline; +pub mod power; pub mod relaunch; pub mod remote_config; pub mod settings; diff --git a/host/src/logging.rs b/host/src/logging.rs index 56d1f1b6..e3628c08 100644 --- a/host/src/logging.rs +++ b/host/src/logging.rs @@ -543,12 +543,51 @@ impl Visit for MessageVisitor { } } +/// Call before a test sets the subscriber that captures its log lines. +/// +/// `tracing` decides once, for the whole process, whether a log line is +/// wanted: the first time any thread runs it. While only one subscriber is +/// registered, it asks only the thread running the line, so a test thread +/// with no subscriber caches the line as unwanted and the test capturing it +/// on another thread never sees it. A second registered subscriber, nobody's +/// default, makes `tracing` ask every live subscriber instead. +#[cfg(test)] +pub(crate) fn prepare_log_capture() { + static ANOTHER: Lazy = + Lazy::new(|| tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default())); + Lazy::force(&ANOTHER); +} + #[cfg(test)] mod tests { use super::*; use std::sync::mpsc; use std::time::Duration; + /// A line another thread runs first, with no subscriber, still reaches + /// the capturing test. + #[test] + fn a_capture_sees_a_line_another_thread_ran_first() { + use tracing_subscriber::prelude::*; + fn line() { + tracing::info!("first run elsewhere"); + } + struct Count(Arc>); + impl tracing_subscriber::Layer for Count { + fn on_event(&self, _: &Event<'_>, _: Context<'_, S>) { + *self.0.lock() += 1; + } + } + prepare_log_capture(); + let count = Arc::new(Mutex::new(0)); + let _capture = tracing::subscriber::set_default( + tracing_subscriber::registry().with(Count(Arc::clone(&count))), + ); + std::thread::spawn(line).join().unwrap(); + line(); + assert_eq!(*count.lock(), 1); + } + #[test] fn mdns_success_announcements_do_not_hide_real_errors() { use tracing_subscriber::prelude::*; @@ -565,6 +604,7 @@ mod tests { .with_writer(move || writer.clone()) .with_filter(MdnsDedupFilter::new()), ); + prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { // The log-to-tracing bridge stores the original module in log.target. tracing::error!(target: "log", { log.target = "mdns_sd::service_daemon" }, @@ -815,6 +855,7 @@ mod tests { .with_ansi(false) .with_writer(move || writer.clone()), ); + prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { crate::diagnostics::install_ffmpeg_logging(); let format = diff --git a/host/src/main.rs b/host/src/main.rs index f6ecd1b9..d0c0a621 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -14,6 +14,10 @@ use tracing_subscriber::prelude::*; use tracing_subscriber::EnvFilter; fn main() -> Result<(), Box> { + // The Stream page's "Fix it" for a Windows Firewall block starts this + // exe as administrator to change the firewall rules for this exe, and + // nothing else: no settings, log, window or single-instance check. + eternal_host::firewall_fix::run_if_requested(); // Before anything below creates the settings folder or a file in it: a // folder that held nothing means a fresh install, which opens first-run // setup. Upgrades never see it. @@ -98,9 +102,10 @@ fn main() -> Result<(), Box> { // Never leave a virtual display attached across runs. If a previous run crashed or was // force-killed while streaming the virtual extended display, the VDD device is still enabled - // and shows up as a phantom monitor. Disabling it unconditionally at startup guarantees the - // managed display only ever exists while EternalMonitor is actively using it. It runs beside - // startup: a slow or hung task used to hold a Mirror stream's first frame for up to 30 s. + // and shows up as a phantom monitor. Disabling it at startup, unless Windows already reports it + // disabled, guarantees the managed display only ever exists while EternalMonitor is actively + // using it. It runs beside startup: a slow or hung task used to hold a Mirror stream's first + // frame for up to 30 s. vdd::disable_in_background(); // Belt-and-suspenders: a panic anywhere (GUI, supervisor, pipeline threads) must still tear @@ -130,16 +135,9 @@ fn main() -> Result<(), Box> { // Best-effort capture source for the banner — reflects the default (primary). The // authoritative per-run source (including a persisted/fallback selection) is logged - // inside the capture loop when the stream starts. - let capture_summary = { - let outputs = capture::enumerate_outputs(); - outputs - .iter() - .find(|o| o.is_primary) - .or_else(|| outputs.first()) - .map(|o| format!("{} ({}x{})", o.device_name, o.width, o.height)) - .unwrap_or_else(|| "unknown".to_string()) - }; + // inside the capture loop when the stream starts. Time-boxed with GPU + // detection: it would wait on the same driver. + let capture_summary = capture::listing::startup_display_label(); // Load persisted settings BEFORE the first pipeline spawns, so generation // 0 already streams with the user's bitrate/fps/encoder/display/codec. @@ -160,6 +158,7 @@ fn main() -> Result<(), Box> { info!(" Listen: {}:{}", listen_ip, listen_port); info!("══════════════════════════════════"); eternal_host::network::log_addresses(); + eternal_host::firewall::watch(listen_port); let auth_token = persisted .ensure_auth_token() @@ -175,7 +174,7 @@ fn main() -> Result<(), Box> { let mut pairing = shared.pairing.lock(); pairing.configure(persisted.require_pairing, auth_token); pairing.log_code(); - pairing.warn_if_off(); + pairing.log_if_off(); } // Each Troubleshooting knob: an environment variable (developers, CI) // wins, then the saved setting, then Automatic. @@ -215,6 +214,8 @@ fn main() -> Result<(), Box> { eternal_host::remote_config::load(); #[cfg(windows)] eternal_host::capture::second_screen::load_memory(&shared); + #[cfg(windows)] + eternal_host::capture::second_screen::check_windows(&shared); eternal_host::remote_config::log_launch(); shared .hevc_enabled @@ -246,7 +247,6 @@ fn main() -> Result<(), Box> { shared: shared.clone(), supervisor_tx: supervisor_tx.clone(), }; - /// Set by the termination-signal handler; the headless loop polls it. static SHUTDOWN_REQUESTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); @@ -266,11 +266,119 @@ fn main() -> Result<(), Box> { } } + // An iPad looking for PCs starts waking the encoder's graphics card, which + // a laptop powers off when idle (see `gpu_wake`). + let wake_shared = shared.clone(); + let mut wake_gate = eternal_host::gpu_wake::WakeGate::default(); + let on_browse = move || { + // The card of the encoder that streams, as its last open published. + let Some(adapter_index) = *wake_shared.wake_adapter.lock() else { + return; + }; + let sessions = wake_shared.session.lock().sessions_started(); + if wake_gate.browse( + std::time::Instant::now(), + wake_shared.client_connected(), + sessions, + ) { + // A HELLO between here and the wake's device takes priority. + let shared = wake_shared.clone(); + eternal_host::gpu_wake::wake(adapter_index, move || { + !shared.client_connected() && shared.session.lock().sessions_started() == sessions + }); + } + }; + let supervisor_thread = std::thread::spawn(move || { eternal_host::supervisor::run(listen_port, shared, gpu_info, supervisor_tx, supervisor_rx); }); - let mdns = discovery::advertise_service(listen_port); + let mdns = discovery::advertise_service(listen_port, on_browse); + + // The quit path, once: after the window closes (or the headless loop + // ends), or from the first-paint watcher when the user quits while the + // window's thread is stuck in the graphics driver and may never return. + let quit_control = gui_control.clone(); + let quit_once: QuitOnce = std::sync::Arc::new(parking_lot::Mutex::new(Some(Box::new( + move |window_error: Option| { + let stop_streaming = || { + eternal_host::diagnostics::log_session_end_at_quit(&quit_control.shared); + quit_control.shared.stop(); + quit_control.request_shutdown(); + + // Never leave the virtual display attached after we exit — it should only exist while + // EternalMonitor is actively using it. No Enable starts once the second screen worker + // is shut down, so this Disable is the last change. If it may still be on, the failure + // reporter turns it off after this process exits. + eternal_host::capture::second_screen::shut_down(&quit_control.shared); + eternal_host::startup::display_off_at_quit(vdd::disable); + }; + // A window error is saved first and shown after the cleanup, as a panic is. + match window_error { + Some(detail) => eternal_host::startup::report_after(&detail, stop_streaming), + None => stop_streaming(), + } + + // Give the supervisor a bounded amount of time to shut down cleanly, + // exiting the moment it finishes instead of always burning the full + // window. If it doesn't finish (blocked on DXGI acquire, etc.), + // force-exit so the process never lingers as an invisible zombie. + let (supervisor_done_tx, supervisor_done_rx) = mpsc::channel(); + std::thread::spawn(move || { + if let Err(error) = supervisor_thread.join() { + error!(error = ?error, "Supervisor thread panicked"); + } + let _ = supervisor_done_tx.send(()); + }); + + // Say goodbye on mDNS while the supervisor winds down, so iPads drop the + // host from their scan list promptly (process::exit skips destructors — + // without this no goodbye is ever sent). + if let Some(mdns) = mdns { + discovery::say_goodbye(mdns); + } + + if supervisor_done_rx + .recv_timeout(std::time::Duration::from_secs(3)) + .is_err() + { + tracing::warn!("Supervisor did not shut down within 3s — forcing exit"); + } + + // Release every key and button the iPad left held, behind whatever the + // last stream queued, unless Windows is stuck. + quit_control + .shared + .input + .finish(std::time::Duration::from_millis(500)); + + // The window is gone and the display is off, so a slow disk keeps no one + // waiting on screen: give the second screen record up to 30 s to land. + eternal_host::capture::second_screen::finish_saving(&quit_control.shared); + + eternal_host::startup::simulate_slow_exit(); + info!("EternalMonitor shutting down"); + // process::exit skips destructors. Flush the log writers and the upload + // outbox first, so a normal quit is told apart from a crash. + logging::flush_before_exit(_session_guard, _uploads, _stdout_guard); + eternal_host::startup::finish(); + std::process::exit(0); + }, + )))); + + // Sleep, shutdown and sign-out end the iPad's session with a goodbye. An + // installer closing the app takes the quit path, as closing the window does. + #[cfg(windows)] + eternal_host::power::watch(gui_control.shared.clone(), { + let quit_once = std::sync::Arc::clone(&quit_once); + Box::new(move || { + gui::quitting_from_outside(); + let quit = quit_once.lock().take(); + if let Some(quit) = quit { + quit(None); + } + }) + }); // Headless mode for automation (the iOS E2E harness runs the host without // a window): stream until SIGTERM/SIGINT, then fall through to the same @@ -317,64 +425,29 @@ fn main() -> Result<(), Box> { } } info!("Termination signal received — shutting down cleanly"); - } else if let Err(e) = gui::run_gui(gui_control.clone()) { + } else if let Err(e) = gui::run_gui(gui_control.clone(), { + let quit_once = std::sync::Arc::clone(&quit_once); + Box::new(move || { + let quit = quit_once.lock().take(); + if let Some(quit) = quit { + quit(None); + } + }) + }) { error!(error = %e, "GUI exited with error"); window_error = Some(format!("Window error: {e}")); } - let stop_streaming = || { - eternal_host::diagnostics::log_session_end_at_quit(&gui_control.shared); - gui_control.shared.stop(); - gui_control.request_shutdown(); - - // Never leave the virtual display attached after we exit — it should only exist while - // EternalMonitor is actively using it. No Enable starts once the second screen worker - // is shut down, so this Disable is the last change. If it may still be on, the failure - // reporter turns it off after this process exits. - eternal_host::capture::second_screen::shut_down(&gui_control.shared); - eternal_host::startup::display_off_at_quit(vdd::disable); - }; - // A window error is saved first and shown after the cleanup, as a panic is. - match window_error { - Some(detail) => eternal_host::startup::report_after(&detail, stop_streaming), - None => stop_streaming(), + // The quit path ends the process. Without it, the first-paint watcher + // took it and is ending the process. + let quit = quit_once.lock().take(); + if let Some(quit) = quit { + quit(window_error); } - - // Give the supervisor a bounded amount of time to shut down cleanly, - // exiting the moment it finishes instead of always burning the full - // window. If it doesn't finish (blocked on DXGI acquire, etc.), - // force-exit so the process never lingers as an invisible zombie. - let (supervisor_done_tx, supervisor_done_rx) = mpsc::channel(); - std::thread::spawn(move || { - if let Err(error) = supervisor_thread.join() { - error!(error = ?error, "Supervisor thread panicked"); - } - let _ = supervisor_done_tx.send(()); - }); - - // Say goodbye on mDNS while the supervisor winds down, so iPads drop the - // host from their scan list promptly (process::exit skips destructors — - // without this no goodbye is ever sent). - if let Some(mdns) = mdns { - discovery::say_goodbye(mdns); - } - - if supervisor_done_rx - .recv_timeout(std::time::Duration::from_secs(3)) - .is_err() - { - tracing::warn!("Supervisor did not shut down within 3s — forcing exit"); + loop { + std::thread::park(); } - - // The window is gone and the display is off, so a slow disk keeps no one - // waiting on screen: give the second screen record up to 30 s to land. - eternal_host::capture::second_screen::finish_saving(&gui_control.shared); - - eternal_host::startup::simulate_slow_exit(); - info!("EternalMonitor shutting down"); - // process::exit skips destructors. Flush the log writers and the upload - // outbox first, so a normal quit is told apart from a crash. - logging::flush_before_exit(_session_guard, _uploads, _stdout_guard); - eternal_host::startup::finish(); - std::process::exit(0); } + +/// The quit path; taken by whichever ends the process first. +type QuitOnce = std::sync::Arc) + Send>>>>; diff --git a/host/src/network.rs b/host/src/network.rs index 0238f108..3217c20e 100644 --- a/host/src/network.rs +++ b/host/src/network.rs @@ -368,6 +368,211 @@ impl Drop for Listing { } } +/// The home network addresses among `adapters`: IPv4 on Wi-Fi and Ethernet +/// adapters that are up, in a private range or link-local. Those on an +/// adapter with a default gateway come first and link-local ones last, so a +/// cap on how many are signed keeps the home network's. +/// An iPad checking that it found this PC at an address gets these signed. +pub fn home_addresses_of(adapters: &[Adapter]) -> Vec { + let mut addresses: Vec<(Ipv4Addr, bool)> = adapters + .iter() + .filter(|adapter| adapter.up && matches!(adapter.kind, Kind::Wifi | Kind::Ethernet)) + .flat_map(|adapter| adapter.ipv4.iter().map(|&ip| (ip, adapter.gateway))) + .filter(|(ip, _)| ip.is_private() || ip.is_link_local()) + .collect(); + addresses.sort_by_key(|&(ip, gateway)| (ip.is_link_local(), !gateway)); + let mut seen = Vec::with_capacity(addresses.len()); + for (ip, _) in addresses { + if !seen.contains(&ip) { + seen.push(ip); + } + } + seen +} + +/// How long one listing of the home network addresses answers. +const HOME_TTL: Duration = Duration::from_secs(2); + +/// The home network addresses from the last listing, listed on a thread of +/// its own at most every 2 s. Asking never waits: an identity check asks +/// under the session lock, and no listing runs while it is held. Addresses +/// older than that are never given out: the PC may have lost them. +pub struct HomeCache { + list: fn() -> Option>, + ttl: Duration, + state: Arc>, +} + +#[derive(Default)] +struct HomeState { + listed: Option<(Instant, Vec)>, + listing: bool, + /// Bumped each time debug builds clear the cache: a listing begun + /// before that is dropped. + cleared: u64, +} + +impl HomeCache { + pub(crate) fn with(list: fn() -> Option>, ttl: Duration) -> Self { + Self { + list, + ttl, + state: Arc::default(), + } + } + + /// The last listing's addresses while it is fresh: it started less than + /// `ttl` ago. A missing or expired one reads as none, so nothing is + /// signed, and starts a listing on its own thread. A listing that is + /// already expired when it ends is dropped. + pub fn get(&self) -> Vec { + let (addresses, start) = { + let mut state = self.state.lock(); + let addresses = self.fresh(&state); + let start = addresses.is_none() && !state.listing; + state.listing |= start; + ( + addresses.unwrap_or_default(), + start.then_some(state.cleared), + ) + }; + if let Some(cleared) = start { + let (list, ttl, listing) = (self.list, self.ttl, HomeListing(self.state.clone())); + // A thread that can't start drops the closure, and with it the guard. + let _ = std::thread::Builder::new() + .name("home-addresses".into()) + .spawn(move || { + let started = Instant::now(); + let addresses = list() + .map(|adapters| home_addresses_of(&adapters)) + .unwrap_or_default(); + let mut state = listing.0.lock(); + if started.elapsed() < ttl && state.cleared == cleared { + state.listed = Some((started, addresses)); + } + }); + } + addresses + } + + /// The last listing's addresses while it is fresh. + fn fresh(&self, state: &HomeState) -> Option> { + state + .listed + .as_ref() + .filter(|(started, _)| started.elapsed() < self.ttl) + .map(|(_, addresses)| addresses.clone()) + } + + /// Debug builds only: the last listing's addresses while it is fresh, + /// without starting a listing. + #[cfg(debug_assertions)] + fn peek(&self) -> Option> { + self.fresh(&self.state.lock()) + } + + /// Debug builds only: forgets the last listing, and any listing under + /// way, so the next one reads the adapters as they are now. + #[cfg(debug_assertions)] + fn clear(&self) { + let mut state = self.state.lock(); + state.listed = None; + state.cleared += 1; + } +} + +/// Ends a listing when dropped, so a worker that panics loses only that +/// refresh. +struct HomeListing(Arc>); + +impl Drop for HomeListing { + fn drop(&mut self) { + self.0.lock().listing = false; + } +} + +fn listed_home() -> Option> { + listed().or_else(interfaces) +} + +/// This PC's home network addresses (see [`home_addresses_of`] and +/// [`HomeCache`]). +pub fn home_addresses() -> Vec { + HOME.get() +} + +/// Debug builds only: the home network addresses of a fresh listing, if +/// there is one, without starting a listing as [`home_addresses`] does. +#[cfg(debug_assertions)] +#[doc(hidden)] +pub fn listed_home_addresses() -> Option> { + HOME.peek() +} + +static HOME: std::sync::LazyLock = + std::sync::LazyLock::new(|| HomeCache::with(listed_home, HOME_TTL)); + +/// What kind of interface a Unix name and flags say: network cards are +/// named en*, eth* or wl*; point-to-point interfaces are tunnels. +#[cfg(unix)] +fn interface_kind(name: &str, flags: libc::c_int) -> Kind { + if flags & libc::IFF_LOOPBACK != 0 { + Kind::Virtual + } else if flags & libc::IFF_POINTOPOINT != 0 { + Kind::Vpn + } else if ["en", "eth", "wl"] + .iter() + .any(|prefix| name.starts_with(prefix)) + { + Kind::Ethernet + } else { + Kind::Other + } +} + +/// Unix interfaces with their IPv4 addresses, from `getifaddrs`. +#[cfg(unix)] +fn interfaces() -> Option> { + let mut list: *mut libc::ifaddrs = std::ptr::null_mut(); + if unsafe { libc::getifaddrs(&mut list) } != 0 { + return None; + } + let mut out: Vec = Vec::new(); + let mut next = list; + while !next.is_null() { + let entry = unsafe { &*next }; + next = entry.ifa_next; + if entry.ifa_addr.is_null() + || i32::from(unsafe { (*entry.ifa_addr).sa_family }) != libc::AF_INET + { + continue; + } + let sin = unsafe { &*entry.ifa_addr.cast::() }; + let ip = Ipv4Addr::from(u32::from_be(sin.sin_addr.s_addr)); + let name = unsafe { std::ffi::CStr::from_ptr(entry.ifa_name) } + .to_string_lossy() + .into_owned(); + let flags = entry.ifa_flags as libc::c_int; + match out.iter_mut().find(|adapter| adapter.name == name) { + Some(adapter) => adapter.ipv4.push(ip), + None => out.push(Adapter { + kind: interface_kind(&name, flags), + name, + up: flags & libc::IFF_UP != 0 && flags & libc::IFF_RUNNING != 0, + gateway: false, + ipv4: vec![ip], + }), + } + } + unsafe { libc::freeifaddrs(list) }; + Some(out) +} + +#[cfg(not(unix))] +fn interfaces() -> Option> { + None +} + /// The local address the OS would use toward a public address. pub fn default_route() -> Option { std::net::UdpSocket::bind("0.0.0.0:0") @@ -379,8 +584,10 @@ pub fn default_route() -> Option { .map(|addr| addr.ip()) } -/// The startup line: address counts by reach, the default route's reach and -/// the kinds of adapters with an address. No address and no adapter name. +/// The startup line: address counts by reach, the default route's reach, the +/// kinds of adapters with an address and the kinds with a home network +/// address, which tell a PC on a cable from one on Wi-Fi. No address and no +/// adapter name. pub fn summary(adapters: Option<&[Adapter]>, route: Option) -> String { let Some(adapters) = adapters else { return "Network: addresses unavailable".to_string(); @@ -394,17 +601,25 @@ pub fn summary(adapters: Option<&[Adapter]>, route: Option) -> String { .find(|address| IpAddr::V4(address.ip) == ip) .map_or("unknown", |address| address.reach.label()), }; - let mut kinds: Vec<&str> = ranked.iter().map(|address| address.kind.label()).collect(); - kinds.sort_unstable(); - kinds.dedup(); + let kinds = |reach: Option| { + let mut kinds: Vec<&str> = ranked + .iter() + .filter(|address| reach.is_none_or(|reach| address.reach == reach)) + .map(|address| address.kind.label()) + .collect(); + kinds.sort_unstable(); + kinds.dedup(); + kinds.join(" ") + }; format!( - "Network: addresses lan={} other={} virtual={} vpn={} link_local={} route={route} adapters=[{}]", + "Network: addresses lan={} other={} virtual={} vpn={} link_local={} route={route} adapters=[{}] lan_adapters=[{}]", count(Reach::Lan), count(Reach::Other), count(Reach::Virtual), count(Reach::Vpn), count(Reach::LinkLocal), - kinds.join(" "), + kinds(None), + kinds(Some(Reach::Lan)), ) } @@ -416,12 +631,15 @@ pub fn log_addresses() { #[cfg(debug_assertions)] static TEST_ADAPTERS: Mutex>> = Mutex::new(None); -/// Debug builds only: has the Address row's hint read `adapters` instead of -/// the PC's, until called with `None`. +/// Debug builds only: has the Address row's hint and the home network +/// addresses read `adapters` instead of the PC's, until called with `None`. +/// The last listing of home network addresses is forgotten each time, so no +/// test hears another's. #[cfg(debug_assertions)] #[doc(hidden)] pub fn set_test_adapters(adapters: Option>) { *TEST_ADAPTERS.lock() = adapters; + HOME.clear(); } fn listed() -> Option> { @@ -897,7 +1115,18 @@ mod tests { assert_eq!( line, "Network: addresses lan=1 other=0 virtual=1 vpn=1 link_local=0 route=vpn \ - adapters=[virtual vpn wifi]" + adapters=[virtual vpn wifi] lan_adapters=[wifi]" + ); + // A PC on a cable, its Wi-Fi without DHCP: which one iPads reach. + let cable = classify(6, "Realtek PCIe GbE Family Controller", "Ethernet"); + let both = [ + adapter(cable, true, [192, 168, 1, 21]), + adapter(wifi, false, [169, 254, 7, 9]), + ]; + assert_eq!( + summary(Some(&both), Some(IpAddr::V4(ip([192, 168, 1, 21])))), + "Network: addresses lan=1 other=0 virtual=0 vpn=0 link_local=1 route=lan \ + adapters=[ethernet wifi] lan_adapters=[ethernet]" ); for private in [ "100.", @@ -913,11 +1142,198 @@ mod tests { assert!(!line.contains(private), "{private} in {line}"); } assert_eq!(summary(None, None), "Network: addresses unavailable"); - assert!(summary(Some(&[]), None).contains("route=none adapters=[]")); + assert!(summary(Some(&[]), None).contains("route=none adapters=[] lan_adapters=[]")); } /// Windows says the list grew every time: three bigger buffers, then no /// list, so no hint. + /// What an identity proof signs: home network addresses on network + /// cards that are up, never a VPN's or a virtual switch's, even in a + /// private range. + #[test] + fn home_addresses_are_on_network_cards_that_are_up() { + let mut down = adapter(Kind::Ethernet, true, [192, 168, 7, 2]); + down.up = false; + let adapters = [ + adapter(Kind::Vpn, false, [10, 8, 0, 2]), + adapter(Kind::Virtual, false, [172, 20, 0, 1]), + adapter(Kind::Wifi, true, [169, 254, 3, 4]), + down, + adapter(Kind::Other, true, [192, 168, 9, 9]), + Adapter { + ipv4: vec![ + ip([192, 168, 1, 20]), + ip([203, 0, 113, 5]), + ip([192, 168, 1, 20]), + ], + ..adapter(Kind::Ethernet, true, [0, 0, 0, 0]) + }, + adapter(Kind::Wifi, true, [10, 0, 0, 31]), + ]; + assert_eq!( + home_addresses_of(&adapters), + [ + ip([192, 168, 1, 20]), + ip([10, 0, 0, 31]), + ip([169, 254, 3, 4]) + ] + ); + assert!(home_addresses_of(&[]).is_empty()); + } + + /// The home network's address goes first, so the 4 signed include it + /// however many other addresses the PC has. + #[test] + fn home_addresses_with_a_gateway_come_first() { + let direct = Adapter { + ipv4: (1..=4).map(|last| ip([10, 0, 9, last])).collect(), + ..adapter(Kind::Ethernet, false, [0, 0, 0, 0]) + }; + let adapters = [ + adapter(Kind::Wifi, true, [169, 254, 3, 4]), + direct, + adapter(Kind::Wifi, true, [192, 168, 1, 20]), + ]; + let addresses = home_addresses_of(&adapters); + assert_eq!(addresses[0], ip([192, 168, 1, 20])); + assert_eq!( + addresses[1..5], + (1..=4).map(|last| ip([10, 0, 9, last])).collect::>()[..] + ); + assert_eq!(addresses.last(), Some(&ip([169, 254, 3, 4]))); + } + + static SLOW_CALLS: AtomicUsize = AtomicUsize::new(0); + + fn slow_home() -> Option> { + SLOW_CALLS.fetch_add(1, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(150)); + tailscale_and_wifi() + } + + /// A listing that took longer than its lifetime is already stale when + /// it arrives: it is dropped, and the addresses stay unknown. + #[test] + fn a_listing_older_than_its_lifetime_is_dropped() { + let cache = HomeCache::with(slow_home, Duration::from_millis(50)); + let deadline = Instant::now() + Duration::from_secs(10); + while SLOW_CALLS.load(Ordering::SeqCst) < 3 { + assert!(cache.get().is_empty(), "a stale listing was signed"); + assert!(Instant::now() < deadline, "no listing finished"); + std::thread::sleep(Duration::from_millis(5)); + } + } + + static HOME_GATE: Mutex<()> = Mutex::new(()); + static HOME_CALLS: AtomicUsize = AtomicUsize::new(0); + static HOME_THREAD: Mutex> = Mutex::new(None); + + fn gated_home() -> Option> { + *HOME_THREAD.lock() = std::thread::current().name().map(str::to_string); + HOME_CALLS.fetch_add(1, Ordering::SeqCst); + let _open = HOME_GATE.lock(); + tailscale_and_wifi() + } + + /// An identity check asks under the session lock: while the adapters are + /// still being listed it gets none at once, from one listing on a thread + /// of its own, and the addresses once it is done. + #[test] + fn home_addresses_never_wait_for_the_listing() { + let cache = Arc::new(HomeCache::with(gated_home, HOME_TTL)); + let closed = HOME_GATE.lock(); + for _ in 0..50 { + let (answer, asked) = std::sync::mpsc::channel(); + let asking = cache.clone(); + std::thread::spawn(move || answer.send(asking.get())); + let addresses = asked + .recv_timeout(Duration::from_secs(1)) + .expect("asking waited for the listing"); + assert!(addresses.is_empty()); + } + let deadline = Instant::now() + Duration::from_secs(10); + while HOME_CALLS.load(Ordering::SeqCst) == 0 { + assert!(Instant::now() < deadline, "no listing started"); + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(HOME_THREAD.lock().as_deref(), Some("home-addresses")); + drop(closed); + let deadline = Instant::now() + Duration::from_secs(10); + while cache.get().is_empty() { + assert!(Instant::now() < deadline, "the addresses never came"); + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(cache.get(), [ip([192, 168, 1, 20])]); + assert_eq!(HOME_CALLS.load(Ordering::SeqCst), 1); + } + + static SWITCHED_GATE: Mutex<()> = Mutex::new(()); + static SWITCHED_CALLS: AtomicUsize = AtomicUsize::new(0); + static SWITCHED_IP: Mutex<[u8; 4]> = Mutex::new([192, 168, 1, 20]); + + fn switched_home() -> Option> { + let listed = *SWITCHED_IP.lock(); + SWITCHED_CALLS.fetch_add(1, Ordering::SeqCst); + let _open = SWITCHED_GATE.lock(); + Some(vec![adapter(Kind::Wifi, true, listed)]) + } + + /// Clearing forgets a fresh listing at once, and drops one that was + /// under way when the adapters changed: a test never hears the + /// addresses of the adapters it replaced. + #[cfg(debug_assertions)] + #[test] + fn a_cleared_cache_never_gives_the_old_addresses() { + let cache = HomeCache::with(switched_home, HOME_TTL); + let deadline = Instant::now() + Duration::from_secs(10); + while cache.get() != [ip([192, 168, 1, 20])] { + assert!(Instant::now() < deadline, "never listed"); + std::thread::sleep(Duration::from_millis(5)); + } + // The get after clearing starts a listing of the old adapters; the + // adapters change and the cache is cleared again while it runs. + let closed = SWITCHED_GATE.lock(); + cache.clear(); + assert!(cache.get().is_empty(), "a cleared listing was given out"); + // The first listing may have published before its worker let go: + // asking again starts the second once it has. + let deadline = Instant::now() + Duration::from_secs(10); + while SWITCHED_CALLS.load(Ordering::SeqCst) < 2 { + assert!(cache.get().is_empty(), "a cleared listing was given out"); + assert!(Instant::now() < deadline, "no listing started"); + std::thread::sleep(Duration::from_millis(5)); + } + *SWITCHED_IP.lock() = [192, 168, 1, 30]; + cache.clear(); + drop(closed); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let addresses = cache.get(); + assert_ne!( + addresses, + [ip([192, 168, 1, 20])], + "the old listing came back" + ); + if addresses == [ip([192, 168, 1, 30])] { + break; + } + assert!(Instant::now() < deadline, "the new addresses never came"); + std::thread::sleep(Duration::from_millis(1)); + } + } + + #[cfg(unix)] + #[test] + fn unix_interfaces_are_told_apart_by_flags_then_name() { + assert_eq!(interface_kind("en0", libc::IFF_UP), Kind::Ethernet); + assert_eq!(interface_kind("wlan0", libc::IFF_UP), Kind::Ethernet); + assert_eq!(interface_kind("utun3", libc::IFF_POINTOPOINT), Kind::Vpn); + assert_eq!(interface_kind("en5", libc::IFF_POINTOPOINT), Kind::Vpn); + assert_eq!(interface_kind("lo0", libc::IFF_LOOPBACK), Kind::Virtual); + assert_eq!(interface_kind("bridge100", libc::IFF_UP), Kind::Other); + assert_eq!(interface_kind("docker0", libc::IFF_UP), Kind::Other); + } + #[test] fn listing_gives_up_after_three_bigger_buffers() { let warned = AtomicBool::new(false); @@ -967,6 +1383,7 @@ mod tests { let warned = AtomicBool::new(false); let levels = Levels::default(); let subscriber = tracing_subscriber::registry().with(levels.clone()); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { for _ in 0..3 { assert_eq!(fill(&warned, |_, _| 232), None); diff --git a/host/src/pairing.rs b/host/src/pairing.rs index 61c6696b..b836e02e 100644 --- a/host/src/pairing.rs +++ b/host/src/pairing.rs @@ -41,6 +41,14 @@ pub fn parse_token(text: &str) -> Option<[u8; 16]> { (token != [0; 16]).then_some(token) } +fn identity_proof(token: &[u8; 16], message: &[u8]) -> [u8; 32] { + let key = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, token); + ring::hmac::sign(&key, message) + .as_ref() + .try_into() + .expect("SHA-256 is 32 bytes") +} + fn random_code(previous: u32) -> Result { // Rejection sampling avoids modulo bias. Exclude 000000 (wire sentinel) // and the preceding code so a rotation always changes what the user sees. @@ -105,10 +113,11 @@ impl Pairing { } } - /// Said at startup and whenever pairing is turned off. - pub fn warn_if_off(&self) { + /// Said at startup and whenever pairing is turned off. The user chose + /// it, so it is not a warning. + pub fn log_if_off(&self) { if !self.required { - warn!("Pairing is off: any iPad on the network can connect without a code"); + info!("Pairing is off: any iPad on the network can connect without a code"); } } @@ -141,6 +150,18 @@ impl Pairing { Ok(()) } + /// The answer to an iPad checking that this is the PC it paired with, at + /// the address it asked: HMAC-SHA256 keyed with the pairing token over + /// the nonce, the port and the addresses this PC receives the stream on + /// (`identity_message`). Only a PC holding the token can make it, and the + /// token never leaves. + pub fn identity_proof(&self, nonce: &[u8; 16], port: u16, addresses: &[[u8; 4]]) -> [u8; 32] { + identity_proof( + &self.token, + &eternal_wire::v2::control::identity_message(nonce, port, addresses), + ) + } + pub fn regenerate_token(&mut self) -> Result<(), getrandom::Error> { let token = new_token()?; self.rotate_code()?; @@ -355,6 +376,73 @@ impl RateLimiter { mod tests { use super::*; + /// The known answer the wire golden vector and the iPad's tests share. + #[test] + fn identity_proof_is_the_shared_known_answer() { + let token = std::array::from_fn(|index| index as u8 + 1); + let nonce = std::array::from_fn(|index| 0xA0 + index as u8); + let pairing = Pairing::new(true, token).unwrap(); + let addresses = [[192, 0, 2, 10], [198, 51, 100, 10]]; + let proof = |nonce: &[u8; 16], port, addresses: &[[u8; 4]]| { + pairing.identity_proof(nonce, port, addresses) + }; + let hex: String = proof(&nonce, 9876, &addresses) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + assert_eq!( + hex, + "18fb81d76bbb96e91101119775ad9766b0c332b2c49441e6f07ec33be1c6d94e" + ); + // Another token, nonce, port or address gives another proof. + let other = Pairing::new(true, [9; 16]).unwrap(); + let signed = proof(&nonce, 9876, &addresses); + assert_ne!(other.identity_proof(&nonce, 9876, &addresses), signed); + assert_ne!(proof(&[0; 16], 9876, &addresses), signed); + assert_ne!(proof(&nonce, 9877, &addresses), signed); + assert_ne!(proof(&nonce, 9876, &addresses[..1]), signed); + assert_ne!( + proof(&nonce, 9876, &[[192, 0, 2, 10], [198, 51, 100, 11]]), + signed + ); + } + + /// Log lines `run` writes on this thread. + fn logged(run: impl FnOnce()) -> String { + let buffer = std::sync::Arc::new(parking_lot::Mutex::new(Vec::new())); + let writer = buffer.clone(); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || Captured(writer.clone())) + .finish(); + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, run); + let text = String::from_utf8_lossy(&buffer.lock()).into_owned(); + text + } + + struct Captured(std::sync::Arc>>); + impl std::io::Write for Captured { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + /// The user turned pairing off: that is a setting, not a warning. + #[test] + fn pairing_off_is_logged_at_info() { + let off = Pairing::new(false, [7; 16]).unwrap(); + let text = logged(|| off.log_if_off()); + assert!(text.contains(" INFO "), "{text}"); + assert!(text.contains("Pairing is off"), "{text}"); + let on = Pairing::new(true, [7; 16]).unwrap(); + assert_eq!(logged(|| on.log_if_off()), ""); + } + fn guess(nonce: u32) -> FailedRequest { FailedRequest { nonce, diff --git a/host/src/pipeline.rs b/host/src/pipeline.rs index 2633a12e..f67fb9a8 100644 --- a/host/src/pipeline.rs +++ b/host/src/pipeline.rs @@ -60,19 +60,14 @@ pub async fn run_pipeline_supervised( let capture_reporter = reporter.clone(); let capture_shared = shared.clone(); - let adapter_index = gpu_info.adapter_index; let capture_thread = std::thread::Builder::new() .name(format!("capture-g{generation}")) .spawn(move || { - let outcome = match capture::run_capture_stage( - frame_producer, - capture_shared, - adapter_index, - generation, - ) { - Ok(()) => StageOutcome::Completed, - Err(e) => StageOutcome::Failed(e.to_string()), - }; + let outcome = + match capture::run_capture_stage(frame_producer, capture_shared, generation) { + Ok(()) => StageOutcome::Completed, + Err(e) => StageOutcome::Failed(e.to_string()), + }; capture_reporter.stage_exited(Stage::Capture, outcome); }); if let Err(e) = capture_thread { @@ -103,13 +98,19 @@ pub async fn run_pipeline_supervised( return; } - let transport_outcome = - match transport::start_sender(nal_rx, audio_rx, listen_port, shared.clone(), supervisor_tx) - .await - { - Ok(()) => StageOutcome::Completed, - Err(e) => transport_failure(&*e), - }; + let transport_outcome = match transport::start_sender( + nal_rx, + audio_rx, + listen_port, + shared.clone(), + supervisor_tx, + generation, + ) + .await + { + Ok(()) => StageOutcome::Completed, + Err(e) => transport_failure(&*e), + }; reporter.stage_exited(Stage::Transport, transport_outcome); stats::PIPELINE_STATS.lock().mark_pipeline_stopped(); diff --git a/host/src/power.rs b/host/src/power.rs new file mode 100644 index 00000000..f337af86 --- /dev/null +++ b/host/src/power.rs @@ -0,0 +1,596 @@ +//! Windows putting the PC to sleep, shutting it down or signing the user +//! out while an iPad streams. +//! +//! The host said goodbye only when its window closed. A PC that slept or +//! shut down mid-stream went silent instead: the iPad showed a lost signal, +//! then tried an address that no longer answered, and no session summary +//! was written, so these drops never reached the diagnostics either. Now the +//! session ends when Windows says the PC is going down: its summary is +//! written first, and an iPad that understands it hears why. +//! +//! Windows waits for none of this for long: about two seconds after a +//! suspend message, and a few seconds after the end-of-session message before +//! it lists the app as holding up shutdown. Each step here is bounded well +//! inside that, and nothing ever asks Windows to wait. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use tracing::{debug, info, warn}; + +use crate::control::SharedControl; +use crate::transport::session::{PowerDown, PowerEvent, SessionEnd}; + +/// Longest a suspend message waits for the goodbye to leave. +pub const SLEEP_WAIT: Duration = Duration::from_millis(500); +/// Longest the end-of-session message waits for it. +pub const END_SESSION_WAIT: Duration = Duration::from_millis(1500); + +/// Wakes the transport to send a goodbye the PC going down left, and counts +/// the goodbyes it sent, so the message can wait for its own. +#[derive(Debug, Default)] +pub struct Signal { + wake: tokio::sync::Notify, + said: AtomicU64, +} + +impl Signal { + /// The transport sent a goodbye. + pub(crate) fn said(&self) { + self.said.fetch_add(1, Ordering::SeqCst); + } + + /// Resolves when a goodbye waits for the transport; a wake that came + /// while the transport was busy is kept for its next look. + pub(crate) async fn woken(&self) { + self.wake.notified().await; + } + + fn count(&self) -> u64 { + self.said.load(Ordering::SeqCst) + } +} + +/// What Windows told the app, from its window messages. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Message { + /// WM_POWERBROADCAST PBT_APMSUSPEND. + Suspend, + /// WM_POWERBROADCAST PBT_APMRESUMEAUTOMATIC or PBT_APMRESUMESUSPEND. + Resume, + /// WM_QUERYENDSESSION: Windows asks whether the session may end. Always + /// yes, and nothing happens yet: another app can still call it off. + QueryEndSession, + /// WM_ENDSESSION: `ending` false when the shutdown was called off. + /// `close_app` (ENDSESSION_CLOSEAPP) without `logoff`: the PC stays up, + /// and Restart Manager closes the app because an installer needs its + /// files, as Setup does on every update. + EndSession { + ending: bool, + logoff: bool, + close_app: bool, + }, + /// WM_WTSSESSION_CHANGE WTS_SESSION_LOGOFF. + SessionLogoff, + /// WM_CLOSE: someone asks the app to close, as Restart Manager does + /// after the end of session when the app still runs. + Close, +} + +/// Whether the app quits once `message` is answered, the way it does when +/// its window is closed: Restart Manager closing it for an installer, or a +/// WM_CLOSE. The hidden window stays meanwhile, so sleep and shutdown are +/// still heard should the quit not end the process. +pub fn quits(message: Message) -> bool { + matches!( + message, + Message::Close + | Message::EndSession { + ending: true, + logoff: false, + close_app: true, + } + ) +} + +/// Longest the end-of-session message waits for the session summary to +/// reach the disk, counted from when the message came. Windows lists the +/// app as holding up shutdown after about five seconds; the goodbye's own +/// wait runs at the same time. +pub const SUMMARY_WAIT: Duration = Duration::from_secs(3); + +/// Acts on `message` and returns what the window answers Windows: 1 (TRUE) +/// for WM_POWERBROADCAST and WM_QUERYENDSESSION, 0 otherwise. +pub fn handle(shared: &SharedControl, message: Message) -> isize { + match message { + Message::Suspend => { + going_down(shared, PowerEvent::Sleep, SLEEP_WAIT); + 1 + } + Message::Resume => { + back_up(shared, "The PC woke"); + 1 + } + Message::QueryEndSession => 1, + Message::EndSession { + ending: true, + logoff: false, + close_app: true, + } => { + closing_app(shared); + 0 + } + Message::EndSession { + ending: true, + logoff, + .. + } => { + let event = if logoff { + PowerEvent::Logoff + } else { + PowerEvent::Shutdown + }; + going_down(shared, event, END_SESSION_WAIT); + 0 + } + Message::EndSession { ending: false, .. } => { + back_up(shared, "The shutdown was called off"); + 0 + } + Message::SessionLogoff => { + going_down(shared, PowerEvent::Logoff, END_SESSION_WAIT); + 0 + } + // The quit ends the stream ([`quits`]). + Message::Close => 0, + } +} + +/// Ends the session for `event`, writes its summary, and waits at most +/// `wait` for the transport to send the goodbye, if there is one. +fn going_down(shared: &SharedControl, event: PowerEvent, wait: Duration) { + let started = Instant::now(); + let before = shared.power.count(); + let (down, told) = { + let mut session = shared.session.lock(); + let down = session.power_down(started, event); + (down, session.power_goodbye_pending()) + }; + let end = match down { + PowerDown::Repeat => { + debug!( + power = event.name(), + "Windows repeated that the PC is going down" + ); + return; + } + PowerDown::NoSession => { + info!( + power = event.name(), + "Windows is taking the PC down; no iPad was streaming" + ); + return; + } + PowerDown::Ended(end) => end, + }; + info!( + power = event.name(), + told, "Windows is taking the PC down — ended the iPad's session" + ); + // A PC that sleeps comes back with this process and its log writers; + // one that shuts down or signs out may end it any moment now. + let write = match event { + PowerEvent::Sleep => Write::Log, + PowerEvent::Shutdown | PowerEvent::Logoff => Write::Disk, + }; + finish( + shared, + end, + write, + told, + before, + started, + wait, + event.name(), + ); +} + +/// Restart Manager closes the app for an installer while the PC stays up: +/// the stream ends as on a quit, with the goodbye every iPad reads (connect +/// again, when an update hands over to its new copy), and the summary on +/// disk, since the process ends right after. +fn closing_app(shared: &SharedControl) { + let started = Instant::now(); + let before = shared.power.count(); + let end = crate::diagnostics::end_stream(shared, crate::diagnostics::quit_stop(shared)); + let told = shared.session.lock().goodbye_pending(); + let Some(end) = end else { + info!("Windows is closing the app for an installer; no iPad was streaming"); + return; + }; + info!( + told, + "Windows is closing the app for an installer — ended the iPad's session" + ); + finish( + shared, + end, + Write::Disk, + told, + before, + started, + END_SESSION_WAIT, + "close_app", + ); +} + +/// Where an ended session's summary goes. +enum Write { + /// Through the log, which outlives a sleep. + Log, + /// Straight to disk, on a thread of its own, waited for until + /// [`SUMMARY_WAIT`] after the message came. + Disk, +} + +/// Writes `end`'s summary and waits for the goodbye the transport was told +/// to send, each within its bound from `started`. +#[allow(clippy::too_many_arguments)] +fn finish( + shared: &SharedControl, + end: SessionEnd, + write: Write, + told: bool, + before: u64, + started: Instant, + wait: Duration, + why: &'static str, +) { + // Only a running transport can send it. + let sending = told && shared.transport_beat.read().is_some(); + if sending { + shared.power.wake.notify_one(); + } + let written = match write { + Write::Log => { + crate::diagnostics::log_session_end(&end, shared); + None + } + Write::Disk => { + let (done, written) = std::sync::mpsc::channel(); + let writer = shared.clone(); + let spawned = std::thread::Builder::new() + .name("summary-now".into()) + .spawn(move || { + crate::diagnostics::log_session_end_now(&end, &writer); + let _ = done.send(()); + }); + spawned.ok().map(|_| written) + } + }; + if sending { + while shared.power.count() == before && started.elapsed() < wait { + std::thread::sleep(Duration::from_millis(5)); + } + info!( + power = why, + sent = shared.power.count() != before, + waited_ms = started.elapsed().as_millis() as u64, + "Goodbye before the PC goes down" + ); + } + if let Some(written) = written { + let left = SUMMARY_WAIT.saturating_sub(started.elapsed()); + if written.recv_timeout(left).is_err() { + warn!( + power = why, + "The session summary did not reach the disk in time" + ); + } + } +} + +fn back_up(shared: &SharedControl, why: &'static str) { + shared.session.lock().power_up(); + info!("{why} — iPads may connect again"); +} + +/// Starts listening for Windows' sleep, shutdown and sign-out messages, on a +/// hidden window of its own. `quit` is the app's quit path, taken on a +/// thread of its own after a message that [`quits`]; it runs once. Later +/// calls do nothing. +#[cfg(windows)] +pub fn watch(shared: SharedControl, quit: Box) { + windows_impl::watch(shared, quit); +} + +/// The hidden window's handle, once it exists, for tests that send it +/// messages as Windows would. +#[cfg(windows)] +pub fn window() -> Option { + windows_impl::window() +} + +#[cfg(windows)] +mod windows_impl { + use std::sync::atomic::{AtomicIsize, Ordering}; + use std::sync::{Mutex, OnceLock}; + + use windows::core::w; + use windows::Win32::Foundation::{HANDLE, HINSTANCE, HWND, LPARAM, LRESULT, WPARAM}; + use windows::Win32::System::LibraryLoader::GetModuleHandleW; + use windows::Win32::System::Power::RegisterSuspendResumeNotification; + use windows::Win32::System::RemoteDesktop::{ + WTSRegisterSessionNotification, NOTIFY_FOR_THIS_SESSION, + }; + use windows::Win32::UI::WindowsAndMessaging::{ + CreateWindowExW, DefWindowProcW, DispatchMessageW, GetMessageW, RegisterClassW, + TranslateMessage, DEVICE_NOTIFY_WINDOW_HANDLE, ENDSESSION_CLOSEAPP, ENDSESSION_LOGOFF, MSG, + PBT_APMRESUMEAUTOMATIC, PBT_APMRESUMESUSPEND, PBT_APMSUSPEND, WINDOW_EX_STYLE, + WINDOW_STYLE, WM_CLOSE, WM_ENDSESSION, WM_POWERBROADCAST, WM_QUERYENDSESSION, + WM_WTSSESSION_CHANGE, WNDCLASSW, WTS_SESSION_LOGOFF, + }; + + use super::Message; + use crate::control::SharedControl; + + static SHARED: OnceLock = OnceLock::new(); + static WINDOW: AtomicIsize = AtomicIsize::new(0); + /// The app's quit path, until a message takes it. + #[allow(clippy::type_complexity)] + static QUIT: Mutex>> = Mutex::new(None); + + pub(super) fn watch(shared: SharedControl, quit: Box) { + if SHARED.set(shared).is_err() { + return; + } + *QUIT.lock().unwrap_or_else(|e| e.into_inner()) = Some(quit); + let started = std::thread::Builder::new() + .name("power-watch".into()) + .spawn(|| { + if let Err(error) = unsafe { run() } { + tracing::warn!( + %error, + "Could not watch for sleep and shutdown; the iPad isn't told when the PC goes down" + ); + } + }); + if let Err(error) = started { + tracing::warn!(%error, "Could not start watching for sleep and shutdown"); + } + } + + pub(super) fn window() -> Option { + Some(WINDOW.load(Ordering::SeqCst)).filter(|&hwnd| hwnd != 0) + } + + unsafe fn run() -> windows::core::Result<()> { + let instance: HINSTANCE = GetModuleHandleW(None)?.into(); + let class = w!("EternalMonitorPowerWatch"); + let window_class = WNDCLASSW { + lpfnWndProc: Some(window_proc), + hInstance: instance, + lpszClassName: class, + ..Default::default() + }; + RegisterClassW(&window_class); + // A hidden top-level window: Windows sends the end-of-session + // messages to top-level windows only, never to message-only ones. + let hwnd = CreateWindowExW( + WINDOW_EX_STYLE::default(), + class, + w!("EternalMonitor power"), + WINDOW_STYLE::default(), + 0, + 0, + 0, + 0, + None, + None, + instance, + None, + )?; + // Sleep on a modern-standby PC reaches only registered windows. + if let Err(error) = + RegisterSuspendResumeNotification(HANDLE(hwnd.0), DEVICE_NOTIFY_WINDOW_HANDLE) + { + tracing::warn!(%error, "Could not register for sleep notices"); + } + if let Err(error) = WTSRegisterSessionNotification(hwnd, NOTIFY_FOR_THIS_SESSION) { + tracing::warn!(%error, "Could not register for sign-out notices"); + } + WINDOW.store(hwnd.0 as isize, Ordering::SeqCst); + let mut message = MSG::default(); + while GetMessageW(&mut message, None, 0, 0).as_bool() { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + Ok(()) + } + + /// The message Windows sent, if it is one of those [`super::handle`] acts on. + fn message(kind: u32, wparam: usize, lparam: isize) -> Option { + Some(match kind { + WM_POWERBROADCAST => match wparam as u32 { + PBT_APMSUSPEND => Message::Suspend, + PBT_APMRESUMEAUTOMATIC | PBT_APMRESUMESUSPEND => Message::Resume, + _ => return None, + }, + WM_QUERYENDSESSION => Message::QueryEndSession, + WM_ENDSESSION => Message::EndSession { + ending: wparam != 0, + logoff: lparam as u32 & ENDSESSION_LOGOFF != 0, + close_app: lparam as u32 & ENDSESSION_CLOSEAPP != 0, + }, + WM_WTSSESSION_CHANGE if wparam as u32 == WTS_SESSION_LOGOFF => Message::SessionLogoff, + WM_CLOSE => Message::Close, + _ => return None, + }) + } + + unsafe extern "system" fn window_proc( + hwnd: HWND, + kind: u32, + wparam: WPARAM, + lparam: LPARAM, + ) -> LRESULT { + match (message(kind, wparam.0, lparam.0), SHARED.get()) { + (Some(message), Some(shared)) => { + let answer = super::handle(shared, message); + if super::quits(message) { + quit(message); + } + LRESULT(answer) + } + _ => DefWindowProcW(hwnd, kind, wparam, lparam), + } + } + + /// Takes the app's quit path on a thread of its own, so the window + /// answers Windows at once; the first time only. The window is never + /// destroyed here: until the process ends, it still hears Windows. + fn quit(message: Message) { + let Some(quit) = QUIT.lock().unwrap_or_else(|e| e.into_inner()).take() else { + tracing::debug!(?message, "Windows asked again for the app to close"); + return; + }; + tracing::info!(?message, "Windows asked the app to close — quitting"); + if let Err(error) = std::thread::Builder::new() + .name("power-quit".into()) + .spawn(quit) + { + tracing::warn!(%error, "Could not start quitting for Windows"); + } + } + + #[cfg(test)] + mod tests { + use super::*; + use windows::Win32::UI::WindowsAndMessaging::ENDSESSION_CRITICAL; + + #[test] + fn window_messages_map_to_what_windows_means() { + let cases = [ + ( + WM_POWERBROADCAST, + PBT_APMSUSPEND as usize, + 0, + Some(Message::Suspend), + ), + ( + WM_POWERBROADCAST, + PBT_APMRESUMEAUTOMATIC as usize, + 0, + Some(Message::Resume), + ), + ( + WM_POWERBROADCAST, + PBT_APMRESUMESUSPEND as usize, + 0, + Some(Message::Resume), + ), + // A power setting change is someone else's business. + (WM_POWERBROADCAST, 0x8013, 0, None), + (WM_QUERYENDSESSION, 0, 0, Some(Message::QueryEndSession)), + ( + WM_ENDSESSION, + 1, + 0, + Some(Message::EndSession { + ending: true, + logoff: false, + close_app: false, + }), + ), + // Restart Manager closing the app for an installer, also + // when it must. + ( + WM_ENDSESSION, + 1, + ENDSESSION_CLOSEAPP as isize, + Some(Message::EndSession { + ending: true, + logoff: false, + close_app: true, + }), + ), + ( + WM_ENDSESSION, + 1, + (ENDSESSION_CLOSEAPP | ENDSESSION_CRITICAL) as i32 as isize, + Some(Message::EndSession { + ending: true, + logoff: false, + close_app: true, + }), + ), + (WM_CLOSE, 0, 0, Some(Message::Close)), + ( + WM_ENDSESSION, + 1, + ENDSESSION_LOGOFF as i32 as isize, + Some(Message::EndSession { + ending: true, + logoff: true, + close_app: false, + }), + ), + ( + WM_ENDSESSION, + 0, + 0, + Some(Message::EndSession { + ending: false, + logoff: false, + close_app: false, + }), + ), + ( + WM_WTSSESSION_CHANGE, + WTS_SESSION_LOGOFF as usize, + 1, + Some(Message::SessionLogoff), + ), + // A lock is not the PC going down. + (WM_WTSSESSION_CHANGE, 7, 1, None), + ]; + for (kind, wparam, lparam, expected) in cases { + assert_eq!( + message(kind, wparam, lparam), + expected, + "{kind:#x} {wparam:#x}" + ); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Only an installer closing the app, or a WM_CLOSE, quits it; the PC + /// going down, or a sign-out, leaves that to Windows. + #[test] + fn closing_the_app_quits_it_and_the_pc_going_down_does_not() { + let end = |ending, logoff, close_app| Message::EndSession { + ending, + logoff, + close_app, + }; + assert!(quits(Message::Close)); + assert!(quits(end(true, false, true))); + for message in [ + end(true, false, false), + end(true, true, false), + end(true, true, true), + end(false, false, true), + Message::Suspend, + Message::Resume, + Message::QueryEndSession, + Message::SessionLogoff, + ] { + assert!(!quits(message), "{message:?}"); + } + } +} diff --git a/host/src/settings.rs b/host/src/settings.rs index d1214233..79bfda13 100644 --- a/host/src/settings.rs +++ b/host/src/settings.rs @@ -24,6 +24,10 @@ pub struct SettingsFile { pub encoder_input: crate::encoder::input::EncoderInput, #[serde(default = "default_true")] pub check_for_updates: bool, + /// On a Beta build: newer Betas are offered too. Off: only stable + /// releases. + #[serde(default = "default_true")] + pub beta_updates: bool, /// DXGI `DeviceName` of the capture display (e.g. `\\.\DISPLAY3`). /// `None` means auto (primary monitor). #[serde(default)] @@ -147,6 +151,7 @@ impl Default for SettingsFile { encoder_override: None, encoder_input: Default::default(), check_for_updates: true, + beta_updates: true, capture_display: None, hevc_enabled: false, stream_audio: true, diff --git a/host/src/stall.rs b/host/src/stall.rs index 9d8b5a20..b42a644c 100644 --- a/host/src/stall.rs +++ b/host/src/stall.rs @@ -40,9 +40,22 @@ pub enum Call { /// Reading the graphics cards and the screens' wiring for the second /// screen's decisions (`capture::second_screen::facts`). GpuFacts, + /// After a lost duplication: asking the duplicated output whether it is + /// still on the desktop, and listing the outputs when it is not. + ListOutputs, + /// The graphics calls before the window's first paint: finding the + /// graphics cards (`gpu::GpuInfo::detect`), listing the screens and the + /// window renderer's adapter list. They share one lane: they wait on the + /// same driver, so after one hangs the others are refused at once + /// instead of each waiting out its own limit. + GpuDetect, + /// The window's adapters listed through wgpu on another thread before + /// the window's thread makes its first graphics call + /// (`gui::renderer::adapter_probe`). + GpuProbe, } -const CALLS: usize = 6; +const CALLS: usize = 9; /// When a call is slow enough to log, and when it is given up on. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -70,6 +83,9 @@ impl Call { Self::VddEnable => "vdd_enable", Self::VddDisable => "vdd_disable", Self::GpuFacts => "gpu_facts", + Self::ListOutputs => "list_outputs", + Self::GpuDetect => "gpu_detect", + Self::GpuProbe => "gpu_probe", } } @@ -99,6 +115,20 @@ impl Call { // adapters, as device creation begins, then asks DXCore and reads // the display configuration; it keeps device creation's limits. Self::GpuFacts => Limits::ms(1_000, 4_000), + // Not timed in the field. Listing creates a DXGI factory and asks + // every adapter for its outputs, which is how device creation + // begins; it keeps device creation's limits. + Self::ListOutputs => Limits::ms(1_000, 4_000), + // 34 startups on 3 PCs (two test PCs, one field PC's uploads): + // median 10 ms, slowest 44 ms, from "Support system information" + // to "Selected encoder". It makes the same DXGI calls as + // GpuFacts and keeps its limits. One AMD laptop's dxgkrnl + // deadlock held it 62 s, and the window with it. + Self::GpuDetect => Limits::ms(1_000, 4_000), + // Not timed in the field yet: on a test PC (RTX 5080) 340-440 ms + // in debug builds, which run wgpu's validation. It makes the + // window's own first calls, and keeps detection's limits. + Self::GpuProbe => Limits::ms(1_000, 4_000), } } } @@ -205,7 +235,8 @@ impl std::fmt::Display for Stall { impl std::error::Error for Stall {} -/// Calls of each kind that were abandoned and still run. +/// Calls of each kind that still run with no caller waiting for them: +/// abandoned, or parked (see [`Pending::park`]). pub(crate) struct Lanes([AtomicUsize; CALLS]); impl Lanes { @@ -225,8 +256,8 @@ pub(crate) fn lanes() -> &'static Lanes { &LANES } -/// Abandoned calls of this kind still running, process-wide: while one -/// runs, the next of its kind is refused. +/// Abandoned or parked calls of this kind still running, process-wide: +/// while one runs, the next of its kind is refused. pub fn abandoned(call: Call) -> usize { LANES.abandoned(call) } @@ -255,8 +286,26 @@ pub(crate) fn run_on( call: Call, limits: Limits, f: F, - progress: impl FnMut(), + mut progress: impl FnMut(), ) -> Result +where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, +{ + start_on(lanes, call, limits, f)? + .wait_up_to(limits.limit, &mut progress) + .expect("the limit ends the wait") +} + +/// Start `f` on its own thread under `limits`, to wait for it with +/// [`Pending::wait_up_to`]. Refused, as by [`run`], while an earlier call of +/// its kind is stuck. +pub(crate) fn start_on( + lanes: &'static Lanes, + call: Call, + limits: Limits, + f: F, +) -> Result, Stall> where T: Send + 'static, F: FnOnce() -> T + Send + 'static, @@ -264,7 +313,7 @@ where let wait = Wait { limits, slice: WAIT_SLICE, - now: Instant::now, + now: Instant::now as fn() -> Instant, }; let delay = simulated_delay(call, limits); let hooks = Hooks { @@ -275,7 +324,7 @@ where }, handoff: || {}, }; - run_in(lanes, call, wait, f, progress, hooks) + start_in(lanes, call, wait, f, hooks) } /// Points on the call's thread: `before` the call (a simulated stall), and @@ -298,6 +347,7 @@ pub(crate) fn no_hooks() -> Hooks { type Outcome = std::thread::Result; /// [`run`] with its lanes, clock and hooks given. +#[cfg(test)] pub(crate) fn run_in( lanes: &'static Lanes, call: Call, @@ -310,7 +360,57 @@ where T: Send + 'static, F: FnOnce() -> T + Send + 'static, { - use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; + let limit = wait.limits.limit; + start_in(lanes, call, wait, f, hooks)? + .wait_up_to(limit, &mut progress) + .expect("the limit ends the wait") +} + +/// Who takes a call's result. Decided under one lock that the call's thread +/// holds from reading it until its result is in the channel, and that the +/// caller holds while it looks in the channel before changing it, so exactly +/// one of them owns the result, and the lane is freed whenever it was taken. +/// Never held across the call itself. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Owner { + /// The caller waits for it. + Waiting, + /// The caller stopped waiting before the limit and takes the result + /// later (see [`Pending::park`]). The lane counts the call meanwhile. + Parked, + /// The caller gave up on it: its thread releases the result. + Nobody, +} + +/// A time-limited call that runs on its own thread, from [`start_in`]. +pub(crate) struct Pending Instant> { + call: Call, + limits: Limits, + slice: Duration, + now: N, + lanes: &'static Lanes, + owner: Arc>, + rx: mpsc::Receiver>, + started: Instant, + /// Its result was returned, or it was given up on. + done: bool, +} + +/// Start `f` on its own thread under `wait`'s limits, or refuse it while an +/// earlier call of its kind still runs with nobody waiting. +pub(crate) fn start_in( + lanes: &'static Lanes, + call: Call, + wait: Wait, + f: F, + hooks: Hooks, +) -> Result, Stall> +where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, + N: Fn() -> Instant, +{ + use std::panic::{catch_unwind, AssertUnwindSafe}; let lane = &lanes.0[call as usize]; let abandoned = lane.load(SeqCst); if abandoned >= MAX_ABANDONED { @@ -321,14 +421,9 @@ where ); return Err(Stall::Refused { call }); } - // Set by the caller when it gives up. The call's thread holds the same - // lock from reading it until its result is in the channel, and the - // caller looks in the channel under it before giving up, so exactly one - // of them owns the result, and the lane is freed whenever it was taken. - // Never held across the call itself. - let gave_up = Arc::new(Mutex::new(false)); + let owner = Arc::new(Mutex::new(Owner::Waiting)); let (tx, rx) = mpsc::channel::>(); - let thread_gave_up = Arc::clone(&gave_up); + let thread_owner = Arc::clone(&owner); let real_start = Instant::now(); let Hooks { before, handoff } = hooks; let spawned = std::thread::Builder::new() @@ -340,13 +435,23 @@ where before(); f() })); - let given_up = thread_gave_up.lock(); + let owner = thread_owner.lock(); handoff(); - if !*given_up { - let _ = tx.send(outcome); - return; + match *owner { + Owner::Waiting => { + let _ = tx.send(outcome); + return; + } + // The caller takes it from the channel. The call no longer + // holds anything up. + Owner::Parked => { + let _ = tx.send(outcome); + lanes.0[call as usize].fetch_sub(1, SeqCst); + return; + } + Owner::Nobody => {} } - drop(given_up); + drop(owner); let panicked = outcome.is_err(); // Released here, before the lane lets a new call start. A drop // that panics must not keep the lane taken either. @@ -364,38 +469,154 @@ where return Err(Stall::Refused { call }); } let started = (wait.now)(); - let finish = |outcome: Outcome, took: Duration| match outcome { - Ok(value) => { - note_finished(call, wait.limits, took); - Ok(value) + Ok(Pending { + call, + limits: wait.limits, + slice: wait.slice, + now: wait.now, + lanes, + owner, + rx, + started, + done: false, + }) +} + +impl Instant> Pending { + /// Wait until the call returns, until `up_to` after it started, or until + /// its limit, calling `progress` every slice meanwhile. Past its limit it + /// is given up on. None: it still runs, after `up_to`. + pub(crate) fn wait_up_to( + &mut self, + up_to: Duration, + progress: &mut impl FnMut(), + ) -> Option> { + loop { + match self.rx.recv_timeout(self.slice) { + Ok(outcome) => { + let took = (self.now)().saturating_duration_since(self.started); + return Some(self.finish(outcome, took)); + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + panic!("{} ended without a result", self.call.name()) + } + } + let after = (self.now)().saturating_duration_since(self.started); + if after >= self.limits.limit { + return Some(self.give_up(after)); + } + if after >= up_to { + return None; + } + progress(); } - Err(panic) => resume_unwind(panic), - }; - loop { - match rx.recv_timeout(wait.slice) { - Ok(outcome) => { - return finish(outcome, (wait.now)().saturating_duration_since(started)); + } + + /// Stop waiting for a call that still runs, to take its result later + /// with [`Self::poll`]. Until it returns, the lane counts it as it counts + /// an abandoned call, so the next of its kind is refused instead of + /// waiting behind it. Its result, if it already returned. + pub(crate) fn park(&mut self) -> Option> { + let mut owner = self.owner.lock(); + if let Ok(outcome) = self.rx.try_recv() { + drop(owner); + let took = (self.now)().saturating_duration_since(self.started); + return Some(self.finish(outcome, took)); + } + if *owner == Owner::Waiting { + *owner = Owner::Parked; + self.lanes.0[self.call as usize].fetch_add(1, SeqCst); + } + None + } + + /// A parked call's result once it returned. Past its limit it is given + /// up on, as when waited for. None while it runs within its limit. + pub(crate) fn poll(&mut self) -> Option> { + let after = (self.now)().saturating_duration_since(self.started); + match self.rx.try_recv() { + Ok(outcome) => Some(self.finish(outcome, after)), + Err(mpsc::TryRecvError::Empty) if after >= self.limits.limit => { + Some(self.give_up(after)) } - Err(mpsc::RecvTimeoutError::Timeout) => {} - Err(mpsc::RecvTimeoutError::Disconnected) => { - panic!("{} ended without a result", call.name()) + Err(mpsc::TryRecvError::Empty) => None, + Err(mpsc::TryRecvError::Disconnected) => { + panic!("{} ended without a result", self.call.name()) } } - let after = (wait.now)().saturating_duration_since(started); - if after >= wait.limits.limit { - let mut gave_up = gave_up.lock(); - // It may have returned since the last look. - if let Ok(outcome) = rx.try_recv() { - drop(gave_up); - return finish(outcome, after); + } + + /// [`Self::wait_up_to`] until `by` on the caller's clock, however long + /// the call took to start. None at once if `by` has passed. + pub(crate) fn wait_until( + &mut self, + by: Instant, + progress: &mut impl FnMut(), + ) -> Option> { + let up_to = by.saturating_duration_since(self.started); + if (self.now)() >= by { + return None; + } + self.wait_up_to(up_to, progress) + } + + /// How long the call has run, by its caller's clock. + pub(crate) fn running_for(&self) -> Duration { + (self.now)().saturating_duration_since(self.started) + } + + fn finish(&mut self, outcome: Outcome, took: Duration) -> Result { + self.done = true; + match outcome { + Ok(value) => { + note_finished(self.call, self.limits, took); + Ok(value) } - *gave_up = true; - lane.fetch_add(1, SeqCst); - drop(gave_up); - log_timed_out(call, wait.limits, after); - return Err(Stall::TimedOut { call, after }); + Err(panic) => std::panic::resume_unwind(panic), + } + } + + fn give_up(&mut self, after: Duration) -> Result { + let mut owner = self.owner.lock(); + // It may have returned since the last look. + if let Ok(outcome) = self.rx.try_recv() { + drop(owner); + return self.finish(outcome, after); + } + // A parked call is counted already. + if *owner == Owner::Waiting { + self.lanes.0[self.call as usize].fetch_add(1, SeqCst); + } + *owner = Owner::Nobody; + drop(owner); + self.done = true; + log_timed_out(self.call, self.limits, after); + Err(Stall::TimedOut { + call: self.call, + after, + }) + } +} + +/// Dropped before its result was taken (a pipeline that ends while an +/// encoder opens in the background): the call's thread releases the result +/// when it returns, and the lane counts the call until then. +impl Drop for Pending { + fn drop(&mut self) { + if self.done { + return; + } + let mut owner = self.owner.lock(); + // Returned already: its result is released here, and a parked + // call's thread freed the lane when it handed it over. + if self.rx.try_recv().is_ok() { + return; + } + if *owner == Owner::Waiting { + self.lanes.0[self.call as usize].fetch_add(1, SeqCst); } - progress(); + *owner = Owner::Nobody; } } @@ -781,6 +1002,9 @@ mod tests { Call::VddEnable, Call::VddDisable, Call::GpuFacts, + Call::ListOutputs, + Call::GpuDetect, + Call::GpuProbe, ] .map(Call::name) .into(); @@ -798,6 +1022,9 @@ mod tests { Call::EncoderOpen, Call::VddEnable, Call::GpuFacts, + Call::ListOutputs, + Call::GpuDetect, + Call::GpuProbe, ] { let limits = call.limits(); assert!(limits.limit >= Duration::from_secs(3), "{call:?}"); @@ -922,6 +1149,182 @@ mod tests { }); } + /// A clock that stands still until the test moves it: the time since + /// the test began, in ms, is the returned atomic. + fn paused_clock() -> (Arc, impl Fn() -> Instant + Clone) { + let start = Instant::now(); + let ms = Arc::new(AtomicU64::new(0)); + let read = Arc::clone(&ms); + (ms, move || start + Duration::from_millis(read.load(SeqCst))) + } + + #[test] + fn a_parked_call_holds_its_lane_and_is_taken_once_it_returns() { + let lanes = lanes(); + let (ms, now) = paused_clock(); + let dropped_on = Arc::new(Mutex::new(None)); + let (release, call) = hang(Device(Arc::clone(&dropped_on))); + let wait = || Wait { + limits: encoder_open_limits("h264_qsv"), + slice: Duration::from_millis(1), + now: now.clone(), + }; + let mut pending = start_in(lanes, Call::EncoderOpen, wait(), call, no_hooks()).unwrap(); + // Waited for until 3 s, then parked: no stall line, and the lane + // counts it. + ms.store(3_000, SeqCst); + assert!(pending + .wait_up_to(Duration::from_secs(3), &mut || {}) + .is_none()); + assert!(pending.park().is_none()); + assert_eq!(lanes.abandoned(Call::EncoderOpen), 1); + assert_eq!(pending.running_for(), Duration::from_secs(3)); + // The next open in the lane is refused at once, as behind a stuck + // one, instead of waiting on FFmpeg's lock behind it. + let refused = run_in( + lanes, + Call::EncoderOpen, + wait(), + || (), + || panic!("a refused call waited"), + no_hooks(), + ); + assert_eq!( + refused, + Err(Stall::Refused { + call: Call::EncoderOpen + }) + ); + ms.store(14_000, SeqCst); + assert!(pending.poll().is_none()); + // It returns within its limit: the lane frees as it hands over, and + // the caller takes the result on its own thread. + release.send(()).unwrap(); + wait_until("the parked call's lane did not free", || { + lanes.abandoned(Call::EncoderOpen) == 0 + }); + let device = pending.poll().expect("returned").expect("not a stall"); + drop(device); + assert_eq!(*dropped_on.lock(), Some(std::thread::current().id())); + drop(pending); + assert_eq!(lanes.abandoned(Call::EncoderOpen), 0); + } + + #[test] + fn a_parked_call_past_its_limit_is_given_up_on_as_one_waited_for() { + let lanes = lanes(); + let (ms, now) = paused_clock(); + let dropped_on = Arc::new(Mutex::new(None)); + let (release, call) = hang(Device(Arc::clone(&dropped_on))); + let mut pending = start_in( + lanes, + Call::EncoderOpen, + Wait { + limits: encoder_open_limits("h264_nvenc"), + slice: Duration::from_millis(1), + now, + }, + call, + no_hooks(), + ) + .unwrap(); + assert!(pending.park().is_none()); + ms.store(14_999, SeqCst); + assert!(pending.poll().is_none()); + ms.store(15_000, SeqCst); + let Some(Err(Stall::TimedOut { call, after })) = pending.poll() else { + panic!("not given up on at the limit") + }; + assert_eq!((call, after), (Call::EncoderOpen, Duration::from_secs(15))); + // Counted once, from the park, until it returns; then released on + // its own thread. + assert_eq!(lanes.abandoned(Call::EncoderOpen), 1); + release.send(()).unwrap(); + wait_until("the given-up call's lane did not free", || { + lanes.abandoned(Call::EncoderOpen) == 0 + }); + wait_until("the late result was not released", || { + dropped_on.lock().is_some() + }); + assert_ne!(*dropped_on.lock(), Some(std::thread::current().id())); + } + + #[test] + fn a_result_in_before_the_park_or_the_look_is_taken() { + let lanes = lanes(); + let (ms, now) = paused_clock(); + let wait = || Wait { + limits: encoder_open_limits("h264_amf"), + slice: Duration::from_millis(1), + now: now.clone(), + }; + // It returned while the caller was about to park it: the park finds + // the result, once the call's thread has let go of the hand-over. + let (inside_tx, inside_rx) = mpsc::channel::<()>(); + let mut pending = start_in( + lanes, + Call::EncoderOpen, + wait(), + || 5, + Hooks { + before: || {}, + handoff: move || { + let _ = inside_tx.send(()); + }, + }, + ) + .unwrap(); + inside_rx.recv_timeout(Duration::from_secs(60)).unwrap(); + assert_eq!(pending.park(), Some(Ok(5))); + assert_eq!(lanes.abandoned(Call::EncoderOpen), 0); + // A parked call that returned before the limit is taken even when + // the caller looks only after the limit. + let (release, call) = hang(6); + let mut pending = start_in(lanes, Call::EncoderOpen, wait(), call, no_hooks()).unwrap(); + assert!(pending.park().is_none()); + release.send(()).unwrap(); + wait_until("the parked call's lane did not free", || { + lanes.abandoned(Call::EncoderOpen) == 0 + }); + ms.store(60_000, SeqCst); + assert_eq!(pending.poll(), Some(Ok(6))); + } + + #[test] + fn a_pending_call_dropped_before_its_result_frees_its_lane_when_it_returns() { + for park in [true, false] { + let lanes = lanes(); + let (_ms, now) = paused_clock(); + let dropped_on = Arc::new(Mutex::new(None)); + let (release, call) = hang(Device(Arc::clone(&dropped_on))); + let mut pending = start_in( + lanes, + Call::EncoderOpen, + Wait { + limits: encoder_open_limits("h264_nvenc"), + slice: Duration::from_millis(1), + now, + }, + call, + no_hooks(), + ) + .unwrap(); + if park { + assert!(pending.park().is_none()); + } + drop(pending); + assert_eq!(lanes.abandoned(Call::EncoderOpen), 1, "park={park}"); + release.send(()).unwrap(); + wait_until("a dropped call kept its lane", || { + lanes.abandoned(Call::EncoderOpen) == 0 + }); + wait_until("the dropped call's result was not released", || { + dropped_on.lock().is_some() + }); + assert_ne!(*dropped_on.lock(), Some(std::thread::current().id())); + } + } + #[test] fn the_stall_line_is_unquoted_for_counting() { let out = Arc::new(Mutex::new(Vec::::new())); @@ -940,6 +1343,7 @@ mod tests { .with_ansi(false) .with_writer(move || Sink(Arc::clone(&writer))) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { log_timed_out( Call::D3d11CreateDevice, @@ -954,6 +1358,7 @@ mod tests { ); // An encoder's line gives that encoder's own limit. out.lock().clear(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default( tracing_subscriber::fmt() .with_ansi(false) diff --git a/host/src/startup/mod.rs b/host/src/startup/mod.rs index 124cd263..5f36924c 100644 --- a/host/src/startup/mod.rs +++ b/host/src/startup/mod.rs @@ -34,8 +34,12 @@ static SHOWN_AT: OnceLock = OnceLock::new(); /// thread holds the heap lock) or `fail-fast` (past the exception filter) to /// fail while the window renderer starts; `early-error` to fail right after the failure reporter /// starts; `panic-later` or, on Windows, `fail-fast-later` to fail 15 s after -/// the window works; or `surface-error` to report a surface error on every -/// frame of the first two seconds. A `-quiet` suffix skips the message box. +/// the window works; `device-lost-later` to lose the window's graphics +/// device 15 s after the window works, with `-fail` or `-hang` to fail or +/// hang every replacement device; `device-lost-early-fail` to lose it at the +/// first frame and fail every replacement; or `surface-error` to report a surface +/// error on every frame of the first two seconds. A `-quiet` suffix skips the +/// message box. /// Only debug builds and builds with the `fault-injection` feature read it; /// a release build ignores it. pub const SIMULATE_ENV: &str = "ETERNAL_SIMULATE_STARTUP_FAILURE"; @@ -60,22 +64,9 @@ fn crash_dir() -> Option { /// threads belong to the pipeline, which the supervisor restarts, so they keep /// the plain log line. Native crashes on any thread end the process. pub fn install_failure_hooks() { - let default_hook = std::panic::take_hook(); - std::panic::set_hook(Box::new(move |info| { - if std::thread::current().name() == Some("main") { - in_report_order( - || save(&panic_detail(info)), - || { - crate::vdd::disable(); - }, - show, - ); - } else { - tracing::error!(panic = %info, backtrace = %std::backtrace::Backtrace::force_capture(), "Host panic"); - crate::vdd::disable(); - } - default_hook(info); - })); + install_panic_hook(|| { + crate::vdd::disable(); + }); #[cfg(windows)] prepare_crash_reporting(); if simulated() == "early-error" { @@ -112,6 +103,35 @@ fn prepare_crash_reporting() { reporter::start(&directory, owner, &launch_id); } +/// The panic hook of [`install_failure_hooks`], ahead of the one before it, +/// with `disable_virtual_display` turning the virtual display off. +fn install_panic_hook(disable_virtual_display: impl Fn() + Send + Sync + 'static) { + let default_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + on_panic(info, &disable_virtual_display); + default_hook(info); + })); +} + +/// What [`install_failure_hooks`] does for a panic, by its thread, with +/// `disable_virtual_display` turning the virtual display off. A panic on +/// egui-wgpu's device replacement thread ends only that try: the window tries +/// again or gives up, and the stream, maybe on the virtual display, goes on. +fn on_panic(info: &std::panic::PanicHookInfo<'_>, disable_virtual_display: impl Fn()) { + match std::thread::current().name() { + Some("main") => { + in_report_order(|| save(&panic_detail(info)), &disable_virtual_display, show) + } + Some(eframe::egui_wgpu::winit::REPLACEMENT_THREAD) => { + tracing::warn!(panic = %info, "Making a new device for the window panicked"); + } + _ => { + tracing::error!(panic = %info, backtrace = %std::backtrace::Backtrace::force_capture(), "Host panic"); + disable_virtual_display(); + } + } +} + /// Save the report, clean up, then ask for the message: the dialog never /// holds the cleanup, and killing a stuck app cannot skip it. fn in_report_order( @@ -388,9 +408,9 @@ fn log_line(summary: &str, detail: &str) -> String { timestamped("ERROR", &format!("{summary}: {detail}")) } -/// A line the reporter adds to the session log about its own work. -#[cfg(windows)] -fn info_line(text: &str) -> String { +/// An INFO line shaped like the tracing ones, for the session log written +/// directly: the reporter's own work, or a summary before the PC goes down. +pub(crate) fn info_line(text: &str) -> String { timestamped(" INFO", text) } @@ -583,6 +603,44 @@ pub(crate) fn simulate_window_failure() { } } +/// [`SIMULATE_ENV`] asks for `device-lost-later`, `device-lost-later-fail` or +/// `device-lost-later-hang` (every replacement device fails or never comes), +/// or `device-lost-early-fail` (at the first frame, before the window works). +#[cfg(any(debug_assertions, feature = "fault-injection"))] +pub(crate) fn device_loss_requested() -> bool { + simulated().starts_with("device-lost-") +} + +/// Fail or hang the choice of a replacement device, when [`SIMULATE_ENV`] +/// asks. It runs on egui-wgpu's replacement thread. +#[cfg(any(debug_assertions, feature = "fault-injection"))] +pub(crate) fn simulate_replacement_failure() -> Result<(), String> { + match simulated().as_str() { + "device-lost-later-fail" | "device-lost-early-fail" => { + Err(format!("Simulated replacement failure from {SIMULATE_ENV}")) + } + "device-lost-later-hang" => loop { + std::thread::park(); + }, + _ => Ok(()), + } +} + +/// True once when [`device_loss_requested`]: at the first frame for +/// `device-lost-early-*`, else 15 s after the window works. The window's +/// graphics device is to be lost then. +#[cfg(any(debug_assertions, feature = "fault-injection"))] +pub(crate) fn simulate_device_loss_now() -> bool { + static DONE: AtomicBool = AtomicBool::new(false); + let early = simulated().starts_with("device-lost-early"); + device_loss_requested() + && (early + || SHOWN_AT + .get() + .is_some_and(|shown| shown.elapsed() > Duration::from_secs(15))) + && !DONE.swap(true, Ordering::SeqCst) +} + /// Count surface errors for the first two seconds of frames, when /// [`SIMULATE_ENV`] asks. pub(crate) fn simulate_surface_error(failures: &std::sync::atomic::AtomicU64) { @@ -856,6 +914,120 @@ mod tests { assert!(detail.contains("mod.rs:"), "{detail}"); } + /// Set for [`panic_hook_child`]: where it notes what happened. + const PANIC_HOOK_CHILD: &str = "EM_TEST_PANIC_HOOK_CHILD"; + const PIPELINE_THREAD: &str = "test pipeline thread"; + + /// Run alone, in its own process, by + /// [`a_panic_making_a_window_device_leaves_the_virtual_display_on`]: the + /// panic hook is process-wide. Installs the host's panic hook with a + /// virtual display "turn off" that notes the thread it ran for, then + /// panics on egui-wgpu's replacement thread and on another background + /// thread. + #[test] + fn panic_hook_child() { + use eframe::egui_wgpu::winit::{Answer, Asked, REPLACEMENT_THREAD, REPLACEMENT_TIMEOUT}; + let Some(notes) = std::env::var_os(PANIC_HOOK_CHILD).map(PathBuf::from) else { + return; + }; + let note = move |line: String| { + use std::io::Write; + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(¬es) + .unwrap(); + writeln!(file, "{line}").unwrap(); + }; + let off = note.clone(); + install_panic_hook(move || { + let thread = std::thread::current() + .name() + .unwrap_or("unnamed") + .to_string(); + off(format!("virtual display off: {thread}")); + }); + + // The painter's replacement thread panics, as in a driver. + let start = Instant::now(); + let mut replacing = + Answer::<()>::ask(REPLACEMENT_THREAD, start, REPLACEMENT_TIMEOUT, || { + panic!("simulated panic making a device") + }) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !matches!(replacing.poll(start), Asked::Ended) { + assert!( + Instant::now() < deadline, + "the painter never saw the try end" + ); + std::thread::sleep(Duration::from_millis(10)); + } + note("replacement ended".into()); + + // Any other background thread. + let _ = std::thread::Builder::new() + .name(PIPELINE_THREAD.into()) + .spawn(|| panic!("simulated pipeline panic")) + .unwrap() + .join(); + note("pipeline ended".into()); + } + + #[test] + fn a_panic_making_a_window_device_leaves_the_virtual_display_on() { + use eframe::egui_wgpu::winit::{Replacement, REPLACEMENT_THREAD}; + let directory = std::env::temp_dir().join(format!( + "eternal-panic-hook-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&directory).unwrap(); + let notes = directory.join("notes.txt"); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "startup::tests::panic_hook_child", + "--exact", + "--nocapture", + "--test-threads=1", + ]) + .env(PANIC_HOOK_CHILD, ¬es) + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&child.stderr); + assert!(child.status.success(), "{stderr}"); + let notes = fs::read_to_string(¬es).unwrap(); + // The worker's panic left the virtual display alone, the other + // thread's turned it off, in order. + assert_eq!( + notes.lines().collect::>(), + [ + "replacement ended", + &format!("virtual display off: {PIPELINE_THREAD}"), + "pipeline ended", + ], + "{stderr}" + ); + assert!(!notes.contains(REPLACEMENT_THREAD)); + // The hook before the host's still ran for both. + assert!( + stderr.contains("simulated panic making a device"), + "{stderr}" + ); + assert!(stderr.contains("simulated pipeline panic"), "{stderr}"); + fs::remove_dir_all(directory).unwrap(); + + // The painter counts the ended try as failed and tries again later. + let start = Instant::now(); + let mut replacement = Replacement::default(); + replacement.lost(start); + replacement.failed(start); + assert_eq!(replacement.wait(start), Some(Duration::from_millis(500))); + } + #[test] fn a_gui_panic_turns_the_virtual_display_off_before_the_message() { let steps = std::cell::RefCell::new(Vec::new()); diff --git a/host/src/startup/port.rs b/host/src/startup/port.rs index ba082fb4..2f29acc9 100644 --- a/host/src/startup/port.rs +++ b/host/src/startup/port.rs @@ -195,6 +195,11 @@ fn note(waiting: &mut Waiting, held: Option) { port, code, "Windows is blocking the stream port; checking again every 2 s" ), + // Its socket is still closing after a restart: no fault. + HeldBy::ThisCopy => info!( + port, + "This copy's previous stream still has the stream port; the stream starts once it lets go" + ), by => warn!( port, holder = category(by), @@ -604,6 +609,7 @@ mod tests { .with_ansi(false) .with_writer(move || writer.clone()) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, run); let bytes = lines.0.lock().unwrap().clone(); String::from_utf8(bytes).unwrap() @@ -825,6 +831,29 @@ mod tests { assert!(log.contains("Windows is blocking"), "{log}"); } + /// This copy's own previous stream, whose socket is still closing, is + /// no fault: said at info. Any other holder is still a warning. + #[test] + fn this_copys_previous_stream_is_said_at_info() { + let _serial = HELD_TESTS.lock().unwrap_or_else(|e| e.into_inner()); + let log = logged(|| { + check_with(9876, Bind::InUse, || Some(HeldBy::ThisCopy), || None); + check_with(9876, Bind::Free, || None, || None); + check_with(9876, Bind::InUse, || Some(HeldBy::OtherCopy), || None); + forget(); + }); + let lines: Vec<&str> = log.lines().collect(); + assert_eq!(lines.len(), 3, "{log}"); + assert!( + lines[0].contains(" INFO ") && lines[0].contains("previous stream still has"), + "{log}" + ); + assert!( + lines[2].contains(" WARN ") && lines[2].contains("Another program"), + "{log}" + ); + } + fn image(path: &'static str) -> impl FnOnce(u32) -> Option<(String, Option)> { started(path, 1) } diff --git a/host/src/stats.rs b/host/src/stats.rs index f74f03ea..c35ab389 100644 --- a/host/src/stats.rs +++ b/host/src/stats.rs @@ -18,6 +18,9 @@ pub struct PipelineStats { // Capture pub capture_fps: f64, + /// The size the stream encodes at: capture's, without an odd last + /// column or row (see [`crate::encoder::encoded_size`]). The iPad's + /// stream config, the GUI and the stats summary report it. pub capture_resolution: (u32, u32), pub capture_frame_count: u64, pub gpu_name: String, @@ -39,10 +42,19 @@ pub struct PipelineStats { /// (CPU) encoding. Surfaced as a warning banner in the GUI so a tester is never silently /// stuck on slow software encoding. pub using_software_fallback: bool, + /// Pictures given to an encoder again to bring out a keyframe it held + /// (see `encoder::holds_last_frame`), for the connected viewer. + pub keyframe_resubmits: u64, /// NVENC refused to open because the NVIDIA driver is older than the /// linked FFmpeg's NVENC needs. The fallback notice then says to update /// it. pub nvidia_driver_outdated: bool, + /// The processor encodes because startup's GPU detection did not return + /// in time. The fallback notice then says the driver did not answer. + pub gpu_detection_stalled: bool, + /// AMD and Intel cards on the PC whose encoder failed to open: the + /// fallback notice says to update their driver. + pub driver_update_vendors: Vec, encode_timestamps: VecDeque, // Transport @@ -51,6 +63,9 @@ pub struct PipelineStats { pub transport_packets_sent: u64, pub transport_fragments_sent: u64, pub transport_retransmits: u64, + /// Pointer positions and images sent to an iPad that draws the pointer. + pub pointer_states_sent: u64, + pub pointer_shapes_sent: u64, pub usb_service_reachable: bool, pub usb_devices: usize, pub usb_link_state: String, @@ -105,7 +120,10 @@ impl PipelineStats { bitrate_bps: 0, codec_name: String::new(), using_software_fallback: false, + keyframe_resubmits: 0, nvidia_driver_outdated: false, + gpu_detection_stalled: false, + driver_update_vendors: Vec::new(), encode_timestamps: VecDeque::with_capacity(128), transport_fps: 0.0, @@ -113,6 +131,8 @@ impl PipelineStats { transport_packets_sent: 0, transport_fragments_sent: 0, transport_retransmits: 0, + pointer_states_sent: 0, + pointer_shapes_sent: 0, usb_service_reachable: false, usb_devices: 0, usb_link_state: "Checking Apple device service".into(), @@ -147,6 +167,8 @@ impl PipelineStats { self.codec_name = previous.codec_name; self.using_software_fallback = previous.using_software_fallback; self.nvidia_driver_outdated = previous.nvidia_driver_outdated; + self.gpu_detection_stalled = previous.gpu_detection_stalled; + self.driver_update_vendors = previous.driver_update_vendors; self.mdns_active = previous.mdns_active; previous.audio } @@ -160,6 +182,9 @@ impl PipelineStats { self.transport_packets_sent = 0; self.transport_fragments_sent = 0; self.transport_retransmits = 0; + self.pointer_states_sent = 0; + self.pointer_shapes_sent = 0; + self.keyframe_resubmits = 0; self.transport_timestamps.clear(); self.bandwidth_samples.clear(); self.bandwidth_bps = 0.0; @@ -207,7 +232,7 @@ impl PipelineStats { pub fn record_capture(&mut self, width: u32, height: u32) { let now = Instant::now(); self.capture_frame_count += 1; - self.capture_resolution = (width, height); + self.capture_resolution = crate::encoder::encoded_size(width, height); self.capture_timestamps.push_back(now); self.capture_fps = Self::calc_fps(&mut self.capture_timestamps, now); } @@ -294,6 +319,16 @@ impl PipelineStats { mod tests { use super::PipelineStats; + #[test] + fn a_new_viewer_starts_counting_pictures_given_again_from_zero() { + let mut stats = PipelineStats { + keyframe_resubmits: 3, + ..PipelineStats::default() + }; + stats.reset_connection_stats(); + assert_eq!(stats.keyframe_resubmits, 0); + } + #[test] fn reset_for_restart_preserves_runtime_configuration() { let mut stats = PipelineStats::new(); diff --git a/host/src/supervisor.rs b/host/src/supervisor.rs index e12e4d0e..076238d8 100644 --- a/host/src/supervisor.rs +++ b/host/src/supervisor.rs @@ -16,6 +16,8 @@ use tracing::{error, info, warn}; use crate::control::{SharedControl, SupervisorCommand}; use crate::gpu::GpuInfo; use crate::stats::PIPELINE_STATS; +use crate::transport::session::{NotStreaming, StageFailure, StreamStop}; +use crate::transport::standby::Standby; /// The current pipeline generation. Stage health reports carry the generation /// they belong to; anything from an older generation is ignored (a "poisoned" @@ -70,6 +72,33 @@ const BACKOFF_MAX: Duration = Duration::from_secs(30); const STORM_WINDOW: Duration = Duration::from_secs(60); const STORM_LIMIT: usize = 6; +/// The supervisor looks at the stage heartbeats every 250 ms. A longer gap +/// between two looks means this process did not run in between: the PC +/// slept or hibernated (Windows' monotonic clock counts the time asleep), or +/// the whole host was starved. Heartbeats stamped before such a gap say +/// nothing about whether a stage is wedged. +const WATCH_GAP_MS: u64 = 2_000; + +/// After such a gap the stages get this long before the watchdog judges +/// them. Waking displays and drivers can hold capture for seconds while +/// nothing is wrong; a stage still silent after it is restarted then. +const WAKE_GRACE_MS: u64 = 10_000; + +/// Lets integration tests trip the storm brake in milliseconds instead of +/// half a minute: the first backoff, in ms. +#[cfg(feature = "test-hooks")] +pub static TEST_BACKOFF_BASE_MS: AtomicU64 = AtomicU64::new(BACKOFF_BASE.as_millis() as u64); + +#[cfg(feature = "test-hooks")] +fn backoff_base() -> Duration { + Duration::from_millis(TEST_BACKOFF_BASE_MS.load(Ordering::SeqCst)) +} + +#[cfg(not(feature = "test-hooks"))] +fn backoff_base() -> Duration { + BACKOFF_BASE +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum SupervisorState { Running { @@ -88,6 +117,8 @@ pub enum SupervisorState { /// Too many automatic restarts in the window; a manual Start is required. Failed { reason: String, + /// The stage that kept failing (a wedge counts as capture). + stage: Stage, }, Stopped, ShutDown, @@ -112,6 +143,10 @@ pub struct Machine { pub state: SupervisorState, restarts: Vec, attempt: u32, + /// When the watchdog last looked at the live generation's heartbeats. + watched_ms: Option, + /// When the watchdog last found such a gap, during [`WAKE_GRACE_MS`]. + woke_ms: Option, } impl Machine { @@ -122,6 +157,8 @@ impl Machine { }, restarts: Vec::new(), attempt: 0, + watched_ms: None, + woke_ms: None, } } @@ -136,6 +173,22 @@ impl Machine { Effect::Restart } }, + SupervisorCommand::RestartStream { generation } => match self.state { + SupervisorState::Running { + generation: live_generation, + } if live_generation == *generation => { + self.attempt = 0; + self.restarts.clear(); + Effect::Restart + } + _ => { + info!( + generation, + "A stream that no longer runs asked to start again: left as it is" + ); + Effect::None + } + }, SupervisorCommand::Stop => match self.state { SupervisorState::Running { .. } | SupervisorState::BackingOff { .. } @@ -222,12 +275,15 @@ impl Machine { STORM_WINDOW.as_secs() ); error!("{message} — giving up until a manual Start"); - self.state = SupervisorState::Failed { reason: message }; + self.state = SupervisorState::Failed { + reason: message, + stage, + }; return Effect::StopPipeline; } self.restarts.push(now); self.attempt += 1; - let backoff = BACKOFF_BASE + let backoff = backoff_base() .saturating_mul(1u32 << (self.attempt - 1).min(6)) .min(BACKOFF_MAX); warn!( @@ -260,6 +316,9 @@ impl Machine { pub fn note_started(&mut self, generation: u64) { self.state = SupervisorState::Running { generation }; + // Joining the old generation can take seconds; that is no sleep. + self.watched_ms = None; + self.woke_ms = None; } /// A successful stretch of running clears the failure budget. @@ -277,6 +336,11 @@ pub struct HealthReporter { } impl HealthReporter { + #[cfg(test)] + pub(crate) fn new(tx: mpsc::Sender, generation: u64) -> Self { + Self { tx, generation } + } + pub fn stage_exited(&self, stage: Stage, outcome: StageOutcome) { let _ = self.tx.send(SupMsg::Health { generation: self.generation, @@ -292,7 +356,7 @@ impl HealthReporter { pub fn run( listen_port: u16, shared: SharedControl, - gpu_info: GpuInfo, + mut gpu_info: GpuInfo, command_tx: mpsc::Sender, command_rx: mpsc::Receiver, ) { @@ -326,11 +390,17 @@ pub fn run( } else { machine.wait_for_port(Instant::now() + crate::startup::port::RECHECK); } + let mut standby = None; + settle(&machine, &shared, &mut standby, listen_port); let mut healthy_since = Instant::now(); + let mut transport_watch = crate::transport::beat::Watch::default(); loop { let message = sup_rx.recv_timeout(Duration::from_millis(250)); let now = Instant::now(); + // A stuck transport stays in its generation's thread, which no join + // can end: this line says where it is stuck. + transport_watch.check_and_log(&shared.transport_beat, crate::clock::host_now_us() / 1000); // A minute of health clears the storm budget. if matches!(machine.state, SupervisorState::Running { .. }) @@ -351,7 +421,7 @@ pub fn run( let ticked = machine.on_tick(now); if ticked != Effect::None { ticked - } else if let Some(reason) = wedge_reason(&shared, &machine) { + } else if let Some(reason) = wedge_reason(&shared, &mut machine) { machine.on_wedged(&reason, now) } else { Effect::None @@ -363,16 +433,34 @@ pub fn run( match effect { Effect::None => {} Effect::StopPipeline => { - shared.stop(); + // Before the join: a HELLO2 the stopping transport still + // takes hears why. + shared + .session + .lock() + .set_not_streaming(not_streaming(&machine.state, &shared)); + stop_pipeline(&shared, &machine.state); bounded_join(&mut pipeline, generation); } Effect::Restart | Effect::Start => { + // A HELLO2 meanwhile is told to ask again in a moment. + shared + .session + .lock() + .set_not_streaming(Some(NotStreaming::Restarting)); shared.stop(); bounded_join(&mut pipeline, generation); + // The check below binds the port: the standby lets go first. + if let Some(standby) = standby.take() { + standby.stop(); + } // Before anything of the new generation starts: a held port - // leaves capture and the encoder closed while it waits. - if !crate::startup::port::check(listen_port) { + // leaves capture and the encoder closed while it waits. One + // the last owner left bound is this host's own. + let ours = shared.stream_socket.lock().is_some(); + if !ours && !crate::startup::port::check(listen_port) { machine.wait_for_port(now + crate::startup::port::RECHECK); + settle(&machine, &shared, &mut standby, listen_port); continue; } crate::audio::restart_stats(&PIPELINE_STATS, |stats| { @@ -381,7 +469,13 @@ pub fn run( }); generation = CURRENT_GENERATION.fetch_add(1, Ordering::SeqCst) + 1; machine.note_started(generation); + settle(&machine, &shared, &mut standby, listen_port); healthy_since = Instant::now(); + if adopt_late_detection(&mut gpu_info, &shared, crate::gpu::take_late_detection) { + let mut stats = PIPELINE_STATS.lock(); + stats.set_gpu_name(gpu_info.name.clone()); + stats.set_codec_name(gpu_info.codec_display_name.clone()); + } pipeline = Some(spawn_generation( generation, listen_port, @@ -394,31 +488,196 @@ pub fn run( Effect::Shutdown => { shared.stop(); bounded_join(&mut pipeline, generation); + // The port closes as the stream ends, as it always did: a + // copy that takes over after this one quits binds it. + shared.stream_socket.lock().take(); break; } } + settle(&machine, &shared, &mut standby, listen_port); // Stopped by the user (or given up) while waiting: no notice. if !matches!(machine.state, SupervisorState::WaitingForPort { .. }) { crate::startup::port::forget(); } } + if let Some(standby) = standby.take() { + standby.stop(); + } + shared.stream_socket.lock().take(); info!("Supervisor loop ended"); } +/// Swap startup's software facts for what a detection given up on at +/// startup found once it returned (`late`), so this and later pipelines +/// encode on the graphics card. True when they changed. +fn take_late_detection(gpu: &mut GpuInfo, late: impl FnOnce() -> Option) -> bool { + if !gpu.detection_stalled { + return false; + } + let Some(found) = late() else { + return false; + }; + info!( + gpu = %found.name, + encoder = %found.encoder_name, + "GPU detection returned after startup went on without it; this pipeline uses its graphics card" + ); + *gpu = found; + true +} + +/// [`take_late_detection`], and the card it found becomes the one to wake +/// for an iPad that looks for PCs, until the new pipeline's encoder opens +/// and says which it uses. +fn adopt_late_detection( + gpu: &mut GpuInfo, + shared: &SharedControl, + late: impl FnOnce() -> Option, +) -> bool { + if !take_late_detection(gpu, late) { + return false; + } + *shared.wake_adapter.lock() = crate::gpu_wake::wake_adapter_for(&gpu.encoder_name, gpu); + true +} + +/// Makes what iPads hear match `machine`: the session's answer to a HELLO2, +/// and the standby responder on the port while no generation streams. +fn settle( + machine: &Machine, + shared: &SharedControl, + standby: &mut Option, + listen_port: u16, +) { + if !standby_answers(&machine.state) { + if let Some(standby) = standby.take() { + standby.stop(); + } + } + // Cleared only once the standby has stopped: it never starts a session. + shared + .session + .lock() + .set_not_streaming(not_streaming(&machine.state, shared)); + if standby_answers(&machine.state) && standby.is_none() { + *standby = Standby::start(listen_port, shared.clone()); + } +} + +/// Whether the standby responder holds the port in `state`. Not while a +/// generation runs, and not while another program holds the port. +fn standby_answers(state: &SupervisorState) -> bool { + matches!( + state, + SupervisorState::BackingOff { .. } + | SupervisorState::Stopped + | SupervisorState::Failed { .. } + ) +} + +/// What a HELLO2 hears in `state`. Nothing starts streaming by itself after +/// Stop or the storm brake: only Start does. +fn not_streaming(state: &SupervisorState, shared: &SharedControl) -> Option { + use eternal_wire::v2::control::FailedReason; + match state { + SupervisorState::Running { .. } => None, + SupervisorState::BackingOff { .. } | SupervisorState::WaitingForPort { .. } => { + Some(NotStreaming::Restarting) + } + SupervisorState::Stopped | SupervisorState::ShutDown => Some(NotStreaming::Stopped), + SupervisorState::Failed { stage, .. } => { + let block = *shared.capture_block.lock(); + Some(NotStreaming::Failed(block.map_or( + match stage { + Stage::Capture | Stage::Audio => FailedReason::Capture, + Stage::Encoder => FailedReason::Encoder, + Stage::Transport => FailedReason::Transport, + }, + crate::capture::recovery::CaptureBlock::failed_reason, + ))) + } + } +} + +/// Stops the pipeline that the machine stopped in `state`, first ending the +/// session as [`stop_when_stopped`] says: its goodbye goes out with the +/// transport, which runs until it sees `running` false. +fn stop_pipeline(shared: &SharedControl, state: &SupervisorState) { + if let Some(why) = stop_when_stopped(state) { + crate::diagnostics::stop_session(shared, why); + } + shared.stop(); +} + +/// How a session ends when the pipeline stops in `state`. Stopped and +/// Failed wait for a manual Start, so the session ends now: kept, it went on +/// unanswered while its iPad lost the signal and every HELLO it sent went +/// unanswered. Stop says the PC ended the session; giving up says connect +/// again, and names the stage in the summary. A backoff or a held port +/// restarts the stream by itself, and its session goes on. +fn stop_when_stopped(state: &SupervisorState) -> Option { + match state { + SupervisorState::Stopped => Some(StreamStop::User), + SupervisorState::Failed { stage, .. } => Some(StreamStop::GaveUp(match stage { + Stage::Capture => StageFailure::Capture, + Stage::Encoder => StageFailure::Encoder, + Stage::Transport | Stage::Audio => StageFailure::Other, + })), + SupervisorState::Running { .. } + | SupervisorState::BackingOff { .. } + | SupervisorState::WaitingForPort { .. } + | SupervisorState::ShutDown => None, + } +} + /// Watchdog rules over the heartbeat atomics. Only meaningful while Running /// and after the pipeline has had a moment to warm up. -fn wedge_reason(shared: &SharedControl, machine: &Machine) -> Option { +fn wedge_reason(shared: &SharedControl, machine: &mut Machine) -> Option { wedge_reason_at(shared, machine, crate::clock::host_now_us() / 1000) } pub(crate) fn wedge_reason_at( shared: &SharedControl, - machine: &Machine, + machine: &mut Machine, now_ms: u64, ) -> Option { if !matches!(machine.state, SupervisorState::Running { .. }) { + machine.watched_ms = None; + machine.woke_ms = None; return None; } + // After a sleep the first look can come before any stage has run again, + // and every heartbeat looks as old as the sleep: 7 of 10 "capture loop + // silent" restarts in the field were PCs waking after 3 min to 3.5 h. + // Give the stages a fresh window instead, with a grace while the + // displays wake; a real wedge still shows when the grace ends. + let last_look = machine.watched_ms.replace(now_ms); + if let Some(gap_ms) = last_look + .map(|last| now_ms.saturating_sub(last)) + .filter(|&gap| gap > WATCH_GAP_MS) + { + info!( + gap_ms, + "The host did not run for a while (the PC slept?) — restarting the pipeline watchdog's clock" + ); + for beat in [ + &shared.hb_capture_loop_ms, + &shared.hb_capture_frame_ms, + &shared.hb_encode_frame_ms, + ] { + if beat.load(Ordering::Relaxed) > 0 { + beat.store(now_ms, Ordering::Relaxed); + } + } + machine.woke_ms = Some(now_ms); + return None; + } + if let Some(woke_ms) = machine.woke_ms { + if now_ms.saturating_sub(woke_ms) < WAKE_GRACE_MS { + return None; + } + machine.woke_ms = None; + } let loop_ms = shared.hb_capture_loop_ms.load(Ordering::Relaxed); let frame_ms = shared.hb_capture_frame_ms.load(Ordering::Relaxed); let encode_ms = shared.hb_encode_frame_ms.load(Ordering::Relaxed); @@ -560,6 +819,62 @@ fn bounded_join(pipeline: &mut Option>, generation: mod tests { use super::*; + #[test] + fn a_late_detection_is_used_by_the_next_pipeline_only_after_a_stall() { + let late = || { + Some(GpuInfo { + name: "late".to_string(), + encoder_name: "h264_amf".to_string(), + ..GpuInfo::software_fallback() + }) + }; + // Startup gave up: the next pipeline takes what it found since. + let mut gpu = GpuInfo { + detection_stalled: true, + ..GpuInfo::software_fallback() + }; + assert!(take_late_detection(&mut gpu, late)); + assert_eq!( + ( + gpu.name.as_str(), + gpu.encoder_name.as_str(), + gpu.detection_stalled + ), + ("late", "h264_amf", false) + ); + // Not returned yet: the processor goes on encoding, still marked. + let mut gpu = GpuInfo { + detection_stalled: true, + ..GpuInfo::software_fallback() + }; + assert!(!take_late_detection(&mut gpu, || None)); + assert!(gpu.detection_stalled); + // A detection that answered at startup is never replaced. + let mut gpu = GpuInfo::software_fallback(); + assert!(!take_late_detection(&mut gpu, late)); + assert_eq!(gpu.name, "Software"); + } + #[test] + fn an_adopted_late_detection_becomes_the_card_to_wake() { + let shared = SharedControl::new(0, 8_000_000); + // Startup gave up and encodes on the CPU: nothing to wake. + let mut gpu = GpuInfo { + detection_stalled: true, + ..GpuInfo::software_fallback() + }; + assert!(!adopt_late_detection(&mut gpu, &shared, || None)); + assert_eq!(*shared.wake_adapter.lock(), None); + let late = || { + Some(GpuInfo { + encoder_name: "h264_amf".to_string(), + adapter_index: 1, + ..GpuInfo::software_fallback() + }) + }; + assert!(adopt_late_detection(&mut gpu, &shared, late)); + assert_eq!(*shared.wake_adapter.lock(), Some(1)); + } + #[test] fn audio_failures_never_restart_or_exhaust_the_video_failure_budget() { let mut machine = Machine::new(7); @@ -629,6 +944,111 @@ mod tests { .unwrap(); } + /// Stop and giving up wait for a manual Start: their session ends with + /// a goodbye. A stream that restarts by itself keeps its session. + #[test] + fn only_a_stop_that_waits_for_start_ends_the_session() { + use crate::transport::link::PeerId; + use crate::transport::session::ConfigSource; + use eternal_wire::v2::control::{ + parse_control, ByeReason, ControlMessage, Hello2, StreamConfig, CAP_DECODE_H264, + }; + struct Config; + impl ConfigSource for Config { + fn stream_config(&self) -> StreamConfig { + StreamConfig { + stream_epoch: 1, + width: 1280, + height: 720, + fps: 60, + codec: 0, + flags: 0, + bitrate_bps: 10_000_000, + } + } + fn host_name(&self) -> String { + "TEST-HOST".into() + } + } + // Stops the pipeline as the run loop does, with an iPad streaming, + // and returns the goodbye left for the transport to send. + let stop = |state: &SupervisorState| { + let shared = SharedControl::new(19899, 10_000_000); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + let peer = PeerId::udp("127.0.0.1:19898".parse().unwrap()); + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 1, + listen_port: 19898, + decoder_caps: CAP_DECODE_H264, + feature_caps: 0, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "stop test".into(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + shared + .session + .lock() + .handle_control(peer, 0, hello, &Config, Instant::now()); + stop_pipeline(&shared, state); + assert!(!shared.running.load(Ordering::SeqCst)); + let goodbye = shared.session.lock().take_goodbye(); + let ended = !shared.client_connected(); + goodbye.map(|(to, bytes)| { + assert_eq!(to, peer); + assert!(ended, "a goodbye ends the session"); + match parse_control(&bytes) { + Ok((_, ControlMessage::Bye(reason))) => reason, + other => panic!("not a goodbye: {other:?}"), + } + }) + }; + + let mut machine = Machine::new(1); + assert_eq!( + machine.on_command(&SupervisorCommand::Stop), + Effect::StopPipeline + ); + assert_eq!(stop(&machine.state), Some(ByeReason::UserDisconnect)); + + let mut machine = Machine::new(1); + let t0 = Instant::now(); + let mut effect = Effect::None; + for _ in 0..=STORM_LIMIT { + effect = machine.on_stage_exit(1, Stage::Encoder, &failed("x"), t0); + if let SupervisorState::BackingOff { .. } = machine.state { + assert_eq!(stop(&machine.state), None, "a backoff keeps the session"); + machine.note_started(1); + } + } + assert_eq!(effect, Effect::StopPipeline); + assert!(matches!( + machine.state, + SupervisorState::Failed { + stage: Stage::Encoder, + .. + } + )); + assert_eq!( + stop_when_stopped(&machine.state), + Some(StreamStop::GaveUp(StageFailure::Encoder)) + ); + assert_eq!(stop(&machine.state), Some(ByeReason::Error)); + + let mut machine = Machine::new(1); + machine.wait_for_port(t0); + assert_eq!(stop(&machine.state), None, "a held port keeps the session"); + } + fn failed(reason: &str) -> StageOutcome { StageOutcome::Failed(reason.to_string()) } @@ -667,6 +1087,35 @@ mod tests { assert!(matches!(machine.state, SupervisorState::Running { .. })); } + /// A stream asks to start again on its own only while it runs: after + /// Stop, the storm brake or a newer stream, nothing starts. + #[test] + fn a_stream_restarts_itself_only_while_it_runs() { + let restart = |generation| SupervisorCommand::RestartStream { generation }; + let mut machine = Machine::new(5); + assert_eq!(machine.on_command(&restart(4)), Effect::None); + assert_eq!(machine.on_command(&restart(5)), Effect::Restart); + machine.note_started(6); + + assert_eq!( + machine.on_command(&SupervisorCommand::Stop), + Effect::StopPipeline + ); + assert_eq!(machine.on_command(&restart(6)), Effect::None); + assert_eq!(machine.state, SupervisorState::Stopped); + + machine.state = SupervisorState::Failed { + reason: "encoder".into(), + stage: Stage::Encoder, + }; + assert_eq!(machine.on_command(&restart(6)), Effect::None); + let t0 = Instant::now(); + machine.note_started(7); + machine.on_stage_exit(7, Stage::Capture, &failed("dxgi died"), t0); + assert_eq!(machine.on_command(&restart(7)), Effect::None); + assert!(matches!(machine.state, SupervisorState::BackingOff { .. })); + } + #[test] fn restart_storm_lands_in_failed_until_manual_start() { let mut machine = Machine::new(1); @@ -710,6 +1159,100 @@ mod tests { assert_eq!(machine.on_command(&SupervisorCommand::Start), Effect::Start); } + /// The PC sleeps for two hours mid-stream. Windows' clock counts the time + /// asleep, so at the supervisor's first look after waking, before capture + /// has run again, every heartbeat looks two hours old. The field logged + /// "capture loop silent for 184184ms" up to 12886316ms this way. + #[test] + fn a_pc_that_slept_is_not_a_wedged_pipeline() { + let shared = SharedControl::new(0, 1); + let mut machine = Machine::new(1); + let beat = |now_ms: u64| { + shared.hb_capture_loop_ms.store(now_ms, Ordering::Relaxed); + shared.hb_capture_frame_ms.store(now_ms, Ordering::Relaxed); + shared.hb_encode_frame_ms.store(now_ms, Ordering::Relaxed); + }; + let mut now_ms = 10_000; + beat(now_ms); + for _ in 0..8 { + now_ms += 250; + beat(now_ms); + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + } + // Asleep: no thread runs, the clock moves on two hours. + now_ms += 2 * 3600 * 1000; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + // The displays wake slowly: capture is stuck in a driver call for + // 6 s, then runs again. No restart. + let woke = now_ms; + while now_ms < woke + 6_000 { + now_ms += 250; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + } + while now_ms < woke + 12_000 { + now_ms += 250; + beat(now_ms); + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + } + + // A real wedge once awake is caught as before: 3 s after the capture + // loop stops. + let stuck = now_ms; + while now_ms < stuck + 3_000 { + now_ms += 250; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + } + now_ms += 250; + assert_eq!( + wedge_reason_at(&shared, &mut machine, now_ms), + Some("capture loop silent for 3250ms".into()) + ); + + // A capture loop that never runs again after waking is restarted + // when the grace ends. + let mut machine = Machine::new(2); + beat(now_ms); + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + now_ms += 250; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + now_ms += 3600 * 1000; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + let woke = now_ms; + while now_ms < woke + WAKE_GRACE_MS - 250 { + now_ms += 250; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + } + now_ms += 250; + assert_eq!( + wedge_reason_at(&shared, &mut machine, now_ms), + Some("capture loop silent for 10000ms".into()) + ); + } + + /// A restart joins the old generation for up to 5 s between two looks; + /// the new generation's watchdog starts from its own first look. + #[test] + fn a_restart_is_not_taken_for_a_sleep() { + let shared = SharedControl::new(0, 1); + let mut machine = Machine::new(1); + let mut now_ms = 10_000; + shared.hb_capture_loop_ms.store(now_ms, Ordering::Relaxed); + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + assert_eq!( + machine.on_wedged("capture loop silent", Instant::now()), + Effect::StopPipeline + ); + now_ms += 100; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + // The join took 5 s, then the new generation started. + now_ms += 5_000; + machine.note_started(2); + shared + .hb_capture_loop_ms + .store(now_ms - 3_500, Ordering::Relaxed); + assert!(wedge_reason_at(&shared, &mut machine, now_ms).is_some()); + } + #[test] fn wedge_detection_restarts() { let mut machine = Machine::new(1); diff --git a/host/src/telemetry/mod.rs b/host/src/telemetry/mod.rs index 7d4f3187..b086a621 100644 --- a/host/src/telemetry/mod.rs +++ b/host/src/telemetry/mod.rs @@ -1863,6 +1863,47 @@ mod tests { assert_eq!(event_with(&stamped, "info", &names).message, stamped); } + /// The Windows Firewall line uploads byte for byte, even when the PC or + /// user is named like one of its tokens; the same names elsewhere go. + #[test] + fn the_firewall_line_is_never_redacted() { + let names = redact::Names::new(Some(r"C:\Users\blocked"), Some("blocked"), Some("public")); + let profiles = |bit| crate::firewall::Profile { + bit, + on: true, + block_all_inbound: false, + }; + let exe = std::path::Path::new(r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"); + let report = crate::firewall::judge( + &[ + profiles(crate::firewall::PRIVATE), + profiles(crate::firewall::PUBLIC), + ], + &[ + crate::firewall::Rule::unscoped( + &exe.to_string_lossy().to_lowercase(), + false, + crate::firewall::PUBLIC, + ), + crate::firewall::Rule::unscoped("", false, crate::firewall::PRIVATE), + ], + exe, + 9876, + true, + |_| None, + ); + let stamped = format!("2026-11-06T02:04:08.196847Z WARN {}", report.summary()); + assert!( + stamped.contains("private:blocked_all_programs public:blocked") + && stamped.contains("all_programs_block_rules=1"), + "{stamped}" + ); + assert_eq!(event_with(&stamped, "warn", &names).message, stamped); + let named = "2026-11-06T02:04:08.196847Z INFO Signed in as blocked on public"; + let uploaded = event_with(named, "info", &names).message; + assert!(!uploaded.contains(" blocked on public"), "{uploaded}"); + } + /// Names that are also the summary's own tokens leave it byte for byte, /// while the same words elsewhere are names. /// A line is redacted as it is queued: a device name the registry lets diff --git a/host/src/telemetry/redact.rs b/host/src/telemetry/redact.rs index 92a17d50..683cfb45 100644 --- a/host/src/telemetry/redact.rs +++ b/host/src/telemetry/redact.rs @@ -124,6 +124,8 @@ const DIAGNOSTIC_KEYS: &[&str] = &[ "avoided", "attached", "advertised_formats", + "capture_size", + "encoded_size", "usb_state", "wire_stream", "running", @@ -136,6 +138,7 @@ const DIAGNOSTIC_KEYS: &[&str] = &[ "ss_rule", // A silent client's session held, kept or ended. "silent_ms", + "stalled_ms", "held_ms", "last_heard", "heard", @@ -164,6 +167,16 @@ const DIAGNOSTIC_KEYS: &[&str] = &[ "choice", // The startup network line. "route", + "lan_adapters", + // The Windows Firewall line. + "profiles", + "allow_rules", + "block_rules", + "all_programs_block_rules", + "scoped_block_rules", + "bypass_rules", + "lan", + "verdict", // The virtual display snapshot. "driver", "tech", @@ -1308,6 +1321,7 @@ mod tests { .with_ansi(false) .with_writer(move || Captured(writer.clone())), ); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, log); let line = String::from_utf8(out.lock().clone()).unwrap(); line.trim_end().to_owned() @@ -1940,6 +1954,11 @@ mod tests { "modes", "INFO Second screen mode mode=2732x2048 requested=the driver's modes", ), + ( + "1680x945", + "INFO Encoding without the capture's odd last column or row \ + capture_size=1680x945 encoded_size=1680x944", + ), ("arm64", "INFO EternalMonitor host 0.3.4 revision=x | OS=Windows arch=arm64 logical_cpus=8"), ( "Client", @@ -1977,6 +1996,10 @@ mod tests { "WARN Client liveness expired — tearing session down peer=192.0.2.7:50000 silent_ms=10000 \ held_ms=7000 last_heard=input late_input=0 device=iPad", ), + ( + "2750", + "INFO The stream's transport loop moved again phase=input stalled_ms=2750", + ), ]; for (name, message) in cases { let line = format!("{STAMP} {message}"); diff --git a/host/src/transport/abr.rs b/host/src/transport/abr.rs index 5735cb48..83a8d6b3 100644 --- a/host/src/transport/abr.rs +++ b/host/src/transport/abr.rs @@ -16,6 +16,16 @@ const LADDER_BPS: [u32; 11] = [ const DOWN_COOLDOWN: Duration = Duration::from_secs(3); const UP_COOLDOWN: Duration = Duration::from_secs(15); +/// The wait after any change before a step up, while the encoder takes a +/// new bitrate only by reopening (AMF, libx265): each step there costs a +/// stutter and an IDR. On a link just below the next rung, a step up is +/// undone by a step down within seconds, so a probe that fails reopens the +/// encoder twice. A probe undone within [`PROBE_HOLDS`] doubles the wait, +/// up to [`REOPEN_UP_COOLDOWN_MAX`]; one that holds that long resets it. +const REOPEN_UP_COOLDOWN: Duration = Duration::from_secs(30); +const REOPEN_UP_COOLDOWN_MAX: Duration = Duration::from_secs(120); +/// A step up held this long, with no step down, worked. +const PROBE_HOLDS: Duration = Duration::from_secs(30); /// This long continuously clean before stepping up. const CLEAN_WINDOW: Duration = Duration::from_secs(15); /// Freeze the controller when reports stop (liveness machinery owns that). @@ -50,6 +60,11 @@ pub struct AbrController { /// Whether the tunnel dropped frames in each of the latest reports, /// newest in bit 0. tunnel_drop_history: u8, + /// The encoder takes a new bitrate only by reopening (see + /// [`Self::encoder_reopens`]). + encoder_reopens: bool, + /// The wait before a step up while it does. + reopen_up_cooldown: Duration, } impl AbrController { @@ -73,6 +88,50 @@ impl AbrController { prev_cumulative: None, prev_tunnel_drops: None, tunnel_drop_history: 0, + encoder_reopens: false, + reopen_up_cooldown: REOPEN_UP_COOLDOWN, + } + } + + /// Whether the open encoder session takes a new bitrate only by + /// reopening it, as the transport reads it before each report. Then a + /// step up waits [`REOPEN_UP_COOLDOWN`] or more after any change instead + /// of [`UP_COOLDOWN`]. Steps down are not held: they relieve a link that + /// is losing frames. + pub fn encoder_reopens(&mut self, reopens: bool) { + self.encoder_reopens = reopens; + } + + /// The wait after a change before a step up. + fn up_cooldown(&self) -> Duration { + if self.encoder_reopens { + self.reopen_up_cooldown + } else { + UP_COOLDOWN + } + } + + /// A step down at `now`. One soon after a step up undoes that probe: + /// the next probe waits twice as long, while the encoder reopens. + fn note_down(&mut self, now: Instant) { + let undoes_probe = matches!( + self.last_change, + Some((when, false)) if now.duration_since(when) < PROBE_HOLDS + ); + if undoes_probe && self.encoder_reopens { + self.reopen_up_cooldown = (self.reopen_up_cooldown * 2).min(REOPEN_UP_COOLDOWN_MAX); + } + self.last_change = Some((now, true)); + } + + /// A step up that has held for [`PROBE_HOLDS`] worked: the next waits + /// the shortest time again. + fn note_probe_held(&mut self, now: Instant) { + if matches!( + self.last_change, + Some((when, false)) if now.duration_since(when) >= PROBE_HOLDS + ) { + self.reopen_up_cooldown = REOPEN_UP_COOLDOWN; } } @@ -123,6 +182,7 @@ impl AbrController { if !self.enabled { return unchanged; } + self.note_probe_held(now); // Interval deltas from cumulative counters. let (delta_received, delta_lost, delta_repaired) = match self.prev_cumulative { @@ -200,7 +260,7 @@ impl AbrController { "ABR stepping bitrate down" ); self.current_bps = target; - self.last_change = Some((now, true)); + self.note_down(now); return AbrDecision { target_bps: target, changed: true, @@ -220,7 +280,7 @@ impl AbrController { { let clean_since = *self.clean_since.get_or_insert(now); let up_cooldown_over = match self.last_change { - Some((when, _)) => now.duration_since(when) >= UP_COOLDOWN, + Some((when, _)) => now.duration_since(when) >= self.up_cooldown(), None => true, }; if now.duration_since(clean_since) >= CLEAN_WINDOW && up_cooldown_over { @@ -246,6 +306,44 @@ impl AbrController { unchanged } + + /// The system had no buffer space for a datagram: one rung down at + /// once, without waiting for the iPad to report the loss. Like any + /// downgrade it waits out the cooldown after the last one, and starts + /// it, so refusals and loss reports together step at most once per + /// [`DOWN_COOLDOWN`]. + pub fn on_send_refused(&mut self, now: Instant) -> AbrDecision { + let unchanged = AbrDecision { + target_bps: self.current_bps, + changed: false, + }; + if !self.enabled { + return unchanged; + } + self.clean_since = None; + if matches!( + self.last_change, + Some((when, true)) if now.duration_since(when) < DOWN_COOLDOWN + ) { + return unchanged; + } + let target = step(self.current_bps, -1, self.ceiling_bps); + if target == self.current_bps { + return unchanged; + } + info!( + from = self.current_bps, + to = target, + reason = "no_buffer_space", + "ABR stepping bitrate down" + ); + self.current_bps = target; + self.note_down(now); + AbrDecision { + target_bps: target, + changed: true, + } + } } /// Nearest ladder rung at or below `bps`, capped by `ceiling`. @@ -580,4 +678,272 @@ mod tests { assert!(stepped_up); assert_eq!(abr.current_bps(), 15_000_000); } + + /// A refused send steps one rung down at once, and the loss reports of + /// the datagrams it lost wait out the cooldown instead of stepping again. + #[test] + fn a_refused_send_steps_down_at_once_and_reports_wait_the_cooldown() { + let mut abr = AbrController::new(20_000_000, true); + let t0 = Instant::now(); + let d = abr.on_send_refused(t0); + assert_eq!((d.target_bps, d.changed), (12_000_000, true)); + let d = abr.on_report(&report(950, 50, 0), None, t0 + Duration::from_secs(1)); + assert!(!d.changed, "the same loss, inside the cooldown"); + assert_eq!(abr.current_bps(), 12_000_000); + + let d = abr.on_send_refused(t0 + Duration::from_secs(2)); + assert!(!d.changed, "a refusal waits out the cooldown too"); + assert_eq!(abr.current_bps(), 12_000_000); + + let mut floor = AbrController::new(4_000_000, true); + assert!(!floor.on_send_refused(t0).changed); + let mut disabled = AbrController::new(20_000_000, false); + assert!(!disabled.on_send_refused(t0).changed); + assert_eq!(disabled.current_bps(), 20_000_000); + } + + /// Episodes of refused sends a little over a second apart each ask for + /// a step, but steps come at most once per cooldown: not 15 Mbps down to + /// the floor in five seconds. + #[test] + fn refusal_episodes_step_at_most_once_per_cooldown() { + let mut abr = AbrController::new(20_000_000, true); + let t0 = Instant::now(); + let mut steps = Vec::new(); + for n in 0..5u64 { + let at = Duration::from_millis(1_100 * n); + if abr.on_send_refused(t0 + at).changed { + steps.push((at.as_millis(), abr.current_bps())); + } + } + assert_eq!(steps, [(0, 12_000_000), (3_300, 10_000_000)]); + } + + /// A receiver report that just stepped down starts the cooldown: a + /// refusal right after it doesn't step again for the same congestion. + #[test] + fn a_refusal_right_after_a_report_downgrade_waits_the_cooldown() { + let mut abr = AbrController::new(20_000_000, true); + let t0 = Instant::now(); + let d = abr.on_report(&report(950, 50, 0), None, t0); + assert_eq!((d.target_bps, d.changed), (12_000_000, true)); + assert!(!abr.on_send_refused(t0 + Duration::from_millis(500)).changed); + assert!( + !abr.on_send_refused(t0 + Duration::from_millis(2_900)) + .changed + ); + let d = abr.on_send_refused(t0 + Duration::from_millis(3_100)); + assert_eq!((d.target_bps, d.changed), (10_000_000, true)); + } + + /// Feeds one clean report a second from `from` to `to` (in s from + /// `t0`), with counters continuing from `counter`. The changes, as + /// (second, target). + fn clean_reports( + abr: &mut AbrController, + t0: Instant, + counter: &mut u32, + seconds: std::ops::Range, + ) -> Vec<(u64, u32)> { + let mut changes = Vec::new(); + for second in seconds { + *counter += 1_000; + let d = abr.on_report( + &report(*counter, 0, 0), + None, + t0 + Duration::from_secs(second), + ); + if d.changed { + changes.push((second, d.target_bps)); + } + } + changes + } + + #[test] + fn a_step_up_waits_30_s_after_a_change_while_the_encoder_reopens() { + // In place: 15 s clean from the first clean report, at 1 s; then a + // rung per 15 s. Reopening: 30 s after the change, then a rung per + // 30 s. + for (reopens, first_up, wait) in [(false, 16, 15), (true, 30, 30)] { + let mut abr = AbrController::new(50_000_000, true); + abr.encoder_reopens(reopens); + let t0 = Instant::now(); + // Loss steps 15 -> 12 Mbps at 0 s; clean from then on. + assert!(abr.on_report(&report(980, 30, 0), None, t0).changed); + let mut counter = 980; + let changes = clean_reports(&mut abr, t0, &mut counter, 1..80); + assert_eq!(changes[0], (first_up, 15_000_000), "reopens={reopens}"); + // Then one rung per wait, the clean window being shorter. + let gaps: Vec = changes + .windows(2) + .map(|pair| pair[1].0 - pair[0].0) + .collect(); + assert!( + gaps.iter().all(|&gap| gap == wait), + "reopens={reopens}: {changes:?}" + ); + } + } + + #[test] + fn a_probe_undone_soon_doubles_the_wait_and_one_that_holds_resets_it() { + let mut abr = AbrController::new(50_000_000, true); + abr.encoder_reopens(true); + let t0 = Instant::now(); + let at = |s: u64| t0 + Duration::from_secs(s); + let mut counter = 0; + // Steps down at 0 s, then probes up once the wait is over, and each + // probe is lost 4 s later, on a link that cannot carry it. + assert!(abr.on_report(&report(980, 30, 0), None, at(0)).changed); + let mut downs = vec![0u64]; + let mut ups = Vec::new(); + let mut second = 1; + while second < 600 { + let since_down = second - downs.last().unwrap(); + let lossy = ups + .last() + .is_some_and(|&up: &u64| up > *downs.last().unwrap() && second - up == 4); + counter += 1_000; + let lost = if lossy { 50 } else { 0 }; + let d = abr.on_report(&report(counter, lost, 0), None, at(second)); + if d.changed { + if lossy { + downs.push(second); + } else { + ups.push(second); + } + } + let _ = since_down; + second += 1; + } + let waits: Vec = ups + .iter() + .map(|up| up - downs.iter().rev().find(|down| *down < up).unwrap()) + .collect(); + assert_eq!(waits, [30, 60, 120, 120, 120, 120], "{ups:?} {downs:?}"); + // A probe that holds 30 s: the next one waits 30 s again. + let mut abr = AbrController::new(50_000_000, true); + abr.encoder_reopens(true); + assert!(abr.on_report(&report(980, 30, 0), None, at(0)).changed); + let mut counter = 980; + let ups = clean_reports(&mut abr, t0, &mut counter, 1..31); + assert_eq!(ups, [(30, 15_000_000)]); + assert!( + abr.on_report(&report(counter + 900, 60, 0), None, at(32)) + .changed + ); + counter += 900; + // Undone at 32 s: the wait doubles to 60 s. + assert_eq!(clean_reports(&mut abr, t0, &mut counter, 33..95)[0].0, 92); + // That probe holds; after 30 s the wait is 30 s again. + let ups = clean_reports(&mut abr, t0, &mut counter, 95..160); + assert_eq!(ups, [(122, 20_000_000), (152, 25_000_000)]); + } + + #[test] + fn steps_down_and_in_place_encoders_are_not_held() { + // A reopening encoder steps down as before: at once, then 3 s apart. + let mut abr = AbrController::new(50_000_000, true); + abr.encoder_reopens(true); + let t0 = Instant::now(); + let at = |ms: u64| t0 + Duration::from_millis(ms); + let (mut counter, mut lost) = (0, 0); + let mut downs = Vec::new(); + for ms in (0..10_000).step_by(1_000) { + counter += 1_000; + lost += 50; + if abr + .on_report(&report(counter, lost, 0), None, at(ms)) + .changed + { + downs.push((ms, abr.current_bps())); + } + } + assert_eq!( + downs, + [ + (0, 12_000_000), + (3_000, 10_000_000), + (6_000, 8_000_000), + (9_000, 6_000_000) + ] + ); + // A refused send too. + let d = abr.on_send_refused(at(12_000)); + assert_eq!((d.target_bps, d.changed), (4_000_000, true)); + // An encoder that takes the bitrate in place keeps the 15 s wait: + // 15 s clean from the first clean report. + abr.encoder_reopens(false); + let ups = clean_reports(&mut abr, at(12_000), &mut counter, 1..17); + assert_eq!(ups, [(16, 6_000_000)]); + } + + /// A link that carries `capacity` bps: each second, the fragments over + /// it are lost. `capacity_at` gives it by second. The steps, by second. + fn on_a_link(reopens: bool, seconds: u64, capacity_at: impl Fn(u64) -> u32) -> Vec<(u64, u32)> { + let mut abr = AbrController::new(50_000_000, true); + abr.encoder_reopens(reopens); + let t0 = Instant::now(); + let (mut received, mut lost) = (0u32, 0u32); + let mut steps = Vec::new(); + for second in 0..seconds { + let rate = abr.current_bps(); + let capacity = capacity_at(second); + // A fragment a kilobit: rate in kbit per second. + let sent = rate / 1_000; + let over = rate.saturating_sub(capacity) / 1_000; + received += sent - over; + lost += over; + let d = abr.on_report( + &report(received, lost, 0), + None, + t0 + Duration::from_secs(second), + ); + if d.changed { + steps.push((second, d.target_bps)); + } + } + steps + } + + #[test] + fn a_link_just_below_the_next_rung_probes_less_often_while_the_encoder_reopens() { + // 11 Mbps: 10 Mbps holds, every probe to 12 Mbps is lost. + let minutes = 30; + let steady = on_a_link(false, minutes * 60, |_| 11_000_000); + let reopening = on_a_link(true, minutes * 60, |_| 11_000_000); + let per_minute = |steps: &[(u64, u32)]| steps.len() as f64 / minutes as f64; + eprintln!( + "steps a minute at 11 Mbps: {:.2} in place, {:.2} reopening", + per_minute(&steady), + per_minute(&reopening) + ); + // In place: a probe and its undoing every 15 s plus a few. + assert!(per_minute(&steady) >= 6.0, "{steady:?}"); + // Reopening: about one probe and its undoing every 2 minutes once + // the wait reaches its cap. + assert!(per_minute(&reopening) <= 1.2, "{reopening:?}"); + // Both settle on 10 Mbps between probes and never stay above the + // link. + for steps in [&steady, &reopening] { + assert!(steps.iter().all(|&(_, bps)| bps <= 12_000_000), "{steps:?}"); + } + // Capacity that comes back: the reopening encoder still climbs, a + // rung per 30 s. + let recovered = on_a_link( + true, + 400, + |second| if second < 60 { 7_000_000 } else { 60_000_000 }, + ); + let after: Vec<_> = recovered + .iter() + .filter(|(second, _)| *second >= 60) + .collect(); + assert_eq!( + after.last().map(|(_, bps)| *bps), + Some(50_000_000), + "{recovered:?}" + ); + } } diff --git a/host/src/transport/audio.rs b/host/src/transport/audio.rs index f765f853..60fe82bd 100644 --- a/host/src/transport/audio.rs +++ b/host/src/transport/audio.rs @@ -123,19 +123,40 @@ impl AudioSender { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use eternal_wire::v2::control::*; use std::sync::atomic::Ordering; use std::time::Instant; - fn session(caps: u16) -> (SharedControl, u32) { + /// A session whose iPad advertised `caps` and answered, as the app does + /// at once, and its id. + pub(crate) fn session(caps: u16) -> (SharedControl, u32) { + let (shared, id) = unanswered_session(caps); + let config = super::super::SharedConfigSource { + shared: &shared, + stream_epoch: 7, + pen_available: false, + listen: "127.0.0.1:0".parse().unwrap(), + }; + shared.session.lock().handle_control( + PeerId::udp("127.0.0.1:19891".parse().unwrap()), + id, + ControlMessage::Ping(Ping { t1_us: 1 }), + &config, + Instant::now(), + ); + (shared, id) + } + + fn unanswered_session(caps: u16) -> (SharedControl, u32) { let shared = SharedControl::new(9876, 15_000_000); shared.pairing.lock().required = false; let config = super::super::SharedConfigSource { shared: &shared, stream_epoch: 7, pen_available: false, + listen: "127.0.0.1:0".parse().unwrap(), }; let peer = PeerId::udp("127.0.0.1:19891".parse().unwrap()); shared.session.lock().handle_control( @@ -166,8 +187,7 @@ mod tests { (shared, id) } - /// Tests that replace the global audio stats run one at a time. - static AUDIO_STATS: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); + use crate::audio::STATS_TEST_LOCK as AUDIO_STATS; fn packet(id: u32, time: u64) -> SessionPacket { SessionPacket { @@ -189,6 +209,11 @@ mod tests { let (legacy, id) = session(0); assert!(crate::audio::wanted_session(&legacy).is_none()); assert!(sender.prepare(packet(id, 100), &legacy, 7, 100).is_none()); + let (unanswered, id) = unanswered_session(FEATURE_WANTS_AUDIO); + assert!(crate::audio::wanted_session(&unanswered).is_none()); + assert!(sender + .prepare(packet(id, 100), &unanswered, 7, 100) + .is_none()); let (shared, id) = session(FEATURE_WANTS_AUDIO); assert_eq!(crate::audio::wanted_session(&shared), Some(id)); shared.audio_enabled.store(false, Ordering::SeqCst); diff --git a/host/src/transport/beat.rs b/host/src/transport/beat.rs new file mode 100644 index 00000000..3fec62a8 --- /dev/null +++ b/host/src/transport/beat.rs @@ -0,0 +1,280 @@ +//! The transport loop's heartbeat, and the watch that notices when it stops. +//! +//! The loop wakes at least every 250 ms (its liveness tick). A loop stuck in +//! one call answers nobody: its iPad loses the heartbeats and gives up, the +//! session is never ended for silence, and new HELLOs wait unread. That +//! happened in the field for close to a minute while Windows switched the +//! virtual display, and once, perhaps, for 18 minutes, with nothing in the +//! log to say where. The loop now marks each step it takes with a [`Phase`], +//! and the supervisor's thread logs one line when the mark goes stale, and +//! one when it moves again. + +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering::Relaxed}; +use std::time::Duration; + +use tracing::{info, warn}; + +/// Where the transport loop was when it last marked its heartbeat. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum Phase { + /// Waiting for its next timer, datagram, frame or audio packet. + Waiting = 0, + /// A heartbeat or liveness tick. + Tick, + /// Handling a control datagram from an iPad, or a USB tunnel event. + Control, + /// Queueing the iPad's touch, pen, mouse or keyboard input for the input + /// thread, which hands it to Windows. + Input, + /// Sending what the session decided: replies, goodbyes, summaries. + Actions, + /// The second screen's next step. + SecondScreen, + /// Sending a video frame, with its pacing waits and the repairs served + /// during them. + Video, + /// Sending PC audio. + Audio, + /// Sending the pointer to an iPad that draws it. + Pointer, +} + +impl Phase { + const ALL: [Phase; 9] = [ + Phase::Waiting, + Phase::Tick, + Phase::Control, + Phase::Input, + Phase::Actions, + Phase::SecondScreen, + Phase::Video, + Phase::Audio, + Phase::Pointer, + ]; + + /// Stable log value. + pub const fn name(self) -> &'static str { + match self { + Phase::Waiting => "waiting", + Phase::Tick => "tick", + Phase::Control => "control", + Phase::Input => "input", + Phase::Actions => "actions", + Phase::SecondScreen => "second_screen", + Phase::Video => "video", + Phase::Audio => "audio", + Phase::Pointer => "pointer", + } + } +} + +/// The last mark: when (ms on the process clock, 0 while no transport runs) +/// and in which phase. +#[derive(Debug, Default)] +pub struct Beat { + at_ms: AtomicU64, + phase: AtomicU8, +} + +impl Beat { + pub fn mark(&self, phase: Phase) { + self.phase.store(phase as u8, Relaxed); + self.at_ms + .store((crate::clock::host_now_us() / 1000).max(1), Relaxed); + } + + /// The transport loop ended. + pub fn stop(&self) { + self.at_ms.store(0, Relaxed); + } + + /// The last mark, unless no transport runs. + pub fn read(&self) -> Option<(u64, Phase)> { + let at = self.at_ms.load(Relaxed); + let phase = Phase::ALL + .get(usize::from(self.phase.load(Relaxed))) + .copied() + .unwrap_or(Phase::Waiting); + (at != 0).then_some((at, phase)) + } +} + +/// A mark this old means the loop is stuck. An iPad gives up 3 s after the +/// last heartbeat it got, plus 5 s. +pub const STALL_AFTER: Duration = Duration::from_secs(2); + +/// A gap this long between two checks means the watch itself did not run: +/// the PC slept, or the whole process was paused. The loop did not run +/// either, through no fault of its own, so the gap is not counted. +const PAUSED_AFTER: Duration = Duration::from_secs(1); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// The loop has not marked for `silent_ms`, stuck in `phase`. + Stalled { phase: Phase, silent_ms: u64 }, + /// The loop marked again, or ended, after it was stuck for `stalled_ms`. + Moving { phase: Phase, stalled_ms: u64 }, +} + +/// Judges the heartbeat at each check. Pure: the caller passes the clock. +#[derive(Debug, Default)] +pub struct Watch { + checked_ms: u64, + /// Counting starts again here after a pause. + resumed_ms: u64, + /// The stale mark the last stall was reported for. + stalled: Option<(u64, Phase)>, +} + +impl Watch { + pub fn check(&mut self, beat: Option<(u64, Phase)>, now_ms: u64) -> Option { + if self.checked_ms != 0 + && now_ms.saturating_sub(self.checked_ms) >= PAUSED_AFTER.as_millis() as u64 + { + self.resumed_ms = now_ms; + } + self.checked_ms = now_ms; + if let Some((at, phase)) = self.stalled { + if beat.is_some_and(|(now_at, _)| now_at == at) { + return None; + } + self.stalled = None; + let moved = beat.map_or(now_ms, |(now_at, _)| now_at); + return Some(Verdict::Moving { + phase, + stalled_ms: moved.saturating_sub(at), + }); + } + let (at, phase) = beat?; + let silent_since = at.max(self.resumed_ms); + if now_ms.saturating_sub(silent_since) < STALL_AFTER.as_millis() as u64 { + return None; + } + self.stalled = Some((at, phase)); + Some(Verdict::Stalled { + phase, + silent_ms: now_ms.saturating_sub(at), + }) + } + + /// [`Self::check`], logged. + pub fn check_and_log(&mut self, beat: &Beat, now_ms: u64) { + match self.check(beat.read(), now_ms) { + Some(Verdict::Stalled { phase, silent_ms }) => warn!( + stall = "transport_loop", + phase = phase.name(), + silent_ms, + "The stream's transport loop stopped: iPads get no heartbeats or answers until it moves" + ), + Some(Verdict::Moving { phase, stalled_ms }) => info!( + phase = phase.name(), + stalled_ms, + "The stream's transport loop moved again" + ), + None => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_loop_that_keeps_marking_is_never_stalled() { + let mut watch = Watch::default(); + for step in 0..100u64 { + let now = 10_000 + step * 250; + assert_eq!(watch.check(Some((now - 200, Phase::Waiting)), now), None); + } + // Nothing runs, nothing to judge. + assert_eq!(watch.check(None, 40_000), None); + } + + #[test] + fn a_stuck_loop_is_reported_once_with_its_phase_then_when_it_moves() { + let mut watch = Watch::default(); + let stuck = Some((10_000, Phase::Input)); + let mut verdicts = Vec::new(); + for now in (10_000..=20_000).step_by(250) { + verdicts.extend(watch.check(stuck, now)); + } + assert_eq!( + verdicts, + [Verdict::Stalled { + phase: Phase::Input, + silent_ms: 2_000 + }] + ); + assert_eq!( + watch.check(Some((20_100, Phase::Actions)), 20_250), + Some(Verdict::Moving { + phase: Phase::Input, + stalled_ms: 10_100 + }) + ); + assert_eq!(watch.check(Some((20_300, Phase::Waiting)), 20_500), None); + } + + #[test] + fn a_stuck_loop_that_ends_is_reported_as_moving() { + let mut watch = Watch::default(); + for now in (5_000..=8_000).step_by(250) { + watch.check(Some((5_000, Phase::SecondScreen)), now); + } + assert_eq!( + watch.check(None, 8_250), + Some(Verdict::Moving { + phase: Phase::SecondScreen, + stalled_ms: 3_250 + }) + ); + } + + /// The PC slept: the loop's mark is minutes old when the watch runs + /// again, but the loop was never given a chance to run. + #[test] + fn a_sleeping_pc_is_not_a_stall() { + let mut watch = Watch::default(); + assert_eq!(watch.check(Some((9_900, Phase::Waiting)), 10_000), None); + let woke = 10_000 + 600_000; + assert_eq!(watch.check(Some((9_900, Phase::Waiting)), woke), None); + // The loop marks again within its tick after the wake. + assert_eq!( + watch.check(Some((woke + 100, Phase::Tick)), woke + 250), + None + ); + // A loop that stays stuck after the wake is still caught. + let mut stuck = Watch::default(); + stuck.check(Some((9_900, Phase::Video)), 10_000); + let mut verdicts = Vec::new(); + for now in (woke..woke + 3_000).step_by(250) { + verdicts.extend(stuck.check(Some((9_900, Phase::Video)), now)); + } + assert_eq!( + verdicts, + [Verdict::Stalled { + phase: Phase::Video, + silent_ms: woke + 2_000 - 9_900 + }] + ); + } + + #[test] + fn the_heartbeat_reads_back_what_was_marked() { + crate::clock::init(); + let beat = Beat::default(); + assert_eq!(beat.read(), None); + beat.mark(Phase::Audio); + let (at, phase) = beat.read().unwrap(); + assert!(at > 0); + assert_eq!(phase, Phase::Audio); + for phase in Phase::ALL { + beat.mark(phase); + assert_eq!(beat.read().unwrap().1, phase); + } + beat.stop(); + assert_eq!(beat.read(), None); + } +} diff --git a/host/src/transport/congestion.rs b/host/src/transport/congestion.rs new file mode 100644 index 00000000..72bcbdc3 --- /dev/null +++ b/host/src/transport/congestion.rs @@ -0,0 +1,200 @@ +//! Windows refuses a UDP send with WSAENOBUFS (10055) when it has no buffer +//! space for the datagram: a queue in the network adapter, its driver or a +//! filter driver is full. macOS and Linux say ENOBUFS. The socket's own send +//! buffer (4 MiB) is not the one that ran out. In the field each refused send +//! logged its own line, and the stream lowered its bitrate only after the +//! iPad reported the losses. A refused send now counts as congestion: video +//! backs off for a moment, the bitrate steps down at once, and the log says +//! one line when an episode starts and one when it ends. + +use std::io; +use std::time::{Duration, Instant}; + +use tracing::{info, warn}; + +/// How long a frame's sending pauses after the system refused a datagram, +/// before that datagram is sent once more. A frame pauses once at most. +pub const BACKOFF: Duration = Duration::from_millis(5); + +/// How long a refused control reply waits before its one retry. +pub const REPLY_RETRY: Duration = Duration::from_millis(2); + +/// An episode ends once sends go this long without a refusal. +pub const EPISODE_QUIET: Duration = Duration::from_secs(1); + +#[cfg(windows)] +const NO_BUFFER_SPACE: i32 = 10055; // WSAENOBUFS +#[cfg(not(windows))] +const NO_BUFFER_SPACE: i32 = libc::ENOBUFS; + +/// Whether the system refused a send for lack of buffer space. +pub fn is_no_buffer_space(error: &io::Error) -> bool { + error.raw_os_error() == Some(NO_BUFFER_SPACE) +} + +/// The error a send the system had no buffer space for returns. +pub fn no_buffer_space() -> io::Error { + io::Error::from_raw_os_error(NO_BUFFER_SPACE) +} + +/// Refused sends from the first until sends go [`EPISODE_QUIET`] without one. +struct Episode { + started: Instant, + last: Instant, + refused: u64, +} + +/// Every refused UDP send of one transport, by episode. +#[derive(Default)] +pub struct SendCongestion { + episode: Option, + step_down: bool, +} + +impl SendCongestion { + /// Notes a send refused for lack of buffer space. The first of an + /// episode is logged, and asks for one step down. + pub fn refused(&mut self, error: &io::Error, now: Instant) { + self.settle(now); + match &mut self.episode { + Some(episode) => { + episode.refused += 1; + episode.last = now; + } + None => { + warn!(link = %"wifi", error = %error, "UDP send failed: no buffer space, backing off"); + self.episode = Some(Episode { + started: now, + last: now, + refused: 1, + }); + self.step_down = true; + } + } + } + + /// Whether an episode started since the last call, once per episode. + pub fn take_step_down(&mut self) -> bool { + std::mem::take(&mut self.step_down) + } + + /// Ends an episode with no refusal for [`EPISODE_QUIET`], logging how + /// many sends it refused and how long it lasted. + pub fn settle(&mut self, now: Instant) { + if self + .episode + .as_ref() + .is_some_and(|episode| now.saturating_duration_since(episode.last) >= EPISODE_QUIET) + { + self.end(); + } + } + + fn end(&mut self) { + if let Some(episode) = self.episode.take() { + info!( + link = %"wifi", + refused = episode.refused, + took_ms = episode.last.duration_since(episode.started).as_millis() as u64, + "UDP sends have buffer space again" + ); + } + } +} + +/// A transport that stops during an episode still says how it went. +impl Drop for SendCongestion { + fn drop(&mut self) { + self.end(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn logged(run: impl FnOnce()) -> Vec { + let out = std::sync::Arc::new(parking_lot::Mutex::new(Vec::::new())); + struct Sink(std::sync::Arc>>); + impl io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.lock().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let writer = std::sync::Arc::clone(&out); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_target(false) + .with_writer(move || Sink(std::sync::Arc::clone(&writer))) + .finish(); + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, run); + let text = String::from_utf8(out.lock().clone()).unwrap(); + text.lines().map(|line| line.trim().to_owned()).collect() + } + + #[test] + fn only_no_buffer_space_counts_as_congestion() { + assert!(is_no_buffer_space(&no_buffer_space())); + #[cfg(windows)] + assert!(is_no_buffer_space(&io::Error::from_raw_os_error(10055))); + #[cfg(target_os = "macos")] + assert!(is_no_buffer_space(&io::Error::from_raw_os_error(55))); + #[cfg(target_os = "linux")] + assert!(is_no_buffer_space(&io::Error::from_raw_os_error(105))); + for other in [ + io::Error::from(io::ErrorKind::ConnectionReset), + io::Error::from(io::ErrorKind::WouldBlock), + io::Error::from_raw_os_error(10054), + ] { + assert!(!is_no_buffer_space(&other), "{other}"); + } + } + + /// Hundreds of refused sends close together are one episode: one line + /// when it starts, one step down, and one summary once it has been quiet. + #[test] + fn an_episode_logs_one_line_and_its_end() { + let start = Instant::now(); + let mut congestion = SendCongestion::default(); + let lines = logged(|| { + for n in 0..300u32 { + congestion.refused(&no_buffer_space(), start + Duration::from_millis(n.into())); + congestion.settle(start + Duration::from_millis(n.into())); + } + assert!(congestion.take_step_down()); + assert!(!congestion.take_step_down(), "one step per episode"); + congestion.settle(start + Duration::from_millis(299) + EPISODE_QUIET / 2); + congestion.settle(start + Duration::from_millis(299) + EPISODE_QUIET); + }); + assert_eq!(lines.len(), 2, "{lines:#?}"); + assert!( + lines[0] + .starts_with("WARN UDP send failed: no buffer space, backing off link=wifi error="), + "{lines:#?}" + ); + assert_eq!( + lines[1], + "INFO UDP sends have buffer space again link=wifi refused=300 took_ms=299" + ); + + // A refusal after the quiet starts a new episode, with its own step. + let later = start + Duration::from_secs(5); + let lines = logged(|| { + congestion.refused(&no_buffer_space(), later); + assert!(congestion.take_step_down()); + drop(congestion); + }); + assert_eq!(lines.len(), 2, "{lines:#?}"); + assert!(lines[0].starts_with("WARN UDP send failed: no buffer space")); + assert_eq!( + lines[1], "INFO UDP sends have buffer space again link=wifi refused=1 took_ms=0", + "a transport that stops ends its episode" + ); + } +} diff --git a/host/src/transport/link.rs b/host/src/transport/link.rs index 2d3e73d8..29520567 100644 --- a/host/src/transport/link.rs +++ b/host/src/transport/link.rs @@ -103,6 +103,10 @@ impl Link for UdpLink { return Err(invalid("UDP destination")); }; check_length(datagram.len())?; + #[cfg(feature = "test-hooks")] + if let Some(error) = super::probe::fail(addr, datagram) { + return Err(error); + } self.socket.send_to(datagram, addr).await?; #[cfg(feature = "test-hooks")] super::probe::sent(addr); @@ -398,10 +402,66 @@ pub fn tunnel_congestion_drops() -> u64 { TUNNEL_CONGESTION.load(Ordering::Relaxed) } +/// What this host saw end a tunnel first. Neither side used to say: a +/// tunnel that closed on both sides at once looked the same whoever closed +/// it. It is the first end seen here, not necessarily what started it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CloseCause { + /// The other end (the iPad, or Apple's device service) ended the stream. + PeerEof, + /// Reading failed, e.g. a reset. + ReadFailed(io::ErrorKind), + /// The other end sent a length no datagram has. + BadLength, + /// Writing failed. + WriteFailed(io::ErrorKind), + /// This host let go of it: its device worker stopped, or the link was + /// dropped. + Host, +} + +impl CloseCause { + pub fn as_str(self) -> &'static str { + match self { + Self::PeerEof => "peer_eof", + Self::ReadFailed(io::ErrorKind::ConnectionReset) => "peer_reset", + Self::ReadFailed(_) => "read_error", + Self::BadLength => "bad_length", + Self::WriteFailed(_) => "write_error", + Self::Host => "host", + } + } + + fn of_read(error: &io::Error) -> Self { + match error.kind() { + io::ErrorKind::UnexpectedEof => Self::PeerEof, + io::ErrorKind::InvalidData => Self::BadLength, + kind => Self::ReadFailed(kind), + } + } +} + +/// How a closed tunnel ended, for its close line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CloseReport { + pub cause: CloseCause, + /// From the preamble to the close. + pub open: Duration, + /// From the last datagram read (or the preamble) to the close. + pub quiet: Duration, +} + +struct Lifetime { + opened: Instant, + last_read: Instant, + closed: Option<(CloseCause, Instant)>, +} + struct WriterState { queue: Mutex, ready: Notify, closed: watch::Sender, + lifetime: Mutex, /// Control datagrams fully written to the stream, in queue order. controls_written: watch::Sender, force_idr: Arc, @@ -422,6 +482,15 @@ impl WriterState { audio.map(|audio| (audio, false)) } + /// Closes the tunnel. Only the first cause counts. + fn close(&self, cause: CloseCause) { + self.lifetime + .lock() + .closed + .get_or_insert((cause, Instant::now())); + let _ = self.closed.send(true); + } + fn wrote(&self, control: bool) { if control { self.controls_written.send_modify(|written| *written += 1); @@ -486,37 +555,48 @@ impl FramedLink { let (mut reader, writer) = tokio::io::split(stream); let (inbound_tx, inbound_rx) = mpsc::channel(128); let (closed, mut read_closed) = watch::channel(false); + let opened = Instant::now(); let state = Arc::new(WriterState { queue: Mutex::new(Outbound::default()), ready: Notify::new(), closed, + lifetime: Mutex::new(Lifetime { + opened, + last_read: opened, + closed: None, + }), controls_written: watch::channel(0).0, force_idr, stats: Arc::new(FramedStats::default()), }); let read_state = Arc::clone(&state); let reading = tokio::spawn(async move { - loop { + let cause = loop { let packet = tokio::select! { biased; - _ = read_closed.changed() => break, + _ = read_closed.changed() => break CloseCause::Host, packet = read_datagram(&mut reader) => packet, }; - let Ok(bytes) = packet else { - break; + let bytes = match packet { + Ok(bytes) => bytes, + Err(error) => break CloseCause::of_read(&error), }; + read_state.lifetime.lock().last_read = Instant::now(); tokio::select! { biased; - _ = read_closed.changed() => break, - sent = inbound_tx.send((peer, bytes)) => if sent.is_err() { break; }, + _ = read_closed.changed() => break CloseCause::Host, + sent = inbound_tx.send((peer, bytes)) => if sent.is_err() { break CloseCause::Host; }, } - } - let _ = read_state.closed.send(true); + }; + read_state.close(cause); }); let write_state = Arc::clone(&state); let writing = tokio::spawn(async move { - let _ = write_loop(writer, &write_state).await; - let _ = write_state.closed.send(true); + let cause = match write_loop(writer, &write_state).await { + Ok(()) => CloseCause::Host, + Err(error) => CloseCause::WriteFailed(error.kind()), + }; + write_state.close(cause); }); Self { peer, @@ -541,7 +621,19 @@ impl FramedLink { } pub fn close_guard(&self) -> CloseGuard { - CloseGuard(self.state.closed.clone()) + CloseGuard(Arc::clone(&self.state)) + } + + /// What this host saw end the tunnel first, how long it was open and + /// how long the other end had been quiet. `None` while it is open. + pub fn close_report(&self) -> Option { + let lifetime = self.state.lifetime.lock(); + let (cause, at) = lifetime.closed?; + Some(CloseReport { + cause, + open: at.saturating_duration_since(lifetime.opened), + quiet: at.saturating_duration_since(lifetime.last_read), + }) } /// Waits until every control datagram queued so far has been written to @@ -581,10 +673,10 @@ impl FramedLink { /// A device worker owns this guard after handing the link to the transport. /// Removing the device or stopping its supervisor closes both tunnel halves. -pub struct CloseGuard(watch::Sender); +pub struct CloseGuard(Arc); impl Drop for CloseGuard { fn drop(&mut self) { - let _ = self.0.send(true); + self.0.close(CloseCause::Host); } } @@ -643,7 +735,7 @@ impl Link for FramedLink { impl Drop for FramedLink { fn drop(&mut self) { - let _ = self.state.closed.send(true); + self.state.close(CloseCause::Host); for task in &self.tasks { task.abort(); } @@ -1027,7 +1119,7 @@ mod tests { .is_err()); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn silent_listener_peer_times_out_and_dropping_the_link_closes_io() { let (server, _silent) = tokio::io::duplex(32); let failed = @@ -1049,11 +1141,94 @@ mod tests { ); } + /// A tunnel says which end this host saw close it first, which is not + /// necessarily the end that started the close. In the field some closed + /// on both sides 9.5-9.8 s after they opened, and neither side said why. + #[tokio::test] + async fn a_closed_tunnel_reports_the_first_end_seen() { + let force_idr = || Arc::new(AtomicBool::new(false)); + let open = || async { + let (host, mut ipad) = tokio::io::duplex(1024); + let link = FramedLink::connect(host, PeerId::usb(7), force_idr()) + .await + .unwrap(); + read_preamble(&mut ipad).await.unwrap(); + (link, ipad) + }; + let closed = |link: &FramedLink| { + let mut closed = link.closed_signal(); + async move { + tokio::time::timeout(Duration::from_secs(1), closed.wait_for(|closed| *closed)) + .await + .unwrap() + .unwrap(); + } + }; + + let (link, ipad) = open().await; + assert_eq!(link.close_report(), None); + drop(ipad); + closed(&link).await; + assert_eq!(link.close_report().unwrap().cause, CloseCause::PeerEof); + + let (link, mut ipad) = open().await; + ipad.write_all(&[0, 0]).await.unwrap(); + closed(&link).await; + assert_eq!(link.close_report().unwrap().cause, CloseCause::BadLength); + + let (link, _ipad) = open().await; + drop(link.close_guard()); + closed(&link).await; + assert_eq!(link.close_report().unwrap().cause, CloseCause::Host); + // A later end changes nothing: the first cause stands. + link.state.close(CloseCause::PeerEof); + assert_eq!(link.close_report().unwrap().cause, CloseCause::Host); + + assert_eq!( + CloseCause::ReadFailed(io::ErrorKind::ConnectionReset).as_str(), + "peer_reset" + ); + assert_eq!( + CloseCause::ReadFailed(io::ErrorKind::ConnectionAborted).as_str(), + "read_error" + ); + } + + /// Nothing in the link times a tunnel out: one whose iPad pings every + /// 2 s stays open for a minute, and when it closes it says how long it + /// was open and how long the iPad had been quiet. + #[tokio::test(start_paused = true)] + async fn a_tunnel_stays_open_until_an_end_closes_it_and_reports_its_timings() { + let (host, mut ipad) = tokio::io::duplex(64 * 1024); + let mut link = FramedLink::connect(host, PeerId::usb(7), Arc::new(AtomicBool::new(false))) + .await + .unwrap(); + read_preamble(&mut ipad).await.unwrap(); + let mut inbound = link.take_inbound().unwrap(); + let start = Instant::now(); + let ping = encode_control(1, 1, &ControlMessage::Ping(Ping { t1_us: 1 })); + while start.elapsed() < Duration::from_secs(60) { + write_datagram(&mut ipad, &ping).await.unwrap(); + assert!(inbound.recv().await.is_some()); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(!link.is_closed(), "closed {:?} in", start.elapsed()); + } + tokio::time::sleep(Duration::from_millis(1_500)).await; + drop(ipad); + tokio::time::timeout(Duration::from_secs(1), inbound.recv()) + .await + .unwrap(); + let report = link.close_report().unwrap(); + assert_eq!(report.cause, CloseCause::PeerEof); + assert_eq!(report.open, Duration::from_millis(61_500)); + assert_eq!(report.quiet, Duration::from_millis(3_500)); + } + /// The frame being written is finished however many newer frames /// overflow behind it. Cutting it short sent the iPad a partial keyframe /// over a lossless cable: lost fragments in its report, and another /// keyframe request. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn bounded_writer_finishes_the_frame_in_flight_and_recovers_with_a_keyframe() { let (host, mut client) = tokio::io::duplex(64); let force_idr = Arc::new(AtomicBool::new(false)); @@ -1105,7 +1280,7 @@ mod tests { /// A link that overflows again right after a keyframe waits before it /// asks for the next one, and still asks once the wait is over, even if /// no other frame comes. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn overflow_right_after_a_keyframe_defers_the_next_request() { let (host, mut client) = tokio::io::duplex(64); let force_idr = Arc::new(AtomicBool::new(false)); diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 70cfb197..9ba55d34 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -1,11 +1,17 @@ pub mod abr; pub mod apple_usb; mod audio; +#[cfg(test)] +pub(crate) use audio::tests::session as test_session; +pub mod beat; +pub mod congestion; pub mod fault; pub mod link; pub mod pacer; +pub mod pointer; pub mod retransmit; pub mod session; +pub mod standby; pub mod usb; pub mod usbmuxd; @@ -15,6 +21,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::mpsc as std_mpsc; use std::time::{Duration, Instant}; +use eternal_wire::v2::control::ControlMessage; use eternal_wire::v2::media::{ media_payload_size, MediaHeader, MAX_FRAG_COUNT, MAX_MEDIA_PAYLOAD, MEDIA_HEADER_SIZE, }; @@ -28,6 +35,7 @@ use crate::control::{CaptureTarget, SharedControl, SupervisorCommand, VddStatus} use crate::encoder::NALUnit; use crate::stats::PIPELINE_STATS; use abr::AbrController; +use beat::Phase; use fault::FaultInjector; use link::{LinkId, PeerId}; use pacer::{Deadline, FramePacer, PacingClock}; @@ -53,8 +61,9 @@ fn now() -> Instant { /// is in flight before moving the clock on. #[cfg(feature = "test-hooks")] pub mod probe { - use std::collections::BTreeMap; + use std::collections::{BTreeMap, VecDeque}; use std::net::SocketAddr; + use std::time::Instant; use parking_lot::Mutex; @@ -76,6 +85,149 @@ pub mod probe { pub fn taken_from(addr: SocketAddr) -> u64 { TAKEN.lock().get(&addr).copied().unwrap_or(0) } + + /// The datagrams a refusal applies to. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum Kind { + Media, + Control, + } + + /// How a failed send fails. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum Fail { + /// As if the system had no buffer space (WSAENOBUFS, ENOBUFS). + NoBufferSpace, + /// Any other error: connection reset. + Other, + } + + struct Failing { + kind: Kind, + after: u64, + fails: VecDeque, + } + + static FAIL: Mutex> = Mutex::new(BTreeMap::new()); + + /// Of the UDP sends of `kind` to `addr`, the next `after` go out and + /// the ones after them fail as `fails` says, in order. + pub fn fail_sends(addr: SocketAddr, kind: Kind, after: u64, fails: &[Fail]) { + let fails = fails.iter().copied().collect(); + FAIL.lock().insert(addr, Failing { kind, after, fails }); + } + + /// As [`fail_sends`], with `count` sends refused for lack of buffer space. + pub fn refuse_sends(addr: SocketAddr, kind: Kind, after: u64, count: usize) { + fail_sends(addr, kind, after, &vec![Fail::NoBufferSpace; count]); + } + + /// Failures [`fail_sends`] still has to give. + pub fn refusals_left(addr: SocketAddr) -> usize { + FAIL.lock() + .get(&addr) + .map_or(0, |failing| failing.fails.len()) + } + + /// How this send of `datagram` to `addr` fails, if it does. + pub(crate) fn fail(addr: SocketAddr, datagram: &[u8]) -> Option { + use eternal_wire::v2::{classify, Classified}; + let kind = match classify(datagram) { + Classified::Media { .. } => Kind::Media, + Classified::Control(_) => Kind::Control, + _ => return None, + }; + let fail = match FAIL.lock().get_mut(&addr).filter(|f| f.kind == kind) { + Some(failing) if failing.after > 0 => { + failing.after -= 1; + None + } + Some(failing) => failing.fails.pop_front(), + None => None, + }; + attempted(addr, datagram, fail); + fail.map(|fail| match fail { + Fail::NoBufferSpace => super::congestion::no_buffer_space(), + Fail::Other => std::io::Error::from(std::io::ErrorKind::ConnectionReset), + }) + } + + /// One video datagram the transport tried to send, as the send saw it. + #[derive(Clone, Copy, Debug)] + pub struct Attempt { + pub at: Instant, + pub frame_seq: u32, + pub frag_index: u16, + pub retransmit: bool, + pub failed: Option, + } + + static RECORDED: Mutex> = Mutex::new(None); + static ATTEMPTS: Mutex> = Mutex::new(Vec::new()); + /// When each pacing wait was asked for, and its deadline. + static WAITS: Mutex> = Mutex::new(Vec::new()); + + /// Records every video send to `addr` from now on, and every pacing + /// wait, clearing what was recorded. + pub fn record(addr: SocketAddr) { + *RECORDED.lock() = Some(addr); + ATTEMPTS.lock().clear(); + WAITS.lock().clear(); + } + + /// The video sends recorded so far, in order. + pub fn attempts() -> Vec { + ATTEMPTS.lock().clone() + } + + /// The pacing waits recorded so far: when each was asked for, and until when. + pub fn waits() -> Vec<(Instant, Instant)> { + WAITS.lock().clone() + } + + fn attempted(addr: SocketAddr, datagram: &[u8], failed: Option) { + if *RECORDED.lock() != Some(addr) { + return; + } + if let Ok((header, _)) = eternal_wire::v2::media::MediaHeader::decode(datagram) { + ATTEMPTS.lock().push(Attempt { + at: Instant::now(), + frame_seq: header.frame_seq, + frag_index: header.frag_index, + retransmit: header.is_retransmit, + failed, + }); + } + } + + pub(crate) fn waited(deadline: Instant) { + if RECORDED.lock().is_some() { + WAITS.lock().push((Instant::now(), deadline)); + } + } +} + +/// What an identity proof signs for a socket on `listen`: that address, or +/// for one on every address, `home()`. None, and no answer, without an +/// address to sign. +fn identity_endpoints( + listen: SocketAddr, + home: impl FnOnce() -> Vec, +) -> Option<(u16, Vec<[u8; 4]>)> { + let std::net::IpAddr::V4(ip) = listen.ip() else { + return None; + }; + let addresses = if ip.is_unspecified() { + home() + } else { + vec![ip] + }; + (!addresses.is_empty()).then(|| { + ( + listen.port(), + addresses.iter().map(|ip| ip.octets()).collect(), + ) + }) } /// Stream parameters advertised in HELLO_ACK and heartbeats. @@ -83,6 +235,8 @@ struct SharedConfigSource<'a> { shared: &'a SharedControl, stream_epoch: u32, pen_available: bool, + /// Where the socket receives the stream. + listen: SocketAddr, } impl ConfigSource for SharedConfigSource<'_> { @@ -115,6 +269,31 @@ impl ConfigSource for SharedConfigSource<'_> { } } + fn identity_endpoints(&self) -> Option<(u16, Vec<[u8; 4]>)> { + identity_endpoints(self.listen, crate::network::home_addresses) + } + + fn notice(&self) -> u8 { + use crate::control::{CaptureTarget, VddStatus}; + let block = *self.shared.capture_block.lock(); + let second_screen_failed = *self.shared.capture_target.lock() + == CaptureTarget::VirtualExtended + && matches!( + *self.shared.vdd_status.lock(), + VddStatus::Failed | VddStatus::TaskFailed + ); + crate::capture::recovery::heartbeat_notice( + block, + second_screen_failed, + &mut self.shared.gpu_refused_since.lock(), + std::time::Instant::now(), + ) + } + + fn needs_windows_11(&self) -> bool { + second_screen::windows_notice(self.shared, Instant::now()) + } + fn host_caps(&self) -> u16 { // `start_sender` numbers sent frames contiguously per epoch. let numbering = eternal_wire::v2::control::HOSTCAP_CONTIGUOUS_FRAME_SEQ; @@ -125,6 +304,7 @@ impl ConfigSource for SharedConfigSource<'_> { }; numbering | eternal_wire::v2::control::HOSTCAP_KEYBOARD + | eternal_wire::v2::control::HOSTCAP_POINTER | if self.pen_available { eternal_wire::v2::control::HOSTCAP_PEN } else { @@ -145,6 +325,53 @@ impl ConfigSource for SharedConfigSource<'_> { } } +/// How often a stream looks at its chosen network while nobody streams. +const NETWORK_CHECK: Duration = Duration::from_secs(10); + +/// With Listen on set to one network: while nobody streams, a stream bound +/// to an address that network no longer gives restarts to listen on the +/// one it gives now. The supervisor does that only while this stream still +/// runs, never once the user stopped it. +fn look_at_network( + shared: &SharedControl, + listen_on: &crate::network::ListenOn, + bind_ip: std::net::Ipv4Addr, + supervisor: &StreamSupervisor, +) { + if *listen_on == crate::network::ListenOn::All + || shared.client_connected() + || crate::network::listen_ip_now(listen_on) == bind_ip + { + return; + } + info!( + listen_on = listen_on.label(), + "The chosen network changed: restarting the stream to listen on it" + ); + shared.stop(); + if let Err(error) = supervisor.restart() { + warn!(error = %error, "Failed to request a pipeline restart for the chosen network"); + } +} + +/// The supervisor a stream answers to, and the pipeline generation the +/// stream belongs to. +struct StreamSupervisor { + tx: std_mpsc::Sender, + generation: u64, +} + +impl StreamSupervisor { + /// Asks the supervisor to restart this stream. It does only while this + /// stream's generation still runs: never one the user stopped, or one a + /// newer stream replaced. + fn restart(&self) -> Result<(), std_mpsc::SendError> { + self.tx.send(SupervisorCommand::RestartStream { + generation: self.generation, + }) + } +} + /// Consumes NAL units from the encoder, fragments each access unit into v2 media /// datagrams (raw Annex B — no FlatBuffer wrapper), and runs the protocol-v2 /// control plane (hello/ack, heartbeats, keyframe requests, liveness) on the @@ -155,7 +382,12 @@ pub async fn start_sender( listen_port: u16, shared: SharedControl, supervisor_tx: std_mpsc::Sender, + generation: u64, ) -> Result<(), Box> { + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation, + }; let listen_on = shared.listen_on.lock().clone(); let bind_ip = crate::network::listen_ip_now(&listen_on); if bind_ip.is_loopback() { @@ -165,42 +397,11 @@ pub async fn start_sender( ); } let bind_addr = SocketAddr::from((bind_ip, listen_port)); - let socket = { - let raw = socket2::Socket::new(socket2::Domain::IPV4, socket2::Type::DGRAM, None)?; - // A 4 MiB send buffer lets the kernel absorb whatever the pacer's - // hard budget doesn't smooth (best effort — some platforms clamp it). - let _ = raw.set_send_buffer_size(4 * 1024 * 1024); - let _ = raw.set_recv_buffer_size(4 * 1024 * 1024); - let _ = raw.set_tos(0xB8); - #[cfg(windows)] - { - use std::os::windows::io::AsRawSocket; - use windows::Win32::Networking::WinSock::{ - WSAGetLastError, WSAIoctl, SIO_UDP_CONNRESET, SOCKET, - }; - let disabled = 0u32; - let mut returned = 0; - let result = unsafe { - WSAIoctl( - SOCKET(raw.as_raw_socket() as usize), - SIO_UDP_CONNRESET, - Some((&disabled as *const u32).cast()), - std::mem::size_of_val(&disabled) as u32, - None, - 0, - &mut returned, - None, - None, - ) - }; - if result != 0 { - warn!(error = ?unsafe { WSAGetLastError() }, "Could not disable UDP connection-reset notifications"); - } - } - raw.set_nonblocking(true)?; - raw.bind(&bind_addr.into())?; - UdpSocket::from_std(raw.into())? + let socket = match take_stream_socket(&shared, bind_addr) { + Some(socket) => socket, + None => bind_stream_socket(bind_addr)?, }; + let socket = UdpSocket::from_std(socket)?; // Never 0 — the receiver treats epoch 0 as invalid. (Wrap after 2^32 runs is // harmless: a fresh session id accompanies any host relaunch.) @@ -210,6 +411,10 @@ pub async fn start_sender( .max(1); let local_addr = socket.local_addr()?; + if bind_ip.is_unspecified() { + // Lists the home network addresses before anyone asks for them. + crate::network::home_addresses(); + } info!(%local_addr, stream_epoch, "UDP transport ready — waiting for a v2 client HELLO"); // The Address row and QR code follow every bind, so a chosen network // that came back shows its address. Resolved before locking: the stats @@ -220,11 +425,14 @@ pub async fn start_sender( .lock() .set_target_addr(shared.target_addr.lock().to_string()); - let input_relay = crate::input::InputRelay::default(); + let input = shared + .input + .begin(std::sync::Arc::clone(&shared.capture_geometry)); let config = SharedConfigSource { shared: &shared, stream_epoch, - pen_available: input_relay.pen_available(), + pen_available: input.pen_available(), + listen: local_addr, }; let abr_enabled = !std::env::var("ETERNAL_ABR").is_ok_and(|v| v.trim() == "0"); @@ -263,18 +471,37 @@ pub async fn start_sender( // Capture numbers skip the frames the encoder never took, and the iPad // held the frame after each skip waiting for one that never existed. let mut frame_seq: u32 = 0; + // The encoder closes its channel when it exits, before it reports why. + // The loop goes on until `running` goes false, so a goodbye the + // supervisor leaves on hearing that report (the storm brake) still goes + // out. + let mut video_open = true; let mut state = LoopState { retransmit_ring: RetransmitRing::default(), fault, - input_relay, + input, abr, + pointer: Default::default(), }; let mut audio_sender = audio::AudioSender::default(); let pacing_clock = PacingClock::new()?; let mut links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); - shared.session.lock().transport_resumed(now()); + // One lock, with `running` read under it: a stop + // (`diagnostics::stop_session`) queues its goodbye and clears `running` + // under the same lock, so this either sees that stop and keeps its + // goodbye, or comes first and the stop's goodbye is kept anyway. + let retired = { + let mut session = shared.session.lock(); + let stopping = !shared.running.load(Ordering::SeqCst); + session.transport_resumed(now(), stopping) + }; + if retired { + debug!("Dropped a goodbye left while the stream was stopped"); + } + let beat = std::sync::Arc::clone(&shared.transport_beat); + let _beat_ends = BeatEnds(&beat); let mut deferred_control = VecDeque::new(); let mut dgram_scratch = [0u8; MAX_DGRAM_SIZE]; let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); @@ -282,71 +509,78 @@ pub async fn start_sender( let mut liveness_tick = tokio::time::interval(Duration::from_millis(250)); liveness_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); // A chosen network can connect, or get a new address, after the stream - // starts. It is looked at again while nobody streams. - let mut network_check = tokio::time::interval(Duration::from_secs(10)); + // bound. It is looked at now, before anything else, and again every + // 10 s while nobody streams. The first look used to be the interval's + // first tick, which could lose to the liveness tick: a stream stopped + // meanwhile then ended without looking. + look_at_network(&shared, &listen_on, bind_ip, &supervisor); + let mut network_check = + tokio::time::interval_at(tokio::time::Instant::now() + NETWORK_CHECK, NETWORK_CHECK); network_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let mut pen_tick = tokio::time::interval(Duration::from_millis(50)); - pen_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - let mut pen_spacing = Deadline::new("pen-spacing")?; + let mut pointer_budget = Deadline::new("pointer-budget")?; loop { - // A pen backlog sends one bounded chunk at the top of every pass, - // whatever the select below keeps ready, so the loop serves one - // branch between chunks and the backlog never waits behind them. - if state - .input_relay - .pen_spaced_until() - .is_some_and(|at| at <= Instant::now()) - { - state - .input_relay - .keep_pen_alive(*shared.capture_geometry.lock()); - } - pen_spacing.arm(state.input_relay.pen_spaced_until()); + // Tokio's timer can wake 15 ms late on Windows, longer than a frame + // budget: the budget's end waits on a high-resolution timer. + let pointer_repeat = { + let (shape, position) = pointer_latest(&shared); + let budget = crate::capture::frame_budget_for(shared.effective_fps()); + pointer_budget.arm(state.pointer.budget_end( + budget, + position, + shape.map(|shape| shape.id), + )); + state.pointer.repeat_due(budget) + }; + beat.mark(Phase::Waiting); tokio::select! { // Service timers and queued control before another paced frame. // Random branch selection can delay a NACK by multiple frame // bursts on platforms with coarse sleep granularity. biased; - // Not while a backlog is held: its chunk went at the top of - // this pass, and a second one would leave nothing between them. - _ = pen_tick.tick(), if state.input_relay.pen_tick_needed() && state.input_relay.pen_spaced_until().is_none() => { - state.input_relay.keep_pen_alive(*shared.capture_geometry.lock()); - } _ = tokio::time::sleep_until(state.fault.next_deadline().unwrap_or_else(|| Instant::now() + Duration::from_secs(3600)).into()), if state.fault.enabled() => { + beat.mark(Phase::Video); let destination = *shared.target_addr.lock(); for datagram in state.fault.drain_due(Instant::now()) { let _ = links.send_to(&datagram, destination).await; } } _ = heartbeat.tick() => { + state.pointer.refresh(now()); + beat.mark(Phase::Tick); let actions = shared.session.lock().tick(&config, true, now()); - act(actions, &mut state, &links, &shared, &config, &supervisor_tx).await; + act(actions, &mut state, &links, &shared, &config, &supervisor).await; } - _ = network_check.tick(), if listen_on != crate::network::ListenOn::All => { - if !shared.client_connected() && crate::network::listen_ip_now(&listen_on) != bind_ip { - info!(listen_on = listen_on.label(), "The chosen network changed: restarting the stream to listen on it"); - shared.stop(); - if let Err(error) = supervisor_tx.send(SupervisorCommand::Restart) { - warn!(error = %error, "Failed to request a pipeline restart for the chosen network"); - } - } + // The PC is going down and left a goodbye: send it now, not at + // the next liveness tick, while Windows still waits. + _ = shared.power.woken() => { + beat.mark(Phase::Tick); + let actions = shared.session.lock().tick(&config, false, now()); + act(actions, &mut state, &links, &shared, &config, &supervisor).await; + } + _ = network_check.tick() => { + beat.mark(Phase::Tick); + look_at_network(&shared, &listen_on, bind_ip, &supervisor); } _ = liveness_tick.tick() => { + beat.mark(Phase::Tick); links.update_stats(); + links.congestion.lock().settle(now()); + ease_congestion(&links, &mut state, &shared, &config).await; if !shared.running.load(Ordering::SeqCst) { info!("Transport loop stopping on running=false"); break; } let actions = shared.session.lock().tick(&config, false, now()); - act(actions, &mut state, &links, &shared, &config, &supervisor_tx).await; + act(actions, &mut state, &links, &shared, &config, &supervisor).await; } result = receive_control(&mut links, &mut deferred_control) => { + beat.mark(Phase::Control); match result { Ok(LinkEvent::Closed(id)) => { let actions = shared.session.lock().link_lost(id); - act(actions, &mut state, &links, &shared, &config, &supervisor_tx).await; + act(actions, &mut state, &links, &shared, &config, &supervisor).await; } Ok(LinkEvent::Datagram(src, datagram)) => { #[cfg(feature = "test-hooks")] @@ -362,10 +596,11 @@ pub async fn start_sender( src, &header, message, + datagram.len(), &config, now(), ); - act(actions, &mut state, &links, &shared, &config, &supervisor_tx).await; + act(actions, &mut state, &links, &shared, &config, &supervisor).await; } Err(error) => { debug!(peer = %src, %error, "Dropped malformed control datagram"); @@ -384,11 +619,30 @@ pub async fn start_sender( } } + // Before video: a position is small and the iPad draws it at once. + _ = shared.pointer.changed() => { + beat.mark(Phase::Pointer); + send_pointer(&mut state.pointer, &links, &shared).await; + } + _ = pointer_budget.wait(), if pointer_budget.armed() => { + beat.mark(Phase::Pointer); + send_pointer(&mut state.pointer, &links, &shared).await; + } + _ = tokio::time::sleep_until(pointer_repeat.unwrap_or_else(Instant::now).into()), if pointer_repeat.is_some() => { + beat.mark(Phase::Pointer); + send_pointer(&mut state.pointer, &links, &shared).await; + } Some(packet) = audio_rx.recv() => { + beat.mark(Phase::Audio); audio_sender.send(packet, &links, &shared, stream_epoch).await; } - nal_opt = nal_rx.recv() => { - let Some(nal) = nal_opt else { break; }; + nal_opt = nal_rx.recv(), if video_open => { + beat.mark(Phase::Video); + let Some(nal) = nal_opt else { + info!("The encoder closed its channel; the transport runs until the stream stops"); + video_open = false; + continue; + }; let Some(session_id) = shared.session.lock().session_id() else { continue; }; let target_addr = *shared.target_addr.lock(); if target_addr.link == LinkId::Udp && target_addr.addr.is_none_or(|a| a.ip().is_unspecified() || a.port() == 0) { @@ -401,10 +655,7 @@ pub async fn start_sender( if awaiting_keyframe { let started = shared.session.lock().client_info().map(|info| info.connected_at); match first_frame_gate(nal.is_keyframe, nal.timestamp, started) { - FirstFrame::Send => { - awaiting_keyframe = false; - shared.session.lock().note_video_sent(session_id, now()); - } + FirstFrame::Send => {} FirstFrame::Skip => continue, // the forced IDR is coming FirstFrame::SkipAndRequestKeyframe => { shared.force_next_idr.store(true, Ordering::SeqCst); @@ -413,11 +664,29 @@ pub async fn start_sender( } } - frame_seq = frame_seq.wrapping_add(1); - let send_start = Instant::now(); let payload = &nal.data; - let payload_size = media_payload_size(shared.max_dgram.load(Ordering::SeqCst) as usize).unwrap_or(MAX_MEDIA_PAYLOAD); + // The PC's packet size, or the smaller one this session's + // client asked for when none of the larger reached it. + let max_dgram = shared.session.lock().max_media_dgram(session_id).map_or( + shared.max_dgram.load(Ordering::SeqCst), + |asked| shared.max_dgram.load(Ordering::SeqCst).min(u32::from(asked)), + ); + let payload_size = media_payload_size(max_dgram as usize).unwrap_or(MAX_MEDIA_PAYLOAD); let frag_count_usize = payload.len().div_ceil(payload_size).max(1); + // Held back until the client answers; the frames after it + // wait for the keyframe its answer asks for. + let wire_bytes = payload.len() + frag_count_usize * MEDIA_HEADER_SIZE; + if !shared.session.lock().may_send_media(session_id, wire_bytes, now()) { + awaiting_keyframe = true; + continue; + } + if awaiting_keyframe { + awaiting_keyframe = false; + shared.session.lock().note_video_sent(session_id, now()); + } + + frame_seq = frame_seq.wrapping_add(1); + let send_start = Instant::now(); if frag_count_usize > usize::from(MAX_FRAG_COUNT) { warn!( seq = nal.sequence, @@ -428,6 +697,9 @@ pub async fn start_sender( } let frag_count = frag_count_usize as u16; let capture_ts_us = crate::clock::instant_to_us(nal.timestamp); + if nal.is_keyframe { + shared.session.lock().note_keyframe_sent(session_id, stream_epoch, frame_seq); + } let mut send_failed = false; let keep_for_repair = shared.session.lock().client_supports_nack(); @@ -458,31 +730,44 @@ pub async fn start_sender( } let mut frame_pacer = FramePacer::new(frag_count_usize, Instant::now()); + let mut backed_off = false; for datagram in &stored { // A busy machine can exhaust the pacing budget mid-frame. // Keep serving audio even when there are no more pauses; // otherwise a large IDR can fill its 60 ms capture queue. audio_sender.send_queued(&mut audio_rx, &links, &shared, stream_epoch).await; - if target_addr.link == LinkId::Udp && state.fault.enabled() { - for due in state.fault.push(datagram, Instant::now()) { - if let Err(e) = links.send_to(&due, target_addr).await { - if !send_failed { - log_send_failed(target_addr.link, &e); - send_failed = true; - } - } - } - } else if let Err(e) = links.send_to(datagram, target_addr).await { - if !send_failed { + let mut refused = false; + let mut note = |sent: std::io::Result<()>| match sent { + Err(e) if congestion::is_no_buffer_space(&e) => refused = true, + Err(e) if !send_failed => { log_send_failed(target_addr.link, &e); send_failed = true; } + _ => {} + }; + if target_addr.link == LinkId::Udp && state.fault.enabled() { + for due in state.fault.push(datagram, Instant::now()) { + note(links.send_to(&due, target_addr).await); + } + } else { + note(links.send_to(datagram, target_addr).await); + } + // The system had no buffer space for it. A frame backs + // off once and sends it again; a refusal after that gives + // up the rest of the frame, which repair recovers. + if refused && backed_off { + break; + } + if refused { + backed_off = true; + ease_congestion(&links, &mut state, &shared, &config).await; } - let pause = frame_pacer.after_send(Instant::now()); + let paced = frame_pacer.after_send(Instant::now()); + let pause = if refused { congestion::BACKOFF } else { paced }; if target_addr.link == LinkId::Udp && pause > Duration::ZERO { - let repairs = repair_while_pacing(&links, &mut state, &shared, &config, &supervisor_tx, - pacing_clock.wait(pause), &mut deferred_control); + let repairs = repair_while_pacing(&links, &mut state, &shared, &config, &supervisor, + pacing_clock.until(Instant::now() + pause), &mut deferred_control); tokio::pin!(repairs); loop { tokio::select! { @@ -494,6 +779,16 @@ pub async fn start_sender( } } } + if refused { + match links.send_to(datagram, target_addr).await { + Err(e) if congestion::is_no_buffer_space(&e) => break, + Err(e) if !send_failed => { + log_send_failed(target_addr.link, &e); + send_failed = true; + } + _ => {} + } + } } let total_bytes = payload.len(); @@ -516,23 +811,92 @@ pub async fn start_sender( "Frame sent" ); } - // Only wakes an otherwise idle loop for the next chunk, which - // goes at the top of the pass. Last, because any branch ready - // before it starts that pass anyway. - _ = pen_spacing.wait(), if pen_spacing.armed() => {} } } // Every way out of the loop comes here, before the links close: an update - // that just stopped the stream must still tell the iPad. + // or a Stop that just stopped the stream must still tell the iPad. + beat.mark(Phase::Actions); let goodbye = shared.session.lock().take_goodbye(); if let Some(goodbye) = goodbye { say_goodbye(&links, goodbye).await; + shared.power.said(); } + park_stream_socket(&shared, links.into_udp_socket()); info!("NAL channel closed, transport sender shutting down"); Ok(()) } +/// The UDP socket iPads send HELLO2 to and the stream is sent from, bound +/// to `bind_addr`. The standby responder binds it the same way. +fn bind_stream_socket(bind_addr: SocketAddr) -> std::io::Result { + let raw = socket2::Socket::new(socket2::Domain::IPV4, socket2::Type::DGRAM, None)?; + // A 4 MiB send buffer lets the kernel absorb whatever the pacer's + // hard budget doesn't smooth (best effort — some platforms clamp it). + let _ = raw.set_send_buffer_size(4 * 1024 * 1024); + let _ = raw.set_recv_buffer_size(4 * 1024 * 1024); + let _ = raw.set_tos(0xB8); + #[cfg(windows)] + { + use std::os::windows::io::AsRawSocket; + use windows::Win32::Networking::WinSock::{ + WSAGetLastError, WSAIoctl, SIO_UDP_CONNRESET, SOCKET, + }; + let disabled = 0u32; + let mut returned = 0; + let result = unsafe { + WSAIoctl( + SOCKET(raw.as_raw_socket() as usize), + SIO_UDP_CONNRESET, + Some((&disabled as *const u32).cast()), + std::mem::size_of_val(&disabled) as u32, + None, + 0, + &mut returned, + None, + None, + ) + }; + if result != 0 { + warn!(error = ?unsafe { WSAGetLastError() }, "Could not disable UDP connection-reset notifications"); + } + } + raw.set_nonblocking(true)?; + raw.bind(&bind_addr.into())?; + Ok(raw.into()) +} + +/// The stream socket the last owner of the port left, if it is bound to +/// `bind_addr`. One bound elsewhere (the chosen network changed) closes. +fn take_stream_socket( + shared: &SharedControl, + bind_addr: SocketAddr, +) -> Option { + shared + .stream_socket + .lock() + .take() + .filter(|socket| socket.local_addr().is_ok_and(|addr| addr == bind_addr)) +} + +/// Leaves the stream socket, still bound, for the next owner of the port. +fn park_stream_socket(shared: &SharedControl, socket: UdpSocket) { + match socket.into_std() { + Ok(socket) => *shared.stream_socket.lock() = Some(socket), + Err(error) => debug!(%error, "Closed the stream port between owners"), + } +} + +/// Marks the transport ended however `start_sender` stops, so the watch +/// never reads a finished loop's last mark as a stuck one. +struct BeatEnds<'a>(&'a beat::Beat); + +impl Drop for BeatEnds<'_> { + fn drop(&mut self) { + self.0.stop(); + } +} + #[derive(Debug, PartialEq, Eq)] pub(crate) enum FirstFrame { Send, @@ -617,15 +981,15 @@ async fn receive_control( } async fn send_repairs( - socket: &UdpSocket, + links: &TransportLinks, ring: &mut RetransmitRing, shared: &SharedControl, nack: &eternal_wire::v2::control::Nack, ) { let target = *shared.target_addr.lock(); - let Some(destination) = target.addr.filter(|_| target.link == LinkId::Udp) else { + if target.link != LinkId::Udp || target.addr.is_none() { return; - }; + } let mut sent = 0; for datagram in ring.resend( nack.stream_epoch, @@ -634,9 +998,8 @@ async fn send_repairs( &nack.missing, Instant::now(), ) { - if socket.send_to(&datagram, destination).await.is_ok() { - #[cfg(feature = "test-hooks")] - probe::sent(destination); + // Through the links, so a refused repair counts in the episode. + if links.send_to(&datagram, target).await.is_ok() { sent += 1; } } @@ -660,7 +1023,7 @@ async fn repair_while_pacing( state: &mut LoopState, shared: &SharedControl, config: &impl ConfigSource, - supervisor_tx: &std_mpsc::Sender, + supervisor: &StreamSupervisor, timer: impl std::future::Future, deferred: &mut DeferredControl, ) { @@ -677,8 +1040,8 @@ async fn repair_while_pacing( #[cfg(feature = "test-hooks")] probe::taken(peer); let actions = shared.session.lock().handle_datagram(PeerId::udp(peer), &header, - message, config, now()); - act(actions, state, links, shared, config, supervisor_tx).await; + message, len, config, now()); + act(actions, state, links, shared, config, supervisor).await; continue; } if deferred.len() < 64 { deferred.push_back(LinkEvent::Datagram(PeerId::udp(peer), bytes[..len].to_vec())); } @@ -691,8 +1054,11 @@ async fn repair_while_pacing( struct LoopState { retransmit_ring: RetransmitRing, fault: FaultInjector, - input_relay: crate::input::InputRelay, + /// Hands input to Windows on its own thread: Windows' input calls can + /// wait seconds while it reconfigures displays. + input: crate::input::Injector, abr: AbrController, + pointer: pointer::PointerSender, } /// Carries out everything the session decided. Every path that feeds the @@ -705,27 +1071,33 @@ async fn act( links: &TransportLinks, shared: &SharedControl, config: &impl ConfigSource, - supervisor_tx: &std_mpsc::Sender, + supervisor: &StreamSupervisor, ) { if actions.new_target.is_some() || actions.client_lost { state.retransmit_ring.clear(); state.fault.clear(); } if actions.new_target.is_some() || actions.client_lost || actions.release_input { - state.input_relay.reset(); + state.input.reset(); } if let Some(nack) = actions.retransmit.take() { - send_repairs(&links.udp.socket, &mut state.retransmit_ring, shared, &nack).await; + send_repairs(links, &mut state.retransmit_ring, shared, &nack).await; } if let Some((session_id, event)) = actions.input.take() { + shared.transport_beat.mark(Phase::Input); if let Some(geometry) = *shared.capture_geometry.lock() { - if actions.input_late { - state.input_relay.relay_late(session_id, &event, geometry); - } else { - state.input_relay.relay(session_id, &event, geometry); - } + state + .input + .relay(session_id, event, geometry, actions.input_late); } } + if actions.force_idr || actions.new_target.is_some() { + state.pointer.keyframe(now()); + } + if let Some(report) = actions.report.as_ref() { + state.pointer.report(report.pointer_shape_id, now()); + shared.note_pointer_report(report.pointer_shape_id); + } if let Some(report) = actions.report.take() { // A slow log sink must not hold the capture/encode stats lock. let retransmits = PIPELINE_STATS.lock().transport_retransmits; @@ -753,33 +1125,146 @@ async fn act( let clamped = state.abr.set_ceiling(ceiling); let tunnel_drops = (shared.target_addr.lock().link != LinkId::Udp).then(link::tunnel_congestion_drops); + state + .abr + .encoder_reopens(shared.encoder_reopens_for_bitrate.load(Ordering::Relaxed)); let reported = state.abr.on_report(&report, tunnel_drops, Instant::now()); let decision = abr::AbrDecision { target_bps: reported.target_bps, changed: clamped.changed || reported.changed, }; - if decision.changed { - shared - .abr_current_bps - .store(decision.target_bps, Ordering::SeqCst); - PIPELINE_STATS.lock().set_bitrate(decision.target_bps); - // One immediate notify; the 1s heartbeat's embedded config - // self-heals any loss. - let notify = shared.session.lock().stream_config_notify(config); - for (destination, datagram) in notify { - let _ = links.send_to(&datagram, destination).await; + apply_bitrate(decision, links, shared, config).await; + } + execute_actions(actions, links, shared, supervisor).await; + send_pointer(&mut state.pointer, links, shared).await; +} + +/// The pointer to send: hidden until the iPad holds its image, while +/// capture still draws it into the video (`client_draws_pointer`). No image +/// and hidden while capture withholds it (`PointerFeed::set_withheld`). +fn pointer_latest( + shared: &SharedControl, +) -> ( + Option>, + Option, +) { + let (shape, position) = shared.pointer.latest(); + let withheld = shared.pointer.withheld(); + let shown = shared.client_draws_pointer() && !withheld; + let position = position.map(|position| crate::capture::pointer::Position { + visible: position.visible && shown, + ..position + }); + (shape.filter(|_| !withheld), position) +} + +/// Sends what [`pointer::PointerSender`] says is due, to a client that +/// draws the pointer. +async fn send_pointer( + sender: &mut pointer::PointerSender, + links: &TransportLinks, + shared: &SharedControl, +) { + let session_id = shared.session.lock().pointer_session_id(); + if session_id.is_some() && sender.session_id() != session_id { + info!( + ?session_id, + "The iPad draws the pointer: sending it apart from the video" + ); + } + sender.serve(session_id); + if session_id.is_none() || (shared.pointer.withheld() && !sender.withhold()) { + return; + } + let budget = crate::capture::frame_budget_for(shared.effective_fps()); + for send in sender.poll(now(), budget, pointer_latest(shared)) { + match send { + pointer::Send::Shape(shape, from) => { + let image = &shape.image; + let chunks = eternal_wire::v2::control::pointer_shape_chunks( + shape.id, + image.source_type, + image.flags, + (image.width, image.height), + image.hot, + &image.bytes, + ); + let total = chunks.len(); + let mut next = from; + let mut refused = false; + for chunk in chunks.into_iter().skip(from) { + let message = ControlMessage::PointerShape(chunk); + let Some((peer, datagram)) = shared.session.lock().encode_for_client(&message) + else { + return; + }; + // A full USB control queue refuses it: resume here later. + if links.send_to(&datagram, peer).await.is_err() { + refused = true; + break; + } + next += 1; + } + if next >= total { + PIPELINE_STATS.lock().pointer_shapes_sent += 1; + } + sender.shape_progress(shape.id, next, total, refused, now()); + } + pointer::Send::State(state) => { + PIPELINE_STATS.lock().pointer_states_sent += 1; + let message = ControlMessage::PointerState(state); + let Some((peer, datagram)) = shared.session.lock().encode_for_client(&message) + else { + return; + }; + // A lost position is replaced by the next, or the repeat. + let _ = links.send_to(&datagram, peer).await; } } } - execute_actions(actions, links, shared, supervisor_tx).await; +} + +/// Steps the bitrate down once for each episode of sends the system had no +/// buffer space for, without waiting for the iPad to report the loss. +async fn ease_congestion( + links: &TransportLinks, + state: &mut LoopState, + shared: &SharedControl, + config: &impl ConfigSource, +) { + if links.congestion.lock().take_step_down() { + let decision = state.abr.on_send_refused(Instant::now()); + apply_bitrate(decision, links, shared, config).await; + } +} + +async fn apply_bitrate( + decision: abr::AbrDecision, + links: &TransportLinks, + shared: &SharedControl, + config: &impl ConfigSource, +) { + if decision.changed { + shared + .abr_current_bps + .store(decision.target_bps, Ordering::SeqCst); + PIPELINE_STATS.lock().set_bitrate(decision.target_bps); + // One immediate notify; the 1s heartbeat's embedded config + // self-heals any loss. + let notify = shared.session.lock().stream_config_notify(config); + for (destination, datagram) in notify { + let _ = links.send_to(&datagram, destination).await; + } + } } async fn execute_actions( mut actions: Actions, links: &TransportLinks, shared: &SharedControl, - supervisor_tx: &std_mpsc::Sender, + supervisor: &StreamSupervisor, ) { + shared.transport_beat.mark(Phase::Actions); // An ended session's summary waits for this pass's second screen step, // which can give the second screen up for it, unless a new session // starts now: second screen news from here on is the new one's. @@ -793,8 +1278,18 @@ async fn execute_actions( } } for (destination, datagram) in &actions.replies { - if let Err(e) = links.send_to(datagram, *destination).await { - warn!(peer = %destination, error = %e, "Failed to send control reply"); + let mut sent = links.send_to(datagram, *destination).await; + // The system had no buffer space for it: one short retry. The + // episode's own lines report it if that fails too. + if sent.as_ref().is_err_and(congestion::is_no_buffer_space) { + tokio::time::sleep(congestion::REPLY_RETRY).await; + sent = links.send_to(datagram, *destination).await; + } + match sent { + Err(e) if !congestion::is_no_buffer_space(&e) => { + warn!(peer = %destination, error = %e, "Failed to send control reply"); + } + _ => {} } } @@ -820,6 +1315,7 @@ async fn execute_actions( if let Some(goodbye) = actions.goodbye { say_goodbye(links, goodbye).await; + shared.power.said(); } if actions.force_idr { @@ -855,6 +1351,7 @@ async fn execute_actions( } } + shared.transport_beat.mark(Phase::SecondScreen); let restart = match second_screen::next_step(shared, now()) { second_screen::Step::Nothing => false, second_screen::Step::Attach => { @@ -889,7 +1386,7 @@ async fn execute_actions( } if restart { shared.stop(); - if let Err(error) = supervisor_tx.send(SupervisorCommand::Restart) { + if let Err(error) = supervisor.restart() { warn!(error = %error, "Failed to request pipeline restart for the virtual display"); } } @@ -922,6 +1419,43 @@ mod tests { CAP_DECODE_H264, FEATURE_SUPPORTS_NACK, }; + /// No iPad came back to the virtual display within the grace period: + /// the stream asks to restart itself, by its own generation, whichever + /// generation the supervisor counts now. + #[test] + fn the_second_screen_release_restarts_only_this_stream() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let shared = SharedControl::new(socket.local_addr().unwrap().port(), 15_000_000); + shared.running.store(true, Ordering::SeqCst); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + *shared.vdd_status.lock() = VddStatus::Active; + // The viewer left a grace period ago. + assert_eq!( + second_screen::next_step(&shared, Instant::now() - second_screen::RELEASE_GRACE), + second_screen::Step::Nothing + ); + let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); + let (supervisor_tx, supervisor_rx) = std_mpsc::channel(); + let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst) + 1000; + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation, + }; + execute_actions(Actions::default(), &links, &shared, &supervisor).await; + assert_eq!( + supervisor_rx.try_recv(), + Ok(SupervisorCommand::RestartStream { generation }) + ); + assert!(!shared.running.load(Ordering::SeqCst)); + assert!(supervisor_rx.try_recv().is_err()); + }); + } + /// The second screen gives up for a session in the same pass that ends /// it: the session's summary names that fallback. #[test] @@ -929,6 +1463,7 @@ mod tests { use tracing_subscriber::prelude::*; let _slot = crate::diagnostics::tests::FALLBACK_SLOT.lock(); let summaries = std::sync::Arc::new(parking_lot::Mutex::new(Vec::::new())); + crate::logging::prepare_log_capture(); let _log = tracing::subscriber::set_default( tracing_subscriber::registry() .with(crate::diagnostics::FallbackWatch) @@ -948,10 +1483,15 @@ mod tests { *shared.vdd_status.lock() = VddStatus::Active; let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation: 1, + }; let config = SharedConfigSource { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let peer = PeerId::udp("127.0.0.1:19893".parse().unwrap()); let hello = ControlMessage::Hello2(Hello2 { @@ -977,7 +1517,7 @@ mod tests { .session .lock() .handle_control(peer, 0, hello, &config, now); - execute_actions(accepted, &links, &shared, &supervisor_tx).await; + execute_actions(accepted, &links, &shared, &supervisor).await; // The virtual display attached and has shown no picture for too long. second_screen::note_attached(&shared, "virtual".into()); second_screen::start_proving(&shared, 1, now - second_screen::PROVE_TIMEOUT); @@ -990,7 +1530,7 @@ mod tests { Instant::now(), ); assert!(bye.ended.is_some()); - execute_actions(bye, &links, &shared, &supervisor_tx).await; + execute_actions(bye, &links, &shared, &supervisor).await; }); let summaries = summaries.lock(); assert_eq!(summaries.len(), 1, "{summaries:?}"); @@ -1004,6 +1544,27 @@ mod tests { crate::diagnostics::tests::assert_contract(&summaries[0]); } + /// A socket on one address signs that address; one on every address + /// signs the home network addresses. With none to sign, or IPv6, the PC + /// answers nothing rather than an empty list. + #[test] + fn identity_endpoints_need_an_address_to_sign() { + let all = SocketAddr::from(([0, 0, 0, 0], 9876)); + let home = || vec![std::net::Ipv4Addr::new(192, 0, 2, 4)]; + assert_eq!( + identity_endpoints(all, home), + Some((9876, vec![[192, 0, 2, 4]])) + ); + assert_eq!(identity_endpoints(all, Vec::new), None); + let one = SocketAddr::from(([198, 51, 100, 3], 9876)); + assert_eq!( + identity_endpoints(one, || panic!("lists nothing")), + Some((9876, vec![[198, 51, 100, 3]])) + ); + let v6 = SocketAddr::from((std::net::Ipv6Addr::LOCALHOST, 9876)); + assert_eq!(identity_endpoints(v6, home), None); + } + #[tokio::test] async fn repairs_current_and_previous_frames_before_the_pacing_wait_ends() { let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); @@ -1016,6 +1577,7 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let hello = ControlMessage::Hello2(Hello2 { proto_min: 2, @@ -1066,16 +1628,21 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation: 1, + }; let mut deferred = DeferredControl::new(); repair_while_pacing( &links, &mut state, &shared, &config, - &supervisor_tx, + &supervisor, tokio::time::sleep(Duration::from_secs(1)), &mut deferred, ) @@ -1134,20 +1701,139 @@ mod tests { LoopState { retransmit_ring: RetransmitRing::default(), fault: FaultInjector::new(0.0, 0.0, Duration::ZERO), - input_relay: crate::input::InputRelay::default(), + input: crate::input::InputThread::default().begin(Default::default()), abr: AbrController::new(15_000_000, true), + pointer: Default::default(), + } + } + + /// Review r2: the USB tunnel's control queue takes 64 datagrams and + /// refuses more while its writer is blocked. A 256x256 image is 194 + /// chunks: they went out until the queue filled, the rest were dropped, + /// and the image counted as sent, so the iPad never had it whole. The + /// chunks now resume where the link refused them. + #[tokio::test] + async fn an_image_larger_than_the_usb_queue_arrives_whole_after_backpressure() { + use crate::capture::pointer::{PointerImage, Position}; + use eternal_wire::v2::control::{parse_control, FEATURE_DRAWS_POINTER}; + let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let shared = SharedControl::new(socket.local_addr().unwrap().port(), 15_000_000); + shared.pairing.lock().required = false; + let config = SharedConfigSource { + shared: &shared, + stream_epoch: 1, + pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), + }; + let mut links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); + let cable = PeerId::usb(5); + let (host_end, mut ipad_end) = tokio::io::duplex(4096); + let link = link::FramedLink::connect( + host_end, + cable, + std::sync::Arc::clone(&shared.force_next_idr), + ) + .await + .unwrap(); + links.adopt_for_test(std::sync::Arc::new(link)); + link::read_preamble(&mut ipad_end).await.unwrap(); + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 1, + listen_port: 0, + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_DRAWS_POINTER, + screen_px_w: 1920, + screen_px_h: 1080, + screen_pt_w: 960, + screen_pt_h: 540, + refresh_hz: 60, + device_name: "cable test".into(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + shared + .session + .lock() + .handle_control(cable, 0, hello, &config, now()); + assert!( + shared.session.lock().pointer_session_id().is_some(), + "a cable counts as answered" + ); + let (side, total_bytes) = (256u16, 256usize * 256 * 4); + shared.pointer.set_shape(PointerImage { + source_type: 2, + flags: 0, + width: side, + height: side, + hot: (0, 0), + bytes: (0..total_bytes).map(|i| i as u8).collect(), + }); + shared.pointer.set_position(Position { + visible: true, + x: 10, + y: 10, + host_time_us: 0, + }); + let chunk_count = total_bytes.div_ceil(eternal_wire::v2::control::POINTER_CHUNK_MAX); + assert!(chunk_count > 64, "{chunk_count} chunks"); + + // Nobody reads the tunnel: the queue fills and refuses the rest. + let mut sender = pointer::PointerSender::default(); + send_pointer(&mut sender, &links, &shared).await; + let mut got = vec![false; chunk_count]; + let mut first = 0; + let deadline = Instant::now() + Duration::from_secs(10); + let mut rounds = 0; + while got.iter().any(|g| !g) { + assert!( + Instant::now() < deadline, + "only {} of {chunk_count} chunks arrived", + got.iter().filter(|g| **g).count() + ); + // Drain what the tunnel holds, then let the sender go on. + while let Ok(Ok(datagram)) = tokio::time::timeout( + Duration::from_millis(20), + link::read_datagram(&mut ipad_end), + ) + .await + { + if let Ok((_, ControlMessage::PointerShape(chunk))) = parse_control(&datagram) { + got[chunk.offset as usize / eternal_wire::v2::control::POINTER_CHUNK_MAX] = + true; + } + } + if rounds == 0 { + first = got.iter().filter(|g| **g).count(); + } + rounds += 1; + tokio::time::sleep(pointer::SHAPE_RETRY).await; + send_pointer(&mut sender, &links, &shared).await; } + assert!( + first < chunk_count, + "the queue refused some at first ({first} of {chunk_count})" + ); + assert!(rounds > 1); } /// A cable pulled while a key and a mouse button are down ends the USB /// session, and both go up, as at every other end of a session. #[cfg(not(windows))] #[tokio::test] + // The slot only keeps other tests out; nothing on this test's runtime + // takes it. + #[allow(clippy::await_holding_lock)] async fn a_usb_session_ended_by_its_cable_releases_held_input() { use crate::input::{ recorder, CaptureGeometry, Injection, KIND_KEY, KIND_MOUSE_ABS, PHASE_BEGAN, }; use eternal_wire::v2::control::{InputEvent, FEATURE_WANTS_INPUT}; + // The session starts and ends through `act`, which uses the fallback slot. + let _slot = crate::diagnostics::tests::FALLBACK_SLOT.lock(); let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); let shared = SharedControl::new(socket.local_addr().unwrap().port(), 15_000_000); shared.pairing.lock().required = false; @@ -1161,9 +1847,14 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation: 1, + }; let mut state = loop_state(); let cable = PeerId::usb(5); let hello = ControlMessage::Hello2(Hello2 { @@ -1188,15 +1879,7 @@ mod tests { .session .lock() .handle_control(cable, 0, hello, &config, now()); - act( - accepted, - &mut state, - &links, - &shared, - &config, - &supervisor_tx, - ) - .await; + act(accepted, &mut state, &links, &shared, &config, &supervisor).await; let session_id = shared.session.lock().session_id().unwrap(); let press = |event_id, kind, keycode| { ControlMessage::InputEvent(InputEvent { @@ -1223,18 +1906,18 @@ mod tests { .session .lock() .handle_control(cable, session_id, event, &config, now()); - act( - pressed, - &mut state, - &links, - &shared, - &config, - &supervisor_tx, - ) - .await; + act(pressed, &mut state, &links, &shared, &config, &supervisor).await; } + // Windows takes input on the input thread. + let injected = |count: usize| { + let started = Instant::now(); + while recorder::peek().len() < count && started.elapsed() < Duration::from_secs(5) { + std::thread::sleep(Duration::from_millis(1)); + } + recorder::take() + }; assert!(matches!( - recorder::take().as_slice(), + injected(3).as_slice(), [ Injection::KeyDown { .. }, Injection::MoveAbs { .. }, @@ -1244,18 +1927,10 @@ mod tests { let unplugged = shared.session.lock().link_lost(cable.link); assert!(unplugged.client_lost); - act( - unplugged, - &mut state, - &links, - &shared, - &config, - &supervisor_tx, - ) - .await; + act(unplugged, &mut state, &links, &shared, &config, &supervisor).await; assert!(shared.session.lock().session_id().is_none()); assert!(matches!( - recorder::take().as_slice(), + injected(2).as_slice(), [Injection::KeyUp { .. }, Injection::LeftUp { .. }] )); } @@ -1267,6 +1942,7 @@ mod tests { shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let hello = ControlMessage::Hello2(Hello2 { proto_min: 2, @@ -1342,17 +2018,22 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation: 1, + }; let mut deferred = DeferredControl::new(); repair_while_pacing( &links, &mut state, &shared, &config, - &supervisor_tx, + &supervisor, async { let _ = paced.await; }, @@ -1405,6 +2086,7 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let held = shared .session @@ -1424,6 +2106,163 @@ mod tests { ); } + /// While the lock screen or a permission prompt keeps Windows from + /// letting capture work, every heartbeat tells the iPad the PC is locked. + /// Once capture works again, heartbeats say nothing more. + #[test] + fn heartbeats_say_the_pc_is_locked_while_a_secure_screen_blocks_capture() { + use crate::capture::recovery::{ + note_capture_resumed, run_until_unblocked, CaptureBlock, CaptureUnavailable, + }; + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_PC_LOCKED}; + let shared = SharedControl::new(9876, 15_000_000); + shared.pairing.lock().required = false; + let now = Instant::now(); + repairable_session(&shared, "127.0.0.1:50000".parse().unwrap(), now); + let config = SharedConfigSource { + shared: &shared, + stream_epoch: 1, + pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), + }; + let notices = || { + let actions = shared.session.lock().tick(&config, true, now); + actions + .replies + .iter() + .filter_map(|(_, bytes)| match parse_control(bytes) { + Ok((_, ControlMessage::Heartbeat(heartbeat))) => Some(heartbeat.notice), + _ => None, + }) + .collect::>() + }; + assert_eq!(notices(), [NOTICE_NONE]); + // Windows answers duplication with E_ACCESSDENIED until the user is + // back on the desktop. + let mut attempts = 0; + let mut while_blocked = Vec::new(); + let result = run_until_unblocked( + &shared, + |_| { + attempts += 1; + if attempts < 3 { + Err(CaptureUnavailable { + block: CaptureBlock::from_hresult(0x8007_0005_u32 as i32).unwrap(), + detail: "Access is denied".into(), + } + .into()) + } else { + Ok(()) + } + }, + |_| { + while_blocked.extend(notices()); + true + }, + ); + assert!(result.is_ok()); + assert_eq!(while_blocked, [NOTICE_PC_LOCKED, NOTICE_PC_LOCKED]); + note_capture_resumed(&shared); + assert_eq!(notices(), [NOTICE_NONE]); + } + + /// On Windows 10 a Second screen session mirrors, and its heartbeats + /// tell the iPad why. Windows 11 starts the second screen with no + /// such notice. + #[test] + fn heartbeats_say_second_screen_needs_windows_11_on_windows_10() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11}; + for (build, step, notice) in [ + ( + 19045, + second_screen::Step::Nothing, + NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11, + ), + (26200, second_screen::Step::Attach, NOTICE_NONE), + ] { + let shared = SharedControl::new(9876, 15_000_000); + shared.pairing.lock().required = false; + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + *shared.vdd_status.lock() = VddStatus::WaitingForClient; + second_screen::note_windows(&shared, Some(build), false); + let now = Instant::now(); + repairable_session(&shared, "127.0.0.1:50000".parse().unwrap(), now); + let config = SharedConfigSource { + shared: &shared, + stream_epoch: 1, + pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), + }; + let notices = || { + let actions = shared.session.lock().tick(&config, true, now); + actions + .replies + .iter() + .filter_map(|(_, bytes)| match parse_control(bytes) { + Ok((_, ControlMessage::Heartbeat(heartbeat))) => Some(heartbeat.notice), + _ => None, + }) + .collect::>() + }; + assert_eq!(notices(), [NOTICE_NONE], "{build}"); + assert_eq!(second_screen::next_step(&shared, now), step, "{build}"); + assert_eq!(notices(), [notice], "{build}"); + } + } + + /// A screen Windows won't share from this graphics card: the heartbeats + /// say so once it has lasted 2 s, and in Second screen mode, once the + /// second screen failed, that the main screen it fell back to can't be + /// shared either. While the second screen still starts, the mirror's + /// notice stays. + #[test] + fn heartbeats_say_which_screen_this_graphics_card_cannot_share() { + use crate::capture::recovery::CaptureBlock; + use crate::control::{CaptureTarget, VddStatus}; + use eternal_wire::v2::control::{ + NOTICE_NONE, NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, NOTICE_UNSUPPORTED_GPU, + }; + let shared = SharedControl::new(9876, 15_000_000); + shared.pairing.lock().required = false; + let now = Instant::now(); + repairable_session(&shared, "127.0.0.1:50000".parse().unwrap(), now); + let config = SharedConfigSource { + shared: &shared, + stream_epoch: 1, + pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), + }; + let notices = || { + let actions = shared.session.lock().tick(&config, true, now); + actions + .replies + .iter() + .filter_map(|(_, bytes)| match parse_control(bytes) { + Ok((_, ControlMessage::Heartbeat(heartbeat))) => Some(heartbeat.notice), + _ => None, + }) + .collect::>() + }; + *shared.capture_block.lock() = Some(CaptureBlock::UnsupportedOnThisGpu); + assert_eq!(notices(), [NOTICE_NONE], "a refusal that just began"); + *shared.gpu_refused_since.lock() = Some(Instant::now() - Duration::from_secs(3)); + assert_eq!(notices(), [NOTICE_UNSUPPORTED_GPU]); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + *shared.vdd_status.lock() = VddStatus::Attaching; + assert_eq!(notices(), [NOTICE_UNSUPPORTED_GPU], "still starting"); + for failed in [VddStatus::Failed, VddStatus::TaskFailed] { + *shared.vdd_status.lock() = failed; + assert_eq!( + notices(), + [NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU], + "{failed:?}" + ); + } + *shared.capture_block.lock() = None; + assert_eq!(notices(), [NOTICE_NONE]); + assert_eq!(*shared.gpu_refused_since.lock(), None); + } + /// A NACK of a session that ended for liveness, while another frame is /// paced, is answered with the goodbye, as any of its late packets is. #[tokio::test] @@ -1442,6 +2281,7 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let expired = shared .session @@ -1464,102 +2304,199 @@ mod tests { ); } - /// `start_sender` itself, with a native pen device that takes every - /// frame (Windows' own off Windows). A stall delivers 64 samples at - /// once while the encoder always has a frame ready, which the loop takes - /// in a branch ahead of the pen's wake-up. The spacing holds back all - /// but the first burst, and the loop sends the rest within a few times - /// the spacing, one chunk at the top of each pass. - #[cfg(not(windows))] + /// Windows holds the input thread inside a still Pencil's refresh. The + /// loop takes the capture rect for every input datagram, so a refresh + /// that kept it while Windows held the call stopped the loop at the + /// iPad's next touch: no heartbeats, no HELLO answers, no goodbye. #[test] - fn the_loop_sends_a_held_pen_backlog_while_video_is_always_ready() { - use crate::input::{native_pen, CaptureGeometry, PenPhase, PenSample}; - const SAMPLES: usize = 64; - const VIDEO_QUEUE: usize = 1 << 16; - let output = CaptureGeometry { + fn a_pen_refresh_windows_holds_never_stops_the_loop() { + use crate::input::{CaptureGeometry, InputThread, Sink, KIND_KEY, PHASE_BEGAN}; + use eternal_wire::v2::control::{HelloStatus, InputEvent, FEATURE_WANTS_INPUT}; + // The loop's session starts through `act`, which uses the fallback slot. + let _slot = crate::diagnostics::tests::FALLBACK_SLOT.lock(); + + /// Holds the first refresh until told to go on. + struct StuckRefresh { + entered: Option>, + hold: Option>, + } + impl Sink for StuckRefresh { + fn relay(&mut self, _: u32, _: &InputEvent, _: CaptureGeometry, _: bool) {} + fn reset(&mut self) {} + fn keep_alive(&mut self, _: Option) { + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(hold) = self.hold.take() { + let _ = hold.recv(); + } + } + fn pen_available(&self) -> bool { + true + } + fn spaced_until(&self) -> Option { + None + } + fn tick_needed(&self) -> bool { + true + } + } + + let ipad = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + ipad.set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let port = std::net::UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .unwrap() + .port(); + let host = SocketAddr::from(([127, 0, 0, 1], port)); + let (entered_tx, entered) = std_mpsc::channel(); + let (go, hold) = std_mpsc::channel(); + let mut shared = SharedControl::new(port, 15_000_000); + shared.input = std::sync::Arc::new(InputThread::with_fake(move || StuckRefresh { + entered: Some(entered_tx), + hold: Some(hold), + })); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + *shared.capture_geometry.lock() = Some(CaptureGeometry { left: 0, top: 0, - width: 1920, - height: 1080, - }; - let taken = native_pen::install(); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let (delivered, queued) = runtime.block_on(async { - let shared = SharedControl::new(0, 15_000_000); - shared.running.store(true, Ordering::SeqCst); - *shared.capture_geometry.lock() = Some(output); - let (nal_tx, nal_rx) = mpsc::channel(VIDEO_QUEUE); - let (_audio_tx, audio_rx) = mpsc::channel(1); + width: 1280, + height: 720, + }); + let (nal_tx, nal_rx) = mpsc::channel(4); + let control = shared.clone(); + let transport = std::thread::spawn(move || { + let (_audio_tx, audio_rx) = mpsc::channel(4); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); - let frame = |sequence| NALUnit { - data: vec![0, 0, 0, 1, 0x41], - sequence, - timestamp: Instant::now(), - encode_duration_us: 0, - is_keyframe: false, - }; - // A full queue, kept full until the loop stops and drops it. - let mut sequence = 0; - while nal_tx.try_send(frame(sequence)).is_ok() { - sequence += 1; - } - let video = std::thread::spawn(move || { - while nal_tx.blocking_send(frame(sequence)).is_ok() { - sequence += 1; - } - sequence - }); - let delivered = Instant::now(); - native_pen::with(|device| { - for x in 0..SAMPLES as u16 { - let sample = PenSample { - phase: if x == 0 { - PenPhase::Down - } else { - PenPhase::Move - }, - x: i32::from(x), - y: 10, - x_norm: x, - y_norm: 10, - pressure: 512, - tilt_x: 0, - tilt_y: 0, - }; - device.deliver(sample, native_pen::SCREEN, Some(output), Instant::now()); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(start_sender( + nal_rx, + audio_rx, + port, + control, + supervisor_tx, + 1, + )) + .unwrap(); + }); + // The refresh ticks from the stream's start; Windows holds it. + entered + .recv_timeout(Duration::from_secs(5)) + .expect("the input thread refreshes the pen"); + + let mut msg_seq = 0; + let mut send = |session_id, message: &ControlMessage| { + msg_seq += 1; + ipad.send_to(&encode_control(session_id, msg_seq, message), host) + .unwrap(); + }; + let heard = |until: Instant| { + let mut messages = Vec::new(); + let mut bytes = [0u8; 2048]; + while Instant::now() < until { + if let Ok((len, _)) = ipad.recv_from(&mut bytes) { + if let Ok((_, message)) = parse_control(&bytes[..len]) { + messages.push(message); + } } - }); + } + messages + }; + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 1, + listen_port: 0, + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_WANTS_INPUT, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "Pen refresh test".into(), + device_id: 1, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + let started = Instant::now(); + let session_id = loop { assert!( - taken.borrow().len() < SAMPLES / 2, - "the spacing must hold the backlog back" + started.elapsed() < Duration::from_secs(5), + "no HELLO answer" ); - let stop = shared.clone(); - let watch = async { - while taken.borrow().len() < SAMPLES && delivered.elapsed() < Duration::from_secs(2) - { - tokio::time::sleep(Duration::from_millis(1)).await; - } - stop.stop(); - }; - let (sent, ()) = tokio::join!( - start_sender(nal_rx, audio_rx, 0, shared, supervisor_tx), - watch + send(0, &hello); + let accepted = heard(Instant::now() + Duration::from_millis(200)) + .into_iter() + .find_map(|message| match message { + ControlMessage::HelloAck(ack) if ack.status == HelloStatus::Ok => { + Some(ack.session_id) + } + _ => None, + }); + if let Some(id) = accepted { + break id; + } + }; + + // The iPad keeps typing while Windows holds the refresh; the loop + // keeps its heartbeats going. + let mut watch = beat::Watch::default(); + let mut verdicts = Vec::new(); + let mut heartbeats = 0; + let until = Instant::now() + Duration::from_secs(3); + let mut event_id = 0; + while Instant::now() < until { + event_id += 1; + send( + session_id, + &ControlMessage::InputEvent(InputEvent { + input_ver: 1, + kind: KIND_KEY, + phase: PHASE_BEGAN, + buttons: 0, + event_id, + x_norm: 0, + y_norm: 0, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: 0x01, + modifiers: 0, + client_time_us: 0, + tilt_x: 0, + tilt_y: 0, + }), ); - sent.unwrap(); - (delivered, video.join().unwrap()) - }); - let taken = taken.borrow(); - let order: Vec<_> = taken.iter().map(|(frame, _)| frame.x_norm).collect(); - assert_eq!(order, (0..SAMPLES as u16).collect::>()); - let drained = taken.last().unwrap().1 - delivered; - // 58 held frames 250 µs apart take 15 ms; the 50 ms tick alone - // would take 500 ms. - assert!(drained < Duration::from_millis(250), "{drained:?}"); - // The loop kept taking video. - assert!(queued > (VIDEO_QUEUE + SAMPLES) as u64, "{queued}"); + heartbeats += heard(Instant::now() + Duration::from_millis(100)) + .iter() + .filter(|message| matches!(message, ControlMessage::Heartbeat(_))) + .count(); + verdicts.extend(watch.check( + shared.transport_beat.read(), + crate::clock::host_now_us() / 1000, + )); + } + let stuck = heartbeats < 2 || !verdicts.is_empty(); + if stuck { + // Let a stuck loop go, so the test ends. + let _ = go.send(()); + } + assert!( + heartbeats >= 2, + "heartbeats while Windows held the refresh: {heartbeats}" + ); + assert_eq!(verdicts, []); + shared.stop(); + drop(nal_tx); + transport.join().unwrap(); + let _ = go.send(()); } #[test] @@ -1583,6 +2520,7 @@ mod tests { .with_writer(move || Sink(std::sync::Arc::clone(&writer))) .finish(); let error = std::io::Error::from(std::io::ErrorKind::ConnectionReset); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { log_send_failed(LinkId::Usb { device_id: 3 }, &error); log_send_failed(LinkId::Usb { device_id: 3 }, &error); diff --git a/host/src/transport/pacer.rs b/host/src/transport/pacer.rs index a44643c9..da28366b 100644 --- a/host/src/transport/pacer.rs +++ b/host/src/transport/pacer.rs @@ -34,22 +34,26 @@ impl PacingClock { } pub async fn wait(&self, duration: Duration) { + self.until(Instant::now() + duration.min(MAX_SPREAD)).await; + } + + /// Waits until `deadline`, or not at all if a wait is already queued. + pub async fn until(&self, deadline: Instant) { + #[cfg(feature = "test-hooks")] + super::probe::waited(deadline); let (done, ready) = tokio::sync::oneshot::channel(); - if self - .tx - .try_send((Instant::now() + duration.min(MAX_SPREAD), done)) - .is_ok() - { + if self.tx.try_send((deadline, done)).is_ok() { let _ = ready.await; } } } /// Wakes the transport at a deadline from a dedicated thread, as -/// `PacingClock` does, for pen frames held back to keep them -/// `FRAME_SPACING` apart. The transport arms it at the top of each loop and -/// selects on `wait`; arming the same deadline again sends nothing, and a -/// superseded deadline only costs an extra flush. +/// `PacingClock` does, for a pointer position waiting for the next frame +/// budget (Tokio's timer can wake 15 ms late on Windows). The transport arms +/// it at the top of each loop and selects on `wait`; arming the same +/// deadline again sends nothing, and the thread serves deadlines in order, +/// so only ones that never move earlier are armed. pub struct Deadline { tx: std::sync::mpsc::Sender, due: std::sync::Arc, @@ -165,21 +169,6 @@ mod tests { ready.await.unwrap(); // dropping the clock never strands a pending wait } - #[tokio::test] - async fn a_deadline_wakes_the_runtime_once_it_has_passed() { - let mut deadline = Deadline::new("test-deadline").unwrap(); - let started = Instant::now(); - let at = started + Duration::from_micros(250); - deadline.arm(Some(at)); - // Arming it again does not queue a second wake. - deadline.arm(Some(at)); - assert!(deadline.armed()); - deadline.wait().await; - assert!(Instant::now() >= at); - assert!(started.elapsed() < Duration::from_secs(1)); - assert!(!deadline.armed()); - } - #[test] fn small_frames_are_never_paced() { let now = Instant::now(); diff --git a/host/src/transport/pointer.rs b/host/src/transport/pointer.rs new file mode 100644 index 00000000..8002e847 --- /dev/null +++ b/host/src/transport/pointer.rs @@ -0,0 +1,681 @@ +//! Decides when the pointer goes to an iPad that draws it (see +//! `capture::pointer`). Pure logic with an injected clock; the transport +//! loop feeds it the latest image and position and sends what it returns. +//! +//! - A position goes at most once a frame budget, the newest one, so a +//! 1000 Hz mouse costs no more than the video would. The budgets are a +//! fixed grid from the session's first send, not counted from the last +//! send: a send that ran late would otherwise delay every later one by +//! as much. One that waits for the next budget goes the moment it starts. +//! - The last position goes once more [`REPEAT_AFTER`] later, and every +//! heartbeat, so a lost one is replaced without the pointer moving. +//! - An image goes when it changes, when a session starts, with every IDR +//! the session asks for, and again when a report the iPad sent +//! [`SHAPE_RESEND_AFTER`] or more after it went still names another one. +//! - An image's chunks resume where the link last refused one (the USB +//! tunnel's control queue holds 64), [`SHAPE_RETRY`] later. It counts as +//! sent once every chunk was accepted. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use eternal_wire::v2::control::{PointerState, POINTER_VISIBLE}; + +use crate::capture::pointer::{Position, Shape}; + +pub const REPEAT_AFTER: Duration = Duration::from_millis(100); +/// Long enough for the image to reach the iPad before a report it sends, +/// so a report already on its way doesn't send the image twice. +pub const SHAPE_RESEND_AFTER: Duration = Duration::from_millis(250); +/// How long a refused image chunk waits before it is offered again. +pub const SHAPE_RETRY: Duration = Duration::from_millis(5); + +#[derive(Debug, PartialEq, Eq)] +pub enum Send { + /// The image, from this chunk on: report what went with + /// [`PointerSender::shape_progress`]. + Shape(Arc, usize), + State(PointerState), +} + +#[derive(Default)] +pub struct PointerSender { + session_id: Option, + /// The last state sent, without its time, and when. + sent: Option<(PointerState, Instant)>, + /// Where the grid of frame budgets starts. + epoch: Option, + repeat_at: Option, + /// The image sent last, whole, and when. + shape_sent: Option<(u32, Instant)>, + /// The image being sent, and its next chunk. + pending: Option<(Arc, usize)>, + /// A chunk was refused: offer the image again from then. + retry_at: Option, + shape_due: bool, + /// The image the iPad's last report said it holds, and when it came. + client_shape: Option<(u32, Instant)>, + /// A visible position went this session. + shown: bool, +} + +impl PointerSender { + /// The session the pointer goes to, or `None`. A new one gets + /// everything again. + pub fn serve(&mut self, session_id: Option) { + if self.session_id != session_id { + *self = Self { + session_id, + shape_due: true, + ..Self::default() + }; + } + } + + pub fn session_id(&self) -> Option { + self.session_id + } + + /// The session's IDR: send the image with it, and the position. + pub fn keyframe(&mut self, now: Instant) { + self.shape_due = true; + self.refresh(now); + } + + /// Send the current position again at the next poll. + pub fn refresh(&mut self, now: Instant) { + self.repeat_at = Some(now); + } + + /// Capture withholds the pointer (`PointerFeed::set_withheld`): no + /// image goes, and only hidden positions to an iPad that was shown + /// one this session. Whether to poll at all. + pub fn withhold(&mut self) -> bool { + self.pending = None; + self.retry_at = None; + self.shown + } + + pub fn report(&mut self, pointer_shape_id: u32, now: Instant) { + self.client_shape = Some((pointer_shape_id, now)); + } + + /// What to send now, the image before the position that names it. + pub fn poll( + &mut self, + now: Instant, + budget: Duration, + (shape, position): (Option>, Option), + ) -> Vec { + let mut out = Vec::new(); + if self.session_id.is_none() { + return out; + } + if let Some(shape) = shape.as_ref() { + let changed = self.shape_sent.is_none_or(|(id, _)| id != shape.id); + let lagging = match (self.client_shape, self.shape_sent) { + (Some((held, reported)), Some((_, sent))) => { + held != shape.id && reported >= sent + SHAPE_RESEND_AFTER + } + _ => false, + }; + if changed || lagging || self.shape_due { + self.shape_due = false; + // Sending it already: carry on from where it got to. + if self + .pending + .as_ref() + .is_none_or(|(sending, _)| sending.id != shape.id) + { + self.pending = Some((Arc::clone(shape), 0)); + self.retry_at = None; + } + } + } + if let Some((sending, next)) = self.pending.as_ref() { + if self.retry_at.is_none_or(|at| now >= at) { + out.push(Send::Shape(Arc::clone(sending), *next)); + } + } + let Some(position) = position else { + return out; + }; + let state = PointerState { + flags: if position.visible { POINTER_VISIBLE } else { 0 }, + shape_id: shape.as_ref().map_or(0, |shape| shape.id), + x: position.x, + y: position.y, + host_time_us: 0, + }; + let epoch = *self.epoch.get_or_insert(now); + let slot = + |at: Instant| at.saturating_duration_since(epoch).as_nanos() / budget.as_nanos().max(1); + let send = match self.sent { + None => true, + Some((sent, at)) if sent != state => slot(now) > slot(at), + Some((_, at)) => self.repeat_at.is_some_and(|due| now >= due) && slot(now) > slot(at), + }; + if send { + // A repeat follows a change, not another repeat. + let changed = self.sent.is_none_or(|(sent, _)| sent != state); + self.repeat_at = changed.then_some(now + REPEAT_AFTER); + self.sent = Some((state, now)); + self.shown |= position.visible; + out.push(Send::State(PointerState { + host_time_us: position.host_time_us, + ..state + })); + } + out + } + + /// The transport sent the pending image's chunks up to `next` of + /// `total`; `refused` when the link refused chunk `next`. The image is + /// sent once every chunk was. + pub fn shape_progress( + &mut self, + id: u32, + next: usize, + total: usize, + refused: bool, + now: Instant, + ) { + let Some((sending, at)) = self.pending.as_mut() else { + return; + }; + if sending.id != id { + return; + } + if next >= total { + self.pending = None; + self.retry_at = None; + self.shape_sent = Some((id, now)); + return; + } + *at = next; + self.retry_at = refused.then_some(now + SHAPE_RETRY); + } + + /// When [`Self::poll`] next has something to send without new input: + /// the start of the next budget, for a newer position that waits for + /// it, or a repeat. + pub fn next_poll( + &self, + budget: Duration, + latest: Option, + shape_id: Option, + ) -> Option { + [ + self.budget_end(budget, latest, shape_id), + self.repeat_due(budget), + ] + .into_iter() + .flatten() + .min() + } + + /// The start of the next budget, while a newer position waits for it. + /// At most one budget away, and later than the one before: the loop + /// waits for it on a high-resolution timer. + pub fn budget_end( + &self, + budget: Duration, + latest: Option, + shape_id: Option, + ) -> Option { + let (sent, _) = self.sent?; + let waiting = latest.is_some_and(|position| { + ( + position.visible, + position.x, + position.y, + shape_id.unwrap_or(0), + ) != ( + sent.flags & POINTER_VISIBLE != 0, + sent.x, + sent.y, + sent.shape_id, + ) + }); + waiting.then(|| self.next_slot(budget)).flatten() + } + + /// When the last position goes again, no earlier than the next budget, + /// or a refused image chunk is offered again. + pub fn repeat_due(&self, budget: Duration) -> Option { + let repeat = self + .repeat_at + .and_then(|due| Some(due.max(self.next_slot(budget)?))); + let retry = self.pending.as_ref().and(self.retry_at); + [repeat, retry].into_iter().flatten().min() + } + + fn next_slot(&self, budget: Duration) -> Option { + self.session_id?; + let (_, at) = self.sent?; + let epoch = self.epoch?; + let budget = budget.max(Duration::from_nanos(1)); + let slot = at.saturating_duration_since(epoch).as_nanos() / budget.as_nanos(); + Some(epoch + budget * (slot as u32 + 1)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capture::pointer::PointerImage; + + const BUDGET: Duration = Duration::from_micros(8_333); + + fn shape(id: u32) -> Arc { + Arc::new(Shape { + id, + image: PointerImage { + source_type: 2, + flags: 0, + width: 1, + height: 1, + hot: (0, 0), + bytes: vec![0; 4], + }, + }) + } + + fn at(x: i32) -> Position { + Position { + visible: true, + x, + y: 0, + host_time_us: x as u64, + } + } + + /// Polls with a link that takes every chunk. + fn poll_all( + sender: &mut PointerSender, + now: Instant, + budget: Duration, + latest: (Option>, Option), + ) -> Vec { + let sent = sender.poll(now, budget, latest); + for send in &sent { + if let Send::Shape(shape, _) = send { + sender.shape_progress(shape.id, 1, 1, false, now); + } + } + sent + } + + fn states(sent: &[Send]) -> Vec { + sent.iter() + .filter_map(|send| match send { + Send::State(state) => Some(state.x), + Send::Shape(..) => None, + }) + .collect() + } + + /// Windows' updates at irregular times: a burst 1 ms apart, then gaps of + /// 30 to 60 ms, as the #183 measurements fed the host. + #[test] + fn positions_go_at_most_once_a_budget_newest_first_and_none_is_lost_at_the_end() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + let mut updates = Vec::new(); + let mut t = 0u64; + let mut x = 0; + for gap in [30, 45, 60, 38] { + for _ in 0..4 { + updates.push((t, x)); + t += 1; + x += 1; + } + t += gap; + } + // Poll at every update and every millisecond between, as the loop's + // wakeups would. + let mut sent: Vec<(u64, i32)> = Vec::new(); + let mut latest = None; + let mut last_end: Option = None; + let mut next = updates.iter().peekable(); + for ms in 0..=t + 200 { + while next.peek().is_some_and(|(at_ms, _)| *at_ms == ms) { + latest = Some(at(next.next().unwrap().1)); + } + let now = start + Duration::from_millis(ms); + for x in states(&poll_all( + &mut sender, + now, + BUDGET, + (Some(shape(1)), latest), + )) { + sent.push((ms, x)); + } + // What the loop's high-resolution timer waits for: within one + // budget, and never earlier than the end it waited for before. + if let Some(end) = sender.budget_end(BUDGET, latest, Some(1)) { + assert!(end > now && end <= now + BUDGET, "{ms} ms"); + assert!(last_end.is_none_or(|last| end >= last), "{ms} ms"); + last_end = Some(end); + } + } + let slot = |ms: u64| Duration::from_millis(ms).as_nanos() / BUDGET.as_nanos(); + for pair in sent.windows(2) { + assert!( + slot(pair[1].0) > slot(pair[0].0), + "two sends in one budget: {pair:?}" + ); + } + // Each burst's last position goes once its budget ends, not at the + // next burst; the burst's first went at once. + for burst in 0..4 { + let first = updates[burst * 4]; + let last = updates[burst * 4 + 3]; + assert!( + sent.contains(&first), + "{first:?} not sent at once: {sent:?}" + ); + let went = sent.iter().find(|s| s.1 == last.1).expect("burst end sent"); + assert!(went.0 - last.0 <= 9, "{last:?} waited until {went:?}"); + } + // The very last position goes once more after REPEAT_AFTER, then + // nothing until a heartbeat refresh. + let final_x = updates.last().unwrap().1; + assert_eq!( + sent.iter().filter(|s| s.1 == final_x).count(), + 2, + "{sent:?}" + ); + } + + /// A mouse reporting about once a budget, a little early or late each + /// time (a 125 Hz mouse at 120 fps). A late send must not delay the + /// sends after it: counted from the last send, every later position + /// waited for the lag of the latest one. + #[test] + fn positions_about_once_a_budget_go_without_a_growing_lag() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + let mut seed = 0x2545_F491u32; + let mut delays = Vec::new(); + let period_us = BUDGET.as_micros() as u64; + for k in 0..600u64 { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + let arrived = k * period_us + u64::from(seed % 1500); + let latest = Some(at(k as i32)); + // Poll at arrival, then every 100 us until it went. + let mut t = arrived; + loop { + let now = start + Duration::from_micros(t); + if states(&poll_all( + &mut sender, + now, + BUDGET, + (Some(shape(1)), latest), + )) + .contains(&(k as i32)) + { + delays.push(t - arrived); + break; + } + t += 100; + assert!(t < arrived + 2 * period_us, "position {k} never went"); + } + } + delays.sort_unstable(); + let mean = delays.iter().sum::() / delays.len() as u64; + let p90 = delays[delays.len() * 9 / 10]; + assert!(mean < 1_000 && p90 < 2_000, "mean {mean} us, p90 {p90} us"); + } + + #[test] + fn a_still_pointer_sends_nothing_until_a_refresh() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + let latest = (Some(shape(1)), Some(at(5))); + assert_eq!( + poll_all(&mut sender, start, BUDGET, latest.clone()).len(), + 2 + ); + assert_eq!( + states(&poll_all( + &mut sender, + start + REPEAT_AFTER, + BUDGET, + latest.clone() + )), + [5] + ); + for ms in 101..2000 { + let now = start + Duration::from_millis(ms); + assert!( + poll_all(&mut sender, now, BUDGET, latest.clone()).is_empty(), + "{ms} ms" + ); + assert_eq!(sender.next_poll(BUDGET, latest.1, Some(1)), None); + } + let now = start + Duration::from_secs(2); + sender.refresh(now); + assert_eq!(sender.next_poll(BUDGET, latest.1, Some(1)), Some(now)); + assert_eq!(states(&poll_all(&mut sender, now, BUDGET, latest)), [5]); + } + + /// Review r4: where capture can't tell where Windows puts the pointer, + /// an iPad that was never shown one gets no pointer messages, and one + /// that was (the mirror before a second screen) only hidden positions. + #[test] + fn a_withheld_pointer_sends_nothing_but_a_hidden_state_to_clear_one() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + assert!(!sender.withhold(), "never shown: nothing to send"); + let hidden = Position { + visible: false, + ..at(5) + }; + poll_all(&mut sender, start, BUDGET, (None, Some(hidden))); + assert!(!sender.withhold(), "a hidden position shows nothing"); + + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + let sent = sender.poll(start, BUDGET, (Some(shape(1)), Some(at(5)))); + assert!(matches!(sent[0], Send::Shape(_, 0)), "{sent:?}"); + assert!(sender.withhold(), "shown: the hidden state is owed"); + let later = start + BUDGET; + let sent = poll_all(&mut sender, later, BUDGET, (None, Some(hidden))); + assert_eq!( + sent, + [Send::State(PointerState { + flags: 0, + shape_id: 0, + x: 5, + y: 0, + host_time_us: 5, + })], + "no image, not even the one under way" + ); + } + + #[test] + fn hiding_the_pointer_is_a_change_that_goes() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + poll_all(&mut sender, start, BUDGET, (None, Some(at(5)))); + let hidden = Position { + visible: false, + ..at(5) + }; + assert_eq!( + sender.next_poll(BUDGET, Some(hidden), None), + Some(start + BUDGET) + ); + let sent = poll_all(&mut sender, start + BUDGET, BUDGET, (None, Some(hidden))); + assert!(matches!(sent[..], [Send::State(state)] if state.flags & POINTER_VISIBLE == 0)); + } + + #[test] + fn images_go_on_change_session_start_keyframes_and_when_a_report_lags() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + let shapes = |sent: &[Send]| { + sent.iter() + .filter_map(|send| match send { + Send::Shape(shape, _) => Some(shape.id), + Send::State(_) => None, + }) + .collect::>() + }; + assert!( + sender + .poll(start, BUDGET, (Some(shape(1)), None)) + .is_empty(), + "no session" + ); + sender.serve(Some(7)); + assert_eq!( + shapes(&poll_all( + &mut sender, + start, + BUDGET, + (Some(shape(1)), None) + )), + [1] + ); + assert!(sender + .poll(start, BUDGET, (Some(shape(1)), None)) + .is_empty()); + assert_eq!( + shapes(&poll_all( + &mut sender, + start, + BUDGET, + (Some(shape(2)), None) + )), + [2] + ); + // A report sent before image 2 could arrive names image 1: no resend. + let soon = start + SHAPE_RESEND_AFTER - Duration::from_millis(1); + sender.report(1, soon); + assert!(poll_all(&mut sender, soon, BUDGET, (Some(shape(2)), None)).is_empty()); + // A later one still names it: image 2 was lost on the way. + let later = start + SHAPE_RESEND_AFTER; + sender.report(1, later); + assert_eq!( + shapes(&poll_all( + &mut sender, + later, + BUDGET, + (Some(shape(2)), None) + )), + [2] + ); + assert!( + sender + .poll(later, BUDGET, (Some(shape(2)), None)) + .is_empty(), + "once a report" + ); + let next = later + Duration::from_millis(500); + sender.report(2, next); + assert!(poll_all(&mut sender, next, BUDGET, (Some(shape(2)), None)).is_empty()); + let much_later = later + Duration::from_secs(5); + assert!(sender + .poll(much_later, BUDGET, (Some(shape(2)), None)) + .is_empty()); + sender.keyframe(much_later); + assert_eq!( + shapes(&poll_all( + &mut sender, + much_later, + BUDGET, + (Some(shape(2)), None) + )), + [2] + ); + // A new session gets the image and position again. + sender.serve(Some(8)); + let sent = poll_all( + &mut sender, + much_later, + BUDGET, + (Some(shape(2)), Some(at(1))), + ); + assert_eq!(shapes(&sent), [2]); + assert_eq!(states(&sent), [1]); + sender.serve(None); + assert!(sender + .poll(much_later, BUDGET, (Some(shape(3)), Some(at(2)))) + .is_empty()); + } + + /// Review r2: over USB the tunnel's control queue takes 64 datagrams and + /// refuses the rest. An image of more chunks than that resumes where the + /// link refused it, also when it is due again meanwhile, and counts as + /// sent only once every chunk went. + #[test] + fn a_refused_image_resumes_where_it_stopped_and_is_sent_only_when_whole() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + let total = 194; + let big = shape(1); + let offered = |sent: &[Send]| { + sent.iter().find_map(|send| match send { + Send::Shape(shape, next) => Some((shape.id, *next)), + Send::State(_) => None, + }) + }; + assert_eq!( + offered(&sender.poll(start, BUDGET, (Some(Arc::clone(&big)), None))), + Some((1, 0)) + ); + // The queue takes 64, then refuses. + sender.shape_progress(1, 64, total, true, start); + assert_eq!(sender.repeat_due(BUDGET), Some(start + SHAPE_RETRY)); + let soon = start + SHAPE_RETRY / 2; + assert_eq!( + offered(&sender.poll(soon, BUDGET, (Some(Arc::clone(&big)), None))), + None, + "waits" + ); + // A keyframe asks for it again: it carries on, it doesn't restart. + sender.keyframe(soon); + let retry = start + SHAPE_RETRY; + assert_eq!( + offered(&sender.poll(retry, BUDGET, (Some(Arc::clone(&big)), None))), + Some((1, 64)) + ); + sender.shape_progress(1, 128, total, true, retry); + // Not sent yet: a report naming nothing doesn't resend from 0. + sender.report(0, retry + SHAPE_RESEND_AFTER); + let later = retry + SHAPE_RETRY; + assert_eq!( + offered(&sender.poll(later, BUDGET, (Some(Arc::clone(&big)), None))), + Some((1, 128)) + ); + sender.shape_progress(1, total, total, false, later); + assert_eq!( + offered(&sender.poll(later, BUDGET, (Some(Arc::clone(&big)), None))), + None, + "whole" + ); + assert_eq!(sender.repeat_due(BUDGET), None); + // A newer image replaces one still going. + let next = later + SHAPE_RETRY; + assert_eq!( + offered(&sender.poll(next, BUDGET, (Some(shape(2)), None))), + Some((2, 0)) + ); + sender.shape_progress(2, 10, total, true, next); + assert_eq!( + offered(&sender.poll(next + SHAPE_RETRY, BUDGET, (Some(shape(3)), None))), + Some((3, 0)) + ); + } +} diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index b6ef5504..b5d6fb3c 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -13,8 +13,10 @@ use super::link::{LinkId, PeerId}; use std::time::{Duration, Instant}; use eternal_wire::v2::control::{ - ByeReason, ControlHeader, ControlMessage, HelloAck, HelloStatus, InputEvent, KeyframeRequest, - Nack, ReceiverReport, StreamConfig, FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_NACK, + ByeReason, ControlHeader, ControlMessage, FailedReason, HelloAck, HelloStatus, InputEvent, + KeyframeRequest, Nack, ReceiverReport, StreamConfig, FEATURE_MEDIA_SIZE, FEATURE_POWER_BYE, + FEATURE_STANDBY_STATUS, FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_KEYFRAME_PER_LOSS, + HOSTCAP_MEDIA_SIZE, HOSTCAP_NACK, MIN_MEDIA_DGRAM, }; use tracing::{info, warn}; @@ -30,6 +32,13 @@ pub const LIVENESS_TIMEOUT: Duration = Duration::from_millis(3000); /// 10 s is longer than an iPad waits for a silent PC (3 s, then 5 s more) /// and than the uplink stalls of a congested Wi-Fi channel. pub const HOLD_LIMIT: Duration = Duration::from_secs(10); + +/// How long the host stays closed to new clients after Windows said it is +/// going to sleep or shutting down ([`Session::power_down`]), unless it says +/// first that it woke or that the shutdown was cancelled. A suspend that +/// fails without a resume message, or a missed one, can't leave the PC +/// unreachable for longer. +pub const POWER_DOWN_HOLD: Duration = Duration::from_secs(30); const USB_TAKEOVER_TIMEOUT: Duration = Duration::from_secs(1); /// How long the late packets of a session ended for liveness get its /// goodbye again, at most one copy per [`ENDED_REPLY_GAP`]. @@ -53,6 +62,51 @@ const REFUSED_FOR: Duration = Duration::from_secs(60); const REFUSED_SUMMARIES: u32 = 20; const REFUSAL_MINUTE: Duration = Duration::from_secs(60); +/// IDENTIFY answers per window: for one asking address, and in all. An +/// iPad's check sends at most 8 and stops at the first answer. +const IDENTIFY_WINDOW: Duration = Duration::from_secs(10); +const IDENTIFY_PER_SOURCE: u32 = 8; +const IDENTIFY_OVERALL: u32 = 32; +/// Asking addresses tracked in a window; more get no answer until it ends. +const IDENTIFY_SOURCES: usize = 16; + +/// Bounds what IDENTIFY costs: each answer is an HMAC and a datagram, and +/// anyone on the network may ask. Only answers count: a request left +/// unanswered for want of addresses to sign uses up nothing. +#[derive(Debug, Default)] +struct IdentifyLimiter { + overall: Option<(Instant, u32)>, + sources: Vec<(std::net::IpAddr, Instant, u32)>, +} + +impl IdentifyLimiter { + /// Whether `ip` may be answered now, counting the answer if so. + fn allow(&mut self, ip: std::net::IpAddr, now: Instant) -> bool { + let over = |since: Instant| now.saturating_duration_since(since) >= IDENTIFY_WINDOW; + self.overall.take_if(|(since, _)| over(*since)); + self.sources.retain(|(_, since, _)| !over(*since)); + let overall = self.overall.get_or_insert((now, 0)); + if overall.1 >= IDENTIFY_OVERALL { + return false; + } + let index = match self.sources.iter().position(|(source, ..)| *source == ip) { + Some(index) => index, + None if self.sources.len() < IDENTIFY_SOURCES => { + self.sources.push((ip, now, 0)); + self.sources.len() - 1 + } + None => return false, + }; + let source = &mut self.sources[index]; + if source.2 >= IDENTIFY_PER_SOURCE { + return false; + } + source.2 += 1; + overall.1 += 1; + true + } +} + /// The refused attempts of the current minute. #[derive(Debug, Clone, Copy)] struct RefusalMinute { @@ -60,8 +114,49 @@ struct RefusalMinute { summarized: u32, left_out: u32, } -/// Honor at most one keyframe request per this window (PLI storm guard). +/// Honor at most one keyframe request per this window (PLI storm guard), +/// besides one for each loss after a recovery ([`ActiveSession::recovered`]). pub const KEYFRAME_REQUEST_MIN_INTERVAL: Duration = Duration::from_millis(500); +/// Smaller media sizes per session whose keyframe skips that window: the +/// iPad asks for two (1,200 then 576 bytes). Any later one is an ordinary, +/// rate-limited keyframe request. +const MEDIA_SIZE_IDRS: u8 = 2; + +/// Media bytes, headers included, a Wi-Fi client's address gets before +/// anything of its session comes back from it: room for the first picture, +/// even a large keyframe of a high-resolution stream. Heartbeats go on. An +/// iPad pings the moment its HELLO_ACK lands, so it answers long before +/// this runs out. +pub const UNANSWERED_MEDIA_BUDGET: usize = 512 * 1024; +/// Media to every client not yet heard counts against one host-wide budget +/// of the same size, which comes back over [`UNANSWERED_FOR`]: a client +/// that moves to another address, or many addresses in turn, get no more. +/// +/// Addresses whose session ended before they answered, and for how long a +/// new session there starts with its budget spent. +const UNANSWERED_KEPT: usize = 64; +const UNANSWERED_FOR: Duration = Duration::from_secs(60); + +/// How long a session is kept alive through restarts with no picture. Past +/// this, its iPad is told to connect again: it shows why while the PC +/// keeps restarting, instead of a frozen picture. Restarts that fail +/// quickly end sooner, at the storm brake. +pub const RESTART_HOLD_LIMIT: Duration = Duration::from_secs(20); + +/// Lets integration tests reach [`RESTART_HOLD_LIMIT`] in milliseconds. +#[cfg(feature = "test-hooks")] +pub static TEST_RESTART_HOLD_MS: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(RESTART_HOLD_LIMIT.as_millis() as u64); + +#[cfg(feature = "test-hooks")] +fn restart_hold_limit() -> Duration { + Duration::from_millis(TEST_RESTART_HOLD_MS.load(std::sync::atomic::Ordering::SeqCst)) +} + +#[cfg(not(feature = "test-hooks"))] +fn restart_hold_limit() -> Duration { + RESTART_HOLD_LIMIT +} /// What the transport must do after feeding the session an event. #[derive(Debug, Default)] @@ -101,6 +196,99 @@ pub struct Actions { pub ended: Option, } +/// The stage whose failures made the PC give up, as far as the summary's +/// `failure=` names it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StageFailure { + Capture, + Encoder, + Other, +} + +/// Why the stream stops for good ([`Session::shut_down`]): what the iPad is +/// told, and how the session's summary ends. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamStop { + /// The app quits. The iPad hears that the PC ended the session. + Quit, + /// The app quits for a restart or an update that hands over to a new + /// copy: the iPad is told to connect again. + Handoff, + /// Stop on the PC. + User, + /// The PC gave up after repeated failures; the iPad may try again. + GaveUp(StageFailure), +} + +impl StreamStop { + fn bye(self) -> ByeReason { + match self { + Self::Quit | Self::User => ByeReason::UserDisconnect, + Self::Handoff | Self::GaveUp(_) => ByeReason::Error, + } + } + + fn reason(self) -> EndReason { + match self { + Self::Quit | Self::Handoff => EndReason::Shutdown, + Self::User => EndReason::Stopped, + Self::GaveUp(stage) => EndReason::Failed(stage), + } + } +} + +/// What Windows is doing to the PC ([`Session::power_down`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PowerEvent { + /// Sleep or hibernate. + Sleep, + /// Shut down or restart. + Shutdown, + /// The user signs out. + Logoff, +} + +impl PowerEvent { + /// Stable log value. + pub const fn name(self) -> &'static str { + match self { + Self::Sleep => "sleep", + Self::Shutdown => "shutdown", + Self::Logoff => "logoff", + } + } + + /// The goodbye for an iPad that understands it ([`FEATURE_POWER_BYE`]). + fn bye(self) -> ByeReason { + match self { + Self::Sleep => ByeReason::HostSleeping, + Self::Shutdown | Self::Logoff => ByeReason::HostPoweringOff, + } + } +} + +/// What [`Session::power_down`] did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PowerDown { + /// The session ended; its summary is the caller's to write. + Ended(SessionEnd), + /// Nobody was streaming. + NoSession, + /// The PC was already going down: Windows sends some of this news twice + /// (the end of the session, then the session change, for a sign-out; + /// the broadcast and the registration, for sleep). Nothing changed. + Repeat, +} + +impl PowerDown { + pub fn ended(self) -> Option { + match self { + Self::Ended(end) => Some(end), + Self::NoSession | Self::Repeat => None, + } + } +} + /// How far a session got and how it ended, for its support summary. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SessionEnd { @@ -114,6 +302,8 @@ pub struct SessionEnd { pub first_video: Option, /// The client's receiver reports counted complete frames. pub frames_received: bool, + /// The client's receiver reports counted any video fragment. + pub frags_received: bool, pub duration: Duration, } @@ -129,6 +319,10 @@ pub enum EndReason { Superseded, /// An update stopped the stream, or the app quit. Shutdown, + /// Stop on the PC. + Stopped, + /// The PC gave up restarting a stage that kept failing. + Failed(StageFailure), /// The session's link (the USB tunnel) closed. LinkLost, /// The pairing credentials changed. @@ -165,17 +359,47 @@ struct ActiveSession { client_clock: Option<(i64, Instant)>, late_input: LateInput, awaiting_takeover: bool, + /// The last IDR granted on [`KEYFRAME_REQUEST_MIN_INTERVAL`]'s schedule. last_keyframe_grant: Option, + /// The stream epoch and frame number of the first keyframe sent since + /// the last IDR was granted, on that schedule or not (see + /// [`ActiveSession::recovered`]). + keyframe_since_grant: Option<(u32, u32)>, last_report: Option, msg_seq_out: u32, /// When the first video went to this session. first_video_at: Option, /// A receiver report counted complete frames. frames_received: bool, + /// A receiver report counted any video fragment. + frags_received: bool, + /// Video frames let through to this session ([`Session::may_send_media`]). + frames_sent: u32, + /// The largest media datagram the client asked for in this session + /// ([`KeyframeRequest::max_media_dgram`]), and how many times it went + /// down. + max_media_dgram: Option, + media_size_lowered: u8, /// A Wi-Fi goodbye while the cable takes over. bye: Option, /// The msg_seq of input packets taken in this session. input_seqs: SeqWindow, + /// Something of this session came back from the client after its + /// HELLO2. Always for USB, where the PC dials the iPad on the cable. + answered: bool, + /// Media bytes sent before it answered (see [`UNANSWERED_MEDIA_BUDGET`]). + unanswered_sent: usize, + /// A frame was held back for the budget: the answer asks for a keyframe. + withheld: bool, + /// The stream config its last heartbeat carried, which heartbeats sent + /// while the PC restarts carry again. + last_stream_config: Option, + /// Since when it has been kept alive through restarts without a frame + /// ([`RESTART_HOLD_LIMIT`]): its last frame, or the restart if none. + /// Cleared by the next frame. + restart_held_since: Option, + /// When the last frame was let through to it. + last_frame_at: Option, } /// What a client sent last, for the liveness log lines. @@ -277,6 +501,30 @@ impl SeqWindow { } } +/// Why no stream runs, for an iPad that asks meanwhile. The supervisor sets +/// it whenever no pipeline generation streams (see +/// [`Session::set_not_streaming`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NotStreaming { + /// Someone clicked Stop: only Start on the PC streams again. + Stopped, + /// The supervisor is waiting out the pause before a restart. + Restarting, + /// The storm brake gave up after repeated errors, until Start. + Failed(FailedReason), +} + +/// Why a HELLO2 can't start a session ([`Session::closed_to_clients`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Closed { + /// Turned away with Busy, logged with this line. + Busy(&'static str), + /// Not answered at all, logged with this line. + Silent(&'static str), + /// Told why, once it passes the pairing check. + NotStreaming(NotStreaming), +} + /// Provides the current stream parameters for HELLO_ACK/heartbeats. pub trait ConfigSource { fn pairing(&self) -> Option<&parking_lot::Mutex> { @@ -288,6 +536,22 @@ pub trait ConfigSource { fn host_caps(&self) -> u16 { 0 } + /// The port this host receives the stream on and its home network + /// addresses, which an identity proof signs. None answers no IDENTIFY. + fn identity_endpoints(&self) -> Option<(u16, Vec<[u8; 4]>)> { + None + } + + /// The heartbeat's notice for the iPad (`NOTICE_*` in the wire crate). + fn notice(&self) -> u8 { + eternal_wire::v2::control::NOTICE_NONE + } + + /// Whether the heartbeat says now that Second screen needs Windows 11, + /// when it says nothing else. + fn needs_windows_11(&self) -> bool { + false + } } pub struct Session { @@ -298,7 +562,21 @@ pub struct Session { /// either closes the host or gives up. Changed only under the session lock, /// together with the check that nobody is streaming. updating: bool, - /// The goodbye for a stream an update stopped, sent by the next tick. + /// The stream stopped for good ([`Self::shut_down`]) and no transport + /// has resumed since: new clients are turned away, as nothing would + /// serve them. + stream_stopped: bool, + /// Windows said the PC is going down, and when ([`Self::power_down`]). + powering_down: Option<(PowerEvent, Instant)>, + /// A session [`Self::power_down`] ended without a goodbye: the next tick + /// reports the client lost, as a goodbye would. + lost_unsaid: bool, + /// The queued goodbye is [`Self::power_down`]'s. + power_bye: bool, + /// No stream runs: HELLO2s are told why instead of starting a session. + not_streaming: Option, + /// The goodbye for a stream an update or a stop ended, sent by the next + /// tick or by the transport on its way out. goodbye: Option<(PeerId, Vec)>, /// That stream's end, reported by the next tick or [`Self::shut_down`]. stopped: Option, @@ -307,9 +585,17 @@ pub struct Session { /// Recently refused HELLO2 nonces: a retransmit is the same attempt. refused: std::collections::VecDeque<(u32, Instant)>, refusal_minute: Option, - /// Injected so tests control randomness; production passes a seeded value - /// derived from process entropy. + /// Wi-Fi addresses whose session ended before they answered (see + /// [`UNANSWERED_FOR`]). + unanswered: std::collections::VecDeque<(std::net::IpAddr, Instant)>, + /// The host-wide media sent to clients before they answered, as of when + /// it was last counted (see [`UNANSWERED_MEDIA_BUDGET`]). + unanswered_media: (usize, Instant), + identify_limiter: IdentifyLimiter, + /// Counts the sessions. Session ids come from a per-launch keyed hash + /// of this count ([`Self::session_id_keys`]). next_session_id: u32, + session_id_keys: std::collections::hash_map::RandomState, legacy_client_warned: bool, } @@ -319,12 +605,21 @@ impl Session { active: None, last_end: None, updating: false, + stream_stopped: false, + powering_down: None, + lost_unsaid: false, + power_bye: false, + not_streaming: None, goodbye: None, stopped: None, ended_for_liveness: None, refused: std::collections::VecDeque::new(), refusal_minute: None, - next_session_id: session_id_seed.max(1), + unanswered: std::collections::VecDeque::new(), + unanswered_media: (0, Instant::now()), + identify_limiter: IdentifyLimiter::default(), + next_session_id: session_id_seed, + session_id_keys: std::collections::hash_map::RandomState::new(), legacy_client_warned: false, } } @@ -351,9 +646,73 @@ impl Session { fn end(&mut self, reason: EndReason, now: Instant) -> Option { let session = self.active.take()?; self.last_end = Some((session.session_id, reason)); + // One entry per address, its time refreshed. Once the memory is + // full the oldest goes; the host-wide budget still counts it. + if let Some(addr) = session.peer.addr.filter(|_| !session.answered) { + self.unanswered.retain(|(ip, at)| { + *ip != addr.ip() && now.saturating_duration_since(*at) < UNANSWERED_FOR + }); + if self.unanswered.len() == UNANSWERED_KEPT { + self.unanswered.pop_front(); + } + self.unanswered.push_back((addr.ip(), now)); + } Some(session.end(reason, now)) } + /// Whether a frame of `bytes` may go to `session_id`'s client now. Until + /// it answers, frames go while they fit in its budget; the first that + /// doesn't is held back, and so is every later one until it answers, + /// which asks for a keyframe to start the picture again. + pub fn may_send_media(&mut self, session_id: u32, bytes: usize, now: Instant) -> bool { + let host_spent = self.unanswered_media_at(now); + let Some(session) = self + .active + .as_mut() + .filter(|session| session.session_id == session_id) + else { + return true; + }; + if session.answered { + session.frames_sent = session.frames_sent.saturating_add(1); + session.restart_held_since = None; + session.last_frame_at = Some(now); + return true; + } + if session.withheld + || session.unanswered_sent + bytes > UNANSWERED_MEDIA_BUDGET + || host_spent + bytes > UNANSWERED_MEDIA_BUDGET + { + session.withheld = true; + return false; + } + // Only a frame that goes out ends a hold through a restart. + session.restart_held_since = None; + session.last_frame_at = Some(now); + session.unanswered_sent += bytes; + session.frames_sent = session.frames_sent.saturating_add(1); + self.unanswered_media = (host_spent + bytes, now); + true + } + + /// The largest media datagram `session_id`'s client asked for, if it + /// asked for one. + pub fn max_media_dgram(&self, session_id: u32) -> Option { + self.active + .as_ref() + .filter(|session| session.session_id == session_id) + .and_then(|session| session.max_media_dgram) + } + + /// The host-wide media to clients not yet heard that still counts at + /// `now`. + fn unanswered_media_at(&self, now: Instant) -> usize { + let (spent, at) = self.unanswered_media; + let back = UNANSWERED_MEDIA_BUDGET as u128 * now.saturating_duration_since(at).as_millis() + / UNANSWERED_FOR.as_millis(); + spent.saturating_sub(usize::try_from(back).unwrap_or(usize::MAX)) + } + /// Starts an update handoff. Atomic with admission: call with the session /// lock held; a client either got in before (and this returns `false`) or /// is refused until [`Self::end_update`]. With `stop_stream` (the user @@ -364,37 +723,147 @@ impl Session { if !stop_stream { return false; } - let (peer, session_id) = (session.peer, session.session_id); - info!(%peer, "Stopping the stream for an update"); - let msg_seq = self.next_msg_seq(); - let bye = ControlMessage::Bye(ByeReason::HostShuttingDown); - self.goodbye = Some(( - peer, - eternal_wire::v2::control::encode_control(session_id, msg_seq, &bye), - )); - self.stopped = self.end(EndReason::Shutdown, Instant::now()); + info!(peer = %session.peer, "Stopping the stream for an update"); + self.stopped = self.end_saying( + ByeReason::HostShuttingDown, + EndReason::Shutdown, + Instant::now(), + ); } self.updating = true; true } - /// The goodbye [`Self::begin_update`] left for a stopped stream, if any. + /// Ends the active session for `reason` and leaves `bye` for the + /// transport to send before its links close. + fn end_saying( + &mut self, + bye: ByeReason, + reason: EndReason, + now: Instant, + ) -> Option { + let session = self.active.as_ref()?; + let (peer, session_id) = (session.peer, session.session_id); + let msg_seq = self.next_msg_seq(); + self.power_bye = false; + self.goodbye = Some(( + peer, + eternal_wire::v2::control::encode_control( + session_id, + msg_seq, + &ControlMessage::Bye(bye), + ), + )); + self.end(reason, now) + } + + /// The goodbye left for a stopped stream ([`Self::begin_update`], + /// [`Self::shut_down`]), if any. pub fn take_goodbye(&mut self) -> Option<(PeerId, Vec)> { + self.power_bye = false; self.goodbye.take() } + /// Whether a stream runs, and why not. While it is set, a HELLO2 that + /// passes the version, update and pairing checks is told why, and no + /// session starts. A session the stream keeps across a restart stays. + pub fn set_not_streaming(&mut self, state: Option) { + self.not_streaming = state; + } + + pub fn not_streaming(&self) -> Option { + self.not_streaming + } + /// The update handoff failed; clients may connect again. pub fn end_update(&mut self) { self.updating = false; } - /// The app is quitting: the end of the stream it stops, or of one an - /// update stopped that no tick reported yet. - pub fn shut_down(&mut self, now: Instant) -> Option { - let active = self.end(EndReason::Shutdown, now); + /// The stream stops for good, for `why`: the end of the session it + /// stops, or of one an update stopped that no tick reported yet. A + /// session it stops is told why by the transport on its way out; the + /// iPad used to hear nothing, and showed a lost signal or an unplugged + /// cable. Until a transport resumes, new clients are turned away. + pub fn shut_down(&mut self, now: Instant, why: StreamStop) -> Option { + self.stream_stopped = true; + let active = self.end_saying(why.bye(), why.reason(), now); self.stopped.take().or(active) } + /// Windows said the PC is going to sleep, shutting down or signing out. + /// The session ends now, and its summary is returned for the caller to + /// write while the PC is still up. An iPad that understands it + /// ([`FEATURE_POWER_BYE`]) is told why by the transport's next pass; an + /// older one is told nothing, as before, and notices the silence. Until + /// [`Self::power_up`] or [`POWER_DOWN_HOLD`], HELLO2s go unanswered, as + /// they would a moment later from a PC that is down. The stream keeps + /// running, so the PC streams again when it wakes. + pub fn power_down(&mut self, now: Instant, event: PowerEvent) -> PowerDown { + if self.going_down(now) { + return PowerDown::Repeat; + } + self.powering_down = Some((event, now)); + let Some(session) = self.active.as_ref() else { + return PowerDown::NoSession; + }; + let end = if session.accepted_hello.feature_caps & FEATURE_POWER_BYE != 0 { + let end = self.end_saying(event.bye(), EndReason::Shutdown, now); + self.power_bye = true; + end + } else { + self.lost_unsaid = true; + self.end(EndReason::Shutdown, now) + }; + end.map_or(PowerDown::NoSession, PowerDown::Ended) + } + + /// Windows said the PC is going down, and hasn't said since that it is + /// back, within [`POWER_DOWN_HOLD`]. + fn going_down(&self, now: Instant) -> bool { + self.powering_down + .is_some_and(|(_, since)| now.saturating_duration_since(since) < POWER_DOWN_HOLD) + } + + /// A goodbye waits for the transport. + pub fn goodbye_pending(&self) -> bool { + self.goodbye.is_some() + } + + /// A goodbye [`Self::power_down`] left waits for the transport. + pub fn power_goodbye_pending(&self) -> bool { + self.power_bye + } + + /// The PC woke, or the shutdown was called off: clients may connect + /// again. A goodbye no transport sent before the PC went down is for a + /// session long over, and is dropped. + pub fn power_up(&mut self) { + self.powering_down = None; + if std::mem::take(&mut self.power_bye) { + self.goodbye = None; + } + } + + /// Why a HELLO2 can't start a session now, if it can't: an update is + /// about to close the host, no stream runs (the supervisor says why), or + /// the stream stopped and its transport is on its way out (the app + /// quits). The first and last are turned away with Busy before anything + /// else is looked at; a retry once the stream runs again works. + fn closed_to_clients(&self, now: Instant) -> Option { + if self.going_down(now) { + Some(Closed::Silent("Client unanswered while the PC goes down")) + } else if self.updating { + Some(Closed::Busy("Client turned away while an update starts")) + } else if let Some(state) = self.not_streaming { + Some(Closed::NotStreaming(state)) + } else if self.stream_stopped { + Some(Closed::Busy("Client turned away while the stream stops")) + } else { + None + } + } + /// The first video went to `session_id`. Later calls change nothing. pub fn note_video_sent(&mut self, session_id: u32, now: Instant) { if let Some(session) = self.active.as_mut() { @@ -404,10 +873,27 @@ impl Session { } } + /// A keyframe went to `session_id`'s client as frame `frame_seq` of + /// `stream_epoch`. + pub fn note_keyframe_sent(&mut self, session_id: u32, stream_epoch: u32, frame_seq: u32) { + if let Some(session) = self.active.as_mut() { + if session.session_id == session_id { + session + .keyframe_since_grant + .get_or_insert((stream_epoch, frame_seq)); + } + } + } + pub fn session_id(&self) -> Option { self.active.as_ref().map(|s| s.session_id) } + /// Changes each time a session starts: a count, from a random start. + pub fn sessions_started(&self) -> u32 { + self.next_session_id + } + pub fn link_lost(&mut self, link: LinkId) -> Actions { let on_link = self .active @@ -452,20 +938,64 @@ impl Session { .as_ref() .filter(|session| { !session.awaiting_takeover + && session.answered && session.info.feature_caps & eternal_wire::v2::control::FEATURE_WANTS_AUDIO != 0 }) .map(|session| session.session_id) } + /// The session's id while its client draws the pointer itself, from + /// when it has answered: until then nothing confirms it received its + /// HELLO_ACK, so it gets no pointer messages (an image can be hundreds + /// of kilobytes), and capture keeps the pointer in the video for it, as + /// for any other client. Pointer messages start once the iPad answers; + /// capture switches then, once, and never again in the session. + pub fn pointer_session_id(&self) -> Option { + self.active + .as_ref() + .filter(|session| { + session.answered + && session.info.feature_caps & eternal_wire::v2::control::FEATURE_DRAWS_POINTER + != 0 + }) + .map(|session| session.session_id) + } + + /// `message` for the connected client, numbered like every other + /// control message the host sends it. + pub fn encode_for_client(&mut self, message: &ControlMessage) -> Option<(PeerId, Vec)> { + let (peer, session_id) = self + .active + .as_ref() + .map(|session| (session.peer, session.session_id))?; + let msg_seq = self.next_msg_seq(); + Some(( + peer, + eternal_wire::v2::control::encode_control(session_id, msg_seq, message), + )) + } + pub fn last_report(&self) -> Option { self.active.as_ref().and_then(|s| s.last_report) } fn allocate_session_id(&mut self) -> u32 { - let id = self.next_session_id; - self.next_session_id = self.next_session_id.wrapping_add(0x9E37_79B9).max(1); - id + use std::hash::BuildHasher; + let taken = [ + self.session_id(), + self.last_end.map(|(id, _)| id), + self.ended_for_liveness + .as_ref() + .map(|ended| ended.session_id), + ]; + loop { + self.next_session_id = self.next_session_id.wrapping_add(1); + let id = self.session_id_keys.hash_one(self.next_session_id) as u32; + if id != 0 && !taken.contains(&Some(id)) { + return id; + } + } } fn next_msg_seq(&mut self) -> u32 { @@ -480,18 +1010,23 @@ impl Session { /// Feed one control datagram off a link, with its header. Input whose /// msg_seq this session took already, or that is too old, is dropped. + /// + /// `datagram_len` is the datagram's size: over Wi-Fi, a refused HELLO2, + /// or one the PC answers while it isn't streaming, gets no answer larger + /// than itself. pub fn handle_datagram( &mut self, source: PeerId, header: &ControlHeader, message: ControlMessage, + datagram_len: usize, config: &impl ConfigSource, now: Instant, ) -> Actions { self.handle( source, header.session_id, - Some(header.msg_seq), + Some((header.msg_seq, datagram_len)), message, config, now, @@ -519,7 +1054,8 @@ impl Session { &mut self, source: PeerId, header_session_id: u32, - msg_seq: Option, + // The datagram's msg_seq and size, when it came off a link. + datagram: Option<(u32, usize)>, message: ControlMessage, config: &impl ConfigSource, now: Instant, @@ -532,7 +1068,10 @@ impl Session { // INPUT_EVENT moved the mouse and clicked, a stray RECEIVER_REPORT // steered the bitrate, and a late BYE from a superseded session tore // down the one that replaced it. - if !matches!(message, ControlMessage::Hello2(_)) { + if !matches!( + message, + ControlMessage::Hello2(_) | ControlMessage::Identify(_) + ) { match self.active.as_ref() { Some(session) if session.session_id == header_session_id && !session.awaiting_takeover => {} @@ -540,13 +1079,22 @@ impl Session { _ => return self.answer_ended(source, header_session_id, now), } } + // Answers over USB stay whole. + let hello_len = datagram + .map(|(_, len)| len) + .filter(|_| source.link == LinkId::Udp); match message { - ControlMessage::Hello2(hello) => self.handle_hello(source, hello, config, now), + ControlMessage::Hello2(hello) => { + self.handle_hello(source, hello, hello_len, config, now) + } + ControlMessage::Identify(request) => self.answer_identify(source, request, config, now), ControlMessage::Bye(reason) => self.handle_bye(source, reason, now), ControlMessage::KeyframeRequest(request) => self.handle_keyframe(source, request, now), ControlMessage::ReceiverReport(report) => self.handle_report(source, report, now), ControlMessage::Ping(ping) => self.handle_ping(source, ping, now), - ControlMessage::InputEvent(event) => self.handle_input(source, event, msg_seq, now), + ControlMessage::InputEvent(event) => { + self.handle_input(source, event, datagram.map(|(seq, _)| seq), now) + } ControlMessage::Nack(nack) => self.handle_nack(source, nack, now), // Host-outbound types arriving inbound: ignore. _ => Actions::default(), @@ -620,6 +1168,7 @@ impl Session { &mut self, source: PeerId, hello: eternal_wire::v2::control::Hello2, + hello_len: Option, config: &impl ConfigSource, now: Instant, ) -> Actions { @@ -628,14 +1177,25 @@ impl Session { // Version gate first. if hello.proto_min > 2 || hello.proto_max < 2 { crate::updates::note_ipad_protocol(hello.proto_min, hello.proto_max); - return self.refuse(source, HelloStatus::VersionUnsupported, &hello, config, now); + return self.refuse( + source, + HelloStatus::VersionUnsupported, + &hello, + hello_len, + config, + now, + ); } - // An update is about to close the host: don't start a stream it would cut. - // The iPad shows Busy ("already streaming"); a retry after the restart works. - if self.updating { - info!(peer = %source, "Client turned away while an update starts"); - return self.refuse(source, HelloStatus::Busy, &hello, config, now); + // Don't start a stream nothing will serve. + let closed = self.closed_to_clients(now); + if let Some(Closed::Silent(why)) = closed { + info!(peer = %source, "{why}"); + return actions; + } + if let Some(Closed::Busy(why)) = closed { + info!(peer = %source, "{why}"); + return self.refuse(source, HelloStatus::Busy, &hello, hello_len, config, now); } // Keep credentials locked through authorization, rotation and ACK creation: @@ -649,6 +1209,11 @@ impl Session { { // Exact replay of an accepted handshake. In particular, losing the // first code-based ACK must not require the now-rotated code. + if let Some(Closed::NotStreaming(state)) = closed { + drop(pairing); + return self + .answer_not_streaming(source, state, &hello, hello_len, config, now); + } actions.replies.push((source, session.ack.clone())); return actions; } @@ -659,7 +1224,7 @@ impl Session { { Some(Ok(grant)) => grant, Some(Err(status)) => { - return self.refuse(source, status, &hello, config, now); + return self.refuse(source, status, &hello, hello_len, config, now); } None => crate::pairing::Grant { token: [0; 16], @@ -667,6 +1232,12 @@ impl Session { new_pairing: false, }, }; + // Only an iPad that may connect learns why the PC isn't streaming. + // A right pairing code isn't spent: it still works after Start. + if let Some(Closed::NotStreaming(state)) = closed { + drop(pairing); + return self.answer_not_streaming(source, state, &hello, hello_len, config, now); + } // The media target: the client's advertised listen port at its source IP // (normally identical to the source port; 0 = malformed, use the source). @@ -687,7 +1258,7 @@ impl Session { if let Some(session) = self.active.as_ref() { if session.same_device(source, hello.device_id) { if matches!(session.peer.link, LinkId::Usb { .. }) && source.link == LinkId::Udp { - return self.refuse(source, HelloStatus::Busy, &hello, config, now); + return self.refuse(source, HelloStatus::Busy, &hello, hello_len, config, now); } // Same device, new connect attempt: supersede in place. info!(peer = %source, "Client reconnected — superseding session in place"); @@ -697,7 +1268,7 @@ impl Session { } else { // A different device while one is streaming: reject. info!(peer = %source, "Second client rejected while a session is active"); - return self.refuse(source, HelloStatus::Busy, &hello, config, now); + return self.refuse(source, HelloStatus::Busy, &hello, hello_len, config, now); } } @@ -708,10 +1279,16 @@ impl Session { .rotate_code() { warn!(%error, "Pairing code rotation failed"); - return self.refuse(source, HelloStatus::Error, &hello, config, now); + return self.refuse(source, HelloStatus::Error, &hello, hello_len, config, now); } } + // A session taken over before it was heard passes on what it spent. + let inherited = self + .active + .as_ref() + .filter(|session| !session.answered) + .map_or(0, |session| session.unanswered_sent); let silent_taken_over = self .active .as_ref() @@ -724,6 +1301,17 @@ impl Session { actions.ended = self.end(EndReason::Superseded, now); } let session_id = self.allocate_session_id(); + // An address whose last session ended before it answered (this + // HELLO2 may have just superseded it) gets no media until it does. + self.unanswered + .retain(|(_, at)| now.saturating_duration_since(*at) < UNANSWERED_FOR); + let answered = source.link != LinkId::Udp; + let unanswered_sent = match source.addr { + Some(addr) if self.unanswered.iter().any(|(ip, _)| *ip == addr.ip()) => { + UNANSWERED_MEDIA_BUDGET + } + _ => inherited, + }; crate::telemetry::note_client_device(&hello.device_name); info!( peer = %source, @@ -760,12 +1348,23 @@ impl Session { late_input: LateInput::default(), awaiting_takeover: false, last_keyframe_grant: None, + keyframe_since_grant: None, last_report: None, msg_seq_out: 0, first_video_at: None, frames_received: false, + frags_received: false, + frames_sent: 0, + max_media_dgram: None, + media_size_lowered: 0, bye: None, input_seqs: SeqWindow::default(), + answered, + unanswered_sent, + withheld: false, + last_stream_config: None, + restart_held_since: None, + last_frame_at: None, }); // Only a new pairing carries the token. iPads keep the token they @@ -830,10 +1429,40 @@ impl Session { } actions.force_idr = session.heard(Heard::Keyframe, now); info!(reason = ?request.reason, "Keyframe request received"); + // Only a client that was told the host takes it asks; anything else + // is ignored, as before. The first smaller sizes restart the picture + // at once: the frames sent so far gave the client no picture. + let asked = request.max_media_dgram; + if asked != 0 && session.info.feature_caps & FEATURE_MEDIA_SIZE != 0 { + let size = asked.clamp(MIN_MEDIA_DGRAM, eternal_wire::v2::MAX_DGRAM_SIZE as u16); + if session.max_media_dgram.is_none_or(|current| size < current) { + session.max_media_dgram = Some(size); + session.media_size_lowered = session.media_size_lowered.saturating_add(1); + info!( + max_dgram = size, + frames_sent = session.frames_sent, + "Media packet size lowered for this session" + ); + if session.media_size_lowered <= MEDIA_SIZE_IDRS { + session.last_keyframe_grant = Some(now); + session.keyframe_since_grant = None; + actions.force_idr = true; + return actions; + } + } + } if session.grant_keyframe(now) { info!(reason = ?request.reason, "Keyframe requested by client — forcing IDR"); actions.force_idr = true; + } else if session.recovered(&request) { + // A new loss: the client already had the last recovery. + session.keyframe_since_grant = None; + info!( + reason = ?request.reason, + "Keyframe requested by client after a recovery — forcing IDR" + ); + actions.force_idr = true; } actions } @@ -845,12 +1474,61 @@ impl Session { actions.force_idr = session.heard(Heard::Report, now); session.last_report = Some(report); session.frames_received |= report.frames_complete > 0; + session.frags_received |= report.frags_received > 0; actions.report = Some(report); } } actions } + /// An iPad checking that this is the PC it paired with, at the address + /// it asked, before it reconnects there. It starts no session and works + /// while another device streams, and through the standby responder while + /// nothing streams. Only Wi-Fi asks, within + /// [`IdentifyLimiter`]'s bounds. + fn answer_identify( + &mut self, + source: PeerId, + request: eternal_wire::v2::control::Identify, + config: &impl ConfigSource, + now: Instant, + ) -> Actions { + let mut actions = Actions::default(); + let (LinkId::Udp, Some(asker)) = (source.link, source.addr) else { + return actions; + }; + let Some(pairing) = config.pairing() else { + return actions; + }; + if request.version != eternal_wire::v2::control::IDENTIFY_VERSION { + return actions; + } + let Some((port, addresses)) = config.identity_endpoints() else { + return actions; + }; + if !self.identify_limiter.allow(asker.ip(), now) { + return actions; + } + let addresses: Vec<[u8; 4]> = addresses + .into_iter() + .take(eternal_wire::v2::control::IDENTITY_MAX_ADDRESSES) + .collect(); + let identity = ControlMessage::Identity(eternal_wire::v2::control::Identity { + version: eternal_wire::v2::control::IDENTIFY_VERSION, + nonce: request.nonce, + port, + proof: pairing + .lock() + .identity_proof(&request.nonce, port, &addresses), + addresses, + }); + actions.replies.push(( + source, + eternal_wire::v2::control::encode_control(0, 1, &identity), + )); + actions + } + fn handle_ping( &mut self, source: PeerId, @@ -886,10 +1564,25 @@ impl Session { /// and reports during the restart gap went unheard, so the gap must not /// count against it: 0.3.1 expired live iPads the moment a slow restart /// finished and made them reconnect. - pub fn transport_resumed(&mut self, now: Instant) { + /// + /// `stopping`: the stream was stopped for good (`running` false, read + /// under the session lock, which [`Self::shut_down`]'s caller holds while + /// it clears it) before this transport got going. Its goodbye is + /// current, and this transport sends it on its way out; clients stay + /// turned away. Otherwise the stream runs again: clients are let in, and + /// a goodbye a stop left while no transport ran is retired, as its + /// session is long over. Both happen in this one call, so a stop that + /// comes after it keeps its goodbye. Returns whether one was retired. + pub fn transport_resumed(&mut self, now: Instant, stopping: bool) -> bool { if let Some(session) = self.active.as_mut() { session.liveness_deadline = session.liveness_deadline.max(now + LIVENESS_TIMEOUT); } + if stopping { + return false; + } + let stale = self.stream_stopped && self.goodbye.take().is_some(); + self.stream_stopped = false; + stale } /// Periodic tick: expire the session when the client has been silent past @@ -907,6 +1600,9 @@ impl Session { actions.goodbye = Some(goodbye); actions.client_lost = self.active.is_none(); } + if std::mem::take(&mut self.lost_unsaid) { + actions.client_lost |= self.active.is_none(); + } actions.ended = self.stopped.take(); let Some(session) = self.active.as_ref() else { return actions; @@ -923,6 +1619,30 @@ impl Session { return actions; } + // Kept alive through a restart and no frame since: past the limit, + // even once the next generation runs, tell the iPad to connect again. + // Not one that said goodbye to hand over to the cable: its takeover + // deadline below ends it as it said. + if let Some(since) = session + .restart_held_since + .filter(|since| now.saturating_duration_since(*since) >= restart_hold_limit()) + .filter(|_| session.bye.is_none()) + { + info!( + session_id = session.session_id, + held_ms = now.saturating_duration_since(since).as_millis() as u64, + "The stream is taking too long to restart: telling the iPad to connect again" + ); + actions.ended = self.end_saying( + ByeReason::Error, + EndReason::Failed(StageFailure::Other), + now, + ); + actions.goodbye = self.take_goodbye(); + actions.client_lost = true; + return actions; + } + if now >= session.liveness_deadline { // A client that had a picture may only have lost its uplink: it // keeps the session, on hold, until the hold limit. One that @@ -960,10 +1680,40 @@ impl Session { if send_heartbeat { let peer = session.peer; let session_id = session.session_id; + let frames_sent = (session.info.feature_caps & FEATURE_MEDIA_SIZE != 0) + .then_some(session.frames_sent); + // Held through a restart, until the first new frame: an iPad + // that understands it says the picture comes back by itself, + // unless the capture has more to say (a locked PC, no display). + // Last, a Second screen held on Windows 10 says why. + let notice = match config.notice() { + eternal_wire::v2::control::NOTICE_NONE + if session.restart_held_since.is_some() + && session.info.feature_caps & FEATURE_STANDBY_STATUS != 0 => + { + eternal_wire::v2::control::NOTICE_RESTARTING + } + eternal_wire::v2::control::NOTICE_NONE if config.needs_windows_11() => { + eternal_wire::v2::control::NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11 + } + notice => notice, + }; + let stream_config = match self.not_streaming { + // No stream: the config of the one the iPad last had. + Some(_) => session + .last_stream_config + .unwrap_or_else(|| self.negotiated_config(config)), + None => self.negotiated_config(config), + }; + if let Some(session) = self.active.as_mut() { + session.last_stream_config = Some(stream_config); + } let msg_seq = self.next_msg_seq(); let heartbeat = ControlMessage::Heartbeat(eternal_wire::v2::control::Heartbeat { host_time_us: crate::clock::host_now_us(), - stream_config: self.negotiated_config(config), + stream_config, + notice, + frames_sent, }); actions.replies.push(( peer, @@ -973,6 +1723,29 @@ impl Session { actions } + /// One heartbeat interval while the PC restarts its stream and the + /// standby responder holds the port. A session kept across the restart + /// stays alive: its iPad gets heartbeats as from a running stream, so it + /// holds its last picture instead of losing the signal, and its silence + /// still expires it. Only an iPad that has answered gets them. Once + /// [`RESTART_HOLD_LIMIT`] has passed since its last frame, the session + /// ends with the goodbye that tells its iPad to connect again, on this + /// tick or the next stream's ([`Self::tick`]) until a frame goes out. + /// Does nothing unless the PC is restarting. + pub fn restart_tick(&mut self, config: &impl ConfigSource, now: Instant) -> Actions { + if self.not_streaming != Some(NotStreaming::Restarting) { + return Actions::default(); + } + let Some(session) = self.active.as_mut() else { + return self.tick(config, false, now); + }; + // Counted from the last frame, so a restart ladder keeps counting. + let last_frame = session.last_frame_at.unwrap_or(now); + session.restart_held_since.get_or_insert(last_frame); + let answered = session.answered; + self.tick(config, answered, now) + } + /// Ends the active session for liveness, with the goodbye that tells its /// client to connect again: `error`, which every 0.3.x iPad answers with /// a reconnect. Late packets of the session get it again @@ -1053,15 +1826,97 @@ impl Session { /// Answers `hello` with a refusal. The first refusal of a nonce ends /// that attempt; a retransmit of it gets the answer again. + /// Tells an iPad the PC isn't streaming, in the words it understands. + /// 0.3.4 and older drop an ACK whose status they don't know. A restart + /// is short and their HELLO2 retries outlast it, so they hear nothing, + /// as before. A stopped or failed PC waits for someone at the PC, so it + /// turns them down with Error: "Your PC turned down the connection", + /// which also ends their reconnect cycle. + /// + /// Over Wi-Fi the answer is never larger than the HELLO2: iPads that + /// understand these answers pad theirs so every answer fits whole. A + /// stopping stream answers here too, until its transport ends, so the + /// bound is here as well as in the standby responder. + fn answer_not_streaming( + &mut self, + source: PeerId, + state: NotStreaming, + hello: &eternal_wire::v2::control::Hello2, + hello_len: Option, + config: &impl ConfigSource, + now: Instant, + ) -> Actions { + let (status, reason) = if hello.feature_caps & FEATURE_STANDBY_STATUS == 0 { + match state { + NotStreaming::Restarting => return Actions::default(), + NotStreaming::Stopped | NotStreaming::Failed(_) => (HelloStatus::Error, ""), + } + } else { + match state { + NotStreaming::Stopped => (HelloStatus::Stopped, ""), + NotStreaming::Restarting => (HelloStatus::Restarting, ""), + NotStreaming::Failed(reason) => (HelloStatus::Failed, reason.token()), + } + }; + let actions = + self.refuse_with_reason(source, status, reason, hello, hello_len, config, now); + if actions.ended.is_some() && !actions.replies.is_empty() { + info!(peer = %source, ?status, reason, "Told an iPad the PC isn't streaming"); + } + actions + } + + /// Answers `hello` with a refusal. The first refusal of a nonce ends + /// that attempt; a retransmit of it gets the answer again. + /// + /// With `hello_len`, the answer is no larger than the HELLO2: without + /// the PC name and host caps if need be, which iPads read only from an + /// accepted HELLO2, or not sent at all. A not-streaming reason stays. + /// Every iPad's HELLO2 fits the bare answer. fn refuse( &mut self, source: PeerId, status: HelloStatus, hello: &eternal_wire::v2::control::Hello2, + hello_len: Option, + config: &impl ConfigSource, + now: Instant, + ) -> Actions { + self.refuse_with_reason(source, status, "", hello, hello_len, config, now) + } + + #[allow(clippy::too_many_arguments)] + fn refuse_with_reason( + &mut self, + source: PeerId, + status: HelloStatus, + reason: &str, + hello: &eternal_wire::v2::control::Hello2, + hello_len: Option, config: &impl ConfigSource, now: Instant, ) -> Actions { - let ack = self.make_ack(status, hello.client_nonce, 0, config); + let msg_seq = self.next_msg_seq(); + let mut answer = self.hello_ack(status, hello.client_nonce, 0, config, [0; 16], reason); + let encode = |answer: &HelloAck| { + eternal_wire::v2::control::encode_control( + 0, + msg_seq, + &ControlMessage::HelloAck(answer.clone()), + ) + }; + let mut ack = encode(&answer); + let fits = |ack: &[u8]| hello_len.is_none_or(|len| ack.len() <= len); + if !fits(&ack) { + answer.host_name.clear(); + answer.host_caps = 0; + ack = encode(&answer); + } + let replies = if fits(&ack) { + vec![(source, ack)] + } else { + Vec::new() + }; self.refused .retain(|(_, at)| now.saturating_duration_since(*at) < REFUSED_FOR); let first = !self @@ -1076,7 +1931,7 @@ impl Session { } let summarized = first && self.summarize_refusal(now); Actions { - replies: vec![(source, ack)], + replies, ended: summarized.then_some(SessionEnd { session_id: None, client_nonce: hello.client_nonce, @@ -1084,6 +1939,7 @@ impl Session { reason: EndReason::Refused(status), first_video: None, frames_received: false, + frags_received: false, duration: Duration::ZERO, }), ..Actions::default() @@ -1121,26 +1977,52 @@ impl Session { } } - fn make_ack( + fn make_ack_with_token( &mut self, status: HelloStatus, client_nonce: u32, session_id: u32, config: &impl ConfigSource, + auth_token: [u8; 16], ) -> Vec { - self.make_ack_with_token(status, client_nonce, session_id, config, [0; 16]) + self.make_reply(status, client_nonce, session_id, config, auth_token, "") } - fn make_ack_with_token( + fn make_reply( &mut self, status: HelloStatus, client_nonce: u32, session_id: u32, config: &impl ConfigSource, auth_token: [u8; 16], + reason: &str, ) -> Vec { let msg_seq = self.next_msg_seq(); - let ack = ControlMessage::HelloAck(HelloAck { + let ack = self.hello_ack(status, client_nonce, session_id, config, auth_token, reason); + eternal_wire::v2::control::encode_control( + session_id, + msg_seq, + &ControlMessage::HelloAck(ack), + ) + } + + fn hello_ack( + &self, + status: HelloStatus, + client_nonce: u32, + session_id: u32, + config: &impl ConfigSource, + auth_token: [u8; 16], + reason: &str, + ) -> HelloAck { + // Only the client that asks hears that the host takes smaller media + // datagrams; older iPads get the reply they always did. + let media_size = status == HelloStatus::Ok + && self.active.as_ref().is_some_and(|session| { + session.session_id == session_id + && session.info.feature_caps & FEATURE_MEDIA_SIZE != 0 + }); + HelloAck { status, accepted_version: 2, client_nonce, @@ -1151,9 +2033,12 @@ impl Session { stream_config: self.negotiated_config(config), host_name: config.host_name(), auth_token, - host_caps: HOSTCAP_NACK | config.host_caps(), - }); - eternal_wire::v2::control::encode_control(session_id, msg_seq, &ack) + host_caps: HOSTCAP_NACK + | HOSTCAP_KEYFRAME_PER_LOSS + | config.host_caps() + | if media_size { HOSTCAP_MEDIA_SIZE } else { 0 }, + reason: reason.to_string(), + } } fn negotiated_config(&self, source: &impl ConfigSource) -> StreamConfig { @@ -1232,11 +2117,28 @@ impl ActiveSession { /// The client was heard: it gets a full liveness window again, and a /// session on hold carries on. Returns whether to force an IDR: reports - /// and keyframe requests sent during the silence may not have arrived. + /// and keyframe requests sent during the silence may not have arrived, + /// and a client that answers after frames were held back from it needs + /// a keyframe to start its picture. fn heard(&mut self, kind: Heard, now: Instant) -> bool { let silent = now.saturating_duration_since(self.heard.0); self.heard = (now, kind); self.liveness_deadline = now + LIVENESS_TIMEOUT; + if !self.answered { + self.answered = true; + info!( + session_id = self.session_id, + answered_ms = now + .saturating_duration_since(self.info.connected_at) + .as_millis() as u64, + heard = kind.as_str(), + withheld = self.withheld, + "Client answered" + ); + if self.withheld { + return self.grant_keyframe(now); + } + } let Some(since) = self.held_since.take() else { return false; }; @@ -1259,10 +2161,26 @@ impl ActiveSession { ); if granted { self.last_keyframe_grant = Some(now); + self.keyframe_since_grant = None; } granted } + /// Whether the client completed the first keyframe sent since the last + /// IDR was granted, or a later frame, before it lost the frame it asks + /// for now. Such a loss is new, so its IDR needn't wait out + /// [`KEYFRAME_REQUEST_MIN_INTERVAL`], and doesn't move it. A request + /// from before that recovery landed names an older frame and waits as + /// before. Only a request that names a frame can show it: the iPad's + /// `GapLoss` does; `DecodeError` and `Resume` name none. + fn recovered(&self, request: &KeyframeRequest) -> bool { + self.keyframe_since_grant.is_some_and(|(epoch, seq)| { + request.stream_epoch == epoch + && request.last_complete_seq != 0 + && request.last_complete_seq.wrapping_sub(seq) < u32::MAX / 2 + }) + } + /// `device=` stays last: uploads cut the iPad's name at the line's end. fn log_expiry(&self, now: Instant) { let (heard_at, heard) = self.heard; @@ -1290,6 +2208,7 @@ impl ActiveSession { .first_video_at .map(|at| at.saturating_duration_since(start)), frames_received: self.frames_received, + frags_received: self.frags_received, duration: now.saturating_duration_since(start), } } @@ -1681,6 +2600,59 @@ mod tests { ); } + /// Only an iPad that said it draws the pointer gets it apart from the + /// video, numbered with the session's other control messages. An older + /// iPad keeps the pointer in the video. + #[test] + fn only_an_ipad_that_draws_the_pointer_gets_it_apart() { + use eternal_wire::v2::control::{parse_control, PointerState, FEATURE_DRAWS_POINTER}; + let mut session = Session::new(1234); + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + assert_eq!(session.pointer_session_id(), None, "nobody connected"); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + assert!(session.is_active()); + assert_eq!(session.pointer_session_id(), None, "an iPad before 0.3.5"); + + let ControlMessage::Hello2(mut draws) = hello(2, 50000) else { + unreachable!() + }; + draws.feature_caps = FEATURE_DRAWS_POINTER; + session.handle_control_authed(peer, ControlMessage::Hello2(draws), &TestConfig, now); + // Review r2: nothing has confirmed it got its HELLO_ACK, so no + // pointer messages yet, and capture keeps drawing the pointer. + assert_eq!( + session.pointer_session_id(), + None, + "a HELLO that only claims the bit gets no pointer messages" + ); + let id = session.session_id().unwrap(); + let answer = session.handle_control( + peer, + id, + ControlMessage::ReceiverReport(ReceiverReport::default()), + &TestConfig, + now, + ); + assert!(answer.replies.is_empty()); + assert_eq!(session.pointer_session_id(), Some(id), "it answered"); + let state = ControlMessage::PointerState(PointerState::default()); + let (to, datagram) = session.encode_for_client(&state).unwrap(); + assert_eq!(to, peer); + let (header, parsed) = parse_control(&datagram).unwrap(); + assert_eq!(parsed, state); + assert_eq!(header.session_id, id); + + session.handle_control_authed( + peer, + ControlMessage::Bye(ByeReason::UserDisconnect), + &TestConfig, + now, + ); + assert_eq!(session.pointer_session_id(), None); + assert!(session.encode_for_client(&state).is_none()); + } + #[test] fn duplicate_nonce_gets_identical_ack_without_new_session() { let mut session = Session::new(1234); @@ -1853,7 +2825,7 @@ mod tests { session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); // The transport was down for 11 s (a capture restart with backoff). let resumed = now + Duration::from_secs(11); - session.transport_resumed(resumed); + session.transport_resumed(resumed, false); assert!(!session.tick(&TestConfig, false, resumed).client_lost); assert!(session.is_active()); // A client that stays silent after the restart still expires. @@ -1903,6 +2875,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 10, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, }); let first = session.handle_control_authed(peer, request.clone(), &TestConfig, now); @@ -1943,6 +2916,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 10, reason, + max_media_dgram: 0, }) }; @@ -1976,6 +2950,148 @@ mod tests { assert!(resume.force_idr, "granted once the window has passed"); } + /// A keyframe request from `peer` at `now` + `millis`: whether it forces an IDR. + fn keyframe_asked( + session: &mut Session, + peer: PeerId, + now: Instant, + millis: u64, + (stream_epoch, last_complete_seq): (u32, u32), + reason: KeyframeReason, + ) -> bool { + session + .handle_control_authed( + peer, + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch, + last_complete_seq, + reason, + max_media_dgram: 0, + }), + &TestConfig, + now + Duration::from_millis(millis), + ) + .force_idr + } + + #[test] + fn a_loss_after_a_recovery_gets_its_keyframe_at_once() { + // Under Wi-Fi loss the next frame is often lost soon after a + // recovery IDR lands. A request naming a frame at or past that + // keyframe is a new loss: it gets its own IDR without waiting out + // the 500 ms window, and doesn't move it. A request from before the + // recovery waits, as does a second one for the same loss. + let mut session = Session::new(1234); + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + let id = session.session_id().unwrap(); + let gap = KeyframeReason::GapLoss; + let ask = |session: &mut Session, millis, last_complete_seq| { + keyframe_asked(session, peer, now, millis, (7, last_complete_seq), gap) + }; + + // The session's opening keyframe, then the first loss. + session.note_keyframe_sent(id, 7, 1); + assert!(ask(&mut session, 0, 10)); + assert!( + !ask(&mut session, 40, 11), + "no keyframe has gone out since the grant" + ); + session.note_keyframe_sent(id, 7, 12); + session.note_keyframe_sent(id, 7, 20); + assert!( + !ask(&mut session, 60, 11), + "a request from before the recovery waits" + ); + assert!(ask(&mut session, 150, 12), "the client had the recovery"); + assert!( + !ask(&mut session, 170, 14), + "the same loss asks again before its keyframe went out" + ); + session.note_keyframe_sent(id, 7, 15); + assert!( + !ask(&mut session, 200, 14), + "a request from before the second recovery waits" + ); + assert!(ask(&mut session, 260, 18), "a third loss"); + // The schedule still runs from the first grant. + assert!(!ask(&mut session, 450, 18)); + assert!( + keyframe_asked( + &mut session, + peer, + now, + 520, + (0, 0), + KeyframeReason::DecodeError + ), + "the window is 500 ms from the last grant on schedule" + ); + } + + #[test] + fn only_a_request_naming_a_frame_of_this_epoch_shows_a_recovery() { + let mut session = Session::new(1234); + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + let id = session.session_id().unwrap(); + assert!(keyframe_asked( + &mut session, + peer, + now, + 0, + (7, 10), + KeyframeReason::GapLoss + )); + session.note_keyframe_sent(id, 7, u32::MAX); + // The decoder's requests name no frame. + for reason in [KeyframeReason::DecodeError, KeyframeReason::Resume] { + assert!(!keyframe_asked( + &mut session, + peer, + now, + 100, + (0, 0), + reason + )); + } + assert!( + !keyframe_asked( + &mut session, + peer, + now, + 100, + (6, 5), + KeyframeReason::GapLoss + ), + "another epoch's frame" + ); + assert!( + !keyframe_asked( + &mut session, + peer, + now, + 100, + (7, u32::MAX - 1), + KeyframeReason::GapLoss + ), + "older, just below the wrap" + ); + assert!( + keyframe_asked( + &mut session, + peer, + now, + 100, + (7, 3), + KeyframeReason::GapLoss + ), + "newer, past the wrap" + ); + } + fn input_event(event_id: u32) -> ControlMessage { ControlMessage::InputEvent(InputEvent { input_ver: 1, @@ -2131,7 +3247,9 @@ mod tests { msg_seq, }; let message = ControlMessage::InputEvent(event); - let actions = session.handle_datagram(peer, &header, message, &TestConfig, now); + let len = + eternal_wire::v2::control::encode_control(session_id, msg_seq, &message).len(); + let actions = session.handle_datagram(peer, &header, message, len, &TestConfig, now); actions .input .map(|(_, event)| event) @@ -2241,107 +3359,646 @@ mod tests { assert!(matches!(message, ControlMessage::Heartbeat(_))); } - struct PairConfig(parking_lot::Mutex); - impl ConfigSource for PairConfig { - fn stream_config(&self) -> StreamConfig { - TestConfig.stream_config() - } - fn host_name(&self) -> String { - TestConfig.host_name() - } - fn pairing(&self) -> Option<&parking_lot::Mutex> { - Some(&self.0) - } - } - fn pair_config(required: bool) -> PairConfig { - PairConfig(parking_lot::Mutex::new( - crate::pairing::Pairing::new(required, [7; 16]).unwrap(), - )) - } - fn pair_hello(nonce: u32) -> Hello2 { + fn media_size_hello(nonce: u32, feature_caps: u16) -> ControlMessage { let ControlMessage::Hello2(mut hello) = hello(nonce, 50000) else { unreachable!() }; - hello.device_id = 17; - hello + hello.feature_caps = feature_caps; + ControlMessage::Hello2(hello) } - fn pair_ack(actions: &Actions) -> HelloAck { - let (_, ControlMessage::HelloAck(ack)) = - eternal_wire::v2::control::parse_control(&actions.replies[0].1).unwrap() - else { - panic!("expected ACK") - }; - ack + + fn sized_request(max_media_dgram: u16) -> ControlMessage { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 7, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram, + }) } - fn pair_send( - session: &mut Session, - config: &PairConfig, - peer: PeerId, - hello: Hello2, - now: Instant, - ) -> Actions { - session.handle_control(peer, 0, ControlMessage::Hello2(hello), config, now) + + fn heartbeat_of(actions: &Actions) -> eternal_wire::v2::control::Heartbeat { + match eternal_wire::v2::control::parse_control(&actions.replies[0].1) + .unwrap() + .1 + { + ControlMessage::Heartbeat(heartbeat) => heartbeat, + other => panic!("{other:?}"), + } } + /// Only an iPad that says it may ask for smaller media datagrams hears + /// that the host takes them, and gets the frame count in heartbeats. + /// Any other iPad gets the reply and heartbeats it always did. #[test] - fn retransmitted_rejected_handshake_is_one_pairing_attempt() { - let config = pair_config(true); - let mut session = Session::new(10); - let peer = addr([192, 0, 2, 1], 50000); + fn media_size_is_offered_only_to_an_ipad_that_asks() { + let peer = addr([10, 0, 0, 5], 50000); let now = Instant::now(); - let mut hello = pair_hello(1); - for retry in 0..8 { - let ack = pair_ack(&pair_send( - &mut session, - &config, + for (feature_caps, offered) in [(FEATURE_MEDIA_SIZE, true), (0, false)] { + let mut session = Session::new(1234); + let accepted = session.handle_control( peer, - hello.clone(), - now + Duration::from_millis(retry * 300), - )); - assert_eq!(ack.status, HelloStatus::Unauthorized); - assert_eq!(ack.auth_token, [0; 16]); + 0, + media_size_hello(1, feature_caps), + &TestConfig, + now, + ); + let ack = parse_ack(&accepted.replies[0].1); + assert_eq!(ack.host_caps & HOSTCAP_MEDIA_SIZE != 0, offered); + assert_eq!( + ack.host_caps & !HOSTCAP_MEDIA_SIZE, + HOSTCAP_NACK | HOSTCAP_KEYFRAME_PER_LOSS + ); + let id = session.session_id().unwrap(); + for _ in 0..3 { + assert!(session.may_send_media(id, 1000, now)); + } + let ticked = session.tick(&TestConfig, true, now + Duration::from_millis(100)); + assert_eq!( + heartbeat_of(&ticked).frames_sent, + offered.then_some(3), + "{feature_caps}" + ); + // The heartbeat an older iPad gets is the plain one. + if !offered { + assert_eq!( + ticked.replies[0].1.len(), + eternal_wire::v2::control::CONTROL_HEADER_SIZE + + 8 + + eternal_wire::v2::control::STREAM_CONFIG_SIZE + ); + } } - hello.client_nonce = 2; - hello.pairing_code = config.0.lock().code(); - assert_eq!( - pair_ack(&pair_send( - &mut session, - &config, - peer, - hello, - now + Duration::from_secs(3) - )) - .status, - HelloStatus::Ok + // A refusal never offers it. + let mut busy = Session::new(1234); + busy.handle_control( + peer, + 0, + media_size_hello(1, FEATURE_MEDIA_SIZE), + &TestConfig, + now, + ); + let refused = busy.handle_control( + addr([10, 0, 0, 6], 50000), + 0, + media_size_hello(2, FEATURE_MEDIA_SIZE), + &TestConfig, + now, ); + let ack = parse_ack(&refused.replies[0].1); + assert_eq!(ack.status, HelloStatus::Busy); + assert_eq!(ack.host_caps & HOSTCAP_MEDIA_SIZE, 0); } + /// A smaller size restarts the picture at once, inside the keyframe + /// rate limit, and holds for the rest of the session: never larger, + /// never under the minimum. A new session starts at the PC's size. #[test] - fn changing_credentials_with_the_same_nonce_still_counts_as_guesses() { - let config = pair_config(true); - let mut session = Session::new(10); - let peer = addr([192, 0, 2, 1], 50000); + fn a_smaller_media_size_lowers_only_this_session_and_forces_an_idr() { + let peer = addr([10, 0, 0, 5], 50000); let now = Instant::now(); - let mut hello = pair_hello(1); - for guess in 1..=5 { - hello.pairing_code = 1_000_000 + guess; - let ack = pair_ack(&pair_send(&mut session, &config, peer, hello.clone(), now)); - assert_eq!( - ack.status, - if guess == 5 { - HelloStatus::RateLimited - } else { - HelloStatus::Unauthorized - } + let mut session = Session::new(1234); + session.handle_control( + peer, + 0, + media_size_hello(1, FEATURE_MEDIA_SIZE), + &TestConfig, + now, + ); + let id = session.session_id().unwrap(); + assert_eq!(session.max_media_dgram(id), None); + + let plain = session.handle_control_authed(peer, sized_request(0), &TestConfig, now); + assert!(plain.force_idr); + assert_eq!(session.max_media_dgram(id), None); + + let at = |millis| now + Duration::from_millis(millis); + for (millis, asked, size, idr) in [ + (100, 1200, 1200, true), + (200, 1200, 1200, false), + (300, 1300, 1200, false), + (400, 576, 576, true), + (450, 100, 576, false), + ] { + let actions = + session.handle_control_authed(peer, sized_request(asked), &TestConfig, at(millis)); + assert_eq!(session.max_media_dgram(id), Some(size), "{asked}"); + assert_eq!(actions.force_idr, idr, "{asked}"); + assert!(actions.replies.is_empty() && actions.new_target.is_none()); + } + assert_eq!(session.max_media_dgram(id.wrapping_add(1)), None); + + // A client that keeps asking for less, a byte at a time, gets two + // keyframes past the rate limit, then no more than any other client. + session.handle_control( + peer, + 0, + media_size_hello(3, FEATURE_MEDIA_SIZE), + &TestConfig, + at(1_000), + ); + let id = session.session_id().unwrap(); + let mut forced = Vec::new(); + for step in 0..825u64 { + let actions = session.handle_control_authed( + peer, + sized_request(1400 - step as u16), + &TestConfig, + at(1_000 + step * 10), ); + if actions.force_idr { + forced.push(step * 10); + } } + assert_eq!(session.max_media_dgram(id), Some(MIN_MEDIA_DGRAM)); + // Two at once, then one per rate-limit window from the second. + assert_eq!(forced[..2], [0, 10]); + assert!(forced[2..] + .windows(2) + .all(|pair| pair[1] - pair[0] >= KEYFRAME_REQUEST_MIN_INTERVAL.as_millis() as u64)); + assert_eq!(forced[2], 510); + assert!(forced.len() <= 2 + 8240 / 500, "{forced:?}"); + + session.handle_control( + peer, + 0, + media_size_hello(2, FEATURE_MEDIA_SIZE), + &TestConfig, + at(500), + ); + let next = session.session_id().unwrap(); + assert_ne!(next, id); + assert_eq!(session.max_media_dgram(next), None); } + /// An iPad that didn't say it may ask has its request read as the + /// keyframe request it always was. #[test] - fn pairing_required_code_rotation_retry_token_and_regeneration() { - let config = pair_config(true); - let mut session = Session::new(10); - let peer = addr([192, 0, 2, 1], 50000); + fn a_media_size_from_an_ipad_that_did_not_offer_is_ignored() { + let peer = addr([10, 0, 0, 5], 50000); + let now = Instant::now(); + let mut session = Session::new(1234); + session.handle_control(peer, 0, media_size_hello(1, 0), &TestConfig, now); + let id = session.session_id().unwrap(); + let first = session.handle_control_authed(peer, sized_request(576), &TestConfig, now); + assert!(first.force_idr); + let second = session.handle_control_authed( + peer, + sized_request(576), + &TestConfig, + now + Duration::from_millis(100), + ); + assert!(!second.force_idr, "rate limited as any keyframe request"); + assert_eq!(session.max_media_dgram(id), None); + } + + struct PairConfig(parking_lot::Mutex); + impl ConfigSource for PairConfig { + fn stream_config(&self) -> StreamConfig { + TestConfig.stream_config() + } + fn host_name(&self) -> String { + TestConfig.host_name() + } + fn pairing(&self) -> Option<&parking_lot::Mutex> { + Some(&self.0) + } + } + fn identity_reply(actions: &Actions) -> eternal_wire::v2::control::Identity { + assert_eq!(actions.replies.len(), 1); + let (header, ControlMessage::Identity(identity)) = + eternal_wire::v2::control::parse_control(&actions.replies[0].1).unwrap() + else { + panic!("expected IDENTITY") + }; + assert_eq!(header.session_id, 0); + identity + } + + fn identify(version: u8, nonce: [u8; 16]) -> ControlMessage { + ControlMessage::Identify(eternal_wire::v2::control::Identify { version, nonce }) + } + + /// A paired host on port 9876 with the home network addresses given. + struct IdentifyConfig(PairConfig, Vec<[u8; 4]>); + impl ConfigSource for IdentifyConfig { + fn stream_config(&self) -> StreamConfig { + TestConfig.stream_config() + } + fn host_name(&self) -> String { + TestConfig.host_name() + } + fn pairing(&self) -> Option<&parking_lot::Mutex> { + self.0.pairing() + } + fn identity_endpoints(&self) -> Option<(u16, Vec<[u8; 4]>)> { + Some((9876, self.1.clone())) + } + } + + /// A paired host on every address whose home network addresses come + /// from `HomeCache`, listed by `held_home`. + struct ListedConfig(PairConfig, crate::network::HomeCache); + impl ConfigSource for ListedConfig { + fn stream_config(&self) -> StreamConfig { + TestConfig.stream_config() + } + fn host_name(&self) -> String { + TestConfig.host_name() + } + fn pairing(&self) -> Option<&parking_lot::Mutex> { + self.0.pairing() + } + fn identity_endpoints(&self) -> Option<(u16, Vec<[u8; 4]>)> { + super::super::identity_endpoints(SocketAddr::from(([0, 0, 0, 0], 9876)), || { + self.1.get() + }) + } + } + + /// Held while the test keeps the PC's adapters from being listed. + static HOME_HELD: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); + + fn held_home() -> Option> { + let _open = HOME_HELD.lock(); + Some(vec![crate::network::Adapter { + name: String::new(), + kind: crate::network::Kind::Wifi, + up: true, + gateway: true, + ipv4: vec![std::net::Ipv4Addr::new(192, 168, 1, 20)], + }]) + } + + /// Once the last listing of the PC's addresses has expired, IDENTIFY gets + /// no answer until a new listing comes, however long that takes: the PC + /// never signs an address it may no longer have. + #[test] + fn identify_signs_no_expired_listing() { + let lifetime = Duration::from_secs(1); + let config = ListedConfig( + pair_config(true), + crate::network::HomeCache::with(held_home, lifetime), + ); + let mut session = Session::new(100); + let start = Instant::now(); + // A new asker and a new limiter window each time, so only the + // listing decides. + let mut asked = 0u32; + let mut ask = |session: &mut Session| { + asked += 1; + let asker = addr([192, 0, 2, (asked % 250) as u8 + 1], 50100); + answered(session, asker, &config, start + IDENTIFY_WINDOW * asked) + }; + let deadline = Instant::now() + Duration::from_secs(10); + while !ask(&mut session) { + assert!(Instant::now() < deadline, "the addresses were never listed"); + std::thread::sleep(Duration::from_millis(5)); + } + + let held = HOME_HELD.lock(); + std::thread::sleep(lifetime + Duration::from_millis(100)); + for _ in 0..30 { + assert!(!ask(&mut session), "signed an expired listing"); + std::thread::sleep(Duration::from_millis(50)); + } + drop(held); + let deadline = Instant::now() + Duration::from_secs(10); + while !ask(&mut session) { + assert!(Instant::now() < deadline, "a fresh listing never came"); + std::thread::sleep(Duration::from_millis(5)); + } + } + + fn identify_config(required: bool) -> IdentifyConfig { + IdentifyConfig( + pair_config(required), + vec![[192, 0, 2, 1], [198, 51, 100, 1]], + ) + } + + fn answered( + session: &mut Session, + asker: PeerId, + config: &impl ConfigSource, + now: Instant, + ) -> bool { + !session + .handle_control(asker, 0, identify(1, [3; 16]), config, now) + .replies + .is_empty() + } + + /// The PC proves it holds its pairing token at its own port and + /// addresses, with pairing on or off, without starting a session. + #[test] + fn identify_is_answered_with_the_token_proof_and_starts_nothing() { + let config = identify_config(true); + let mut session = Session::new(100); + let now = Instant::now(); + let asker = addr([192, 0, 2, 9], 50100); + let actions = session.handle_control(asker, 0, identify(1, [3; 16]), &config, now); + let identity = identity_reply(&actions); + assert_eq!(actions.replies[0].0, asker); + assert_eq!(identity.nonce, [3; 16]); + assert_eq!(identity.port, 9876); + assert_eq!(identity.addresses, config.1); + let proof = + |pairing: &crate::pairing::Pairing| pairing.identity_proof(&[3; 16], 9876, &config.1); + assert_eq!(identity.proof, proof(&config.0 .0.lock())); + assert_ne!( + identity.proof, + proof(&crate::pairing::Pairing::new(true, [8; 16]).unwrap()) + ); + assert!(session.session_id().is_none()); + assert!(actions.new_target.is_none()); + + // At most four addresses go out, and the proof signs those four. + let many = IdentifyConfig( + pair_config(false), + (1..=6).map(|last| [192, 0, 2, last]).collect(), + ); + let identity = + identity_reply(&session.handle_control(asker, 0, identify(1, [3; 16]), &many, now)); + assert_eq!(identity.addresses, many.1[..4]); + assert_eq!( + identity.proof, + many.0 + .0 + .lock() + .identity_proof(&[3; 16], 9876, &many.1[..4]) + ); + } + + /// A layout this PC doesn't know gets no answer, as from a PC before + /// 0.3.5; so does a host without pairing state or addresses to sign, + /// and a request over a cable. + #[test] + fn identify_of_another_version_is_not_answered() { + let mut session = Session::new(100); + let now = Instant::now(); + let asker = addr([192, 0, 2, 9], 50100); + let actions = + session.handle_control(asker, 0, identify(2, [3; 16]), &identify_config(true), now); + assert!(actions.replies.is_empty()); + assert!(!answered(&mut session, asker, &TestConfig, now)); + assert!(!answered(&mut session, asker, &pair_config(true), now)); + assert!(!answered( + &mut session, + PeerId::usb(1), + &identify_config(true), + now + )); + assert!(answered(&mut session, asker, &identify_config(true), now)); + } + + /// While no stream runs, the standby responder hands an IDENTIFY to the + /// session: it is answered over Wi-Fi as while streaming, never over a + /// cable, with an answer no larger than the request, and it starts + /// nothing. + #[test] + fn identify_while_not_streaming_is_answered_as_while_streaming() { + let config = identify_config(true); + let request = eternal_wire::v2::control::encode_control(0, 1, &identify(1, [3; 16])); + for state in [ + NotStreaming::Stopped, + NotStreaming::Restarting, + NotStreaming::Failed(FailedReason::Encoder), + ] { + let mut session = Session::new(100); + session.set_not_streaming(Some(state)); + let mut ask = |peer: PeerId| { + let (header, message) = eternal_wire::v2::control::parse_control(&request).unwrap(); + session.handle_datagram( + peer, + &header, + message, + request.len(), + &config, + Instant::now(), + ) + }; + assert!(ask(PeerId::usb(1)).replies.is_empty(), "{state:?}"); + let actions = ask(addr([192, 0, 2, 9], 50100)); + let identity = identity_reply(&actions); + assert_eq!(identity.nonce, [3; 16]); + assert_eq!(actions.replies[0].1.len(), request.len()); + assert_eq!(request.len(), 84); + assert!(actions.new_target.is_none()); + assert!(session.session_id().is_none()); + assert_eq!(session.not_streaming(), Some(state)); + } + } + + /// Requests that get no answer, while the PC has no addresses to sign, + /// use up none of the asker's answers. + #[test] + fn unanswered_identify_uses_up_no_answers() { + let mut session = Session::new(100); + let now = Instant::now(); + let asker = addr([192, 0, 2, 9], 50100); + for _ in 0..IDENTIFY_OVERALL { + assert!(!answered(&mut session, asker, &pair_config(true), now)); + } + let config = identify_config(true); + let answers = (0..IDENTIFY_PER_SOURCE * 2) + .filter(|_| answered(&mut session, asker, &config, now)) + .count(); + assert_eq!(answers, IDENTIFY_PER_SOURCE as usize); + } + + /// A burst from one address gets 8 answers in 10 s, and the PC 32 in + /// all from at most 16 addresses; each count starts over when its + /// window ends. + #[test] + fn identify_answers_are_bounded_per_asker_and_in_all() { + let config = identify_config(true); + let mut session = Session::new(100); + let start = Instant::now(); + let asker = addr([192, 0, 2, 9], 50100); + let burst = (0..20) + .filter(|_| answered(&mut session, asker, &config, start)) + .count(); + assert_eq!(burst, IDENTIFY_PER_SOURCE as usize); + // Another port on the same address shares its count. + assert!(!answered( + &mut session, + addr([192, 0, 2, 9], 50101), + &config, + start + )); + // Others still get theirs, up to the PC's 32. + let others: usize = (10..13) + .map(|last| { + (0..20) + .filter(|_| { + answered(&mut session, addr([192, 0, 2, last], 50100), &config, start) + }) + .count() + }) + .sum(); + assert_eq!(others, (IDENTIFY_OVERALL - IDENTIFY_PER_SOURCE) as usize); + assert!(!answered( + &mut session, + addr([192, 0, 2, 40], 50100), + &config, + start + )); + + let later = start + IDENTIFY_WINDOW - Duration::from_millis(1); + assert!(!answered(&mut session, asker, &config, later)); + let next = start + IDENTIFY_WINDOW; + assert!(answered(&mut session, asker, &config, next)); + assert!(answered( + &mut session, + addr([192, 0, 2, 40], 50100), + &config, + next + )); + + // Past 16 addresses in a window, a new one waits for the next. + let mut session = Session::new(100); + for last in 0..IDENTIFY_SOURCES as u8 { + assert!(answered( + &mut session, + addr([198, 51, 100, last], 50100), + &config, + start + )); + } + assert!(!answered(&mut session, asker, &config, start)); + assert!(answered(&mut session, asker, &config, next)); + } + + /// While another iPad streams, IDENTIFY is answered within the same + /// bounds, and the stream goes on whatever arrives. + #[test] + fn identify_while_streaming_is_bounded_and_leaves_the_stream() { + let mut session = Session::new(100); + let now = Instant::now(); + let streaming = addr([192, 0, 2, 5], 50000); + session.handle_control( + streaming, + 0, + ControlMessage::Hello2(pair_hello(1)), + &pair_config(false), + now, + ); + let live = session.session_id(); + assert!(live.is_some()); + + let config = identify_config(false); + let asker = addr([192, 0, 2, 9], 50100); + let mut answers = 0; + for index in 0..40u8 { + let actions = session.handle_control(asker, 0, identify(1, [index; 16]), &config, now); + assert!( + actions.new_target.is_none() && !actions.client_lost && actions.goodbye.is_none() + ); + if !actions.replies.is_empty() { + let identity = identity_reply(&actions); + assert_eq!(identity.nonce, [index; 16]); + assert_eq!(actions.replies[0].0, asker, "never to the streaming iPad"); + answers += 1; + } + } + assert_eq!(answers, IDENTIFY_PER_SOURCE); + // The streaming iPad's own IDENTIFY counts on its own address. + assert!(answered(&mut session, streaming, &config, now)); + assert_eq!(session.session_id(), live, "the stream goes on"); + let actions = session.tick(&config, true, now + Duration::from_millis(100)); + assert!(!actions.client_lost); + assert_eq!(session.session_id(), live); + } + + fn pair_config(required: bool) -> PairConfig { + PairConfig(parking_lot::Mutex::new( + crate::pairing::Pairing::new(required, [7; 16]).unwrap(), + )) + } + fn pair_hello(nonce: u32) -> Hello2 { + let ControlMessage::Hello2(mut hello) = hello(nonce, 50000) else { + unreachable!() + }; + hello.device_id = 17; + hello + } + fn pair_ack(actions: &Actions) -> HelloAck { + let (_, ControlMessage::HelloAck(ack)) = + eternal_wire::v2::control::parse_control(&actions.replies[0].1).unwrap() + else { + panic!("expected ACK") + }; + ack + } + fn pair_send( + session: &mut Session, + config: &PairConfig, + peer: PeerId, + hello: Hello2, + now: Instant, + ) -> Actions { + session.handle_control(peer, 0, ControlMessage::Hello2(hello), config, now) + } + + #[test] + fn retransmitted_rejected_handshake_is_one_pairing_attempt() { + let config = pair_config(true); + let mut session = Session::new(10); + let peer = addr([192, 0, 2, 1], 50000); + let now = Instant::now(); + let mut hello = pair_hello(1); + for retry in 0..8 { + let ack = pair_ack(&pair_send( + &mut session, + &config, + peer, + hello.clone(), + now + Duration::from_millis(retry * 300), + )); + assert_eq!(ack.status, HelloStatus::Unauthorized); + assert_eq!(ack.auth_token, [0; 16]); + } + hello.client_nonce = 2; + hello.pairing_code = config.0.lock().code(); + assert_eq!( + pair_ack(&pair_send( + &mut session, + &config, + peer, + hello, + now + Duration::from_secs(3) + )) + .status, + HelloStatus::Ok + ); + } + + #[test] + fn changing_credentials_with_the_same_nonce_still_counts_as_guesses() { + let config = pair_config(true); + let mut session = Session::new(10); + let peer = addr([192, 0, 2, 1], 50000); + let now = Instant::now(); + let mut hello = pair_hello(1); + for guess in 1..=5 { + hello.pairing_code = 1_000_000 + guess; + let ack = pair_ack(&pair_send(&mut session, &config, peer, hello.clone(), now)); + assert_eq!( + ack.status, + if guess == 5 { + HelloStatus::RateLimited + } else { + HelloStatus::Unauthorized + } + ); + } + } + + #[test] + fn pairing_required_code_rotation_retry_token_and_regeneration() { + let config = pair_config(true); + let mut session = Session::new(10); + let peer = addr([192, 0, 2, 1], 50000); let now = Instant::now(); let mut hello = pair_hello(1); let rejected = pair_send(&mut session, &config, peer, hello.clone(), now); @@ -2554,6 +4211,7 @@ mod tests { reason: EndReason::Goodbye(ByeReason::UserDisconnect), first_video: Some(Duration::from_millis(40)), frames_received: true, + frags_received: false, duration: Duration::from_secs(2), } ); @@ -2648,33 +4306,153 @@ mod tests { (ended(&stopped).client_nonce, ended(&stopped).reason), (4, EndReason::Shutdown) ); - assert_eq!(session.shut_down(now), None, "reported once"); + assert_eq!( + session.shut_down(now, StreamStop::Quit), + None, + "reported once" + ); + assert_eq!(session.take_goodbye(), None, "nobody left to tell"); session.end_update(); + // The stream runs again: clients are let in. + session.transport_resumed(now, false); session.handle_control(usb, 0, identified_hello(5, 77), &TestConfig, now); - let quit = session.shut_down(now + Duration::from_secs(3)).unwrap(); + let quit = session + .shut_down(now + Duration::from_secs(3), StreamStop::Quit) + .unwrap(); assert_eq!((quit.client_nonce, quit.reason), (5, EndReason::Shutdown)); assert_eq!(quit.duration, Duration::from_secs(3)); assert!(!session.is_active()); } + /// A HELLO2 that arrives after the stream stopped, before its transport + /// noticed (up to a liveness tick), used to open a session nothing would + /// serve: no heartbeats, and the PC counted it as connected. It is turned + /// away until a transport resumes. #[test] - fn a_refused_hello_is_one_attempt_without_a_session() { + fn a_hello_after_the_stream_stopped_is_turned_away_until_it_runs_again() { let now = Instant::now(); let peer = addr([10, 0, 0, 5], 50000); let mut session = Session::new(500); - let ControlMessage::Hello2(mut old) = hello(9, 50000) else { - unreachable!() + assert_eq!(session.shut_down(now, StreamStop::User), None); + let refused = session.handle_control(peer, 0, identified_hello(7, 77), &TestConfig, now); + assert_eq!(parse_ack(&refused.replies[0].1).status, HelloStatus::Busy); + assert!(!session.is_active()); + + session.transport_resumed(now, false); + let admitted = session.handle_control(peer, 0, identified_hello(8, 77), &TestConfig, now); + assert_eq!(admitted.new_target, Some(peer)); + assert!(session.is_active()); + } + + /// A transport starting after a stop retires the stop's goodbye only if + /// the stream runs again; one starting while the stream is stopping keeps + /// it, and keeps clients out. A stop after the start keeps its goodbye. + #[test] + fn a_starting_transport_retires_only_a_goodbye_from_before_the_stream_ran_again() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let admit = |session: &mut Session, nonce| { + session.handle_control(peer, 0, identified_hello(nonce, 77), &TestConfig, now) }; - old.proto_min = 3; - old.proto_max = 3; - let refused = session.handle_control( - peer, - 0, - ControlMessage::Hello2(old.clone()), - &TestConfig, - now, - ); + + // Stopped while no transport ran; Start brought the stream back. + let mut session = Session::new(500); + admit(&mut session, 1); + session.shut_down(now, StreamStop::User).unwrap(); + assert!( + session.transport_resumed(now, false), + "a stale goodbye is retired" + ); + assert_eq!(session.take_goodbye(), None); + assert_eq!(admit(&mut session, 2).new_target, Some(peer)); + + // Stopped while this transport was starting: the goodbye is current. + let mut session = Session::new(500); + admit(&mut session, 3); + session.shut_down(now, StreamStop::User).unwrap(); + assert!(!session.transport_resumed(now, true)); + assert!( + session.take_goodbye().is_some(), + "the current goodbye is kept" + ); + let refused = admit(&mut session, 4); + assert_eq!(parse_ack(&refused.replies[0].1).status, HelloStatus::Busy); + + // Stopped once the transport ran: kept, for the transport to send. + let mut session = Session::new(500); + admit(&mut session, 5); + assert!(!session.transport_resumed(now, false), "nothing to retire"); + session.shut_down(now, StreamStop::User).unwrap(); + assert!(session.take_goodbye().is_some()); + } + + /// Quitting or Stop ends the stream for good: the iPad hears why, from + /// the transport on its way out, instead of waiting out a lost signal. + /// A restart tells it to connect again. + #[test] + fn a_stopped_stream_tells_its_ipad() { + let now = Instant::now(); + let usb = PeerId::usb(17); + let wifi = addr([10, 0, 0, 5], 50000); + for (peer, why, bye, reason) in [ + ( + usb, + StreamStop::Quit, + ByeReason::UserDisconnect, + EndReason::Shutdown, + ), + ( + wifi, + StreamStop::Handoff, + ByeReason::Error, + EndReason::Shutdown, + ), + ( + wifi, + StreamStop::User, + ByeReason::UserDisconnect, + EndReason::Stopped, + ), + ( + usb, + StreamStop::GaveUp(StageFailure::Encoder), + ByeReason::Error, + EndReason::Failed(StageFailure::Encoder), + ), + ] { + let mut session = Session::new(500); + session.handle_control(peer, 0, identified_hello(6, 77), &TestConfig, now); + let id = session.session_id().unwrap(); + let end = session.shut_down(now, why).unwrap(); + assert_eq!(end.reason, reason); + let (to, bytes) = session.take_goodbye().expect("a goodbye"); + let (header, message) = eternal_wire::v2::control::parse_control(&bytes).unwrap(); + assert_eq!( + (to, header.session_id, message), + (peer, id, ControlMessage::Bye(bye)) + ); + assert_eq!(session.take_goodbye(), None, "sent once"); + } + } + + #[test] + fn a_refused_hello_is_one_attempt_without_a_session() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(500); + let ControlMessage::Hello2(mut old) = hello(9, 50000) else { + unreachable!() + }; + old.proto_min = 3; + old.proto_max = 3; + let refused = session.handle_control( + peer, + 0, + ControlMessage::Hello2(old.clone()), + &TestConfig, + now, + ); assert_eq!( ended(&refused), SessionEnd { @@ -2684,6 +4462,7 @@ mod tests { reason: EndReason::Refused(HelloStatus::VersionUnsupported), first_video: None, frames_received: false, + frags_received: false, duration: Duration::ZERO, } ); @@ -2775,6 +4554,7 @@ mod tests { use tracing_subscriber::prelude::*; let counts = LeftOut::default(); let seen = counts.0.clone(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(tracing_subscriber::registry().with(counts), || { session.tick(&config, false, now + Duration::from_secs(30)); session.tick(&config, false, now + REFUSAL_MINUTE); @@ -2814,6 +4594,193 @@ mod tests { ); } + fn power_hello(nonce: u32, feature_caps: u16) -> ControlMessage { + let ControlMessage::Hello2(mut hello) = hello(nonce, 50000) else { + unreachable!() + }; + hello.device_id = 77; + hello.feature_caps = feature_caps; + ControlMessage::Hello2(hello) + } + + /// Windows taking the PC down ends the session at once, with the summary + /// a quit writes. An iPad that understands it is told why by the next + /// pass; the reason says sleep or power off. + #[test] + fn the_pc_going_down_tells_an_ipad_that_understands_it() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + for (event, bye) in [ + (PowerEvent::Sleep, ByeReason::HostSleeping), + (PowerEvent::Shutdown, ByeReason::HostPoweringOff), + (PowerEvent::Logoff, ByeReason::HostPoweringOff), + ] { + let mut session = Session::new(500); + session.handle_control(peer, 0, power_hello(1, FEATURE_POWER_BYE), &TestConfig, now); + let id = session.session_id().unwrap(); + let end = session + .power_down(now, event) + .ended() + .expect("the session ended"); + assert_eq!((end.client_nonce, end.reason), (1, EndReason::Shutdown)); + assert!(!session.is_active()); + assert!(session.power_goodbye_pending()); + let next = session.tick(&TestConfig, false, now); + assert_eq!(goodbye_of(&next), (peer, id, bye), "{event:?}"); + assert!(next.client_lost); + assert_eq!(next.ended, None, "the summary was the caller's to write"); + assert!(!session.power_goodbye_pending()); + } + } + + /// An iPad from before these reasons drops a goodbye it can't read, so + /// it gets none, as before. Its session still ends with a summary, and + /// the next pass lets the transport forget it. + #[test] + fn an_older_ipad_gets_no_goodbye_when_the_pc_goes_down() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(500); + session.handle_control( + peer, + 0, + power_hello(1, FEATURE_SUPPORTS_NACK), + &TestConfig, + now, + ); + let end = session.power_down(now, PowerEvent::Sleep).ended().unwrap(); + assert_eq!(end.reason, EndReason::Shutdown); + assert!(!session.power_goodbye_pending()); + let next = session.tick(&TestConfig, false, now); + assert_eq!(next.goodbye, None); + assert!(next.client_lost, "the transport forgets it"); + assert!(!session.tick(&TestConfig, false, now).client_lost, "once"); + } + + /// While the PC goes down a HELLO2 gets no answer, as it wouldn't a + /// moment later: a session started now would be cut without a word. + /// Waking opens it again, and so does a cancelled shutdown, or the hold + /// running out when Windows never said it woke. + #[test] + fn a_pc_going_down_answers_no_hello_until_it_is_back() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(500); + assert_eq!( + session.power_down(now, PowerEvent::Sleep), + PowerDown::NoSession, + "nobody streamed" + ); + let unanswered = + session.handle_control(peer, 0, power_hello(2, FEATURE_POWER_BYE), &TestConfig, now); + assert!(unanswered.replies.is_empty() && unanswered.new_target.is_none()); + assert!(!session.is_active()); + session.power_up(); + let admitted = + session.handle_control(peer, 0, power_hello(3, FEATURE_POWER_BYE), &TestConfig, now); + assert_eq!(admitted.new_target, Some(peer)); + + let mut session = Session::new(500); + session.power_down(now, PowerEvent::Shutdown); + let later = now + POWER_DOWN_HOLD; + let admitted = session.handle_control( + peer, + 0, + power_hello(4, FEATURE_POWER_BYE), + &TestConfig, + later, + ); + assert_eq!(admitted.new_target, Some(peer), "the hold ran out"); + } + + /// Windows tells some of it twice: the end of the session, then the + /// session change, for a sign-out. The repeat changes nothing: the hold + /// keeps its start, and nothing is ended or told again. + #[test] + fn news_of_the_pc_going_down_counts_once() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(500); + session.handle_control(peer, 0, power_hello(1, FEATURE_POWER_BYE), &TestConfig, now); + assert!(session + .power_down(now, PowerEvent::Logoff) + .ended() + .is_some()); + let again = now + Duration::from_secs(20); + assert_eq!( + session.power_down(again, PowerEvent::Logoff), + PowerDown::Repeat + ); + assert_eq!( + session.power_down(again, PowerEvent::Sleep), + PowerDown::Repeat + ); + assert!(session.take_goodbye().is_some(), "told once"); + assert_eq!(session.take_goodbye(), None); + // The hold still runs from the first news. + let admitted = session.handle_control( + peer, + 0, + power_hello(2, FEATURE_POWER_BYE), + &TestConfig, + now + POWER_DOWN_HOLD, + ); + assert_eq!(admitted.new_target, Some(peer)); + // After waking, the next news counts again. + session.power_up(); + assert_eq!( + session + .power_down(now + POWER_DOWN_HOLD, PowerEvent::Sleep) + .ended() + .map(|end| end.client_nonce), + Some(2) + ); + } + + /// A goodbye no transport sent before the PC slept is for a session + /// that is over by the time it wakes: dropped. Any other goodbye stays. + #[test] + fn waking_drops_only_the_goodbye_the_pc_going_down_left() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(500); + session.handle_control(peer, 0, power_hello(1, FEATURE_POWER_BYE), &TestConfig, now); + session.power_down(now, PowerEvent::Sleep).ended().unwrap(); + session.power_up(); + assert_eq!(session.take_goodbye(), None); + + let mut session = Session::new(500); + session.power_down(now, PowerEvent::Sleep); + session.power_up(); + session.handle_control(peer, 0, power_hello(2, FEATURE_POWER_BYE), &TestConfig, now); + session.shut_down(now, StreamStop::Quit).unwrap(); + session.power_up(); + assert!(session.take_goodbye().is_some(), "a quit's goodbye stays"); + } + + /// The PC woke after hours: the session ended when it went to sleep, so + /// the first pass after waking ends nothing again. It used to expire the + /// session then, as a silence as long as the sleep. + #[test] + fn waking_ends_nothing_a_second_time() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + for caps in [FEATURE_POWER_BYE, 0] { + let mut session = Session::new(500); + session.handle_control(peer, 0, power_hello(1, caps), &TestConfig, now); + session.power_down(now, PowerEvent::Sleep).ended().unwrap(); + session.tick(&TestConfig, false, now); + let woke = now + Duration::from_secs(3 * 3600); + session.power_up(); + session.transport_resumed(woke, false); + let first = session.tick(&TestConfig, true, woke); + assert_eq!( + (first.ended, first.goodbye, first.client_lost), + (None, None, false) + ); + } + } + /// The goodbye's destination, session id and reason. fn goodbye_of(actions: &Actions) -> (PeerId, u32, ByeReason) { let (to, bytes) = actions.goodbye.clone().expect("a goodbye"); @@ -2905,6 +4872,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 10, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, }); let ping = ControlMessage::Ping(eternal_wire::v2::control::Ping { t1_us: 1 }); let nack = ControlMessage::Nack(Nack { @@ -3312,4 +5280,1292 @@ mod tests { assert_eq!(late, 0, "{ppm} ppm"); } } + + /// Feeds `message` as the datagram a Wi-Fi or USB link delivers. + fn datagram( + session: &mut Session, + peer: PeerId, + session_id: u32, + message: ControlMessage, + now: Instant, + ) -> Actions { + let bytes = eternal_wire::v2::control::encode_control(session_id, 1, &message); + let (header, message) = eternal_wire::v2::control::parse_control(&bytes).unwrap(); + session.handle_datagram(peer, &header, message, bytes.len(), &TestConfig, now) + } + + fn ping() -> ControlMessage { + ControlMessage::Ping(eternal_wire::v2::control::Ping { t1_us: 1 }) + } + + /// Until something of its session comes back, a Wi-Fi client gets its + /// first picture and heartbeats, but no more media. Its answer asks for + /// a keyframe, as frames were held back, and the stream goes on. + #[test] + fn a_wifi_client_gets_its_first_picture_then_waits_to_be_heard() { + let mut session = Session::new(1234); + let now = Instant::now(); + let ipad = addr([10, 0, 0, 5], 50000); + session.handle_control(ipad, 0, identified_hello(1, 77), &TestConfig, now); + let id = session.session_id().unwrap(); + + assert!( + session.may_send_media(id, 300 * 1024, now), + "the first keyframe" + ); + assert!(session.may_send_media(id, UNANSWERED_MEDIA_BUDGET - 300 * 1024, now)); + assert!(!session.may_send_media(id, 1, now), "the budget is spent"); + let heartbeat = session.tick(&TestConfig, true, now + Duration::from_secs(1)); + assert_eq!(heartbeat.replies.len(), 1, "heartbeats go on"); + + let answer = datagram(&mut session, ipad, id, ping(), now + Duration::from_secs(1)); + assert!(answer.force_idr, "a keyframe starts the picture again"); + assert!(session.may_send_media(id, 4 * UNANSWERED_MEDIA_BUDGET, now)); + } + + /// A frame that doesn't fit is held back, and so is every one after it + /// until the client answers, even one that would fit: a frame after a + /// gap can't be decoded. + #[test] + fn nothing_follows_a_held_back_frame_until_the_answer() { + let mut session = Session::new(1234); + let now = Instant::now(); + let ipad = addr([10, 0, 0, 5], 50000); + session.handle_control(ipad, 0, identified_hello(1, 77), &TestConfig, now); + let id = session.session_id().unwrap(); + assert!(!session.may_send_media(id, UNANSWERED_MEDIA_BUDGET + 1, now)); + assert!(!session.may_send_media(id, 100, now)); + datagram(&mut session, ipad, id, ping(), now); + assert!(session.may_send_media(id, 100, now)); + } + + /// Whatever the iPad sends first counts, and an answer before the + /// budget ran out needs no keyframe. A packet with another session's + /// id, from another address, or a repeat of the HELLO2 is no answer. + #[test] + fn any_packet_of_the_session_from_its_address_is_its_answer() { + let answers = [ + ping(), + ControlMessage::ReceiverReport(ReceiverReport::default()), + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 7, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram: 0, + }), + ControlMessage::Nack(Nack { + stream_epoch: 7, + frame_seq: 1, + frag_count: 2, + missing: vec![1], + }), + ]; + for answer in answers { + let mut session = Session::new(1234); + let now = Instant::now(); + let ipad = addr([10, 0, 0, 5], 50000); + session.handle_control(ipad, 0, identified_hello(1, 77), &TestConfig, now); + let id = session.session_id().unwrap(); + datagram(&mut session, ipad, id, identified_hello(1, 77), now); + datagram(&mut session, ipad, id.wrapping_add(1), ping(), now); + datagram(&mut session, addr([10, 0, 0, 6], 50000), id, ping(), now); + assert!(!session.may_send_media(id, UNANSWERED_MEDIA_BUDGET + 1, now)); + let heard = datagram(&mut session, ipad, id, answer.clone(), now); + assert!( + session.may_send_media(id, UNANSWERED_MEDIA_BUDGET + 1, now), + "{answer:?}" + ); + + let mut early = Session::new(1234); + early.handle_control(ipad, 0, identified_hello(1, 77), &TestConfig, now); + let id = early.session_id().unwrap(); + assert!(early.may_send_media(id, 1000, now)); + let heard_early = datagram(&mut early, ipad, id, answer.clone(), now); + assert_eq!( + heard_early.force_idr, + heard.force_idr && matches!(answer, ControlMessage::KeyframeRequest(_)), + "{answer:?}" + ); + assert!(heard.force_idr, "{answer:?}"); + } + } + + /// The PC opens USB tunnels itself, to an iPad on the cable: they + /// stream from the start. + #[test] + fn usb_streams_before_the_ipad_is_heard() { + let mut session = Session::new(1234); + let usb = PeerId::usb(17); + let actions = datagram( + &mut session, + usb, + 0, + identified_hello(1, 77), + Instant::now(), + ); + let id = session.session_id().unwrap(); + assert_eq!(actions.new_target, Some(usb)); + assert!(session.may_send_media(id, 10 * UNANSWERED_MEDIA_BUDGET, Instant::now())); + } + + /// A new session from an address whose last one ended before it was + /// heard, by a new HELLO2 or the liveness timeout, gets no media until + /// it is, for a minute. An address that was heard starts afresh. + #[test] + fn an_address_that_was_never_heard_gets_no_new_budget_for_a_minute() { + let mut session = Session::new(1234); + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let ipad = addr([10, 0, 0, 5], 50000); + datagram(&mut session, ipad, 0, identified_hello(1, 77), at(0)); + let first = session.session_id().unwrap(); + datagram(&mut session, ipad, 0, identified_hello(2, 77), at(100)); + let second = session.session_id().unwrap(); + assert_ne!(first, second); + assert!( + !session.may_send_media(second, 1, at(100)), + "superseded unheard" + ); + + let expired = session.tick(&TestConfig, false, at(3200)); + assert_eq!(expired.ended.unwrap().reason, EndReason::Liveness); + datagram(&mut session, ipad, 0, identified_hello(3, 77), at(5000)); + let third = session.session_id().unwrap(); + assert!( + !session.may_send_media(third, 1, at(5000)), + "expired unheard" + ); + session.tick(&TestConfig, false, at(8200)); + assert_eq!(session.last_end(), Some((third, EndReason::Liveness))); + + datagram(&mut session, ipad, 0, identified_hello(4, 77), at(68_000)); + let early = session.session_id().unwrap(); + assert!( + !session.may_send_media(early, 1, at(68_000)), + "not yet a minute" + ); + session.tick(&TestConfig, false, at(71_200)); + + datagram(&mut session, ipad, 0, identified_hello(5, 77), at(132_000)); + let fresh = session.session_id().unwrap(); + assert!( + session.may_send_media(fresh, 1000, at(132_000)), + "a minute later" + ); + + datagram(&mut session, ipad, fresh, ping(), at(132_010)); + datagram(&mut session, ipad, 0, identified_hello(6, 77), at(133_000)); + let reconnect = session.session_id().unwrap(); + assert!( + session.may_send_media(reconnect, 1000, at(133_000)), + "it was heard" + ); + } + + /// A HELLO2 with the same device id from another address takes over a + /// session that was never heard, with what it already spent. + #[test] + fn an_address_change_with_the_same_device_id_brings_no_new_budget() { + let mut session = Session::new(1234); + let now = Instant::now(); + datagram( + &mut session, + addr([10, 0, 0, 5], 50000), + 0, + identified_hello(1, 77), + now, + ); + let first = session.session_id().unwrap(); + assert!(session.may_send_media(first, UNANSWERED_MEDIA_BUDGET, now)); + + let later = now + Duration::from_millis(50); + let moved = addr([10, 0, 0, 9], 50000); + let actions = datagram(&mut session, moved, 0, identified_hello(2, 77), later); + assert_eq!(actions.new_target, Some(moved)); + let second = session.session_id().unwrap(); + assert_ne!(first, second); + assert!(!session.may_send_media(second, 1, later)); + } + + /// Media to clients not yet heard counts against one host-wide budget, + /// whatever their address, which comes back over a minute. It still + /// holds once more addresses than the memory keeps have come and gone. + #[test] + fn the_host_budget_covers_every_address_even_with_the_memory_full() { + let mut session = Session::new(1234); + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let first = addr([10, 0, 0, 5], 50000); + datagram(&mut session, first, 0, identified_hello(1, 77), at(0)); + let id = session.session_id().unwrap(); + assert!(session.may_send_media(id, UNANSWERED_MEDIA_BUDGET, at(0))); + session.tick(&TestConfig, false, at(3200)); + assert_eq!(session.last_end(), Some((id, EndReason::Liveness))); + + // Every slot of the memory taken by other addresses. + for host in 0..UNANSWERED_KEPT as u8 { + let other = addr([10, 0, 1, host], 50000); + datagram( + &mut session, + other, + 0, + identified_hello(10 + u32::from(host), 88), + at(4000 + u64::from(host)), + ); + } + session.tick(&TestConfig, false, at(7200)); + assert!(session.session_id().is_none()); + + datagram(&mut session, first, 0, identified_hello(2, 77), at(9000)); + let back = session.session_id().unwrap(); + assert!(!session.may_send_media(back, 200 * 1024, at(9000))); + session.tick(&TestConfig, false, at(12_200)); + + let fresh = addr([10, 0, 2, 1], 50000); + datagram(&mut session, fresh, 0, identified_hello(3, 99), at(13_000)); + let other = session.session_id().unwrap(); + assert!( + !session.may_send_media(other, 200 * 1024, at(13_000)), + "another address" + ); + session.tick(&TestConfig, false, at(16_200)); + + datagram(&mut session, fresh, 0, identified_hello(4, 99), at(80_000)); + let refilled = session.session_id().unwrap(); + assert!( + session.may_send_media(refilled, UNANSWERED_MEDIA_BUDGET, at(80_000)), + "a minute after the last spend" + ); + } + + /// An address that keeps ending sessions before it is heard holds one + /// entry in the memory, refreshed in place, and can't push another + /// address out of it. + #[test] + fn a_churning_address_keeps_one_entry_in_the_memory() { + let mut session = Session::new(1234); + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let first = addr([10, 0, 0, 5], 50000); + datagram(&mut session, first, 0, identified_hello(1, 77), at(0)); + session.tick(&TestConfig, false, at(3200)); + + let churn = addr([10, 0, 0, 6], 50000); + for nonce in 0..2 * UNANSWERED_KEPT as u32 { + datagram( + &mut session, + churn, + 0, + identified_hello(10 + nonce, 88), + at(4000 + u64::from(nonce)), + ); + } + session.tick(&TestConfig, false, at(7200)); + assert_eq!(session.unanswered.len(), 2); + + datagram(&mut session, first, 0, identified_hello(2, 77), at(8000)); + let back = session.session_id().unwrap(); + assert!(!session.may_send_media(back, 1000, at(8000))); + } + + /// An iPad's HELLO2 as shipped since 0.3.0: the generic "iPad" name and + /// its device id. + fn shipped_hello(nonce: u32, device_id: u64) -> ControlMessage { + let ControlMessage::Hello2(mut hello) = identified_hello(nonce, device_id) else { + unreachable!() + }; + hello.device_name = "iPad".to_string(); + ControlMessage::Hello2(hello) + } + + /// A refusal over Wi-Fi is never larger than the HELLO2 it answers. It + /// leaves out the PC name and host caps when they don't fit (iPads read + /// only the status and version of a refusal), and a HELLO2 too small + /// for even that gets no answer. USB answers stay whole. + #[test] + fn a_refusal_is_never_larger_than_its_hello() { + let mut session = Session::new(1234); + let now = Instant::now(); + datagram( + &mut session, + addr([10, 0, 0, 5], 50000), + 0, + shipped_hello(1, 77), + now, + ); + + let other = addr([10, 0, 0, 6], 50000); + let shipped = shipped_hello(2, 88); + let shipped_len = eternal_wire::v2::control::encode_control(0, 1, &shipped).len(); + assert_eq!(shipped_len, 71); + let busy = datagram(&mut session, other, 0, shipped, now); + let (to, reply) = &busy.replies[0]; + assert_eq!(*to, other); + assert!( + reply.len() <= shipped_len, + "{} > {shipped_len}", + reply.len() + ); + let ack = parse_ack(reply); + assert_eq!( + (ack.status, ack.client_nonce, ack.accepted_version), + (HelloStatus::Busy, 2, 2) + ); + assert_eq!((ack.host_name.as_str(), ack.host_caps), ("", 0)); + + let ControlMessage::Hello2(mut named) = shipped_hello(3, 88) else { + unreachable!() + }; + named.device_name = "A much longer iPad name here".to_string(); + let named = ControlMessage::Hello2(named); + let named_len = eternal_wire::v2::control::encode_control(0, 1, &named).len(); + let busy = datagram(&mut session, other, 0, named, now); + let reply = &busy.replies[0].1; + assert!(reply.len() <= named_len); + assert_eq!(parse_ack(reply).host_name, "TEST-HOST", "it fits whole"); + + let ControlMessage::Hello2(mut bare) = hello(4, 50000) else { + unreachable!() + }; + bare.device_name.clear(); + let bare = ControlMessage::Hello2(bare); + assert_eq!( + eternal_wire::v2::control::encode_control(0, 1, &bare).len(), + 38 + ); + let unanswered = datagram(&mut session, other, 0, bare.clone(), now); + assert!(unanswered.replies.is_empty()); + assert_eq!( + unanswered.ended.unwrap().reason, + EndReason::Refused(HelloStatus::Busy), + "still one refused attempt" + ); + + let usb = datagram(&mut session, PeerId::usb(17), 0, bare, now); + assert_eq!(parse_ack(&usb.replies[0].1).host_name, "TEST-HOST"); + } + + /// The golden HELLO2 of a 0.3.0 iPad, byte for byte, as the second + /// device: a refusal that fits it, which that iPad's parser reads (the + /// `hello_ack_busy_bare` vector). As the only one: a session that + /// streams once its first ping arrives. + #[test] + fn a_shipped_ipads_golden_hello_is_answered_within_its_size() { + let golden = include_str!("../../../proto/testdata/v2_vectors.txt") + .lines() + .find_map(|line| line.strip_prefix("hello2_v030 ")) + .unwrap(); + let bytes: Vec = (0..golden.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&golden[at..at + 2], 16).unwrap()) + .collect(); + let feed = |session: &mut Session, peer: PeerId, now: Instant| { + let (header, message) = eternal_wire::v2::control::parse_control(&bytes).unwrap(); + session.handle_datagram(peer, &header, message, bytes.len(), &TestConfig, now) + }; + let now = Instant::now(); + let ipad = addr([10, 0, 0, 6], 50000); + + let mut session = Session::new(1234); + datagram( + &mut session, + addr([10, 0, 0, 5], 50000), + 0, + shipped_hello(1, 77), + now, + ); + let busy = feed(&mut session, ipad, now); + let reply = &busy.replies[0].1; + assert!( + reply.len() <= bytes.len(), + "{} > {}", + reply.len(), + bytes.len() + ); + assert_eq!(parse_ack(reply).status, HelloStatus::Busy); + + let mut session = Session::new(1234); + let accepted = feed(&mut session, ipad, now); + let ack = parse_ack(&accepted.replies[0].1); + assert_eq!( + (ack.status, ack.host_name.as_str()), + (HelloStatus::Ok, "TEST-HOST") + ); + datagram(&mut session, ipad, ack.session_id, ping(), now); + assert!(session.may_send_media(ack.session_id, 10 * UNANSWERED_MEDIA_BUDGET, now)); + } + + /// Session ids come from a keyed hash, so two instances with the same + /// seed hand out different ids. + #[test] + fn session_ids_differ_between_instances() { + let ipad = addr([10, 0, 0, 5], 50000); + let ids = |session: &mut Session| -> Vec { + (1..=4) + .map(|nonce| { + session.handle_control( + ipad, + 0, + identified_hello(nonce, 77), + &TestConfig, + Instant::now(), + ); + session.session_id().unwrap() + }) + .collect() + }; + let first = ids(&mut Session::new(1234)); + let steps: Vec = first.windows(2).map(|p| p[1].wrapping_sub(p[0])).collect(); + assert!(steps.windows(2).any(|p| p[0] != p[1]), "{first:?}"); + assert_ne!(first, ids(&mut Session::new(1234)), "same seed, other ids"); + } + + fn paired_hello(nonce: u32, feature_caps: u16) -> Hello2 { + let mut hello = pair_hello(nonce); + hello.auth_token = [7; 16]; + hello.feature_caps = feature_caps; + hello + } + + fn ask_while( + session: &mut Session, + config: &PairConfig, + state: NotStreaming, + hello: Hello2, + ) -> Actions { + session.set_not_streaming(Some(state)); + pair_send( + session, + config, + addr([192, 0, 2, 1], 50000), + hello, + Instant::now(), + ) + } + + /// A paired iPad that asks while no stream runs hears why, and nothing + /// starts: no session, no media target, no pairing token. + #[test] + fn a_new_ipad_hears_why_the_pc_is_not_streaming() { + let config = pair_config(true); + let mut session = Session::new(10); + for (state, status, reason) in [ + (NotStreaming::Stopped, HelloStatus::Stopped, ""), + (NotStreaming::Restarting, HelloStatus::Restarting, ""), + ( + NotStreaming::Failed(FailedReason::DisplayChanging), + HelloStatus::Failed, + "display_changing", + ), + ] { + let actions = ask_while( + &mut session, + &config, + state, + paired_hello(1, FEATURE_STANDBY_STATUS), + ); + let ack = pair_ack(&actions); + assert_eq!((ack.status, ack.reason.as_str()), (status, reason)); + assert_eq!((ack.session_id, ack.auth_token), (0, [0; 16])); + assert!(actions.new_target.is_none() && !actions.force_idr); + assert!(!session.is_active()); + } + // Streaming again: the same HELLO2 connects. + session.set_not_streaming(None); + let actions = pair_send( + &mut session, + &config, + addr([192, 0, 2, 1], 50000), + paired_hello(1, FEATURE_STANDBY_STATUS), + Instant::now(), + ); + assert_eq!(pair_ack(&actions).status, HelloStatus::Ok); + assert!(session.is_active()); + } + + /// 0.3.4 and older drop statuses past RateLimited. While the PC restarts + /// they hear nothing and keep retrying, as before; a stopped or failed + /// PC turns them down with Error, a card they have. + #[test] + fn an_old_ipad_gets_only_statuses_it_knows() { + let config = pair_config(true); + let mut session = Session::new(10); + let restarting = ask_while( + &mut session, + &config, + NotStreaming::Restarting, + paired_hello(1, 0), + ); + assert!(restarting.replies.is_empty() && restarting.ended.is_none()); + for state in [ + NotStreaming::Stopped, + NotStreaming::Failed(FailedReason::Encoder), + ] { + let ack = pair_ack(&ask_while(&mut session, &config, state, paired_hello(2, 0))); + assert_eq!(ack.status, HelloStatus::Error); + assert!(ack.reason.is_empty()); + } + assert!(!session.is_active()); + } + + /// Only an iPad that may connect learns the stream state. Wrong codes + /// count toward the rate limit here too, and the right code is not + /// spent, so it still works once the stream starts. + #[test] + fn pairing_still_decides_who_hears_it() { + let config = pair_config(true); + let mut session = Session::new(10); + let mut unpaired = pair_hello(1); + unpaired.feature_caps = FEATURE_STANDBY_STATUS; + for state in [ + NotStreaming::Stopped, + NotStreaming::Restarting, + NotStreaming::Failed(FailedReason::Capture), + ] { + let ack = pair_ack(&ask_while(&mut session, &config, state, unpaired.clone())); + assert_eq!(ack.status, HelloStatus::Unauthorized); + assert!(ack.reason.is_empty()); + } + let mut statuses = Vec::new(); + for attempt in 1..=5 { + let mut guess = unpaired.clone(); + guess.client_nonce = 10 + attempt; + guess.pairing_code = 1_000_000 + attempt; + statuses.push( + pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Stopped, + guess, + )) + .status, + ); + } + assert_eq!(statuses.last(), Some(&HelloStatus::RateLimited)); + + let config = pair_config(true); + let code = config.0.lock().code(); + let mut right = unpaired; + right.client_nonce = 20; + right.pairing_code = code; + let ack = pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Stopped, + right.clone(), + )); + assert_eq!( + (ack.status, ack.auth_token), + (HelloStatus::Stopped, [0; 16]) + ); + assert_eq!(config.0.lock().code(), code, "the code stays valid"); + session.set_not_streaming(None); + let accepted = pair_send( + &mut session, + &config, + addr([192, 0, 2, 1], 50000), + right, + Instant::now(), + ); + assert_eq!(pair_ack(&accepted).status, HelloStatus::Ok); + } + + /// Stop ends the session and closes the PC to clients; the supervisor + /// then says why, and a HELLO2 hears it instead of Busy. A quit, which + /// no supervisor state follows, still turns it away with Busy. + #[test] + fn a_stop_is_answered_with_why_not_busy() { + let config = pair_config(true); + let mut session = Session::new(10); + let now = Instant::now(); + session.shut_down(now, StreamStop::Quit); + let busy = pair_send( + &mut session, + &config, + addr([192, 0, 2, 1], 50000), + paired_hello(1, FEATURE_STANDBY_STATUS), + now, + ); + assert_eq!(pair_ack(&busy).status, HelloStatus::Busy); + session.shut_down(now, StreamStop::User); + let ack = pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Stopped, + paired_hello(2, FEATURE_STANDBY_STATUS), + )); + assert_eq!(ack.status, HelloStatus::Stopped); + } + + #[test] + fn the_version_and_update_refusals_come_first() { + let config = pair_config(true); + let mut session = Session::new(10); + let mut old = paired_hello(1, FEATURE_STANDBY_STATUS); + old.proto_min = 1; + old.proto_max = 1; + let ack = pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Stopped, + old, + )); + assert_eq!(ack.status, HelloStatus::VersionUnsupported); + assert!(session.begin_update(false)); + let ack = pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Stopped, + paired_hello(2, FEATURE_STANDBY_STATUS), + )); + assert_eq!(ack.status, HelloStatus::Busy); + } + + /// A session kept across a restart stays, and its iPad's repeated + /// HELLO2 hears that the PC is restarting instead of the old OK. + #[test] + fn a_session_kept_across_a_restart_stays() { + let config = pair_config(true); + let mut session = Session::new(10); + let hello = paired_hello(1, FEATURE_STANDBY_STATUS); + let peer = addr([192, 0, 2, 1], 50000); + let accepted = pair_send(&mut session, &config, peer, hello.clone(), Instant::now()); + assert_eq!(pair_ack(&accepted).status, HelloStatus::Ok); + let ack = pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Restarting, + hello, + )); + assert_eq!(ack.status, HelloStatus::Restarting); + assert!(session.is_active()); + } + + /// The stream that is stopping still takes HELLO2s until its transport + /// ends. Its not-streaming answers are never larger than the HELLO2 + /// either, as the standby's aren't. + #[test] + fn a_not_streaming_answer_is_never_larger_than_the_hello() { + let config = pair_config(false); + let mut session = Session::new(10); + let peer = addr([192, 0, 2, 1], 50000); + let mut small = pair_hello(1); + small.device_name = String::new(); + small.device_id = 0; + let unpadded = + eternal_wire::v2::control::encode_control(0, 1, &ControlMessage::Hello2(small.clone())); + // The same small HELLO2 claiming the bit, without its padding. + let mut claimed = unpadded.clone(); + claimed[26..28].copy_from_slice(&FEATURE_STANDBY_STATUS.to_le_bytes()); + let mut padded_hello = small; + padded_hello.client_nonce = 2; + padded_hello.feature_caps = FEATURE_STANDBY_STATUS; + let padded = + eternal_wire::v2::control::encode_control(0, 2, &ControlMessage::Hello2(padded_hello)); + session.set_not_streaming(Some(NotStreaming::Failed(FailedReason::Encoder))); + let mut answer = |datagram: &[u8]| { + let (header, message) = eternal_wire::v2::control::parse_control(datagram).unwrap(); + session.handle_datagram( + peer, + &header, + message, + datagram.len(), + &config, + Instant::now(), + ) + }; + for small in [&unpadded, &claimed] { + assert_eq!(small.len(), 38); + let actions = answer(small); + assert!( + actions.replies.is_empty(), + "{} bytes answered", + actions.replies[0].1.len() + ); + } + let actions = answer(&padded); + let (_, reply) = &actions.replies[0]; + assert!(reply.len() <= padded.len()); + let (_, ControlMessage::HelloAck(ack)) = + eternal_wire::v2::control::parse_control(reply).unwrap() + else { + panic!("expected an ACK"); + }; + assert_eq!( + (ack.status, ack.reason.as_str()), + (HelloStatus::Failed, "encoder") + ); + } + + /// Every answer to a Wi-Fi HELLO2 is bounded by its size, the version + /// and pairing refusals too, while no stream runs and while one does. A + /// guess at the code carries the HELLO2's extension, so it hears the + /// short refusal; a padded HELLO2 hears whole answers. Over USB the + /// answers stay whole. + #[test] + fn every_hello_answer_is_bounded_while_not_streaming() { + use eternal_wire::v2::control::{encode_control, parse_control}; + let peer = addr([192, 0, 2, 1], 50000); + let send = |session: &mut Session, config: &PairConfig, hello: &Hello2| { + let datagram = encode_control(0, 1, &ControlMessage::Hello2(hello.clone())); + let (header, message) = parse_control(&datagram).unwrap(); + let actions = session.handle_datagram( + peer, + &header, + message, + datagram.len(), + config, + Instant::now(), + ); + for (_, reply) in &actions.replies { + assert!( + reply.len() <= datagram.len(), + "{} > {}", + reply.len(), + datagram.len() + ); + } + actions + .replies + .first() + .map(|(_, reply)| match parse_control(reply) { + Ok((_, ControlMessage::HelloAck(ack))) => ack.status, + other => panic!("not an ACK: {other:?}"), + }) + }; + let small = |nonce: u32, caps: u16| { + let mut hello = pair_hello(nonce); + hello.device_name = String::new(); + hello.device_id = 0; + hello.feature_caps = caps; + hello + }; + + let config = pair_config(true); + let mut session = Session::new(10); + session.set_not_streaming(Some(NotStreaming::Stopped)); + // The version check. + let mut old_version = small(1, 0); + old_version.proto_min = 1; + old_version.proto_max = 1; + assert_eq!(send(&mut session, &config, &old_version), None); + old_version.feature_caps = FEATURE_STANDBY_STATUS; + assert_eq!( + send(&mut session, &config, &old_version), + Some(HelloStatus::VersionUnsupported) + ); + // The pairing checks: no code, then wrong codes until rate limited. + assert_eq!(send(&mut session, &config, &small(2, 0)), None); + for attempt in 0..6 { + let mut guess = small(10 + attempt, 0); + guess.pairing_code = 1_000_000 + attempt; + let expected = if attempt < 3 { + HelloStatus::Unauthorized + } else { + HelloStatus::RateLimited + }; + assert_eq!( + send(&mut session, &config, &guess), + Some(expected), + "attempt {attempt}" + ); + } + let mut padded = small(20, FEATURE_STANDBY_STATUS); + padded.pairing_code = 1_000_020; + assert_eq!( + send(&mut session, &config, &padded), + Some(HelloStatus::RateLimited) + ); + + // Streaming: the same bound, and an attempt with no answer that + // fits is still one refused attempt. + let config = pair_config(true); + let mut session = Session::new(10); + assert_eq!(send(&mut session, &config, &small(30, 0)), None); + let mut guess = small(31, 0); + guess.pairing_code = 1_000_031; + assert_eq!( + send(&mut session, &config, &guess), + Some(HelloStatus::Unauthorized) + ); + + // Over USB, stopped or streaming, the answers stay whole. + let usb = PeerId::usb(17); + let over_usb = |session: &mut Session, hello: &Hello2| { + let datagram = encode_control(0, 1, &ControlMessage::Hello2(hello.clone())); + let (header, message) = parse_control(&datagram).unwrap(); + let actions = session.handle_datagram( + usb, + &header, + message, + datagram.len(), + &config, + Instant::now(), + ); + assert!(actions.replies[0].1.len() > datagram.len()); + pair_ack(&actions) + }; + let mut session = Session::new(10); + let mut old_version = small(40, 0); + old_version.proto_min = 1; + old_version.proto_max = 1; + let refused = over_usb(&mut session, &old_version); + assert_eq!(refused.status, HelloStatus::VersionUnsupported); + assert_eq!(refused.host_name, "TEST-HOST"); + let mut paired = small(41, 0); + paired.auth_token = [7; 16]; + session.set_not_streaming(Some(NotStreaming::Stopped)); + let stopped = over_usb(&mut session, &paired); + assert_eq!( + (stopped.status, stopped.host_name.as_str()), + (HelloStatus::Error, "TEST-HOST") + ); + } + + /// The standby responder's config: no stream, epoch 0. + struct RestartConfig; + impl ConfigSource for RestartConfig { + fn stream_config(&self) -> StreamConfig { + StreamConfig::default() + } + fn host_name(&self) -> String { + "TEST-HOST".to_string() + } + } + + fn heartbeats(actions: &Actions, peer: PeerId, session_id: u32) -> Vec { + actions + .replies + .iter() + .filter(|(to, _)| *to == peer) + .filter_map( + |(_, bytes)| match eternal_wire::v2::control::parse_control(bytes) { + Ok((header, ControlMessage::Heartbeat(hb))) + if header.session_id == session_id => + { + Some(hb.stream_config) + } + _ => None, + }, + ) + .collect() + } + + /// A streaming session whose iPad has answered, then the PC restarts. + fn restarting_session(peer: PeerId, now: Instant) -> (Session, u32) { + let mut session = Session::new(1234); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + session.handle_control_authed(peer, report(10), &TestConfig, now); + let id = session.session_id().unwrap(); + assert_eq!( + heartbeats(&session.tick(&TestConfig, true, now), peer, id).len(), + 1 + ); + session.set_not_streaming(Some(NotStreaming::Restarting)); + (session, id) + } + + /// While the PC restarts, the kept session gets a heartbeat each + /// interval, with the stream config it last had, and stays alive for as + /// long as its iPad keeps reporting: longer than the iPad's liveness. + #[test] + fn a_restart_keeps_the_ipads_session_alive_with_heartbeats() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let (mut session, id) = restarting_session(peer, now); + for second in 1..=12 { + let at = now + Duration::from_secs(second); + session.handle_control_authed(peer, report(10), &RestartConfig, at); + let actions = session.restart_tick(&RestartConfig, at); + assert!( + !actions.client_lost && actions.goodbye.is_none(), + "second {second}" + ); + let configs = heartbeats(&actions, peer, id); + assert_eq!(configs.len(), 1, "second {second}"); + assert_eq!( + configs[0].stream_epoch, 7, + "the last stream's config, not epoch 0" + ); + } + assert_eq!(session.session_id(), Some(id)); + } + + /// Only an iPad that answered its session gets them, and only while the + /// PC restarts: not after Stop or the storm brake, nor while streaming. + #[test] + fn restart_heartbeats_go_only_to_an_answered_session_while_restarting() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + let id = session.session_id().unwrap(); + session.set_not_streaming(Some(NotStreaming::Restarting)); + let actions = session.restart_tick(&RestartConfig, now + Duration::from_millis(500)); + assert!(heartbeats(&actions, peer, id).is_empty(), "never answered"); + + let (mut session, id) = restarting_session(peer, now); + for state in [ + None, + Some(NotStreaming::Stopped), + Some(NotStreaming::Failed(FailedReason::Encoder)), + ] { + session.set_not_streaming(state); + let actions = session.restart_tick(&RestartConfig, now + Duration::from_secs(1)); + assert!(actions.replies.is_empty(), "{state:?}"); + } + assert!(session.session_id() == Some(id)); + let mut empty = Session::new(1); + empty.set_not_streaming(Some(NotStreaming::Restarting)); + assert!(empty.restart_tick(&RestartConfig, now).replies.is_empty()); + } + + /// An iPad that goes silent during the restart is still expired, with + /// the goodbye that tells it to connect again. + #[test] + fn a_silent_ipad_still_expires_during_a_restart() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let (mut session, _) = restarting_session(peer, now); + let mut lost = false; + for second in 1..=12 { + let actions = session.restart_tick(&RestartConfig, now + Duration::from_secs(second)); + lost |= actions.client_lost; + } + assert!(lost && !session.is_active()); + } + + /// Past the hold limit without a frame, the session ends with the + /// goodbye that tells its iPad to connect again, and its HELLO2 then + /// hears that the PC is restarting. A frame resets the limit. + #[test] + fn a_restart_that_outlasts_the_hold_limit_ends_the_session() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let (mut session, id) = restarting_session(peer, now); + session.restart_tick(&RestartConfig, now); + // A frame went out in between (a generation streamed briefly). + let half = now + restart_hold_limit() / 2; + session.set_not_streaming(None); + assert!(session.may_send_media(id, 1000, half)); + session.set_not_streaming(Some(NotStreaming::Restarting)); + session.handle_control_authed(peer, report(10), &RestartConfig, half); + session.restart_tick(&RestartConfig, half); + let before = half + restart_hold_limit() - Duration::from_millis(1); + session.handle_control_authed(peer, report(10), &RestartConfig, before); + assert!( + !session.restart_tick(&RestartConfig, before).client_lost, + "the frame reset it" + ); + + let past = half + restart_hold_limit(); + let actions = session.restart_tick(&RestartConfig, past); + assert!(actions.client_lost && !session.is_active()); + assert_eq!( + actions.ended.unwrap().reason, + EndReason::Failed(StageFailure::Other) + ); + let (to, bytes) = actions.goodbye.expect("a goodbye"); + assert_eq!(to, peer); + assert!(matches!( + eternal_wire::v2::control::parse_control(&bytes), + Ok((header, ControlMessage::Bye(ByeReason::Error))) if header.session_id == id + )); + let mut again = match hello(2, 50000) { + ControlMessage::Hello2(hello) => hello, + _ => unreachable!(), + }; + again.feature_caps |= FEATURE_STANDBY_STATUS; + let answer = + session.handle_control(peer, 0, ControlMessage::Hello2(again), &RestartConfig, past); + assert_eq!( + parse_ack(&answer.replies[0].1).status, + HelloStatus::Restarting + ); + } + + /// The supervisor clears the restart before the new generation sends a + /// frame. A slow start still ends at the hold limit, counted from the + /// last frame, on the stream's own tick. + #[test] + fn the_hold_limit_holds_after_the_stream_resumes_without_a_frame() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let (mut session, id) = restarting_session(peer, now); + assert!(session.may_send_media(id, 1000, now)); + session.restart_tick(&RestartConfig, now + Duration::from_secs(1)); + // The next generation starts; it hasn't sent a frame yet. + session.set_not_streaming(None); + session.transport_resumed(now + Duration::from_secs(2), false); + let before = now + restart_hold_limit() - Duration::from_millis(1); + session.handle_control_authed(peer, report(10), &TestConfig, before); + assert!(!session.tick(&TestConfig, true, before).client_lost); + let at = now + restart_hold_limit(); + session.handle_control_authed(peer, report(10), &TestConfig, at); + let actions = session.tick(&TestConfig, true, at); + assert!( + actions.client_lost && !session.is_active(), + "still held past the limit" + ); + assert!(matches!( + eternal_wire::v2::control::parse_control(&actions.goodbye.expect("a goodbye").1), + Ok((_, ControlMessage::Bye(ByeReason::Error))) + )); + } + + /// A frame held back for an iPad that hasn't answered never went out, + /// so it neither resets the hold limit nor counts as the last frame. + #[test] + fn a_held_back_frame_does_not_reset_the_hold_limit() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + let id = session.session_id().unwrap(); + session.set_not_streaming(Some(NotStreaming::Restarting)); + session.restart_tick(&RestartConfig, now); + let held = |session: &Session| { + let active = session.active.as_ref().unwrap(); + (active.restart_held_since, active.last_frame_at) + }; + let before = held(&session); + assert!(before.0.is_some()); + let later = now + Duration::from_millis(500); + assert!(!session.may_send_media(id, UNANSWERED_MEDIA_BUDGET + 1, later)); + assert_eq!( + held(&session), + before, + "a frame that never went out counted" + ); + } + + /// An iPad that said goodbye to hand over to the cable during the pause + /// ends as it said, on its takeover deadline, not with the hold limit's + /// goodbye. + #[test] + fn an_ipads_own_goodbye_during_the_pause_ends_its_session_as_it_said() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + session.handle_control_authed(peer, identified_hello(1, 77), &TestConfig, now); + session.handle_control_authed(peer, report(10), &TestConfig, now); + session.set_not_streaming(Some(NotStreaming::Restarting)); + session.restart_tick(&RestartConfig, now); + let limit = now + restart_hold_limit(); + let bye_at = limit - Duration::from_millis(500); + session.handle_control_authed(peer, report(10), &RestartConfig, bye_at); + session.handle_control_authed( + peer, + ControlMessage::Bye(ByeReason::Superseded), + &RestartConfig, + bye_at, + ); + let mut ended = None; + for step in 0..20 { + let at = limit + Duration::from_millis(100 * step); + let actions = session.restart_tick(&RestartConfig, at); + assert!(actions.goodbye.is_none(), "told to connect again at {step}"); + if let Some(end) = actions.ended { + ended = Some(end.reason); + break; + } + } + assert_eq!(ended, Some(EndReason::Goodbye(ByeReason::Superseded))); + } + + fn notices(actions: &Actions) -> Vec { + actions + .replies + .iter() + .filter_map( + |(_, bytes)| match eternal_wire::v2::control::parse_control(bytes) { + Ok((_, ControlMessage::Heartbeat(hb))) => Some(hb.notice), + _ => None, + }, + ) + .collect() + } + + /// While a session is held through a restart, its heartbeats say so to + /// an iPad that understands it, on the standby's ticks and on the next + /// stream's until its first frame; then they stop saying it. + #[test] + fn a_held_session_hears_that_the_pc_is_restarting() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_RESTARTING}; + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + let ControlMessage::Hello2(mut hello) = hello(1, 50000) else { + unreachable!() + }; + hello.feature_caps |= FEATURE_STANDBY_STATUS; + session.handle_control_authed(peer, ControlMessage::Hello2(hello), &TestConfig, now); + session.handle_control_authed(peer, report(10), &TestConfig, now); + let id = session.session_id().unwrap(); + assert_eq!( + notices(&session.tick(&TestConfig, true, now)), + [NOTICE_NONE] + ); + + session.set_not_streaming(Some(NotStreaming::Restarting)); + let at = now + Duration::from_secs(1); + assert_eq!( + notices(&session.restart_tick(&RestartConfig, at)), + [NOTICE_RESTARTING] + ); + // The next stream runs, with no frame yet. + session.set_not_streaming(None); + let at = now + Duration::from_secs(2); + session.handle_control_authed(peer, report(10), &TestConfig, at); + assert_eq!( + notices(&session.tick(&TestConfig, true, at)), + [NOTICE_RESTARTING] + ); + // Its first frame. + assert!(session.may_send_media(id, 1000, at)); + assert_eq!(notices(&session.tick(&TestConfig, true, at)), [NOTICE_NONE]); + } + + /// A 0.3.4 iPad, without the feature bit, gets the heartbeats with no + /// notice, as from a running stream. + #[test] + fn an_old_ipad_held_through_a_restart_gets_no_notice() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let (mut session, _) = restarting_session(peer, now); + let at = now + Duration::from_secs(1); + assert_eq!( + notices(&session.restart_tick(&RestartConfig, at)), + [eternal_wire::v2::control::NOTICE_NONE] + ); + } + + /// The capture's own notice says more than "restarting": a stream that + /// comes back into a locked PC, one with no display, or one whose + /// graphics card Windows won't capture from, says that, and + /// "restarting" shows only when nothing else does. + #[test] + fn a_capture_notice_comes_before_the_restarting_one() { + use eternal_wire::v2::control::{ + NOTICE_NONE, NOTICE_NO_DISPLAY, NOTICE_PC_LOCKED, NOTICE_RESTARTING, + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, NOTICE_UNSUPPORTED_GPU, + }; + struct Blocked(u8); + impl ConfigSource for Blocked { + fn stream_config(&self) -> StreamConfig { + TestConfig.stream_config() + } + fn host_name(&self) -> String { + TestConfig.host_name() + } + fn notice(&self) -> u8 { + self.0 + } + } + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + let ControlMessage::Hello2(mut hello) = hello(1, 50000) else { + unreachable!() + }; + hello.feature_caps |= FEATURE_STANDBY_STATUS; + session.handle_control_authed(peer, ControlMessage::Hello2(hello), &TestConfig, now); + session.handle_control_authed(peer, report(10), &TestConfig, now); + session.set_not_streaming(Some(NotStreaming::Restarting)); + let at = now + Duration::from_secs(1); + assert_eq!( + notices(&session.restart_tick(&RestartConfig, at)), + [NOTICE_RESTARTING] + ); + // The next stream runs into a blocked capture, before its first frame. + session.set_not_streaming(None); + let at = now + Duration::from_secs(2); + for notice in [ + NOTICE_PC_LOCKED, + NOTICE_NO_DISPLAY, + NOTICE_UNSUPPORTED_GPU, + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, + ] { + assert_ne!(notice, NOTICE_NONE); + session.handle_control_authed(peer, report(10), &Blocked(notice), at); + assert_eq!(notices(&session.tick(&Blocked(notice), true, at)), [notice]); + // Unblocked, still no frame: restarting again. + assert_eq!( + notices(&session.tick(&TestConfig, true, at)), + [NOTICE_RESTARTING], + "after notice {notice}" + ); + } + } + + /// The Windows 10 Second screen note comes last: a capture notice and + /// "restarting" each say more. Alone, it is said. + #[test] + fn the_windows_11_note_comes_after_the_capture_and_restarting_notices() { + use eternal_wire::v2::control::{ + NOTICE_NONE, NOTICE_NO_DISPLAY, NOTICE_PC_LOCKED, NOTICE_RESTARTING, + NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11, NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, + NOTICE_UNSUPPORTED_GPU, + }; + struct Windows10(u8); + impl ConfigSource for Windows10 { + fn stream_config(&self) -> StreamConfig { + TestConfig.stream_config() + } + fn host_name(&self) -> String { + TestConfig.host_name() + } + fn notice(&self) -> u8 { + self.0 + } + fn needs_windows_11(&self) -> bool { + true + } + } + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + let ControlMessage::Hello2(mut hello) = hello(1, 50000) else { + unreachable!() + }; + hello.feature_caps |= FEATURE_STANDBY_STATUS; + session.handle_control_authed(peer, ControlMessage::Hello2(hello), &TestConfig, now); + session.handle_control_authed(peer, report(10), &TestConfig, now); + let id = session.session_id().unwrap(); + let alone = Windows10(NOTICE_NONE); + assert_eq!( + notices(&session.tick(&alone, true, now)), + [NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11] + ); + for notice in [ + NOTICE_PC_LOCKED, + NOTICE_NO_DISPLAY, + NOTICE_UNSUPPORTED_GPU, + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, + ] { + assert_eq!( + notices(&session.tick(&Windows10(notice), true, now)), + [notice] + ); + } + // Held through a restart, before the next stream's first frame. + session.set_not_streaming(Some(NotStreaming::Restarting)); + let at = now + Duration::from_secs(1); + assert_eq!( + notices(&session.restart_tick(&RestartConfig, at)), + [NOTICE_RESTARTING] + ); + session.set_not_streaming(None); + let at = now + Duration::from_secs(2); + session.handle_control_authed(peer, report(10), &alone, at); + assert_eq!( + notices(&session.tick(&alone, true, at)), + [NOTICE_RESTARTING] + ); + assert!(session.may_send_media(id, 1000, at)); + assert_eq!( + notices(&session.tick(&alone, true, at)), + [NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11] + ); + } } diff --git a/host/src/transport/standby.rs b/host/src/transport/standby.rs new file mode 100644 index 00000000..c4247f32 --- /dev/null +++ b/host/src/transport/standby.rs @@ -0,0 +1,328 @@ +//! Answers iPads while no pipeline generation streams. +//! +//! The stream port used to be open only inside a running pipeline, so a +//! HELLO2 sent while the stream waited to restart, after Stop, or after the +//! storm brake gave up went unanswered, and the iPad said the PC didn't +//! answer. Between generations the supervisor now runs this responder on +//! the same port and the USB tunnels. It answers each HELLO2 through the +//! session, which says why nothing streams ([`Session::set_not_streaming`]) +//! after the same version, update and pairing checks as always. It answers +//! an IDENTIFY through the session too, so an iPad that finds the PC at a +//! new address can check it is the saved PC while nothing streams. It never +//! starts a session or a stream, and lets go of the port before the +//! supervisor checks it for the next generation. +//! +//! While the PC restarts its stream, a session kept across the restart +//! stays alive here: its iPad gets heartbeats as from a running stream, and +//! its reports and pings are heard as there (see +//! [`Session::restart_tick`]). It used to lose the signal after three +//! seconds, while a restart can wait up to half a minute. +//! +//! [`Session::restart_tick`]: super::session::Session::restart_tick +//! +//! [`Session::set_not_streaming`]: super::session::Session::set_not_streaming + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use eternal_wire::v2::control::{parse_control, ControlMessage}; +use eternal_wire::v2::{classify, Classified}; +use tokio::net::UdpSocket; +use tokio::sync::oneshot; +use tracing::{debug, info, warn}; + +use super::link::{LinkId, PeerId}; +use super::usb::{LinkEvent, TransportLinks}; +use crate::control::SharedControl; + +/// How often the responder tries again to bind a port it couldn't: the +/// generation that just ended may hold it a moment longer. +const BIND_RETRY: Duration = Duration::from_millis(250); +/// How often the responder looks for a goodbye left for a stopped session. +const GOODBYE_CHECK: Duration = Duration::from_millis(250); +/// How often the responder checks the chosen network's address, so it +/// follows the network as the stream does, a little sooner. +const NETWORK_CHECK: Duration = Duration::from_secs(3); +/// How long [`Standby::stop`] waits for the port to close. +const STOP_TIMEOUT: Duration = Duration::from_secs(2); + +/// The running responder. [`Standby::stop`] closes the port. +pub struct Standby { + stop: oneshot::Sender<()>, + thread: std::thread::JoinHandle<()>, +} + +impl Standby { + /// Starts answering on `listen_port`, on its own thread. + pub fn start(listen_port: u16, shared: SharedControl) -> Option { + let (stop, stopped) = oneshot::channel(); + std::thread::Builder::new() + .name("standby".into()) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build(); + match runtime { + Ok(runtime) => runtime.block_on(run(listen_port, shared, stopped)), + Err(error) => warn!(%error, "Could not answer iPads while the stream is off"), + } + }) + .inspect_err(|error| warn!(%error, "Could not answer iPads while the stream is off")) + .ok() + .map(|thread| Self { stop, thread }) + } + + /// Stops answering, and returns once the port and the USB tunnels are + /// free for the next generation. + pub fn stop(self) { + let _ = self.stop.send(()); + let deadline = Instant::now() + STOP_TIMEOUT; + while !self.thread.is_finished() { + if Instant::now() >= deadline { + warn!("The standby responder didn't let go of the stream port in time"); + return; + } + std::thread::sleep(Duration::from_millis(5)); + } + let _ = self.thread.join(); + } +} + +async fn run(listen_port: u16, shared: SharedControl, mut stopped: oneshot::Receiver<()>) { + loop { + let (mut links, bound) = tokio::select! { + opened = open(listen_port, &shared) => opened, + _ = &mut stopped => return, + }; + debug!(%bound, "Answering iPads while no stream runs"); + if !serve(&mut links, bound, &shared, &mut stopped).await { + super::park_stream_socket(&shared, links.into_udp_socket()); + debug!("Stopped answering iPads while no stream runs"); + return; + } + // The chosen network went away, came back or changed its address: + // listen where iPads can reach the PC now, as the stream would. + info!(was = %bound, "The chosen network changed: answering iPads on its new address"); + } +} + +/// Answers until the supervisor stops the responder (false), or until the +/// chosen network's address is no longer `bound` (true). +async fn serve( + links: &mut TransportLinks, + bound: SocketAddr, + shared: &SharedControl, + stopped: &mut oneshot::Receiver<()>, +) -> bool { + let mut goodbyes = tokio::time::interval(GOODBYE_CHECK); + let mut heartbeat = tokio::time::interval(super::session::HEARTBEAT_INTERVAL); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut network_check = tokio::time::interval(NETWORK_CHECK); + network_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + biased; + _ = &mut *stopped => return false, + _ = network_check.tick() => { + // Every network listens on 0.0.0.0, so a change to or from + // it shows here too. + let listen_on = shared.listen_on.lock().clone(); + if crate::network::listen_ip_now(&listen_on) != bound.ip() { + return true; + } + } + _ = heartbeat.tick() => { + let config = config(shared, bound); + let actions = shared.session.lock().restart_tick(&config, super::now()); + carry_out(links, shared, actions).await; + } + // A Stop during a restart pause ends a session no transport + // can tell: say it here. + _ = goodbyes.tick() => { + let goodbye = shared.session.lock().take_goodbye(); + if let Some(goodbye) = goodbye { + super::say_goodbye(links, goodbye).await; + } + } + event = links.receive() => match event { + Ok(LinkEvent::Datagram(peer, datagram)) => { + answer(links, shared, bound, peer, &datagram).await; + } + // A tunnel that closes now is gone for the next generation + // too; a session it carried ends on its liveness there. + Ok(LinkEvent::Closed(_)) => {} + Err(error) => { + debug!(%error, "Standby receive failed"); + tokio::time::sleep(BIND_RETRY).await; + } + }, + } + } +} + +/// Takes the stream port from the generation that just ended, or binds it +/// as the transport does, retrying while that generation still holds it. +/// Returns the links and the address they listen on. +async fn open(listen_port: u16, shared: &SharedControl) -> (TransportLinks, SocketAddr) { + let mut logged = false; + loop { + let listen_on = shared.listen_on.lock().clone(); + let ip = crate::network::listen_ip_now(&listen_on); + let bind_addr = SocketAddr::from((ip, listen_port)); + let socket = match super::take_stream_socket(shared, bind_addr) { + Some(socket) => Ok(socket), + None => super::bind_stream_socket(bind_addr), + }; + match socket.and_then(UdpSocket::from_std) { + Ok(socket) => { + if logged { + info!(%bind_addr, "Answering iPads while the stream is off"); + } + if ip.is_unspecified() { + // Lists the home network addresses before anyone asks + // for them, as the stream does. + crate::network::home_addresses(); + } + // The Address row and QR code follow, as for the stream. + let shown = crate::gui::shown_address(ip, listen_port); + crate::stats::PIPELINE_STATS.lock().set_listen_addr(shown); + let links = TransportLinks::new(socket, Arc::clone(&shared.force_next_idr)); + return (links, bind_addr); + } + Err(error) if !logged => { + logged = true; + warn!(%bind_addr, %error, "Can't answer iPads while the stream is off yet"); + } + Err(_) => {} + } + tokio::time::sleep(BIND_RETRY).await; + } +} + +fn config(shared: &SharedControl, bound: SocketAddr) -> super::SharedConfigSource<'_> { + super::SharedConfigSource { + shared, + stream_epoch: 0, + pen_available: false, + listen: bound, + } +} + +/// Sends what the session decided while no stream runs: heartbeats and +/// pongs to the kept session, its goodbye, and the summary of a session +/// that ended. +async fn carry_out( + links: &TransportLinks, + shared: &SharedControl, + actions: super::session::Actions, +) { + if let Some(end) = &actions.ended { + crate::diagnostics::log_session_end(end, shared); + } + for (destination, datagram) in &actions.replies { + if let Err(error) = links.send_to(datagram, *destination).await { + debug!(peer = %destination, %error, "Could not reach the iPad while the stream restarts"); + } + } + if let Some(goodbye) = actions.goodbye { + super::say_goodbye(links, goodbye).await; + } +} + +/// Answers a HELLO2 or an IDENTIFY, and hears the kept session's reports, +/// pings, keyframe requests and goodbye, so it stays alive while the PC +/// restarts (see [`super::session::Session::restart_tick`]). Input and +/// repair requests wait for the next generation. +async fn answer( + links: &TransportLinks, + shared: &SharedControl, + bound: SocketAddr, + peer: PeerId, + datagram: &[u8], +) { + match classify(datagram) { + Classified::Control(_) => {} + Classified::LegacyHello => { + shared.session.lock().note_legacy_hello(peer); + return; + } + _ => return, + } + let Ok((header, message)) = parse_control(datagram) else { + return; + }; + let config = config(shared, bound); + if matches!( + message, + ControlMessage::ReceiverReport(_) + | ControlMessage::Ping(_) + | ControlMessage::KeyframeRequest(_) + | ControlMessage::Bye(_) + ) { + let actions = { + let mut session = shared.session.lock(); + if session.not_streaming() != Some(super::session::NotStreaming::Restarting) { + return; + } + // Only the kept session's own messages get anywhere: the + // session checks their id and source, as the stream does. + session.handle_datagram( + peer, + &header, + message, + datagram.len(), + &config, + super::now(), + ) + }; + // A keyframe request asks for an IDR, which the next generation + // starts with anyway. + carry_out(links, shared, actions).await; + return; + } + if !matches!( + message, + ControlMessage::Hello2(_) | ControlMessage::Identify(_) + ) { + return; + } + let actions = { + let mut session = shared.session.lock(); + // The supervisor clears it only after stopping this responder; + // nothing answered here starts a session. + if session.not_streaming().is_none() { + return; + } + session.handle_datagram( + peer, + &header, + message, + datagram.len(), + &config, + super::now(), + ) + }; + if let Some(end) = &actions.ended { + crate::diagnostics::log_session_end(end, shared); + } + for (destination, reply) in &actions.replies { + // Over Wi-Fi an answer is never larger than the request it answers. + // iPads that understand these answers pad their HELLO2 so every + // answer fits (`STANDBY_HELLO2_LEN`); an older one gets the short + // refusal, or today's silence. An IDENTIFY is as long as its answer. + // Answers over USB stay whole. + if destination.link == LinkId::Udp && reply.len() > datagram.len() { + debug!( + peer = %destination, + reply = reply.len(), + request = datagram.len(), + "Left a request unanswered: the answer is larger than it" + ); + continue; + } + if let Err(error) = links.send_to(reply, *destination).await { + debug!(peer = %destination, %error, "Could not answer an iPad while the stream is off"); + } + } +} diff --git a/host/src/transport/usb.rs b/host/src/transport/usb.rs index 424a2810..f3288a13 100644 --- a/host/src/transport/usb.rs +++ b/host/src/transport/usb.rs @@ -14,9 +14,10 @@ use tokio::task::{AbortHandle, JoinSet}; use tracing::{debug, info, warn}; use super::apple_usb::{self, AppleUsbState}; -use super::link::{FramedLink, Link, LinkId, PeerId, UdpLink}; +use super::congestion::{is_no_buffer_space, SendCongestion}; +use super::link::{CloseReport, FramedLink, Link, LinkId, PeerId, UdpLink}; use super::usbmuxd::{self, BoxedTunnel, Client}; -use crate::stats::PIPELINE_STATS; +use crate::stats::{PipelineStats, PIPELINE_STATS}; static NEXT_GENERATION: AtomicU64 = AtomicU64::new(1); static HUB: OnceLock = OnceLock::new(); @@ -30,6 +31,26 @@ pub enum LinkEvent { struct UsbEvent { generation: u64, link: FramedLink, + opened: Opened, +} + +/// How a tunnel came to be, for its connected line. +#[derive(Clone, Debug, Default)] +struct Opened { + /// The service's Connect and the preamble. + connect: Duration, + /// Since the device was listed, or since its last tunnel closed. + waited: Duration, + /// Connects that failed in that time, and the last one's error. + failed: u32, + last_error: Option, +} + +impl Opened { + fn failed_with(&mut self, error: String) { + self.failed += 1; + self.last_error = Some(error); + } } /// What the hub reports to the pipeline generation that currently owns it. @@ -161,15 +182,20 @@ fn subscriber(state: &Mutex) -> Option> { /// Owns tunnels for the hub: forwards their datagrams to whichever pipeline /// generation is subscribed and drops them while none is (a restart gap). +/// Each tunnel's opening and close are logged here, once, whether or not a +/// pipeline is subscribed. async fn run_hub(mut events: mpsc::Receiver, state: Arc>) { while let Some(UsbEvent { generation, mut link, + opened, }) = events.recv().await { + log_connected(generation, &opened); let id = link.id(); let mut inbound = link.take_inbound().expect("new USB link owns its reader"); let link = Arc::new(link); + let closing = Arc::clone(&link); state .lock() .links @@ -190,6 +216,8 @@ async fn run_hub(mut events: mpsc::Receiver, state: Arc, usb: HashMap, events: mpsc::Receiver, - token: u64, + _subscription: HubToken, +} + +/// The hub's events go to this owner until it is dropped. +struct HubToken(u64); + +impl Drop for HubToken { + fn drop(&mut self) { + UsbHub::get().unsubscribe(self.0); + } } impl TransportLinks { @@ -243,17 +282,24 @@ impl TransportLinks { let subscription = UsbHub::get().subscribe(&force_idr); let mut links = Self { udp: UdpLink::new(socket), + congestion: Mutex::default(), usb: HashMap::new(), events: subscription.events, - token: subscription.token, + _subscription: HubToken(subscription.token), }; - for (id, generation, link) in subscription.open { - info!(?id, generation, "USB tunnel kept across pipeline restart"); + for (_, generation, link) in subscription.open { + info!(generation, "USB tunnel kept across pipeline restart"); links.adopt(generation, link); } links } + /// Tests: a link over a stream they hold, as if its cable connected. + #[cfg(test)] + pub(crate) fn adopt_for_test(&mut self, link: Arc) { + self.adopt(0, link); + } + fn adopt(&mut self, generation: u64, link: Arc) { // Drops already counted by an earlier generation are not new. let reported_drops = link.stats().frames_dropped.load(Ordering::Relaxed); @@ -270,7 +316,15 @@ impl TransportLinks { pub async fn send_to(&self, bytes: &[u8], peer: PeerId) -> io::Result<()> { match peer.link { - LinkId::Udp => self.udp.send(peer, bytes).await, + LinkId::Udp => { + let sent = self.udp.send(peer, bytes).await; + if let Err(error) = &sent { + if is_no_buffer_space(error) { + self.congestion.lock().refused(error, super::now()); + } + } + sent + } id => match self.usb.get(&id) { Some(active) => active.link.send(peer, bytes).await, None => Err(io::Error::new( @@ -316,7 +370,6 @@ impl TransportLinks { }; match event { Some(HubEvent::Connected { generation, link }) => { - info!(id = ?link.id(), generation, "USB tunnel connected"); self.adopt(generation, link); } Some(HubEvent::Datagram { @@ -343,7 +396,6 @@ impl TransportLinks { PIPELINE_STATS.lock().usb_link_state = "Waiting for the iPad app".into(); } - info!(?id, generation, "USB tunnel closed"); return Ok(LinkEvent::Closed(id)); } } @@ -356,14 +408,42 @@ impl TransportLinks { } } } -} -impl Drop for TransportLinks { - fn drop(&mut self) { - UsbHub::get().unsubscribe(self.token); + /// Lets go of the USB tunnels and returns the UDP socket, for the next + /// owner of the stream port. + pub fn into_udp_socket(self) -> UdpSocket { + self.udp.socket } } +/// How long the service's Connect and the preamble took, how long the +/// worker waited for the tunnel, and the connects that failed meanwhile. +fn log_connected(generation: u64, opened: &Opened) { + info!( + generation, + connect_ms = opened.connect.as_millis() as u64, + waited_ms = opened.waited.as_millis() as u64, + failed = opened.failed, + last_error = opened.last_error.as_deref().unwrap_or("none"), + "USB tunnel connected" + ); +} + +/// What the host saw end the tunnel first, after how long, and how long +/// the iPad had been quiet: a timer shows as the same `open_ms` every time. +/// `seen_first` is the first end this host saw, not necessarily what +/// started it: the iPad or Apple's service closing shows as `peer_eof`. +fn log_closed(generation: u64, report: Option) { + let ms = |duration: Duration| duration.as_millis() as u64; + info!( + generation, + seen_first = %report.map_or("unknown", |report| report.cause.as_str()), + open_ms = report.map_or(0, |report| ms(report.open)), + quiet_ms = report.map_or(0, |report| ms(report.quiet)), + "USB tunnel closed" + ); +} + #[derive(Clone)] enum Source { Direct(String), @@ -401,14 +481,28 @@ impl Source { async fn device_worker(source: Source, events: mpsc::Sender, force_idr: Arc) { let peer = source.peer(); let mut backoff = 1; + let mut opened = Opened::default(); + let mut waiting_since = tokio::time::Instant::now(); loop { + let started = tokio::time::Instant::now(); match source.connect().await { Ok(stream) => match FramedLink::connect(stream, peer, Arc::clone(&force_idr)).await { Ok(link) => { let mut closed = link.closed_signal(); let _close_on_exit = link.close_guard(); let generation = NEXT_GENERATION.fetch_add(1, Ordering::Relaxed); - if events.send(UsbEvent { generation, link }).await.is_err() { + opened.connect = started.elapsed(); + opened.waited = started.saturating_duration_since(waiting_since); + let opened = std::mem::take(&mut opened); + if events + .send(UsbEvent { + generation, + link, + opened, + }) + .await + .is_err() + { return; } loop { @@ -420,10 +514,17 @@ async fn device_worker(source: Source, events: mpsc::Sender, force_idr } } backoff = 1; + waiting_since = tokio::time::Instant::now(); + } + Err(error) => { + debug!(%peer, %error, "USB framing handshake failed"); + opened.failed_with(format!("framing handshake: {error}")); } - Err(error) => debug!(%peer, %error, "USB framing handshake failed"), }, - Err(error) => debug!(%peer, %error, backoff, "USB app unavailable; retrying"), + Err(error) => { + debug!(%peer, %error, backoff, "USB app unavailable; retrying"); + opened.failed_with(error.to_string()); + } } tokio::time::sleep(Duration::from_secs(backoff)).await; backoff = (backoff + 1).min(5); @@ -467,21 +568,103 @@ async fn supervise(events: mpsc::Sender, force_idr: Arc) { } let client = Arc::new(Client::new(usbmuxd::Endpoint::default())); - let mut workers = JoinSet::new(); - let mut handles: HashMap = HashMap::new(); let mut poll = tokio::time::interval(Duration::from_secs(2)); poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + supervise_devices(client, events, force_idr, &PIPELINE_STATS, poll).await; +} + +/// When the supervisor polls Apple's device list next, and the time it +/// judges that poll by. +trait PollClock { + fn next(&mut self) -> impl std::future::Future + Send; +} + +impl PollClock for tokio::time::Interval { + async fn next(&mut self) -> tokio::time::Instant { + self.tick().await; + tokio::time::Instant::now() + } +} + +/// How long Apple's device list may keep failing before the service counts +/// as down and the devices' workers stop, closing their tunnels. A single +/// failed poll used to close every live tunnel, though the service and the +/// tunnels through it were fine. +const LIST_OUTAGE: Duration = Duration::from_secs(10); + +/// Device-list polls that failed in a row, and when the first one did. +#[derive(Debug, Default)] +struct ListFailures { + count: u32, + since: Option, +} + +impl ListFailures { + /// Whether a poll keeps the running workers, whatever it listed: its + /// list failed, but the failures have not lasted [`LIST_OUTAGE`] yet. + /// Only a list that answers can show a device gone. + fn keep_workers(&mut self, listed: bool, workers: usize, now: tokio::time::Instant) -> bool { + if listed { + if self.count > 0 { + info!( + failed_polls = self.count, + "Apple's device list answers again" + ); + } + *self = Self::default(); + return false; + } + self.count = self.count.saturating_add(1); + let since = *self.since.get_or_insert(now); + if workers == 0 { + return false; + } + if now.saturating_duration_since(since) < LIST_OUTAGE { + info!( + failed_polls = self.count, + "Apple's device list failed — keeping the USB tunnels" + ); + return true; + } + info!( + failed_polls = self.count, + failing_ms = now.saturating_duration_since(since).as_millis() as u64, + "Apple's device list keeps failing — closing the USB tunnels" + ); + false + } +} + +/// Polls Apple's device list and runs a worker for each listed device. What +/// a poll finds goes to `stats`, which the PC window reads. +async fn supervise_devices( + client: Arc, + events: mpsc::Sender, + force_idr: Arc, + stats: &Mutex, + mut poll: impl PollClock, +) { + let mut workers = JoinSet::new(); + let mut handles: HashMap = HashMap::new(); let mut unreachable = AppleUsbState::default(); + let mut failures = ListFailures::default(); loop { tokio::select! { - _ = poll.tick() => { + now = poll.next() => { let devices = client.list_devices().await; let available = devices.is_ok(); + let list_error = devices.as_ref().err().map(ToString::to_string); let devices = devices.unwrap_or_default(); - let ids: HashSet<_> = devices.iter().map(|device| device.device_id).collect(); + let kept = failures.keep_workers(available, handles.len(), now); + let ids: HashSet<_> = if kept { + handles.keys().copied().collect() + } else { + devices.iter().map(|device| device.device_id).collect() + }; // Without the service the cable is invisible to us; ask // Windows whether an iPad is attached so the GUI can say why. - let cabled = if available { + // A poll that keeps the tunnels shows nothing new: they work. + let cabled = if available || kept { AppleUsbState::default() } else { tokio::task::spawn_blocking(apple_usb::probe).await.unwrap_or_default() @@ -494,8 +677,8 @@ async fn supervise(events: mpsc::Sender, force_idr: Arc) { } } unreachable = cabled; - { - let mut stats = PIPELINE_STATS.lock(); + if !kept { + let mut stats = stats.lock(); stats.usb_service_reachable = available; stats.usb_devices = devices.len(); stats.usb_cabled_devices = cabled.cabled_devices; @@ -505,7 +688,17 @@ async fn supervise(events: mpsc::Sender, force_idr: Arc) { else if stats.usb_link_state != "Connected" { stats.usb_link_state = "Waiting for the iPad app".into(); } } handles.retain(|id, task| { - if !ids.contains(id) { task.abort(); false } else { !task.is_finished() } + if ids.contains(id) { + return !task.is_finished(); + } + // Stopping the worker closes its tunnel, if it has one. + info!( + reason = %if available { "device_gone" } else { "list_failed" }, + error = list_error.as_deref().unwrap_or("none"), + "USB device worker stopped" + ); + task.abort(); + false }); for id in ids { handles.entry(id).or_insert_with(|| workers.spawn(device_worker( @@ -542,6 +735,172 @@ mod tests { assert_eq!(direct_value(true, || None), None); } + /// Every tunnel's opening and close are logged once, by the hub, whether + /// or not a pipeline is subscribed: with none (a restart gap) they used + /// to go unlogged. The lines carry the tunnel generation, not the device. + #[tokio::test(start_paused = true)] + async fn the_hub_logs_each_tunnel_once_with_or_without_a_pipeline() { + use tracing_subscriber::prelude::*; + #[derive(Clone, Default)] + struct Lines(Arc>>); + impl io::Write for Lines { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let lines = Lines::default(); + let writer = lines.clone(); + crate::logging::prepare_log_capture(); + let _capture = tracing::subscriber::set_default( + tracing_subscriber::registry().with( + tracing_subscriber::fmt::layer() + .with_ansi(false) + .without_time() + .with_target(false) + .with_writer(move || writer.clone()), + ), + ); + let state = Arc::new(Mutex::new(HubState::default())); + let (events, hub_events) = mpsc::channel(4); + tokio::spawn(run_hub(hub_events, Arc::clone(&state))); + let open = |generation| async move { + let (host, mut ipad) = tokio::io::duplex(64 * 1024); + let link = FramedLink::connect(host, PeerId::usb(3), Arc::new(AtomicBool::new(false))) + .await + .unwrap(); + super::super::link::read_preamble(&mut ipad).await.unwrap(); + let opened = Opened { + connect: Duration::from_millis(5), + waited: Duration::from_millis(1_200), + failed: 2, + last_error: Some("usbmuxd Result 3".into()), + }; + let event = UsbEvent { + generation, + link, + opened, + }; + (event, ipad) + }; + let gone = |state: Arc>| async move { + for _ in 0..1000 { + if state.lock().links.is_empty() { + return; + } + tokio::task::yield_now().await; + } + panic!("the hub kept a closed tunnel"); + }; + + // No pipeline: a restart gap. + let (event, ipad) = open(41).await; + events.send(event).await.unwrap(); + tokio::time::sleep(Duration::from_millis(9_600)).await; + drop(ipad); + gone(Arc::clone(&state)).await; + + // A subscriber (standing in for a pipeline) gets the events, and the + // hub does not log the lines again. This checks the hub only: the + // pipeline's TransportLinks no longer logs these lines at all. + let (pipeline, mut hub) = mpsc::channel(8); + state.lock().subscriber = Some((1, pipeline)); + let (event, ipad) = open(42).await; + events.send(event).await.unwrap(); + assert!(matches!( + hub.recv().await, + Some(HubEvent::Connected { generation: 42, .. }) + )); + tokio::time::sleep(Duration::from_millis(1_500)).await; + drop(ipad); + assert!(matches!( + hub.recv().await, + Some(HubEvent::Closed { generation: 42, .. }) + )); + + let text = String::from_utf8(lines.0.lock().clone()).unwrap(); + let lines: Vec<&str> = text.lines().map(str::trim).collect(); + let connected = |generation| { + format!( + "INFO USB tunnel connected generation={generation} connect_ms=5 waited_ms=1200 \ + failed=2 last_error=\"usbmuxd Result 3\"" + ) + }; + assert_eq!( + lines, + [ + connected(41), + "INFO USB tunnel closed generation=41 seen_first=peer_eof open_ms=9600 \ + quiet_ms=9600" + .into(), + connected(42), + "INFO USB tunnel closed generation=42 seen_first=peer_eof open_ms=1500 \ + quiet_ms=1500" + .into(), + ] + ); + } + + /// The connected line says how long the worker waited for the tunnel and + /// how many connects failed meanwhile: Apple's service refuses a Connect + /// while the app isn't listening, and a tunnel the iPad closed starts + /// the count again. + #[tokio::test] + async fn a_tunnel_says_how_it_was_opened() { + let service = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = Arc::new(Client::new(usbmuxd::Endpoint::Tcp( + service.local_addr().unwrap(), + ))); + // Answers each Connect with `result`, and returns the connection. + let answer = |result: u64| { + let service = &service; + async move { + let (mut stream, _) = service.accept().await.unwrap(); + let (tag, _) = usbmuxd::read_message(&mut stream).await.unwrap(); + let reply = plist::Value::Dictionary(plist::Dictionary::from_iter([ + ("MessageType".to_string(), plist::Value::from("Result")), + ("Number".to_string(), plist::Value::from(result)), + ])); + tokio::io::AsyncWriteExt::write_all( + &mut stream, + &usbmuxd::encode_message(tag, &reply).unwrap(), + ) + .await + .unwrap(); + stream + } + }; + let (events, mut opened) = mpsc::channel(4); + let worker = tokio::spawn(device_worker( + Source::Device(client, 3), + events, + Arc::new(AtomicBool::new(false)), + )); + answer(3).await; + let mut ipad = answer(0).await; + let first = opened.recv().await.unwrap(); + assert_eq!(first.opened.failed, 1, "{:?}", first.opened); + assert_eq!(first.opened.last_error.as_deref(), Some("usbmuxd Result 3")); + assert!(first.opened.waited >= Duration::from_secs(1)); + + super::super::link::read_preamble(&mut ipad).await.unwrap(); + drop(ipad); + let _ipad = answer(0).await; + let second = opened.recv().await.unwrap(); + assert_eq!( + first.link.close_report().map(|report| report.cause), + Some(super::super::link::CloseCause::PeerEof) + ); + assert_eq!(second.opened.failed, 0, "{:?}", second.opened); + assert_eq!(second.opened.last_error, None); + assert!(second.opened.waited >= Duration::from_secs(1)); + assert!(second.generation > first.generation); + worker.abort(); + } + /// `cargo test --release` runs this; other builds may use the endpoint. #[cfg(not(any(debug_assertions, feature = "usb-direct")))] #[test] @@ -550,4 +909,280 @@ mod tests { assert_eq!(direct_endpoint(), None); std::env::remove_var("ETERNAL_USB_DIRECT"); } + + /// A failed device list keeps the workers that run, until the failures + /// last [`LIST_OUTAGE`]: then the service counts as down and they stop, + /// as every failed poll used to stop them. Each step is logged with the + /// count of failed polls. + #[test] + fn failed_device_lists_keep_the_workers_for_a_while() { + use tracing_subscriber::prelude::*; + #[derive(Clone, Default)] + struct Lines(Arc>>); + impl io::Write for Lines { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let lines = Lines::default(); + let writer = lines.clone(); + let subscriber = tracing_subscriber::registry().with( + tracing_subscriber::fmt::layer() + .with_ansi(false) + .without_time() + .with_target(false) + .with_writer(move || writer.clone()), + ); + let start = tokio::time::Instant::now(); + let at = |secs: u64| start + Duration::from_secs(secs); + let mut failures = ListFailures::default(); + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, || { + // One failed poll between good ones. + assert!(!failures.keep_workers(true, 1, at(0))); + assert!(failures.keep_workers(false, 1, at(2))); + assert!(!failures.keep_workers(true, 1, at(4))); + // Failures in a row, until they have lasted the outage. + for secs in [6, 8, 10, 12, 14] { + assert!(failures.keep_workers(false, 1, at(secs)), "{secs} s"); + } + assert!(!failures.keep_workers(false, 1, at(16))); + // Nothing left to keep, nothing more to say. + assert!(!failures.keep_workers(false, 0, at(18))); + assert!(!failures.keep_workers(true, 0, at(20))); + // Without workers a failure keeps nothing, but still counts. + assert!(!failures.keep_workers(false, 0, at(22))); + assert!(!failures.keep_workers(false, 1, at(32))); + }); + let text = String::from_utf8(lines.0.lock().clone()).unwrap(); + let lines: Vec<&str> = text.lines().map(str::trim).collect(); + let keeping = |n: u32| { + format!("INFO Apple's device list failed — keeping the USB tunnels failed_polls={n}") + }; + assert_eq!( + lines, + [ + keeping(1), + "INFO Apple's device list answers again failed_polls=1".into(), + keeping(1), + keeping(2), + keeping(3), + keeping(4), + keeping(5), + "INFO Apple's device list keeps failing — closing the USB tunnels \ + failed_polls=6 failing_ms=10000" + .into(), + "INFO Apple's device list answers again failed_polls=7".into(), + "INFO Apple's device list keeps failing — closing the USB tunnels \ + failed_polls=2 failing_ms=10000" + .into(), + ] + ); + } + + /// Apple's device service whose list fails on demand: it drops the + /// request unanswered. It lists one USB iPad otherwise, and hands every + /// Connect's stream to the test as that iPad's end of a tunnel. + struct FlakyService { + failing: Arc, + failed: Arc, + tunnels: mpsc::Receiver, + client: Arc, + } + + impl FlakyService { + async fn start() -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = Arc::new(Client::new(usbmuxd::Endpoint::Tcp( + listener.local_addr().unwrap(), + ))); + let failing = Arc::new(AtomicBool::new(false)); + let failed = Arc::new(AtomicU64::new(0)); + let (tunnel_tx, tunnels) = mpsc::channel(4); + let state = (failing.clone(), failed.clone()); + tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + let (failing, failed) = state.clone(); + let tunnel_tx = tunnel_tx.clone(); + tokio::spawn(async move { + let (tag, request) = usbmuxd::read_message(&mut stream).await.unwrap(); + let kind = request.as_dictionary().unwrap()["MessageType"] + .as_string() + .unwrap() + .to_string(); + let reply = if kind == "ListDevices" { + if failing.load(Ordering::SeqCst) { + failed.fetch_add(1, Ordering::SeqCst); + return; + } + let device = plist::Dictionary::from_iter([ + ("DeviceID".to_string(), plist::Value::from(3u64)), + ( + "Properties".to_string(), + plist::Value::Dictionary(plist::Dictionary::from_iter([( + "ConnectionType".to_string(), + plist::Value::from("USB"), + )])), + ), + ]); + plist::Dictionary::from_iter([( + "DeviceList".to_string(), + plist::Value::Array(vec![plist::Value::Dictionary(device)]), + )]) + } else { + plist::Dictionary::from_iter([ + ("MessageType".to_string(), plist::Value::from("Result")), + ("Number".to_string(), plist::Value::from(0u64)), + ]) + }; + let bytes = + usbmuxd::encode_message(tag, &plist::Value::Dictionary(reply)).unwrap(); + tokio::io::AsyncWriteExt::write_all(&mut stream, &bytes) + .await + .unwrap(); + if kind == "Connect" { + let _ = tunnel_tx.send(stream).await; + } + }); + } + }); + Self { + failing, + failed, + tunnels, + client, + } + } + + /// The iPad's end of the next tunnel, past its preamble. + async fn tunnel(&mut self) -> tokio::net::TcpStream { + let mut ipad = tokio::time::timeout(Duration::from_secs(20), self.tunnels.recv()) + .await + .expect("a tunnel") + .unwrap(); + super::super::link::read_preamble(&mut ipad).await.unwrap(); + ipad + } + } + + /// Poll times a test hands the supervisor one at a time. It hears when + /// each poll has finished: the supervisor asks for the next one. + struct Polls { + finished: mpsc::Sender<()>, + at: mpsc::Receiver, + } + + impl PollClock for Polls { + async fn next(&mut self) -> tokio::time::Instant { + let _ = self.finished.send(()).await; + self.at.recv().await.expect("the test's next poll") + } + } + + /// The test's side of [`Polls`]. + struct Poller { + at: mpsc::Sender, + polled: mpsc::Receiver<()>, + start: tokio::time::Instant, + } + + impl Poller { + /// Polls `secs` after the start, and returns once that poll has + /// finished. + async fn poll(&mut self, secs: u64) { + let at = self.start + Duration::from_secs(secs); + self.at.send(at).await.unwrap(); + self.polled.recv().await.unwrap(); + } + } + + /// A device list that fails once leaves the live tunnel open; one that + /// keeps failing closes it once the failures have lasted + /// [`LIST_OUTAGE`], and once the list answers again the iPad gets a new + /// tunnel. While the tunnel is kept, the PC window goes on showing USB + /// as it was, not "Open Apple Devices". A single failed list used to + /// close the tunnel at once. Each poll's time is the test's, so how the + /// machine schedules the test changes nothing. + #[tokio::test] + async fn a_failed_device_list_keeps_the_tunnel_until_the_service_is_down() { + let mut service = FlakyService::start().await; + let stats: &'static Mutex = + Box::leak(Box::new(Mutex::new(PipelineStats::default()))); + let (finished, mut polled) = mpsc::channel(1); + let (at, polls) = mpsc::channel(1); + let (events, mut opened) = mpsc::channel(4); + let supervisor = tokio::spawn(supervise_devices( + Arc::clone(&service.client), + events, + Arc::new(AtomicBool::new(false)), + stats, + Polls { + finished, + at: polls, + }, + )); + polled.recv().await.unwrap(); + let mut poller = Poller { + at, + polled, + start: tokio::time::Instant::now(), + }; + let shown = || { + let stats = stats.lock(); + ( + stats.usb_service_reachable, + stats.usb_devices, + stats.usb_link_state.clone(), + ) + }; + let streaming = (true, 1, "Waiting for the iPad app".to_string()); + + poller.poll(0).await; + let _ipad = service.tunnel().await; + let first = opened.recv().await.unwrap(); + assert_eq!(shown(), streaming); + + // One failed list between good ones. + service.failing.store(true, Ordering::SeqCst); + poller.poll(2).await; + assert_eq!(service.failed.load(Ordering::SeqCst), 1); + assert!(!first.link.is_closed(), "one failed list closed the tunnel"); + assert_eq!(shown(), streaming, "a kept tunnel shown as unavailable"); + service.failing.store(false, Ordering::SeqCst); + poller.poll(4).await; + assert!(!first.link.is_closed()); + assert!(service.tunnels.try_recv().is_err(), "a second tunnel"); + + // Failures in a row: kept until they have lasted the outage. + service.failing.store(true, Ordering::SeqCst); + for secs in [6, 8, 10, 12, 14] { + poller.poll(secs).await; + assert!(!first.link.is_closed(), "closed at {secs} s"); + assert_eq!(shown(), streaming, "at {secs} s"); + } + poller.poll(16).await; + let mut closed = first.link.closed_signal(); + tokio::time::timeout(Duration::from_secs(20), closed.wait_for(|closed| *closed)) + .await + .expect("the tunnel closes once the service is down") + .unwrap(); + assert_eq!( + shown(), + (false, 0, "Apple device service unavailable".to_string()) + ); + + service.failing.store(false, Ordering::SeqCst); + poller.poll(18).await; + let _ipad = service.tunnel().await; + let second = opened.recv().await.unwrap(); + assert!(second.generation > first.generation); + assert!(!second.link.is_closed()); + assert_eq!(shown(), streaming); + supervisor.abort(); + } } diff --git a/host/src/updates.rs b/host/src/updates.rs index 6755c848..fc19dc71 100644 --- a/host/src/updates.rs +++ b/host/src/updates.rs @@ -6,6 +6,13 @@ //! iPad's while its TestFlight build awaits Apple. Before //! installing, the installer's SHA-256 must match both that manifest and the //! GitHub release body, which are published by different steps. +//! +//! A Beta build (`x.y.z-beta.…`) also reads the nightly's fixed `beta` +//! GitHub release (scripts/nightly.py) and is offered the newer of that Beta +//! and the stable release. Stable builds never see a Beta. A Beta needs a +//! manifest signed with a pinned Ed25519 key that GitHub does not hold +//! (scripts/sign_beta_manifest.py): the release body and the installer's +//! digest are both written by whoever can publish releases. use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::sync::{mpsc, Arc}; @@ -22,6 +29,24 @@ use crate::remote_config::Rollout; pub const WEBSITE_DOWNLOAD: &str = "https://eternalmonitor.dev/download.html"; pub use eternal_updater::SETUP_FLAGS; const INSTALLER_NAME: &str = "EternalMonitor-Setup.exe"; +/// The nightly's fixed release and its installer (scripts/nightly.py). +const BETA_TAG: &str = "beta"; +const BETA_INSTALLER_NAME: &str = "EternalMonitor-Setup-beta.exe"; +/// The Beta's signed manifest and its raw Ed25519 signature, assets of the +/// same release. +const BETA_MANIFEST_NAME: &str = "beta-manifest.txt"; +const BETA_SIGNATURE_NAME: &str = "beta-manifest.sig"; +const MAX_MANIFEST_BYTES: u64 = 4 * 1024; +/// The Ed25519 public key Beta manifests are signed with, raw. +const BETA_PUBLIC_KEY: [u8; 32] = [ + 0x82, 0xc2, 0xaf, 0x21, 0xc6, 0x85, 0xd2, 0xc0, 0x42, 0xc5, 0x1f, 0x72, 0xa3, 0x4c, 0x78, 0xbd, + 0xb5, 0xd0, 0x04, 0xba, 0xd8, 0x9e, 0x75, 0xa8, 0x30, 0x6a, 0x43, 0x22, 0x4d, 0x16, 0xec, 0x5a, +]; +/// What the popup says under a Beta's title. +const BETA_NOTE: &str = "A Beta build with the newest fixes, for testing."; +/// A new Beta comes out most nights. Its notification and popup come at +/// most this often, and its Later covers every Beta for this long. +const BETA_INTERVAL: u64 = 24 * 60 * 60; const CHECK_INTERVAL: u64 = 24 * 60 * 60; /// A relaunch checks again only when the last check is this old, so a crash /// loop or repeated restarts cannot hammer the website. @@ -56,6 +81,8 @@ pub struct Endpoints { /// Hosts the download may start at or be redirected to. pub download_hosts: Vec, pub https_only: bool, + /// The Ed25519 public key a Beta manifest must be signed with. + pub beta_key: [u8; 32], } impl Endpoints { @@ -72,6 +99,7 @@ impl Endpoints { "release-assets.githubusercontent.com".into(), ], https_only: true, + beta_key: BETA_PUBLIC_KEY, } } @@ -97,6 +125,7 @@ impl Endpoints { installer_prefix: format!("{base}/releases/download/"), download_hosts: vec![host], https_only: false, + beta_key: BETA_PUBLIC_KEY, }) } } @@ -117,6 +146,23 @@ pub fn newer_stable(candidate: &str, current: &str) -> Option { (release > current).then(|| release.to_string()) } +/// Whether `version` is a Beta channel build: `x.y.z-beta.…`, either the +/// nightly's `-beta..` or a hand-made `-beta.1`. +pub fn is_beta(version: &str) -> bool { + semver::Version::parse(version).is_ok_and(|version| version.pre.as_str().starts_with("beta.")) +} + +/// Whether `candidate` is newer than `current` by semver, previews included. +/// Build metadata is ignored; malformed versions are never newer. +fn newer(candidate: &str, current: &str) -> bool { + let parse = |version: &str| { + let mut version = semver::Version::parse(version).ok()?; + version.build = semver::BuildMetadata::EMPTY; + Some(version) + }; + matches!((parse(candidate), parse(current)), (Some(a), Some(b)) if a > b) +} + /// The Windows release named by the manifest, already validated. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct Release { @@ -139,6 +185,12 @@ fn recommended() -> bool { true } +impl Release { + fn is_beta(&self) -> bool { + is_beta(&self.version) + } +} + /// Unknown fields are ignored, so a newer manifest still parses here. #[derive(Deserialize)] struct Manifest { @@ -280,6 +332,164 @@ fn published_sha256(body: &str) -> Option { Some(pattern.captures(body)?[1].to_ascii_lowercase()) } +/// The `beta` release as GitHub lists it, checked: the Beta its body names, +/// the installer's size, and where its signed manifest and signature are. +#[derive(Debug, PartialEq, Eq)] +pub struct BetaListing { + pub release: Release, + pub size: u64, + manifest: String, + signature: String, +} + +/// The Beta channel's fixed `beta` release as the GitHub API returns it. +/// scripts/nightly.py writes the body: a line `Version , from the +/// Beta channel…` and the installer's SHA-256 in a code block. GitHub's own +/// digest of the installer must match it, and the signed manifest's assets +/// must be there; [`check_beta_manifest`] checks those against the listing. +pub fn parse_beta(data: &[u8], endpoints: &Endpoints) -> Result { + let bad = |why: &str| UpdateError::BadManifest(why.into()); + let unsafe_download = |why: &str| UpdateError::UnsafeDownload(why.into()); + let github: GithubRelease = + serde_json::from_slice(data).map_err(|error| bad(&error.to_string()))?; + if github.tag_name != BETA_TAG || github.draft || !github.prerelease { + return Err(unsafe_download("release is not the published Beta")); + } + let body = github.body.as_deref().unwrap_or_default(); + let version = only_match(r"(?m)^Version (\S+), from the Beta channel", body) + .and_then(|version| semver::Version::parse(&version).ok()) + .filter(|version| is_beta(&version.to_string()) && version.build.is_empty()) + .ok_or_else(|| bad("Beta has no version"))?; + let sha256 = only_match( + r"(?m)^SHA-256 of EternalMonitor-Setup-beta\.exe:\s*```\s*([0-9A-Fa-f]{64})\s*```", + body, + ) + .ok_or_else(|| bad("Beta has no SHA-256"))? + .to_ascii_lowercase(); + // Each asset must be the release's own file of that name. + let asset = |name: &str| { + let url = format!("{}{BETA_TAG}/{name}", endpoints.installer_prefix); + match github.assets.iter().find(|asset| asset.name == name) { + Some(asset) if asset.browser_download_url == url => Ok(asset), + Some(_) => Err(unsafe_download("unexpected Beta asset URL")), + None => Err(bad("Beta asset missing")), + } + }; + let installer = asset(BETA_INSTALLER_NAME)?; + if installer.digest.as_deref().map(str::to_ascii_lowercase) != Some(format!("sha256:{sha256}")) + { + return Err(unsafe_download("GitHub digest is missing or differs")); + } + if installer.size == 0 || installer.size > MAX_INSTALLER_BYTES { + return Err(unsafe_download("installer size out of range")); + } + Ok(BetaListing { + release: Release { + version: version.to_string(), + installer: installer.browser_download_url.clone(), + sha256, + recommended: false, + note: Some(BETA_NOTE.into()), + rollout: None, + }, + size: installer.size, + manifest: asset(BETA_MANIFEST_NAME)?.browser_download_url.clone(), + signature: asset(BETA_SIGNATURE_NAME)?.browser_download_url.clone(), + }) +} + +/// The first group of `pattern`'s one match in `text`. None, or two, is +/// `None`. +fn only_match(pattern: &str, text: &str) -> Option { + let pattern = regex::Regex::new(pattern).ok()?; + let mut matches = pattern.captures_iter(text); + let first = matches.next()?[1].to_string(); + matches.next().is_none().then_some(first) +} + +/// What `beta-manifest.txt` says, signed outside GitHub. +#[derive(Debug, PartialEq, Eq)] +pub struct BetaManifest { + pub version: String, + pub sha256: String, + pub size: u64, +} + +impl BetaManifest { + /// The canonical text, version 1: UTF-8, every line ending in LF. + pub fn text(&self) -> String { + format!( + "eternalmonitor-beta-manifest 1\nversion={}\nsha256={}\nsize={}\nasset={BETA_INSTALLER_NAME}\n", + self.version, self.sha256, self.size + ) + } + + /// Only the canonical text parses: its fields must render back to the + /// same bytes, so no spacing, case or line ending varies. + pub fn parse(data: &[u8]) -> Option { + let text = std::str::from_utf8(data).ok()?; + // The header and asset lines are checked by rendering back. + let mut lines = text.strip_suffix('\n')?.split('\n').skip(1); + let mut field = |key: &str| { + let line = lines.next()?; + line.strip_prefix(key)? + .strip_prefix('=') + .map(str::to_string) + }; + let manifest = Self { + version: field("version")?, + sha256: field("sha256")?, + size: field("size")?.parse().ok()?, + }; + let valid = semver::Version::parse(&manifest.version).is_ok_and(|version| { + version.to_string() == manifest.version && version.build.is_empty() + }) && is_beta(&manifest.version) + && is_sha256(&manifest.sha256) + && manifest.sha256 == manifest.sha256.to_ascii_lowercase(); + (valid && manifest.text().as_bytes() == data).then_some(manifest) + } +} + +/// The manifest must carry a valid signature by `key` and name the Beta, +/// hash and size the listing does. Anything else means no offer. +pub fn check_beta_manifest( + listing: &BetaListing, + manifest: &[u8], + signature: &[u8], + key: &[u8; 32], +) -> Result<(), UpdateError> { + use ring::signature::{UnparsedPublicKey, ED25519}; + UnparsedPublicKey::new(&ED25519, key) + .verify(manifest, signature) + .map_err(|_| UpdateError::UnsafeDownload("Beta signature is not valid".into()))?; + let manifest = BetaManifest::parse(manifest) + .ok_or_else(|| UpdateError::BadManifest("malformed Beta manifest".into()))?; + let release = &listing.release; + if (&manifest.version, &manifest.sha256, manifest.size) + != (&release.version, &release.sha256, listing.size) + { + return Err(UpdateError::UnsafeDownload( + "Beta manifest differs from the release".into(), + )); + } + Ok(()) +} + +/// Checks the verified Beta release against the one chosen and returns the +/// installer's size. A Beta published since then is refused: its hash is +/// not the one the helper will check. +fn check_beta_release(listing: &BetaListing, release: &Release) -> Result { + let published = &listing.release; + if (&published.version, &published.installer, &published.sha256) + != (&release.version, &release.installer, &release.sha256) + { + return Err(UpdateError::UnsafeDownload( + "the Beta release changed".into(), + )); + } + Ok(listing.size) +} + fn is_sha256(value: &str) -> bool { value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) } @@ -383,6 +593,68 @@ pub fn fetch_manifest(endpoints: &Endpoints) -> Result { parse_manifest(&data, endpoints) } +const GITHUB_JSON: &str = "application/vnd.github+json"; + +/// The newest Beta, offered only with a valid signed manifest. +pub fn fetch_beta(endpoints: &Endpoints) -> Result { + verified_beta(endpoints).map(|listing| listing.release) +} + +/// The `beta` release, once its signed manifest checks out. +fn verified_beta(endpoints: &Endpoints) -> Result { + let data = get_json(&format!("{}{BETA_TAG}", endpoints.release_api), GITHUB_JSON)?; + let listing = parse_beta(&data, endpoints)?; + let manifest = get_asset(&listing.manifest, endpoints)?; + let signature = get_asset(&listing.signature, endpoints)?; + check_beta_manifest(&listing, &manifest, &signature, &endpoints.beta_key)?; + Ok(listing) +} + +/// A manifest file, following redirects only to allowed hosts. One that +/// isn't served or is too large is a bad Beta, not a network problem, so the +/// saved Beta goes too. +fn get_asset(url: &str, endpoints: &Endpoints) -> Result, UpdateError> { + let bad = |why: String| UpdateError::BadManifest(why); + let network = |error: ureq::Error| match error { + ureq::Error::StatusCode(status) => bad(format!("Beta file: HTTP {status}")), + ureq::Error::BodyExceedsLimit(_) => bad("Beta file too large".into()), + error => UpdateError::Network(error.to_string()), + }; + let unsafe_download = |why: String| UpdateError::UnsafeDownload(why); + let agent = agent(Duration::from_secs(10)); + let mut url = url.to_string(); + for _ in 0..=MAX_REDIRECTS { + if !allowed_url(&url, endpoints) { + return Err(unsafe_download(format!("host not allowed: {url}"))); + } + let mut response = agent + .get(&url) + .header("User-Agent", USER_AGENT) + .call() + .map_err(network)?; + if response.status().is_redirection() { + let location = response + .headers() + .get("location") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| unsafe_download("redirect without a location".into()))?; + url = resolve_redirect(&url, location) + .ok_or_else(|| unsafe_download(format!("bad redirect: {location}")))?; + continue; + } + if !response.status().is_success() { + return Err(bad(format!("Beta file: HTTP {}", response.status()))); + } + return response + .body_mut() + .with_config() + .limit(MAX_MANIFEST_BYTES) + .read_to_vec() + .map_err(network); + } + Err(unsafe_download("too many redirects".into())) +} + // ── The failure message ───────────────────────────────────────────────────── /// Whether the failure message can offer a newer version, and how that was @@ -512,9 +784,14 @@ pub fn download( progress: impl FnMut(u64, u64), ) -> Result { let io = |error: std::io::Error| UpdateError::Network(error.to_string()); - let api = format!("{}v{}", endpoints.release_api, release.version); - let github = get_json(&api, "application/vnd.github+json")?; - let size = check_github_release(&github, release)?; + let size = if release.is_beta() { + // The signed manifest is checked again, right before the download. + check_beta_release(&verified_beta(endpoints)?, release)? + } else { + let api = format!("{}v{}", endpoints.release_api, release.version); + let github = get_json(&api, GITHUB_JSON)?; + check_github_release(&github, release)? + }; // Earlier updates are finished; their folders can go. let _ = std::fs::remove_dir_all(root); std::fs::create_dir_all(root).map_err(io)?; @@ -866,6 +1143,17 @@ struct Saved { /// showing for that version, whatever the clock does later. #[serde(default)] rollout_reached: Option, + /// The newest Beta, as last read. Offered only to a Beta build. + #[serde(default)] + beta: Option, + /// When a notification last announced a Beta. + #[serde(default)] + beta_notified: Option, + /// The highest Beta whose signed manifest checked out. An older signed + /// Beta served later is refused, so replaying one can't hold testers + /// back on it. + #[serde(default)] + beta_highest: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -936,6 +1224,9 @@ pub enum Command { pub struct State { current: String, auto: bool, + /// A Beta build also reads the Beta channel unless this is off + /// (Settings). + beta_updates: bool, manual_check: bool, saved: Saved, /// The release being downloaded, waited for or installed. The helper @@ -988,7 +1279,7 @@ fn take_pending(current: &str, saved: &mut Saved) -> Option { Some(UpdateResult::Setup { exit, .. }) => Some(*exit), _ => None, }; - if newer_stable(&to, current).is_none() { + if !newer(&to, current) { // Setup replaces the app before its last steps, so the new version // can run after Setup failed. return Some(match exit { @@ -1092,6 +1383,7 @@ impl State { let mut state = Self { current: current.into(), auto, + beta_updates: true, manual_check: false, saved, chosen: None, @@ -1134,11 +1426,67 @@ impl State { .is_none_or(|last| now.saturating_sub(last) >= RECHECK_OFFER || last > now) } + /// The newer of the stable release and, on a Beta build, the Beta, if + /// it is newer than this version. fn available(&self) -> Option<&Release> { - self.saved + let stable = self + .saved .latest .as_ref() - .filter(|latest| newer_stable(&latest.version, &self.current).is_some()) + .filter(|latest| newer_stable(&latest.version, &self.current).is_some()); + let beta = self.saved.beta.as_ref().filter(|beta| { + self.beta_channel() && beta.is_beta() && newer(&beta.version, &self.current) + }); + match (stable, beta) { + (Some(stable), Some(beta)) if newer(&beta.version, &stable.version) => Some(beta), + (stable, beta) => stable.or(beta), + } + } + + /// This is a Beta build and Beta updates are on. + fn beta_channel(&self) -> bool { + self.beta_updates && is_beta(&self.current) + } + + /// The Beta channel's answer, taken before [`Self::finish_check`] of the + /// same check. Offline, the saved Beta stays; a Beta that fails its + /// checks, or is older than one verified before, is not offered, and + /// neither is the one saved before it. + fn finish_beta(&mut self, result: Result) { + match result { + Ok(release) => match &self.saved.beta_highest { + Some(highest) if newer(highest, &release.version) => { + tracing::warn!(version = %release.version, %highest, "Older Beta refused"); + self.saved.beta = None; + } + _ => { + self.saved.beta_highest = Some(release.version.clone()); + self.saved.beta = Some(release); + } + }, + Err(error @ UpdateError::Network(_)) => { + tracing::info!(%error, "Beta check unavailable"); + } + Err(error) => { + tracing::warn!(%error, "Beta release refused"); + self.saved.beta = None; + } + } + } + + /// A Beta is not announced while a stream runs, nor within a day of + /// the last Beta notification. + fn beta_quiet(&mut self, now: u64, version: &str) -> bool { + // A time ahead of the clock counts as now, as for the popup. + if let Some(at) = self.saved.beta_notified.as_mut() { + *at = (*at).min(now); + } + is_beta(version) + && (self.streaming + || self + .saved + .beta_notified + .is_some_and(|at| now - at < BETA_INTERVAL)) } /// The available release is staged and this install's turn has not come: @@ -1230,17 +1578,24 @@ impl State { || self.snoozed(now) || self.held_back(now) || self.saved.notified.as_deref() == Some(&version) + || self.beta_quiet(now, &version) { return None; } self.saved.notified = Some(version.clone()); + if is_beta(&version) { + self.saved.beta_notified = Some(now); + } Some(version) } + /// Later covers its version, and on a Beta every newer Beta, for a day. fn snoozed(&self, now: u64) -> bool { match (&self.saved.later, self.available()) { (Some((version, at)), Some(latest)) => { - *version == latest.version && now.saturating_sub(*at) < LATER_INTERVAL && *at <= now + (*version == latest.version || (is_beta(version) && latest.is_beta())) + && now.saturating_sub(*at) < LATER_INTERVAL + && *at <= now } _ => false, } @@ -1278,7 +1633,8 @@ impl State { } /// Once per version while the banner offers it, then at most every - /// three days after it is closed. Never when automatic checks are off. + /// three days after it is closed; a Beta at most once a day, whichever + /// Beta was closed. Never when automatic checks are off. fn popup(&mut self, now: u64) -> Option { self.reach_rollout(now); // Closed "in the future" means the clock was set back: count it as @@ -1291,10 +1647,10 @@ impl State { return None; } let release = self.available()?; - let closed = - self.saved.popup.as_ref().is_some_and(|(version, at)| { - *version == release.version && now - at < POPUP_INTERVAL - }); + let closed = self.saved.popup.as_ref().is_some_and(|(version, at)| { + (*version == release.version && now - at < POPUP_INTERVAL) + || (is_beta(version) && release.is_beta() && now - at < BETA_INTERVAL) + }); (!closed).then(|| Popup { version: release.version.clone(), recommended: release.recommended, @@ -1727,8 +2083,8 @@ fn wait_while_checks_held() { impl Updater { /// `gate` is the process's handoff gate, shared with the window's - /// restart. - pub fn start(auto: bool, session: SharedSession, gate: Gate, wake: Wake) -> Self { + /// restart. `beta`: a Beta build also offers newer Betas. + pub fn start(auto: bool, beta: bool, session: SharedSession, gate: Gate, wake: Wake) -> Self { let path = state_path(); let saved = load(path.as_deref()); let mut state = State::resume( @@ -1738,6 +2094,7 @@ impl Updater { PREVIOUS.lock().take(), ); state.gate = gate; + state.beta_updates = beta; let state = Arc::new(Mutex::new(state)); save(path.as_deref(), &state.lock().saved); let (tx, rx) = mpsc::channel(); @@ -1832,6 +2189,17 @@ impl Updater { } } + /// Turning Beta updates back on reads the Beta channel again. + pub fn set_beta(&self, beta: bool) { + let check = self.with(|state| { + state.beta_updates = beta; + state.auto && state.beta_channel() + }); + if check { + self.send(Some(Command::Check { manual: false })); + } + } + pub fn check_now(&self) { let checking = self.with(|state| { let busy = state.check == CheckStatus::Checking; @@ -1908,20 +2276,35 @@ impl Worker { }; match command { Command::Check { manual } => { - self.state.lock().begin_check(manual); + let beta = { + let mut state = self.state.lock(); + state.begin_check(manual); + state.beta_channel() + }; (self.wake)(None); wait_while_checks_held(); + let beta = beta.then(|| fetch_beta(&self.endpoints)); let result = get_json(&self.endpoints.manifest, "application/json").and_then(|data| { // Per-PC settings apply from the next app run. crate::remote_config::save_fetched(&data); parse_manifest(&data, &self.endpoints) }); - let notify = - self.update(|state| state.finish_check(now_secs(), manual, result)); + let notify = self.update(|state| { + if let Some(beta) = beta { + state.finish_beta(beta); + } + state.finish_check(now_secs(), manual, result) + }); (self.wake)(notify.as_deref()); } Command::Download(release) => { + // A Beta published since the check replaces the offer + // here, rather than failing the download's hash check. + if release.is_beta() { + let latest = fetch_beta(&self.endpoints); + self.update(|state| state.finish_beta(latest)); + } if !self.update(|state| state.download_starts(&release)) { (self.wake)(None); continue; diff --git a/host/src/updates/tests.rs b/host/src/updates/tests.rs index 6665c6de..87911b82 100644 --- a/host/src/updates/tests.rs +++ b/host/src/updates/tests.rs @@ -58,8 +58,10 @@ fn versions_compare_as_semver_and_ignore_previews_build_metadata_and_garbage() { assert_eq!(newer_stable("0.4.0", "not a version"), None); } +/// The stable comparison never names a Beta; a Beta build reads the Beta +/// channel separately (see the Beta channel tests at the end). #[test] -fn beta_channel_builds_are_never_offered_and_get_the_next_stable_release() { +fn newer_stable_never_names_a_beta_and_a_beta_gets_the_next_stable_release() { // scripts/nightly.py versions the nightly Beta channel -beta... let beta = "0.3.4-beta.20261030.12"; assert_eq!(newer_stable(beta, "0.3.3"), None); @@ -1716,6 +1718,7 @@ fn logged(run: impl FnOnce()) -> String { .with_ansi(false) .with_writer(move || sink.clone()) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, run); let text = String::from_utf8(out.lock().clone()).unwrap(); text @@ -1826,14 +1829,17 @@ fn the_failure_message_check_is_validated_and_bounded() { } fn pending_saved(from: &str, to: &str, result: Option<&str>) -> (Saved, PathBuf) { + // Each call its own folder: the clock alone repeats for calls in the same + // microsecond, and one test's update then reads or removes another's + // result. + static FOLDERS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let dir = std::env::temp_dir().join(format!( "eternal-update-result-{}-{}", std::process::id(), - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() + FOLDERS.fetch_add(1, std::sync::atomic::Ordering::Relaxed) )); + // A failed run of an earlier process with this id may have left one. + let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join(eternal_updater::RESULT_FILE); if let Some(text) = result { @@ -2125,3 +2131,990 @@ fn the_update_funnel_is_logged_without_identifiers() { "{log}" ); } + +// ── The Beta channel ──────────────────────────────────────────────────────── + +/// The newest Beta in these tests, and the Beta build they run as. +const BETA: &str = "0.3.5-beta.20261107.26"; +const OLD_BETA: &str = "0.3.5-beta.20261105.20"; + +fn beta_installer() -> String { + "https://github.com/sidebandstudio/EternalMonitor/releases/download/beta/EternalMonitor-Setup-beta.exe".into() +} + +fn beta(version: &str) -> Release { + Release { + version: version.into(), + installer: beta_installer(), + sha256: SHA.into(), + recommended: false, + note: Some(BETA_NOTE.into()), + rollout: None, + } +} + +/// The beta release's body exactly as scripts/nightly.py publishes it, for +/// `version` and `sha` (test_nightly.py checks the fixture is nightly.py's). +fn beta_body(version: &str, sha: &str) -> String { + let fixture = + include_str!("../../../scripts/fixtures/beta-release-body.md").replace("\r\n", "\n"); + assert_eq!(fixture.matches("0.3.5-beta.20261107.26").count(), 1); + assert_eq!(fixture.matches(&"ab".repeat(32)).count(), 1); + fixture + .replace("0.3.5-beta.20261107.26", version) + .replace(&"ab".repeat(32), sha) +} + +/// The fixed `beta` release as the GitHub API returns it, with one installer +/// at `url` whose GitHub digest is `digest`, and the manifest and signature +/// beside it. `extra` overrides release fields; its `asset` entry overrides +/// the installer's, and `drop` lists assets to leave out. +fn beta_github( + body: &str, + url: &str, + size: u64, + digest: &str, + extra: serde_json::Value, +) -> Vec { + let mut asset = serde_json::json!({ + "name": "EternalMonitor-Setup-beta.exe", + "size": size, + "browser_download_url": url, + "digest": format!("sha256:{digest}"), + }); + let mut release = serde_json::json!({ + "tag_name": "beta", "draft": false, "prerelease": true, "body": body, + }); + let mut drop = Vec::new(); + for (key, value) in extra.as_object().unwrap() { + if key == "asset" { + for (key, value) in value.as_object().unwrap() { + asset[key] = value.clone(); + } + } else if key == "drop" { + drop = value.as_array().unwrap().clone(); + } else { + release[key] = value.clone(); + } + } + let folder = url.rsplit_once('/').unwrap().0; + let mut assets = vec![asset]; + for name in [BETA_MANIFEST_NAME, BETA_SIGNATURE_NAME] { + if !drop.contains(&name.into()) { + assets.push(serde_json::json!({ + "name": name, "size": 64, "browser_download_url": format!("{folder}/{name}"), + })); + } + } + release["assets"] = serde_json::json!(assets); + release.to_string().into_bytes() +} + +fn beta_state(saved: Saved) -> State { + State::new(OLD_BETA, true, saved) +} + +#[test] +fn beta_versions_order_by_semver_across_beta_stable_and_dated_forms() { + for version in [ + BETA, + "0.3.4-beta.1", + "1.0.0-beta.2", + "0.3.5-beta.20261107.26+ci.1", + ] { + assert!(is_beta(version), "{version}"); + } + for version in [ + "0.3.5", + "0.3.5-rc.1", + "0.3.5-betas.1", + "0.3.5-beta", + "v0.3.5-beta.1", + "0.3-beta.1", + "", + "garbage", + ] { + assert!(!is_beta(version), "{version}"); + } + let newer_than = [ + // A later night, or a later run the same night. + ("0.3.5-beta.20261108.27", BETA), + (BETA, "0.3.5-beta.20261107.25"), + // Numbers compare as numbers, not text. + ("0.3.5-beta.20261107.100", BETA), + // Dated builds sort above the hand-made -beta.1, below the release. + (BETA, "0.3.5-beta.1"), + ("0.3.5", BETA), + (BETA, "0.3.4"), + ("0.3.6-beta.20261201.40", "0.3.5"), + ]; + for (candidate, current) in newer_than { + assert!(newer(candidate, current), "{candidate} > {current}"); + assert!(!newer(current, candidate), "{current} < {candidate}"); + } + for (candidate, current) in [ + (BETA, BETA), + // A higher run on an earlier night is older. + ("0.3.5-beta.20261106.30", BETA), + // Build metadata does not make a version newer. + ("0.3.5-beta.20261107.26+ci.2", BETA), + ("garbage", BETA), + (BETA, "garbage"), + ] { + assert!(!newer(candidate, current), "{candidate} vs {current}"); + } +} + +#[test] +fn the_beta_release_names_its_version_and_hash_in_the_body_nightly_writes() { + let endpoints = Endpoints::production(); + let body = beta_body(BETA, &SHA.to_uppercase()); + // A body edited on Windows may have CRLF line endings. + for body in [body.clone(), body.replace('\n', "\r\n")] { + let data = beta_github(&body, &beta_installer(), 9, SHA, serde_json::json!({})); + let listing = parse_beta(&data, &endpoints).unwrap(); + assert_eq!((listing.release, listing.size), (beta(BETA), 9)); + } + let shown = Popup { + version: BETA.into(), + recommended: false, + note: Some(BETA_NOTE.into()), + }; + assert_eq!(shown.message(), "An update is available."); +} + +#[test] +fn a_beta_with_a_missing_or_malformed_version_or_hash_is_refused() { + let endpoints = Endpoints::production(); + let url = beta_installer(); + let none = || serde_json::json!({}); + let body = beta_body(BETA, SHA); + let no_version = body.replace( + "Version 0.3.5-beta.20261107.26, from the Beta channel", + "From the Beta channel", + ); + let two_versions = format!("Version 0.3.5-beta.20261108.27, from the Beta channel\n{body}"); + let no_hash = body.replace(SHA, ""); + let two_hashes = format!( + "{body}\nSHA-256 of EternalMonitor-Setup-beta.exe:\n\n```\n{}\n```\n", + "0".repeat(64) + ); + let malformed = [ + beta_github(&beta_body("0.3.5", SHA), &url, 9, SHA, none()), + beta_github(&beta_body("0.3.5-rc.1", SHA), &url, 9, SHA, none()), + beta_github(&beta_body("0.3.5-beta.1+ci.2", SHA), &url, 9, SHA, none()), + beta_github(&beta_body("0.3.5-beta.01", SHA), &url, 9, SHA, none()), + beta_github( + &beta_body("v0.3.5-beta.20261107.26", SHA), + &url, + 9, + SHA, + none(), + ), + beta_github(&no_version, &url, 9, SHA, none()), + beta_github(&two_versions, &url, 9, SHA, none()), + beta_github(&no_hash, &url, 9, SHA, none()), + beta_github(&beta_body(BETA, &SHA[..63]), &url, 9, SHA, none()), + beta_github( + &beta_body(BETA, &SHA.replace('b', "g")), + &url, + 9, + SHA, + none(), + ), + beta_github(&two_hashes, &url, 9, SHA, none()), + beta_github(&body, &url, 9, SHA, serde_json::json!({ "body": null })), + // The installer and the signed manifest's files must be there. + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "asset": { "name": "EternalMonitor-Setup.exe" } }), + ), + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "drop": ["beta-manifest.txt"] }), + ), + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "drop": ["beta-manifest.sig"] }), + ), + b"rate limited".to_vec(), + ]; + for data in malformed { + assert!( + matches!( + parse_beta(&data, &endpoints), + Err(UpdateError::BadManifest(_)) + ), + "{}", + String::from_utf8_lossy(&data) + ); + } + let other = "0".repeat(64); + let unsafe_releases = [ + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "tag_name": "v0.3.5" }), + ), + beta_github(&body, &url, 9, SHA, serde_json::json!({ "draft": true })), + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "prerelease": false }), + ), + // GitHub's digest of the installer must be there and agree. + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "asset": { "digest": null } }), + ), + beta_github(&body, &url, 9, &other, none()), + // Mid-publish: the new installer is up, the body still names the old one. + beta_github(&beta_body(BETA, &other), &url, 9, SHA, none()), + beta_github(&body, &url.replace("https:", "http:"), 9, SHA, none()), + beta_github(&body, &production_installer("0.3.5"), 9, SHA, none()), + beta_github(&body, &url, 0, SHA, none()), + beta_github(&body, &url, MAX_INSTALLER_BYTES + 1, SHA, none()), + ]; + for data in unsafe_releases { + assert!( + matches!( + parse_beta(&data, &endpoints), + Err(UpdateError::UnsafeDownload(_)) + ), + "{}", + String::from_utf8_lossy(&data) + ); + } +} + +#[test] +fn a_stable_install_is_never_offered_a_beta() { + for current in ["0.3.4", "0.3.4-rc.1"] { + let mut s = State::new(current, true, Saved::default()); + assert!(!s.beta_channel(), "{current}"); + // Even with a Beta saved (an install that was a Beta before). + s.finish_beta(Ok(beta(BETA))); + assert_eq!(s.finish_check(1000, false, Ok(release("0.3.3"))), None); + assert_eq!(s.notice(1000), None); + assert_eq!(s.popup(1000), None); + assert!(s.update_now(false).is_none()); + // The next stable release is offered as before. + assert_eq!( + s.finish_check(1000 + DAY, false, Ok(release("0.3.5"))), + Some("0.3.5".into()) + ); + assert_eq!( + s.notice(1000 + DAY), + Some(Notice::Available("0.3.5".into())) + ); + } +} + +#[test] +fn a_beta_install_is_offered_the_newer_beta_and_never_an_older_one() { + let mut s = beta_state(Saved::default()); + assert!(s.beta_channel()); + s.finish_beta(Ok(beta(BETA))); + assert_eq!( + s.finish_check(1000, false, Ok(release("0.3.4"))), + Some(BETA.into()) + ); + assert_eq!(s.check, CheckStatus::Available); + assert_eq!(s.notice(1000), Some(Notice::Available(BETA.into()))); + let popup = s.popup(1000).unwrap(); + assert_eq!(popup.title(), format!("EternalMonitor {BETA} is ready")); + assert_eq!(popup.note.as_deref(), Some(BETA_NOTE)); + match s.update_now(false) { + Some(Command::Download(release)) => assert_eq!(release, beta(BETA)), + _ => panic!("Update now downloads the Beta"), + } + + for version in [ + OLD_BETA, + "0.3.5-beta.20261104.30", + "0.3.4-beta.20261020.3", + "0.3.5-beta.1", + ] { + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(version))); + assert_eq!( + s.finish_check(1000, false, Ok(release("0.3.4"))), + None, + "{version}" + ); + assert_eq!(s.check, CheckStatus::UpToDate, "{version}"); + assert_eq!(s.notice(1000), None, "{version}"); + assert!(s.update_now(false).is_none(), "{version}"); + } +} + +#[test] +fn a_newer_stable_release_wins_over_the_beta() { + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + assert_eq!( + s.finish_check(1000, false, Ok(release("0.3.5"))), + Some("0.3.5".into()) + ); + assert_eq!(s.notice(1000), Some(Notice::Available("0.3.5".into()))); + assert_eq!( + s.available().map(|release| release.installer.clone()), + Some(production_installer("0.3.5")) + ); + // A Beta of the release after it is newer still. + s.finish_beta(Ok(beta("0.3.6-beta.20261201.40"))); + s.finish_check(1000 + DAY, false, Ok(release("0.3.5"))); + assert_eq!( + s.notice(1000 + DAY), + Some(Notice::Available("0.3.6-beta.20261201.40".into())) + ); +} + +#[test] +fn a_refused_beta_is_not_offered_and_offline_keeps_the_last_one() { + let endpoints = Endpoints::production(); + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + s.finish_check(1000, false, Ok(release("0.3.4"))); + // Offline: the saved Beta is still offered. + s.finish_beta(Err(UpdateError::Network("offline".into()))); + s.finish_check( + 1000 + DAY, + false, + Err(UpdateError::Network("offline".into())), + ); + assert_eq!(s.notice(1000 + DAY), Some(Notice::Available(BETA.into()))); + // A release with no hash replaces it: neither is offered. + let no_hash = beta_github( + &beta_body(BETA, ""), + &beta_installer(), + 9, + SHA, + serde_json::json!({}), + ); + s.finish_beta(parse_beta(&no_hash, &endpoints).map(|listing| listing.release)); + s.finish_check(1000 + 2 * DAY, false, Ok(release("0.3.4"))); + assert_eq!(s.saved.beta, None); + assert_eq!(s.check, CheckStatus::UpToDate); + assert_eq!(s.notice(1000 + 2 * DAY), None); + // So does one whose installer GitHub hashes differently. + s.finish_beta(Ok(beta(BETA))); + let swapped = beta_github( + &beta_body(BETA, SHA), + &beta_installer(), + 9, + &"0".repeat(64), + serde_json::json!({}), + ); + s.finish_beta(parse_beta(&swapped, &endpoints).map(|listing| listing.release)); + assert_eq!(s.saved.beta, None); +} + +#[test] +fn turning_beta_updates_off_offers_only_stable_releases() { + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + s.finish_check(1000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(1000), Some(Notice::Available(BETA.into()))); + s.beta_updates = false; + assert!(!s.beta_channel()); + assert_eq!(s.notice(1000), None); + assert_eq!( + s.finish_check(1000 + DAY, false, Ok(release("0.3.5"))), + Some("0.3.5".into()) + ); + assert_eq!( + s.notice(1000 + DAY), + Some(Notice::Available("0.3.5".into())) + ); +} + +#[test] +fn betas_are_announced_at_most_once_a_day_and_never_during_a_stream() { + const HOUR: u64 = 60 * 60; + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + assert_eq!( + s.finish_check(1000, false, Ok(release("0.3.4"))), + Some(BETA.into()) + ); + s.popup_closed(1000); + // A newer Beta the same day is on the banner, without a notification or + // the popup. + let next = "0.3.5-beta.20261107.27"; + s.finish_beta(Ok(beta(next))); + assert_eq!( + s.finish_check(1000 + HOUR, false, Ok(release("0.3.4"))), + None + ); + assert_eq!(s.notice(1000 + HOUR), Some(Notice::Available(next.into()))); + assert_eq!(s.popup(1000 + DAY - 1), None); + // A day on, both come back, also after a restart. + let mut s = beta_state(s.saved.clone()); + assert!(s.popup(1000 + DAY).is_some()); + assert_eq!( + s.finish_check(1000 + DAY, false, Ok(release("0.3.4"))), + Some(next.into()) + ); + // Not while a stream runs; a check after it announces it. + let third = "0.3.5-beta.20261109.28"; + s.finish_beta(Ok(beta(third))); + s.streaming = true; + assert_eq!( + s.finish_check(1000 + 2 * DAY, false, Ok(release("0.3.4"))), + None + ); + s.streaming = false; + assert_eq!( + s.finish_check(1000 + 2 * DAY + HOUR, false, Ok(release("0.3.4"))), + Some(third.into()) + ); + // Announced "in the future" (the clock went back) counts as now. + s.saved.beta_notified = Some(u64::MAX / 2); + s.finish_beta(Ok(beta("0.3.5-beta.20261120.40"))); + assert_eq!(s.finish_check(20 * DAY, false, Ok(release("0.3.4"))), None); + assert_eq!(s.saved.beta_notified, Some(20 * DAY)); + assert_eq!( + s.finish_check(21 * DAY, false, Ok(release("0.3.4"))), + Some("0.3.5-beta.20261120.40".into()) + ); + // A stable release keeps its own rules, streaming or not. + s.streaming = true; + assert_eq!( + s.finish_check(21 * DAY + HOUR, false, Ok(release("0.3.5"))), + Some("0.3.5".into()) + ); +} + +#[test] +fn later_on_a_beta_covers_newer_betas_for_a_day_but_not_a_stable_release() { + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + s.finish_check(1000, false, Ok(release("0.3.4"))); + s.later(1000); + assert_eq!(s.notice(1000), None); + let next = "0.3.5-beta.20261108.27"; + s.finish_beta(Ok(beta(next))); + assert_eq!(s.finish_check(2000, false, Ok(release("0.3.4"))), None); + assert_eq!(s.notice(1000 + DAY - 1), None); + assert_eq!(s.notice(1000 + DAY), Some(Notice::Available(next.into()))); + s.later(1000 + DAY); + s.finish_check(2000 + DAY, false, Ok(release("0.3.5"))); + assert_eq!( + s.notice(2000 + DAY), + Some(Notice::Available("0.3.5".into())) + ); +} + +#[test] +fn relaunch_after_a_beta_update_reports_whether_it_finished() { + let saved = Saved { + pending: Some((OLD_BETA.into(), BETA.into())), + beta: Some(beta(BETA)), + ..Saved::default() + }; + let updated = State::new(BETA, true, saved.clone()); + assert_eq!(updated.notice(0), Some(Notice::Updated(BETA.into()))); + let failed = State::new(OLD_BETA, true, saved); + assert!( + matches!(failed.notice(0), Some(Notice::Failed(_))), + "{:?}", + failed.notice(0) + ); +} + +/// A published, test-only seed (scripts/test_sign_beta_manifest.py signs +/// with it too). +const TEST_SEED: [u8; 32] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, +]; + +fn test_key() -> ring::signature::Ed25519KeyPair { + ring::signature::Ed25519KeyPair::from_seed_unchecked(&TEST_SEED).unwrap() +} + +fn test_public_key() -> [u8; 32] { + use ring::signature::KeyPair; + test_key().public_key().as_ref().try_into().unwrap() +} + +/// What the fake GitHub publishes for the Beta: `version` with `published` +/// as its installer, which downloads as `served`; the manifest and +/// signature, or `None` for no such asset. With `status`, the listed +/// manifest's download answers that HTTP status instead. +#[derive(Clone)] +struct BetaSite { + version: String, + published: Vec, + served: Vec, + manifest: Option>, + signature: Option>, + status: Option, +} + +impl BetaSite { + /// Signed with the test key. + fn signed(version: &str, published: &[u8]) -> Self { + let mut site = Self { + version: version.into(), + published: published.to_vec(), + served: published.to_vec(), + manifest: None, + signature: None, + status: None, + }; + site.sign(BetaManifest { + version: version.into(), + sha256: sha_of(published), + size: published.len() as u64, + }); + site + } + + fn sign(&mut self, manifest: BetaManifest) { + let text = manifest.text().into_bytes(); + self.signature = Some(test_key().sign(&text).as_ref().to_vec()); + self.manifest = Some(text); + } +} + +/// Serves `site` like GitHub, and a latest.json naming 0.3.4, until the +/// test ends. The endpoints trust the test key. Counts reads of the Beta. +fn serve_beta(site: Arc>) -> (Endpoints, Arc) { + use std::sync::atomic::{AtomicUsize, Ordering}; + let reads = Arc::new(AtomicUsize::new(0)); + let counted = reads.clone(); + let base = serve(Box::new(move |base, path| { + let site = site.lock().clone(); + let folder = format!("{base}/releases/download/beta"); + let found = |data: Option>| match data { + Some(data) => (200, vec![], data), + None => (404, vec![], Vec::new()), + }; + match path { + "/latest.json" => ( + 200, + vec![], + manifest( + "0.3.4", + &format!("{base}/releases/download/v0.3.4/EternalMonitor-Setup.exe"), + SHA, + ), + ), + "/repos/sidebandstudio/EternalMonitor/releases/tags/beta" => { + counted.fetch_add(1, Ordering::SeqCst); + let sha = sha_of(&site.published); + let mut drop = Vec::new(); + if site.manifest.is_none() { + drop.push(BETA_MANIFEST_NAME); + } + if site.signature.is_none() { + drop.push(BETA_SIGNATURE_NAME); + } + let listing = beta_github( + &beta_body(&site.version, &sha), + &format!("{folder}/EternalMonitor-Setup-beta.exe"), + site.published.len() as u64, + &sha, + serde_json::json!({ "drop": drop }), + ); + (200, vec![], listing) + } + // Downloads redirect to an asset host, as on GitHub. + "/releases/download/beta/EternalMonitor-Setup-beta.exe" => ( + 302, + vec![("Location".into(), "/assets/installer".into())], + Vec::new(), + ), + "/releases/download/beta/beta-manifest.txt" => ( + 302, + vec![("Location".into(), "/assets/manifest".into())], + Vec::new(), + ), + "/assets/installer" => (200, vec![], site.served), + "/assets/manifest" => match site.status { + Some(status) => (status, vec![], Vec::new()), + None => found(site.manifest), + }, + "/releases/download/beta/beta-manifest.sig" => found(site.signature), + _ => (404, vec![], Vec::new()), + } + })); + let endpoints = Endpoints { + beta_key: test_public_key(), + ..Endpoints::local(&base).unwrap() + }; + (endpoints, reads) +} + +fn fetch_site(site: BetaSite) -> (Endpoints, Result) { + let (endpoints, _) = serve_beta(Arc::new(Mutex::new(site))); + let result = fetch_beta(&endpoints); + (endpoints, result) +} + +#[test] +fn the_beta_manifest_parses_only_in_its_canonical_form() { + let fixture = include_str!("../../../scripts/fixtures/beta-manifest.txt").replace("\r\n", "\n"); + let parsed = BetaManifest::parse(fixture.as_bytes()).unwrap(); + assert_eq!( + parsed, + BetaManifest { + version: BETA.into(), + sha256: "ab".repeat(32), + size: 63_301_683, + } + ); + assert_eq!(parsed.text(), fixture); + let variants = [ + fixture.replace('\n', "\r\n"), + fixture.trim_end().to_string(), + format!("{fixture}\n"), + format!("{fixture}note=x\n"), + fixture.replace("manifest 1", "manifest 2"), + fixture.replace(&"ab".repeat(32), &"AB".repeat(32)), + fixture.replace(&"ab".repeat(32), &"ab".repeat(31)), + fixture.replace("size=63301683", "size=063301683"), + fixture.replace("size=63301683", "size= 63301683"), + fixture.replace("size=63301683", "size=-1"), + fixture.replace("Setup-beta.exe", "Setup.exe"), + fixture.replace(BETA, "0.3.5"), + fixture.replace(BETA, "0.3.5-beta.01"), + fixture.replace(BETA, &format!("{BETA}+ci.1")), + fixture.replace("version=", "version = "), + fixture.replace("sha256=", "sha256:"), + ]; + for text in variants { + assert_eq!(BetaManifest::parse(text.as_bytes()), None, "{text:?}"); + } + assert_eq!(BetaManifest::parse(b"\xff\xfe"), None); +} + +/// Ed25519 is deterministic: ring signs the fixture to the bytes OpenSSL +/// made with the same test seed, the way scripts/sign_beta_manifest.py signs. +#[test] +fn a_manifest_signed_by_the_release_script_verifies_here() { + let manifest = + include_str!("../../../scripts/fixtures/beta-manifest.txt").replace("\r\n", "\n"); + let signature = include_bytes!("../../../scripts/fixtures/beta-manifest.test.sig"); + assert_eq!(test_key().sign(manifest.as_bytes()).as_ref(), signature); + let listing = BetaListing { + release: Release { + sha256: "ab".repeat(32), + ..beta(BETA) + }, + size: 63_301_683, + manifest: String::new(), + signature: String::new(), + }; + assert_eq!( + check_beta_manifest(&listing, manifest.as_bytes(), signature, &test_public_key()), + Ok(()) + ); + // Not by the production key. + assert!(matches!( + check_beta_manifest(&listing, manifest.as_bytes(), signature, &BETA_PUBLIC_KEY), + Err(UpdateError::UnsafeDownload(_)) + )); +} + +#[test] +fn the_pinned_key_is_the_published_public_key() { + assert_eq!( + hex(&BETA_PUBLIC_KEY), + "82c2af21c685d2c042c51f72a34c78bdb5d004bad89e75a8306a43224d16ec5a" + ); + assert_eq!(Endpoints::production().beta_key, BETA_PUBLIC_KEY); + assert_eq!( + Endpoints::local("http://127.0.0.1:1").unwrap().beta_key, + BETA_PUBLIC_KEY + ); +} + +#[test] +fn a_beta_is_offered_only_with_a_valid_signed_manifest() { + let payload = b"MZ fake beta installer"; + let (_, fetched) = fetch_site(BetaSite::signed(BETA, payload)); + let fetched = fetched.unwrap(); + assert_eq!( + (fetched.version.as_str(), fetched.sha256.as_str()), + (BETA, sha_of(payload).as_str()) + ); + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(fetched)); + s.finish_check(1000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(1000), Some(Notice::Available(BETA.into()))); + + let good = BetaSite::signed(BETA, payload); + let manifest = |version: &str, sha256: String, size: usize| BetaManifest { + version: version.into(), + sha256, + size: size as u64, + }; + let mut flipped = good.clone(); + flipped.manifest.as_mut().unwrap()[40] ^= 1; + let mut other_sha = good.clone(); + other_sha.sign(manifest(BETA, "0".repeat(64), payload.len())); + let mut other_size = good.clone(); + other_size.sign(manifest(BETA, sha_of(payload), payload.len() + 1)); + let mut other_version = good.clone(); + other_version.sign(manifest( + "0.3.5-beta.20261108.27", + sha_of(payload), + payload.len(), + )); + let mut crlf = good.clone(); + let text = String::from_utf8(good.manifest.clone().unwrap()) + .unwrap() + .replace('\n', "\r\n"); + crlf.signature = Some(test_key().sign(text.as_bytes()).as_ref().to_vec()); + crlf.manifest = Some(text.into_bytes()); + let mut short = good.clone(); + short.signature.as_mut().unwrap().pop(); + let mut no_signature = good.clone(); + no_signature.signature = None; + let mut no_manifest = good.clone(); + no_manifest.manifest = None; + // The nightly published a new build that nobody has signed yet. + let mut unsigned_build = good.clone(); + unsigned_build.version = "0.3.5-beta.20261108.27".into(); + unsigned_build.published = b"MZ next beta installer".to_vec(); + let cases = [ + ("flipped byte", flipped), + ("manifest sha differs from the body", other_sha), + ("manifest size differs from the asset", other_size), + ("manifest version differs from the body", other_version), + ("signed, but not canonical", crlf), + ("63-byte signature", short), + ("missing signature", no_signature), + ("missing manifest", no_manifest), + ("unsigned new build", unsigned_build), + ]; + for (name, site) in cases { + let (_, fetched) = fetch_site(site); + assert!( + matches!( + fetched, + Err(UpdateError::UnsafeDownload(_) | UpdateError::BadManifest(_)) + ), + "{name}: {fetched:?}" + ); + // No offer, and the Beta saved before is not offered either. + let mut s = beta_state(Saved { + beta: Some(beta(BETA)), + ..Saved::default() + }); + s.finish_beta(fetched); + s.finish_check(1000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(1000), None, "{name}"); + } + + // Signed with another key: the production key does not trust the test key. + let (endpoints, _) = serve_beta(Arc::new(Mutex::new(good))); + let production = Endpoints { + beta_key: BETA_PUBLIC_KEY, + ..endpoints + }; + assert!( + matches!(fetch_beta(&production), Err(UpdateError::UnsafeDownload(why)) if why.contains("signature")), + ); +} + +#[test] +fn a_signed_beta_that_is_not_newer_is_not_offered() { + for version in [OLD_BETA, "0.3.5-beta.20261104.30", "0.3.4-beta.1"] { + let (_, fetched) = fetch_site(BetaSite::signed(version, b"MZ old beta installer")); + let mut s = beta_state(Saved::default()); + s.finish_beta(fetched); + assert_eq!( + s.saved.beta.as_ref().map(|r| r.version.as_str()), + Some(version) + ); + assert_eq!( + s.finish_check(1000, false, Ok(release("0.3.4"))), + None, + "{version}" + ); + assert_eq!(s.notice(1000), None, "{version}"); + } +} + +#[test] +fn a_beta_downloads_and_is_verified_like_a_stable_release() { + let payload = b"MZ fake beta installer".to_vec(); + let (endpoints, release) = fetch_site(BetaSite::signed(BETA, &payload)); + let release = release.unwrap(); + let path = download_case(&endpoints, &release).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), payload); + assert!(eternal_updater::open_verified(&path, &release.sha256).is_ok()); + std::fs::remove_dir_all(path.parent().unwrap().parent().unwrap()).unwrap(); + + // The downloaded file must be the one the signed manifest names. + let mut swapped = BetaSite::signed(BETA, &payload); + swapped.served = b"MZ evil beta installer".to_vec(); + let (endpoints, release) = fetch_site(swapped); + let error = download_case(&endpoints, &release.unwrap()).unwrap_err(); + assert!( + matches!(&error, UpdateError::UnsafeDownload(why) if why.contains("SHA-256")), + "{error}" + ); + + // The release now names another Beta than the one chosen. + let (endpoints, release) = fetch_site(BetaSite::signed(BETA, &payload)); + let chosen = Release { + version: OLD_BETA.into(), + ..release.unwrap() + }; + let error = download_case(&endpoints, &chosen).unwrap_err(); + assert!( + matches!(&error, UpdateError::UnsafeDownload(why) if why.contains("changed")), + "{error}" + ); +} + +/// Through the worker against a fake website and GitHub: a stable install +/// never asks for the Beta, a Beta install offers it, and a Beta published +/// after the check replaces the offer before anything downloads. +#[test] +fn the_worker_reads_the_beta_only_for_a_beta_build_and_again_before_downloading() { + use std::sync::atomic::Ordering; + let site = Arc::new(Mutex::new(BetaSite::signed( + BETA, + b"MZ fake beta installer", + ))); + let (endpoints, beta_reads) = serve_beta(site.clone()); + let run = |state: &Arc>, command: Command| { + let (tx, rx) = mpsc::channel(); + tx.send(command).unwrap(); + drop(tx); + Worker { + state: state.clone(), + path: None, + endpoints: endpoints.clone(), + session: Arc::new(Mutex::new(crate::transport::session::Session::new(1))), + wake: Arc::new(|_| {}), + } + .run(rx); + }; + + let stable = Arc::new(Mutex::new(State::new("0.3.4", true, Saved::default()))); + run(&stable, Command::Check { manual: false }); + assert_eq!(stable.lock().check, CheckStatus::UpToDate); + assert_eq!( + beta_reads.load(Ordering::SeqCst), + 0, + "a stable install never asks" + ); + + let state = Arc::new(Mutex::new(beta_state(Saved::default()))); + run(&state, Command::Check { manual: false }); + assert_eq!(beta_reads.load(Ordering::SeqCst), 1); + assert_eq!( + state.lock().notice(now_secs()), + Some(Notice::Available(BETA.into())) + ); + let Some(Command::Download(chosen)) = state.lock().update_now(false) else { + panic!("Update now downloads the Beta"); + }; + let next = "0.3.5-beta.20261108.27"; + *site.lock() = BetaSite::signed(next, b"MZ next beta installer"); + run(&state, Command::Download(chosen)); + assert_eq!(beta_reads.load(Ordering::SeqCst), 2); + let state = state.lock(); + assert_eq!(state.phase, Phase::Idle, "nothing downloaded"); + assert_eq!( + state.notice(now_secs()), + Some(Notice::Available(next.into())) + ); +} + +#[test] +fn a_manifest_file_that_is_not_served_or_too_large_drops_the_beta() { + let good = BetaSite::signed(BETA, b"MZ fake beta installer"); + let mut large = good.clone(); + large.manifest = Some(vec![b'a'; MAX_MANIFEST_BYTES as usize + 1]); + let mut missing = good.clone(); + missing.status = Some(404); + let mut failing = good.clone(); + failing.status = Some(500); + for (name, site) in [("too large", large), ("404", missing), ("500", failing)] { + let (_, fetched) = fetch_site(site); + assert!( + matches!(fetched, Err(UpdateError::BadManifest(_))), + "{name}: {fetched:?}" + ); + let mut s = beta_state(Saved { + beta: Some(beta(BETA)), + ..Saved::default() + }); + s.finish_beta(fetched); + assert_eq!(s.saved.beta, None, "{name}"); + } +} + +#[test] +fn the_download_checks_the_signed_manifest_again() { + let payload = b"MZ fake beta installer"; + let site = Arc::new(Mutex::new(BetaSite::signed(BETA, payload))); + let (endpoints, _) = serve_beta(site.clone()); + let release = fetch_beta(&endpoints).unwrap(); + // Between the check and the download, the manifest stops verifying. + site.lock().manifest.as_mut().unwrap()[40] ^= 1; + let error = download_case(&endpoints, &release).unwrap_err(); + assert!( + matches!(&error, UpdateError::UnsafeDownload(why) if why.contains("signature")), + "{error}" + ); + let mut unsigned = BetaSite::signed(BETA, payload); + unsigned.signature = None; + *site.lock() = unsigned; + assert!(matches!( + download_case(&endpoints, &release), + Err(UpdateError::BadManifest(_)) + )); +} + +#[test] +fn an_older_signed_beta_than_one_verified_before_is_refused() { + let older = "0.3.5-beta.20261106.22"; + let next = "0.3.5-beta.20261108.27"; + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + s.finish_check(1000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(1000), Some(Notice::Available(BETA.into()))); + // A replayed older signed Beta, still newer than this build: no offer, + // also after a restart. + s.finish_beta(Ok(beta(older))); + s.finish_check(2000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(2000), None); + assert_eq!(s.saved.beta_highest.as_deref(), Some(BETA)); + let mut s = beta_state(s.saved.clone()); + s.finish_beta(Ok(beta(older))); + s.finish_check(3000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(3000), None); + // The same Beta again, or a newer one, is offered. + s.finish_beta(Ok(beta(BETA))); + s.finish_check(4000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(4000), Some(Notice::Available(BETA.into()))); + s.finish_beta(Ok(beta(next))); + s.finish_check(5000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(5000), Some(Notice::Available(next.into()))); + assert_eq!(s.saved.beta_highest.as_deref(), Some(next)); +} diff --git a/host/src/vdd.rs b/host/src/vdd.rs index 23e3cb38..2cbd3f4e 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -214,6 +214,28 @@ impl DriverTasks { self.gate.lock().health } + /// The Disable at startup and at quit: on its own turn, no task runs + /// when this host has nothing recorded (the state is unknown, and no + /// Enable ran since the last completed Disable) and `already_off` then + /// says the display is off. The skip records nothing, so the state stays + /// unknown and the next Disable decides afresh. After an Enable, even + /// one that timed out and may still be running, the Disable always runs. + fn disable_unless_off( + &self, + on_turn: impl FnOnce(), + already_off: impl FnOnce() -> bool, + run: impl FnOnce(bool, &mut dyn FnMut()) -> TaskOutcome, + ) -> bool { + let skip = || { + let idle = { + let gate = self.gate.lock(); + gate.known_enabled.is_none() && !gate.enable_ran + }; + idle && already_off() + }; + self.set_enabled_unless(false, Purpose::Cleanup, || {}, on_turn, skip, run) + } + #[cfg(test)] fn set_enabled( &self, @@ -228,11 +250,25 @@ impl DriverTasks { /// As `set_enabled`, calling `on_turn` once this request holds the turn: /// every request made after that queues behind it. fn set_enabled_on_turn( + &self, + enabled: bool, + purpose: Purpose, + progress: impl FnMut(), + on_turn: impl FnOnce(), + run: impl FnOnce(bool, &mut dyn FnMut()) -> TaskOutcome, + ) -> bool { + self.set_enabled_unless(enabled, purpose, progress, on_turn, || false, run) + } + + /// As `set_enabled_on_turn`, returning true without running or recording + /// anything when `skip`, asked on the turn, says so. + fn set_enabled_unless( &self, enabled: bool, purpose: Purpose, mut progress: impl FnMut(), on_turn: impl FnOnce(), + skip: impl FnOnce() -> bool, run: impl FnOnce(bool, &mut dyn FnMut()) -> TaskOutcome, ) -> bool { use std::sync::atomic::Ordering::SeqCst; @@ -259,6 +295,9 @@ impl DriverTasks { progress(); }; on_turn(); + if skip() { + return true; + } // The gate guard ends with this statement. Logging below reads the // health through the same mutex, which deadlocked when the guard // lived through the whole match. @@ -461,20 +500,7 @@ fn request_on_turn( #[cfg(windows)] { TASKS.set_enabled_on_turn(enabled, purpose, progress, on_turn, |enabled, progress| { - let mut outcome = TaskOutcome::Failed(None); - // Read here, after the gate decided and before the task runs. - let display_off = TASKS.gate.lock().known_enabled == Some(false); - crate::startup::display::change(enabled, progress, |progress| { - outcome = crate::display_guard::around( - enabled, - crate::startup::ProcessId::current(), - log_screens, - display_off, - || run_task(if enabled { TASK_ENABLE } else { TASK_DISABLE }, progress), - |outcome| *outcome == TaskOutcome::Done, - ); - outcome == TaskOutcome::Done - }); + let outcome = run_on_turn(enabled, purpose, progress); *ran = Some(outcome); outcome }) @@ -487,18 +513,65 @@ fn request_on_turn( } } +/// Run the driver task for `enabled`, once the gate let a request run. +#[cfg(windows)] +fn run_on_turn(enabled: bool, purpose: Purpose, progress: &mut dyn FnMut()) -> TaskOutcome { + let mut outcome = TaskOutcome::Failed(None); + // Read here, after the gate decided and before the task runs. + let display_off = TASKS.gate.lock().known_enabled == Some(false); + crate::startup::display::change(enabled, progress, |progress| { + let change = crate::display_guard::Change { + enabled, + owner: crate::startup::ProcessId::current(), + log: log_screens, + display_off, + // Start-up, quit and panics: nothing streams it after. + cleanup: !enabled && purpose == Purpose::Cleanup, + }; + outcome = crate::display_guard::around( + change, + || run_task(if enabled { TASK_ENABLE } else { TASK_DISABLE }, progress), + |outcome| *outcome == TaskOutcome::Done, + // Left on: the next turn off tries again. + || TaskOutcome::Failed(None), + ); + outcome == TaskOutcome::Done + }); + outcome +} + +/// `run_on_turn` for the Disable at startup, at quit and in a panic. +#[cfg(windows)] +fn run_cleanup(enabled: bool, progress: &mut dyn FnMut()) -> TaskOutcome { + run_on_turn(enabled, Purpose::Cleanup, progress) +} + /// Disable the virtual display device (best effort — failures are non-fatal). /// For startup, quit and panics: a hung or refused earlier task does not stop /// it, and it waits for an Enable still running, within the task's bounds. -/// True when the display is known to be off. +/// A host that never turned the display on runs no task when Windows reports +/// it disabled, so an idle quit doesn't wait on Task Scheduler. True when the +/// display is known or read to be off. pub fn disable() -> bool { - request(false, Purpose::Cleanup, || {}, &mut None) + #[cfg(windows)] + { + TASKS.disable_unless_off(|| {}, device_off, run_cleanup) + } + #[cfg(not(windows))] + { + request(false, Purpose::Cleanup, || {}, &mut None) + } } /// `disable` at startup, without holding up the first frame: the Disable /// runs on its own thread, and this returns once it holds the turn, so /// every later request (an Enable, the quit's Disable) still runs after it. /// A hung task no longer keeps a Mirror stream black for up to 30 s. +/// +/// A display Windows already reports disabled, as at almost every start, +/// runs no task, and its state stays unknown. That Disable was the task that +/// hung most often in the field, and every later request waited behind it +/// and failed. pub fn disable_in_background() { let (held, turn) = std::sync::mpsc::channel(); let spawned = std::thread::Builder::new() @@ -507,7 +580,10 @@ pub fn disable_in_background() { let on_turn = move || { let _ = held.send(()); }; - request_on_turn(false, Purpose::Cleanup, || {}, on_turn, &mut None) + #[cfg(windows)] + TASKS.disable_unless_off(on_turn, device_off, run_cleanup); + #[cfg(not(windows))] + request_on_turn(false, Purpose::Cleanup, || {}, on_turn, &mut None); }); match spawned { Ok(_) => { @@ -519,6 +595,64 @@ pub fn disable_in_background() { } } +/// How long the Disable at startup, at quit or in a panic waits for +/// Configuration Manager's read of the device before it runs the task. +#[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. +const DEVICE_READ_LIMIT: Duration = Duration::from_secs(2); + +/// Whether Configuration Manager reads the device disabled, or confirms +/// there is none, said in the log. +#[cfg(windows)] +fn device_off() -> bool { + off_within(crate::vdd_device::query_confirmed, DEVICE_READ_LIMIT) +} + +/// Whether `query`, run on its own thread, reads the device disabled, or +/// confirms there is no device (`Missing`), within `limit`. A read that isn't +/// certain (`None`), doesn't answer in time, or whose thread can't start +/// counts as not disabled, so the Disable task runs under its own limit; a +/// late answer is dropped. +#[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. +fn off_within( + query: impl FnOnce() -> Option + Send + 'static, + limit: Duration, +) -> bool { + let (answer, answered) = std::sync::mpsc::channel(); + let spawned = std::thread::Builder::new() + .name("vdd-device-read".into()) + .spawn(move || { + let _ = answer.send(query()); + }); + match spawned.ok().map(|_| answered.recv_timeout(limit)) { + Some(Ok(Some(crate::vdd_device::DeviceState::Missing))) => { + tracing::info!("No virtual display device — not running the Disable task"); + true + } + Some(Ok(Some(device))) => { + let off = reports_disabled(device); + if off { + tracing::info!("Virtual display already off — not running the Disable task"); + } + off + } + Some(Ok(None)) => false, + _ => { + tracing::warn!( + limit_ms = limit.as_millis() as u64, + "Could not read the virtual display's state in time — running the Disable task" + ); + false + } + } +} + +/// Whether the device reads disabled (problem 22): a Disable has nothing to +/// turn off. +#[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. +fn reports_disabled(device: crate::vdd_device::DeviceState) -> bool { + device == crate::vdd_device::DeviceState::Problem(22) +} + /// Disable the virtual display after `host` ended abnormally, if it owned the /// display and no other host of this install runs. For the failure reporter. #[cfg(windows)] @@ -531,14 +665,20 @@ pub(crate) fn disable_for_ended( || crate::startup::display::other_host(host, reporters), || { // The host's layout from before its turn on, which it kept on disk. - crate::display_guard::around( - false, - host, - crate::startup::reporter_log, + let change = crate::display_guard::Change { + enabled: false, + owner: host, + log: crate::startup::reporter_log, // Only for a turn on. - false, + display_off: false, + // The host ended. + cleanup: true, + }; + crate::display_guard::around( + change, || run_task(TASK_DISABLE, &mut || {}), |outcome| *outcome == TaskOutcome::Done, + || TaskOutcome::Failed(None), ) == TaskOutcome::Done }, ) @@ -572,6 +712,263 @@ mod tests { assert_eq!(gate.before(false, Purpose::Attempt, now), Gate::Run); } + #[test] + fn a_display_windows_reports_disabled_at_startup_runs_no_disable_task() { + use crate::vdd_device::DeviceState; + use std::cell::Cell; + let tasks = DriverTasks::new(); + let off = || reports_disabled(DeviceState::Problem(22)); + assert!(tasks.disable_unless_off( + || {}, + off, + |_, _| { panic!("ran the Disable task for a display that is off") } + )); + // Nothing is recorded: the state stays unknown. + { + let gate = tasks.gate.lock(); + assert_eq!(gate.known_enabled, None); + assert!(!gate.enable_ran); + assert_eq!(gate.health, VddTaskHealth::Ok); + } + // An unknown state never holds up an Enable: it runs at once, and + // the Disable after it runs too. + let ran = Cell::new(Vec::new()); + let run = |enabled: bool, _: &mut dyn FnMut()| { + let mut calls = ran.take(); + calls.push(enabled); + ran.set(calls); + TaskOutcome::Done + }; + assert!(tasks.set_enabled(true, Purpose::Attempt, || {}, run)); + assert!(tasks.set_enabled(false, Purpose::Cleanup, || {}, run)); + assert_eq!(ran.take(), vec![true, false]); + + // Any other state may be a display left on, or is unknown: the + // Disable runs as before. + for device in [ + DeviceState::Ready, + DeviceState::NoMonitor, + DeviceState::NotStarted, + DeviceState::Problem(43), + DeviceState::NoDriver, + DeviceState::Missing, + ] { + let tasks = DriverTasks::new(); + let ran = Cell::new(false); + tasks.disable_unless_off( + || {}, + || reports_disabled(device), + |_, _| { + ran.set(true); + TaskOutcome::Done + }, + ); + assert!(ran.get(), "no Disable for {device:?}"); + } + } + + /// Review r1: an Enable task the previous host gave up on can still be + /// running, or not yet started, while the device reads disabled. + /// Recording "off" then let it add the monitor, and every later Disable, + /// the quit's too, found the display "already off": a phantom monitor + /// stayed. The startup skip now records nothing. + #[test] + fn an_enable_left_running_by_the_previous_host_is_turned_off_at_quit() { + use crate::vdd_device::DeviceState; + use std::cell::Cell; + let tasks = DriverTasks::new(); + // The Enable is still on its way, and Windows has not added the + // display yet. + let enable_running = Cell::new(true); + let display_on = Cell::new(false); + let device = || { + if display_on.get() { + DeviceState::Ready + } else { + DeviceState::Problem(22) + } + }; + assert!(tasks.disable_unless_off( + || {}, + || reports_disabled(device()), + |_, _| { panic!("ran the startup Disable for a display that reads disabled") } + )); + // The surviving Enable ends whenever it ends; then the host quits, + // and the quit reads the device as `disable` does. + if enable_running.replace(false) { + display_on.set(true); + } + let disabled = tasks.disable_unless_off( + || {}, + || reports_disabled(device()), + |enabled, _| { + assert!(!enabled); + display_on.set(false); + TaskOutcome::Done + }, + ); + assert!(disabled); + assert!(!display_on.get(), "a phantom monitor stayed after quit"); + } + + /// Review r3: with the startup skip recording nothing, an idle quit ran + /// a Disable task, up to the task's 30 s limit when Task Scheduler + /// stalled, holding up exit and the update helper. + #[test] + fn an_idle_quit_runs_the_disable_task_only_when_the_display_is_on() { + use crate::vdd_device::DeviceState; + use std::cell::Cell; + let never = |_: bool, _: &mut dyn FnMut()| -> TaskOutcome { + panic!("ran the Disable task for a display that reads disabled") + }; + // Started and quit without a second screen: the device reads + // disabled both times, and no task runs. + let tasks = DriverTasks::new(); + assert!(tasks.disable_unless_off( + || {}, + || reports_disabled(DeviceState::Problem(22)), + never + )); + assert!(tasks.disable_unless_off( + || {}, + || reports_disabled(DeviceState::Problem(22)), + never + )); + assert_eq!(tasks.gate.lock().known_enabled, None); + + // A display that is on, or unreadable, is turned off as before. + for device in [ + DeviceState::Ready, + DeviceState::NoMonitor, + DeviceState::Missing, + ] { + let tasks = DriverTasks::new(); + let ran = Cell::new(false); + tasks.disable_unless_off( + || {}, + || reports_disabled(device), + |enabled, _| { + assert!(!enabled); + ran.set(true); + TaskOutcome::Done + }, + ); + assert!(ran.get(), "no Disable at quit for {device:?}"); + } + + // After an Enable this host ran, completed or timed out, the device + // may read disabled while the display is still coming: the Disable + // runs whatever it reads. + for enable in [TaskOutcome::Done, TaskOutcome::TimedOut] { + let tasks = DriverTasks::new(); + tasks.set_enabled(true, Purpose::Attempt, || {}, |_, _| enable); + let ran = Cell::new(false); + let read = Cell::new(false); + tasks.disable_unless_off( + || {}, + || { + read.set(true); + reports_disabled(DeviceState::Problem(22)) + }, + |_, _| { + ran.set(true); + TaskOutcome::Done + }, + ); + assert!( + ran.get(), + "no Disable at quit after an Enable that {enable:?}" + ); + assert!( + !read.get(), + "read the device after an Enable that {enable:?}" + ); + } + } + + /// Configuration Manager's read of the device had no limit: a read that + /// hung held the startup, quit and panic Disables, and every request + /// behind them, for as long as it hung. + #[test] + fn a_device_read_that_hangs_runs_the_disable_task_after_its_limit() { + use crate::vdd_device::DeviceState; + use std::cell::Cell; + let (release, held) = std::sync::mpsc::channel::<()>(); + let (done, finished) = std::sync::mpsc::channel(); + let started = Instant::now(); + std::thread::spawn(move || { + let tasks = DriverTasks::new(); + let ran = Cell::new(false); + let hung = move || { + let _ = held.recv(); + Some(DeviceState::Problem(22)) + }; + tasks.disable_unless_off( + || {}, + || off_within(hung, DEVICE_READ_LIMIT), + |enabled, _| { + assert!(!enabled); + ran.set(true); + TaskOutcome::Done + }, + ); + let _ = done.send(ran.get()); + }); + let ran = finished + .recv_timeout(DEVICE_READ_LIMIT + Duration::from_secs(5)) + .expect("the Disable waited on a hung device read past its limit"); + assert!(ran, "a hung device read skipped the Disable task"); + assert!(started.elapsed() >= DEVICE_READ_LIMIT); + // The read answers late; nothing waits for it. + let _ = release.send(()); + } + + #[test] + fn a_device_read_that_answers_disabled_at_once_keeps_the_skip() { + use crate::vdd_device::DeviceState; + let tasks = DriverTasks::new(); + let started = Instant::now(); + assert!(tasks.disable_unless_off( + || {}, + || off_within(|| Some(DeviceState::Problem(22)), DEVICE_READ_LIMIT), + |_, _| panic!("ran the Disable task for a display that reads disabled") + )); + assert!(started.elapsed() < DEVICE_READ_LIMIT); + for device in [Some(DeviceState::Ready), None] { + assert!(!off_within(move || device, DEVICE_READ_LIMIT), "{device:?}"); + } + } + + /// Windows 10, where Setup installs no driver: Windows confirms there is + /// no device (`Missing`), so the Disable at startup and quit runs no + /// task, and no missing task is logged at every launch. A read that isn't + /// certain (`None`) still runs it. + #[test] + fn a_confirmed_absent_device_runs_no_disable_task() { + use crate::vdd_device::DeviceState; + use std::cell::Cell; + let tasks = DriverTasks::new(); + assert!(tasks.disable_unless_off( + || {}, + || off_within(|| Some(DeviceState::Missing), DEVICE_READ_LIMIT), + |_, _| panic!("ran the Disable task with no device to turn off") + )); + let ran = Cell::new(false); + tasks.disable_unless_off( + || {}, + || off_within(|| None, DEVICE_READ_LIMIT), + |_, _| { + ran.set(true); + TaskOutcome::Missing + }, + ); + assert!( + ran.get(), + "an uncertain device read skipped the Disable task" + ); + assert_eq!(tasks.health(), VddTaskHealth::TasksMissing); + } + #[test] fn a_hung_task_is_not_waited_out_again_until_the_cooldown_ends() { let mut gate = TaskGate::new(); @@ -819,6 +1216,7 @@ mod tests { .with_max_level(tracing::Level::DEBUG) .with_writer(std::io::sink) .finish(); + crate::logging::prepare_log_capture(); let ran = tracing::subscriber::with_default(subscriber, || { tasks.set_enabled( true, diff --git a/host/src/vdd_device.rs b/host/src/vdd_device.rs index 75c207da..86694fee 100644 --- a/host/src/vdd_device.rs +++ b/host/src/vdd_device.rs @@ -51,6 +51,42 @@ pub enum VddProblem { NotExtended, } +/// A device's monitor children, as Configuration Manager listed them. +#[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Monitors { + /// The list was read and is empty. + None, + /// At least one child started without a problem. + Started, + /// Every child was read, and none has started yet. + Pending, + /// The list, or a child's status, couldn't be read. + Unread, +} + +impl Monitors { + /// From every child's status flags (`None` when a child's couldn't be + /// read), or `None` when the children couldn't be listed. A started + /// child counts even when another couldn't be read. + #[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. + fn from_children(children: Option<&[Option]>) -> Self { + let Some(children) = children else { + return Self::Unread; + }; + let started = |flags: &u32| flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0; + if children.iter().flatten().any(started) { + Self::Started + } else if children.is_empty() { + Self::None + } else if children.iter().any(Option::is_none) { + Self::Unread + } else { + Self::Pending + } + } +} + impl DeviceState { /// The Enable task's result, as vdd-toggle.ps1 exits. pub fn from_task_result(code: i64) -> Option { @@ -65,23 +101,37 @@ impl DeviceState { } /// The device as Configuration Manager reports it: its status flags and - /// problem number, whether a driver is bound, and whether its monitor - /// child is started (`None` without a child). + /// problem number, whether a driver is bound, and its monitor children. + /// A running device without a started monitor reads as `NoMonitor`, for + /// the snapshot and the notice once the display did not appear. #[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. - fn from_devnode(status: u32, problem: u32, has_driver: bool, monitor: Option) -> Self { + fn from_devnode(status: u32, problem: u32, has_driver: bool, monitors: Monitors) -> Self { if status & DN_HAS_PROBLEM != 0 { Self::Problem(problem) } else if !has_driver { Self::NoDriver } else if status & DN_STARTED == 0 { Self::NotStarted - } else if monitor == Some(true) { + } else if monitors == Monitors::Started { Self::Ready } else { Self::NoMonitor } } + /// As `from_devnode`, for a decision made while the display may still be + /// arriving: `NoMonitor` only when the device runs and Windows reported + /// no child at all. A child that has not started, or children that could + /// not all be read, are `None`: the monitor may be on its way. + #[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. + fn confirmed(status: u32, problem: u32, has_driver: bool, monitors: Monitors) -> Option { + let state = Self::from_devnode(status, problem, has_driver, monitors); + match (state, monitors) { + (Self::NoMonitor, Monitors::Pending | Monitors::Unread) => None, + (state, _) => Some(state), + } + } + /// With more than one device, the one closest to showing a display. #[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. fn best(states: impl IntoIterator) -> Self { @@ -140,83 +190,324 @@ pub fn from_task_stderr(stderr: &str) -> Option { /// Read the driver's device from Configuration Manager. #[cfg(windows)] pub fn query() -> DeviceState { - use windows::core::{w, PCWSTR}; - use windows::Win32::Devices::DeviceAndDriverInstallation::{ - CM_Get_Child, CM_Get_DevNode_Registry_PropertyW, CM_Get_DevNode_Status, - CM_Get_Device_ID_ListW, CM_Get_Device_ID_List_SizeW, CM_Locate_DevNodeW, - CM_DEVNODE_STATUS_FLAGS, CM_DRP_DRIVER, CM_DRP_HARDWAREID, CM_GETIDLIST_FILTER_ENUMERATOR, - CM_GETIDLIST_FILTER_PRESENT, CM_LOCATE_DEVNODE_NORMAL, CM_PROB, CR_BUFFER_SMALL, - CR_SUCCESS, - }; + let reads = read_devices().unwrap_or_default(); + DeviceState::best(reads.into_iter().filter_map(|read| { + let (flags, problem, has_driver, monitors) = read?; + Some(DeviceState::from_devnode( + flags, problem, has_driver, monitors, + )) + })) +} - let status = |devinst: u32| { - let (mut status, mut problem) = (CM_DEVNODE_STATUS_FLAGS(0), CM_PROB(0)); - (unsafe { CM_Get_DevNode_Status(&mut status, &mut problem, devinst, 0) } == CR_SUCCESS) - .then_some((status.0, problem.0)) +/// As `query`, for the attach wait: `NoMonitor` only when every running +/// device of the driver has confirmed no monitor child (see +/// `DeviceState::confirmed`). `None` when that isn't certain: the devices, +/// one of them or its children couldn't be read, or a monitor hasn't +/// started yet. `Missing` only when every present root device was read and +/// none is the driver's, as on Windows 10, where Setup installs none. +#[cfg(windows)] +pub fn query_confirmed() -> Option { + confirmed_from(read_devices()?) +} + +/// One device of the driver as read: its status flags and problem code, +/// whether a driver is bound, and its monitor children. +type DeviceRead = (u32, u32, bool, Monitors); + +/// `query_confirmed` from each driver device's read (`None` when its status +/// couldn't be read). A device with a monitor wins; otherwise anything +/// uncertain makes the whole read uncertain. +#[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. +fn confirmed_from(reads: Vec>) -> Option { + let states: Vec> = reads + .into_iter() + .map(|read| { + let (flags, problem, has_driver, monitors) = read?; + DeviceState::confirmed(flags, problem, has_driver, monitors) + }) + .collect(); + if states.contains(&Some(DeviceState::Ready)) { + return Some(DeviceState::Ready); + } + let states: Option> = states.into_iter().collect(); + Some(DeviceState::best(states?)) +} + +/// Each present device of the driver: its status flags and problem code, +/// whether a driver is bound, and its monitor children; `None` for one whose +/// status couldn't be read. `None` when the present devices couldn't be +/// listed. +#[cfg(windows)] +fn read_devices() -> Option>> { + use windows::Win32::Devices::DeviceAndDriverInstallation::CM_DRP_DRIVER; + let devnodes = root_devnodes()?; + Some(reads_from(&devnodes, |devinst, (flags, problem)| { + let has_driver = matches!(registry_property(devinst, CM_DRP_DRIVER), Ok(Some(_))); + let children = children(devinst); + ( + flags, + problem, + has_driver, + Monitors::from_children(children.as_deref()), + ) + })) +} + +/// `read_devices` from the present root devices, with `read` reading one +/// device of the driver whose status was read. A root device that couldn't +/// be read may be the driver's, with a monitor of its own: it stays, as a +/// read that isn't certain. Only devices known to be another driver's are +/// left out. +#[cfg(any(windows, test))] +fn reads_from( + devnodes: &[Devnode], + mut read: impl FnMut(u32, (u32, u32)) -> DeviceRead, +) -> Vec> { + devnodes + .iter() + .filter_map(|devnode| match *devnode { + Devnode::Other => None, + Devnode::Unread | Devnode::Driver { status: None, .. } => Some(None), + Devnode::Driver { + devinst, + status: Some(status), + } => Some(Some(read(devinst, status))), + }) + .collect() +} + +/// The status flags of every child of `devinst` (`None` for one whose +/// status couldn't be read), or `None` when the children couldn't be listed. +#[cfg(windows)] +fn children(devinst: u32) -> Option>> { + use windows::Win32::Devices::DeviceAndDriverInstallation::{ + CM_Get_Child, CM_Get_Sibling, CR_NO_SUCH_DEVNODE, CR_SUCCESS, }; - let property = |devinst: u32, which: u32| -> Option> { - let mut bytes = 0u32; - let result = - unsafe { CM_Get_DevNode_Registry_PropertyW(devinst, which, None, None, &mut bytes, 0) }; - if result != CR_BUFFER_SMALL || bytes == 0 { + /// More children than a virtual display ever has: a list this long is + /// not one to trust. + const MOST: usize = 64; + let mut statuses = Vec::new(); + let mut child = 0u32; + match unsafe { CM_Get_Child(&mut child, devinst, 0) } { + CR_NO_SUCH_DEVNODE => return Some(statuses), + CR_SUCCESS => {} + _ => return None, + } + loop { + statuses.push(devnode_status(child).map(|(flags, _)| flags)); + if statuses.len() > MOST { return None; } - let mut buffer = vec![0u16; bytes.div_ceil(2) as usize]; - (unsafe { - CM_Get_DevNode_Registry_PropertyW( + let mut next = 0u32; + match unsafe { CM_Get_Sibling(&mut next, child, 0) } { + CR_NO_SUCH_DEVNODE => return Some(statuses), + CR_SUCCESS => child = next, + _ => return None, + } + } +} + +/// One present root device, as Configuration Manager read it. +#[cfg(any(windows, test))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Devnode { + /// Its hardware IDs were read (or it has none) and don't name the driver. + Other, + /// The driver's device, with its status flags and problem code, `None` + /// when they couldn't be read. + Driver { + devinst: u32, + status: Option<(u32, u32)>, + }, + /// It couldn't be located, or its hardware IDs couldn't be read: it may + /// be the driver's. + Unread, +} + +/// The driver as Configuration Manager reads it (`vdd_settings::DriverRun`), +/// from the present root devices (`None` when they couldn't be listed): +/// off when every device of the driver reads as not started, or there is +/// none; running with the start `start` reads for its one running device; +/// unreadable when any read failed, a device started with a problem, or +/// more than one runs. +#[cfg(any(windows, test))] +pub(crate) fn run_from( + devnodes: Option<&[Devnode]>, + start: impl FnOnce(u32) -> Option, +) -> crate::vdd_settings::DriverRun { + use crate::vdd_settings::DriverRun; + let Some(devnodes) = devnodes else { + return DriverRun::Unreadable; + }; + let mut running = None; + for devnode in devnodes { + match *devnode { + Devnode::Other => {} + Devnode::Unread | Devnode::Driver { status: None, .. } => return DriverRun::Unreadable, + Devnode::Driver { devinst, - which, - None, - Some(buffer.as_mut_ptr().cast()), - &mut bytes, - 0, - ) - } == CR_SUCCESS) - .then_some(buffer) + status: Some((flags, _)), + } => { + if flags & DN_STARTED == 0 { + continue; + } + if flags & DN_HAS_PROBLEM != 0 || running.replace(devinst).is_some() { + return DriverRun::Unreadable; + } + } + } + } + match running { + None => DriverRun::Off, + Some(devinst) => start(devinst).map_or(DriverRun::Unreadable, DriverRun::Running), + } +} + +/// The driver as Configuration Manager reads it now (`run_from`). +#[cfg(windows)] +pub fn run_state() -> crate::vdd_settings::DriverRun { + run_from(root_devnodes().as_deref(), driver_start) +} + +/// The start of the driver running on `devinst`: this boot and the +/// device's last arrival. +#[cfg(windows)] +fn driver_start(devinst: u32) -> Option { + use windows::Win32::Devices::DeviceAndDriverInstallation::{ + CM_Get_DevNode_PropertyW, CR_SUCCESS, + }; + use windows::Win32::Devices::Properties::{ + DEVPKEY_Device_LastArrivalDate, DEVPROPTYPE, DEVPROP_TYPE_FILETIME, + }; + let (mut arrival, mut kind, mut size) = (0u64, DEVPROPTYPE(0), 8u32); + let result = unsafe { + CM_Get_DevNode_PropertyW( + devinst, + &DEVPKEY_Device_LastArrivalDate, + &mut kind, + Some((&mut arrival as *mut u64).cast()), + &mut size, + 0, + ) + }; + if result != CR_SUCCESS || kind != DEVPROP_TYPE_FILETIME || size != 8 || arrival == 0 { + return None; + } + Some(crate::vdd_settings::DriverInstance { + boot_id: boot_id()?, + arrival, + }) +} + +/// Windows' count of boots, which the kernel raises at each one. +#[cfg(windows)] +fn boot_id() -> Option { + use windows::core::w; + use windows::Win32::System::Registry::{RegGetValueW, HKEY_LOCAL_MACHINE, RRF_RT_REG_DWORD}; + let (mut value, mut bytes) = (0u32, 4u32); + unsafe { + RegGetValueW( + HKEY_LOCAL_MACHINE, + w!(r"SYSTEM\CurrentControlSet\Control\Session Manager\Memory Management\PrefetchParameters"), + w!("BootId"), + RRF_RT_REG_DWORD, + None, + Some((&mut value as *mut u32).cast()), + Some(&mut bytes), + ) + } + .is_ok() + .then_some(value) +} + +/// The present root devices (`Devnode`), or `None` when Windows can't list +/// them. A device that can't be located or whose hardware IDs can't be read +/// is kept as `Unread`, never dropped. +#[cfg(windows)] +fn root_devnodes() -> Option> { + use windows::core::{w, PCWSTR}; + use windows::Win32::Devices::DeviceAndDriverInstallation::{ + CM_Get_Device_ID_ListW, CM_Get_Device_ID_List_SizeW, CM_Locate_DevNodeW, CM_DRP_HARDWAREID, + CM_GETIDLIST_FILTER_ENUMERATOR, CM_GETIDLIST_FILTER_PRESENT, CM_LOCATE_DEVNODE_NORMAL, + CR_SUCCESS, }; let flags = CM_GETIDLIST_FILTER_ENUMERATOR | CM_GETIDLIST_FILTER_PRESENT; let mut len = 0u32; if unsafe { CM_Get_Device_ID_List_SizeW(&mut len, w!("ROOT"), flags) } != CR_SUCCESS { - return DeviceState::Missing; + return None; } let mut ids = vec![0u16; len as usize]; if unsafe { CM_Get_Device_ID_ListW(w!("ROOT"), &mut ids, flags) } != CR_SUCCESS { - return DeviceState::Missing; + return None; } - let states = ids - .split(|&unit| unit == 0) - .filter(|id| !id.is_empty()) - .filter_map(|id| { - let id: Vec = id.iter().copied().chain(Some(0)).collect(); - let mut devinst = 0u32; - if unsafe { - CM_Locate_DevNodeW(&mut devinst, PCWSTR(id.as_ptr()), CM_LOCATE_DEVNODE_NORMAL) - } != CR_SUCCESS - { - return None; - } - let hardware_ids = property(devinst, CM_DRP_HARDWAREID)?; - hardware_ids - .split(|&unit| unit == 0) - .any(|hwid| String::from_utf16_lossy(hwid).eq_ignore_ascii_case(HARDWARE_ID)) - .then_some(devinst) - }) - .filter_map(|devinst| { - let (flags, problem) = status(devinst)?; - let mut child = 0u32; - let monitor = - (unsafe { CM_Get_Child(&mut child, devinst, 0) } == CR_SUCCESS).then(|| { - status(child).is_some_and(|(flags, _)| { - flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0 - }) - }); - let has_driver = property(devinst, CM_DRP_DRIVER).is_some(); - Some(DeviceState::from_devnode( - flags, problem, has_driver, monitor, - )) - }); - DeviceState::best(states) + Some( + ids.split(|&unit| unit == 0) + .filter(|id| !id.is_empty()) + .map(|id| { + let id: Vec = id.iter().copied().chain(Some(0)).collect(); + let mut devinst = 0u32; + if unsafe { + CM_Locate_DevNodeW(&mut devinst, PCWSTR(id.as_ptr()), CM_LOCATE_DEVNODE_NORMAL) + } != CR_SUCCESS + { + return Devnode::Unread; + } + match registry_property(devinst, CM_DRP_HARDWAREID) { + Err(()) => Devnode::Unread, + Ok(Some(hardware_ids)) + if hardware_ids.split(|&unit| unit == 0).any(|hwid| { + String::from_utf16_lossy(hwid).eq_ignore_ascii_case(HARDWARE_ID) + }) => + { + Devnode::Driver { + devinst, + status: devnode_status(devinst), + } + } + Ok(_) => Devnode::Other, + } + }) + .collect(), + ) +} + +#[cfg(windows)] +fn devnode_status(devinst: u32) -> Option<(u32, u32)> { + use windows::Win32::Devices::DeviceAndDriverInstallation::{ + CM_Get_DevNode_Status, CM_DEVNODE_STATUS_FLAGS, CM_PROB, CR_SUCCESS, + }; + let (mut status, mut problem) = (CM_DEVNODE_STATUS_FLAGS(0), CM_PROB(0)); + (unsafe { CM_Get_DevNode_Status(&mut status, &mut problem, devinst, 0) } == CR_SUCCESS) + .then_some((status.0, problem.0)) +} + +/// A device's registry property: `Ok(None)` when it has none +/// (`CR_NO_SUCH_VALUE`), `Err` when the read failed. +#[cfg(windows)] +fn registry_property(devinst: u32, which: u32) -> Result>, ()> { + use windows::Win32::Devices::DeviceAndDriverInstallation::{ + CM_Get_DevNode_Registry_PropertyW, CR_BUFFER_SMALL, CR_NO_SUCH_VALUE, CR_SUCCESS, + }; + let mut bytes = 0u32; + match unsafe { CM_Get_DevNode_Registry_PropertyW(devinst, which, None, None, &mut bytes, 0) } { + CR_NO_SUCH_VALUE => return Ok(None), + CR_BUFFER_SMALL if bytes > 0 => {} + _ => return Err(()), + } + let mut buffer = vec![0u16; bytes.div_ceil(2) as usize]; + (unsafe { + CM_Get_DevNode_Registry_PropertyW( + devinst, + which, + None, + Some(buffer.as_mut_ptr().cast()), + &mut bytes, + 0, + ) + } == CR_SUCCESS) + .then_some(Some(buffer)) + .ok_or(()) } /// Set the mode of the virtual display `device` (a GDI name like @@ -298,6 +589,122 @@ pub fn query() -> DeviceState { mod tests { use super::*; + /// Setup installs no driver on Windows 10: a list where every root + /// device was read, and none is the driver's, reads as a confirmed + /// `Missing`. A device that couldn't be read keeps it uncertain. + #[test] + fn only_a_fully_read_list_without_the_driver_confirms_it_missing() { + let unreachable = |_, _| unreachable!("no device of the driver"); + assert_eq!( + confirmed_from(reads_from(&[Devnode::Other, Devnode::Other], unreachable)), + Some(DeviceState::Missing) + ); + for uncertain in [ + Devnode::Unread, + Devnode::Driver { + devinst: 7, + status: None, + }, + ] { + assert_eq!( + confirmed_from(reads_from(&[Devnode::Other, uncertain], unreachable)), + None, + "{uncertain:?}" + ); + } + let off = |_, (flags, problem)| (flags, problem, true, Monitors::None); + let driver = Devnode::Driver { + devinst: 7, + status: Some((DN_HAS_PROBLEM, 22)), + }; + assert_eq!( + confirmed_from(reads_from(&[Devnode::Other, driver], off)), + Some(DeviceState::Problem(22)) + ); + } + + /// Review r6: a root device whose location or hardware IDs couldn't be + /// read may be the driver's, so the driver's state is unreadable, not + /// off, and an Enable after it keeps no record. One whose hardware IDs + /// were read and don't name the driver is not the driver's. + #[test] + fn a_device_that_could_not_be_read_makes_the_driver_unreadable() { + use crate::vdd_settings::{DriverCursor, DriverInstance, DriverRun}; + let start = DriverInstance { + boot_id: 205, + arrival: 30, + }; + let starts = |devinst| (devinst == 7).then_some(start); + let off = Devnode::Driver { + devinst: 7, + status: Some((0, 22)), + }; + let on = Devnode::Driver { + devinst: 7, + status: Some((DN_STARTED, 0)), + }; + let unread = [Devnode::Other, off, Devnode::Unread]; + let before = run_from(Some(&unread), starts); + assert_eq!(before, DriverRun::Unreadable); + assert_eq!( + crate::vdd_settings::record_after_enable( + before, + run_from(Some(&[Devnode::Other, on]), starts), + Some(""), + DriverCursor::Software, + ), + None, + "no record after an unreadable before" + ); + + for (devnodes, run) in [ + (None, DriverRun::Unreadable), + (Some(&[][..]), DriverRun::Off), + (Some(&[Devnode::Other][..]), DriverRun::Off), + (Some(&[Devnode::Other, off][..]), DriverRun::Off), + (Some(&[Devnode::Unread][..]), DriverRun::Unreadable), + (Some(&[Devnode::Unread, on][..]), DriverRun::Unreadable), + (Some(&[Devnode::Other, on][..]), DriverRun::Running(start)), + ( + Some( + &[Devnode::Driver { + devinst: 7, + status: None, + }][..], + ), + DriverRun::Unreadable, + ), + ( + Some( + &[Devnode::Driver { + devinst: 7, + status: Some((DN_STARTED | DN_HAS_PROBLEM, 43)), + }][..], + ), + DriverRun::Unreadable, + ), + ( + Some( + &[ + on, + Devnode::Driver { + devinst: 8, + status: Some((DN_STARTED, 0)), + }, + ][..], + ), + DriverRun::Unreadable, + ), + ] { + assert_eq!(run_from(devnodes, starts), run, "{devnodes:?}"); + } + assert_eq!( + run_from(Some(&[on]), |_| None), + DriverRun::Unreadable, + "its start couldn't be read" + ); + } + #[test] fn the_enable_task_result_names_the_device_state() { assert_eq!(DeviceState::from_task_result(0), None); @@ -341,39 +748,164 @@ mod tests { let started = DN_STARTED; let problem = DN_HAS_PROBLEM; assert_eq!( - DeviceState::from_devnode(problem, 22, true, None), + DeviceState::from_devnode(problem, 22, true, Monitors::None), DeviceState::Problem(22) ); assert_eq!( - DeviceState::from_devnode(problem, 43, true, None), + DeviceState::from_devnode(problem, 43, true, Monitors::None), DeviceState::Problem(43) ); assert_eq!( - DeviceState::from_devnode(problem, 28, false, None), + DeviceState::from_devnode(problem, 28, false, Monitors::None), DeviceState::Problem(28) ); assert_eq!( - DeviceState::from_devnode(started, 0, false, None), + DeviceState::from_devnode(started, 0, false, Monitors::None), DeviceState::NoDriver ); assert_eq!( - DeviceState::from_devnode(0, 0, true, None), + DeviceState::from_devnode(0, 0, true, Monitors::None), DeviceState::NotStarted ); assert_eq!( - DeviceState::from_devnode(started, 0, true, None), + DeviceState::from_devnode(started, 0, true, Monitors::None), DeviceState::NoMonitor ); assert_eq!( - DeviceState::from_devnode(started, 0, true, Some(false)), + DeviceState::from_devnode(started, 0, true, Monitors::Pending), DeviceState::NoMonitor ); assert_eq!( - DeviceState::from_devnode(started, 0, true, Some(true)), + DeviceState::from_devnode(started, 0, true, Monitors::Started), DeviceState::Ready ); } + /// Review r1 of #233: the attach wait ends early on `NoMonitor`, so only + /// a confirmed absence may read as one. Every child counts, not only the + /// first: Windows lists further children as its siblings. + #[test] + fn only_a_confirmed_absence_of_monitors_reads_as_no_monitor_while_waiting() { + let (started, problem) = (DN_STARTED, DN_HAS_PROBLEM); + let starting = 0; + // The children as listed. + assert_eq!(Monitors::from_children(None), Monitors::Unread); + assert_eq!(Monitors::from_children(Some(&[])), Monitors::None); + assert_eq!( + Monitors::from_children(Some(&[Some(starting)])), + Monitors::Pending + ); + assert_eq!(Monitors::from_children(Some(&[None])), Monitors::Unread); + assert_eq!( + Monitors::from_children(Some(&[Some(started | problem)])), + Monitors::Pending + ); + assert_eq!( + Monitors::from_children(Some(&[Some(starting), Some(started)])), + Monitors::Started, + "a monitor among the siblings" + ); + assert_eq!( + Monitors::from_children(Some(&[None, Some(started)])), + Monitors::Started + ); + assert_eq!( + Monitors::from_children(Some(&[Some(starting), None])), + Monitors::Unread + ); + + // A running device: only no children at all is a confirmed absence. + let running = |monitors| DeviceState::confirmed(started, 0, true, monitors); + assert_eq!(running(Monitors::None), Some(DeviceState::NoMonitor)); + assert_eq!(running(Monitors::Started), Some(DeviceState::Ready)); + assert_eq!(running(Monitors::Pending), None); + assert_eq!(running(Monitors::Unread), None); + // The snapshot after a timeout still names the device as before. + for monitors in [Monitors::None, Monitors::Pending, Monitors::Unread] { + assert_eq!( + DeviceState::from_devnode(started, 0, true, monitors), + DeviceState::NoMonitor + ); + } + // States that don't depend on the children are certain. + assert_eq!( + DeviceState::confirmed(0, 0, true, Monitors::Unread), + Some(DeviceState::NotStarted) + ); + assert_eq!( + DeviceState::confirmed(problem, 22, true, Monitors::Unread), + Some(DeviceState::Problem(22)) + ); + + // Across the driver's devices. + let confirmed_none = Some((started, 0, true, Monitors::None)); + assert_eq!( + confirmed_from(vec![confirmed_none]), + Some(DeviceState::NoMonitor) + ); + assert_eq!( + confirmed_from(vec![None]), + None, + "a status that couldn't be read" + ); + assert_eq!( + confirmed_from(vec![ + confirmed_none, + Some((started, 0, true, Monitors::Pending)) + ]), + None + ); + assert_eq!( + confirmed_from(vec![None, Some((started, 0, true, Monitors::Started))]), + Some(DeviceState::Ready) + ); + assert_eq!(confirmed_from(vec![]), Some(DeviceState::Missing)); + } + + /// Review r2 of #233: a root device that couldn't be read may be the + /// driver's, with a monitor still arriving. Next to a running device + /// with no children, the read isn't a confirmed absence. + #[test] + fn an_unread_root_device_keeps_the_read_uncertain() { + let running_childless = |_, (flags, problem)| (flags, problem, true, Monitors::None); + let alone = reads_from( + &[ + Devnode::Other, + Devnode::Driver { + devinst: 7, + status: Some((DN_STARTED, 0)), + }, + ], + running_childless, + ); + assert_eq!(confirmed_from(alone), Some(DeviceState::NoMonitor)); + let with_unread = reads_from( + &[ + Devnode::Unread, + Devnode::Driver { + devinst: 7, + status: Some((DN_STARTED, 0)), + }, + ], + running_childless, + ); + assert_eq!(confirmed_from(with_unread), None); + let unread_status = reads_from( + &[ + Devnode::Driver { + devinst: 8, + status: None, + }, + Devnode::Driver { + devinst: 7, + status: Some((DN_STARTED, 0)), + }, + ], + running_childless, + ); + assert_eq!(confirmed_from(unread_status), None); + } + #[test] fn the_device_closest_to_a_display_wins() { assert_eq!(DeviceState::best([]), DeviceState::Missing); @@ -420,7 +952,7 @@ mod tests { let state = query(); println!("vdd_device={state} problem={:?}", state.problem()); // DXGI lists a virtual display under the adapter that renders it. - for output in crate::capture::enumerate_outputs() { + for output in crate::capture::listing::outputs(&crate::control::SharedControl::new(0, 1)) { println!( "output {} {}x{} at {},{} on {}", output.device_name, diff --git a/host/src/vdd_settings.rs b/host/src/vdd_settings.rs index 9f132195..1813ad8a 100644 --- a/host/src/vdd_settings.rs +++ b/host/src/vdd_settings.rs @@ -17,6 +17,8 @@ use std::io; use std::ops::Range; use std::path::Path; +use serde::{Deserialize, Serialize}; + /// A display mode for the virtual display's mode list. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct VddMode { @@ -126,10 +128,356 @@ pub fn render(modes: &[VddMode], gpu: Option<&str>) -> String { )); } xml.push_str(" \n"); + xml.push_str(CURSOR); xml.push_str("\n"); xml } +/// The driver's hardware cursor, as its own sample file sets it. Without +/// `` the driver leaves it off, and Windows draws the pointer into +/// the virtual display's image, so Desktop Duplication has no pointer to +/// report apart from it and an iPad can't draw it. With it on, the second +/// screen reports the pointer like any other screen. The driver reads a +/// missing size as -1, so the whole block is written. +const CURSOR: &str = " \n true\n \ + 128\n 128\n \ + true\n \ + 2\n \n"; + +/// The hardware cursor a driver gives Windows when it starts, as its +/// settings set it. Windows draws a pointer image larger than this into +/// the picture itself (measured on the reference PC with a 160 px pointer +/// and a 128 px cap). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct CursorCaps { + pub max_x: u32, + pub max_y: u32, + /// `AlphaCursorSupport`, and the XOR level as the driver reads it. + pub alpha: bool, + pub xor: i32, +} + +impl CursorCaps { + /// Whether a pointer image of this size goes to the hardware cursor. + pub fn takes(&self, (width, height): (u32, u32)) -> bool { + width <= self.max_x && height <= self.max_y + } +} + +/// What the running driver does with the pointer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum DriverCursor { + /// It gives Windows a hardware cursor with these caps, which takes + /// every pointer type up to their size. + Hardware(CursorCaps), + /// It has none: Windows draws the pointer into the picture. + Software, + /// This host can't tell. Then the pointer is handled as in 0.3.4 + /// (`capture::PointerPlace::Unknown`). + Unknown, +} + +/// The cursor a settings document gives the driver, read as the driver +/// reads it (`EnabledQuery` and `GetIntegerSetting` in its `Driver.cpp`): +/// for each setting, the first element of that name whose first child is +/// text decides. Only the text `true` turns a switch on, and numbers are +/// read as `std::stoi` reads them, a missing or unreadable one as -1. A +/// document that is not well-formed is `Unknown`. +/// +/// Only a cursor that takes every pointer type counts as `Hardware`: with +/// alpha off, or XOR support off (the driver reads the level but, in +/// 25.5.2, always gives full support), Windows' choice for the types it +/// doesn't take isn't documented, and so is a size the driver can't use. +pub fn cursor_set_by(xml: &str) -> DriverCursor { + use quick_xml::events::Event; + const NAMES: [&[u8]; 5] = [ + b"HardwareCursor", + b"AlphaCursorSupport", + b"CursorMaxX", + b"CursorMaxY", + b"XorCursorSupportLevel", + ]; + let mut reader = quick_xml::Reader::from_str(xml); + reader.config_mut().trim_text(false); + let mut values: [Option; 5] = Default::default(); + // The setting whose first child is next, and whether a setting's + // element is open. + let (mut current, mut inside) = (None, false); + loop { + match reader.read_event() { + Ok(Event::Start(tag)) => { + current = NAMES + .iter() + .position(|name| tag.local_name().as_ref() == *name); + inside |= current.is_some(); + } + Ok(Event::End(_)) => (current, inside) = (None, false), + // XmlLite expands `t` and `&` into the text; quick-xml + // reports them apart. Where that could change a setting, the + // answer is not known. + Ok(Event::GeneralRef(_)) if inside => return DriverCursor::Unknown, + // XmlLite reads white space alone as another kind of node. + Ok(Event::Text(text)) if text.iter().all(u8::is_ascii_whitespace) => current = None, + Ok(Event::Text(text)) => { + if let Some(slot) = current.take().filter(|&slot| values[slot].is_none()) { + values[slot] = Some(String::from_utf8_lossy(text.as_ref()).into_owned()); + } + } + Ok(Event::Eof) => break, + Err(_) => return DriverCursor::Unknown, + Ok(_) => current = None, + } + } + let switch = |slot: usize| values[slot].as_deref() == Some("true"); + let number = |slot: usize| values[slot].as_deref().map_or(-1, stoi); + if !switch(0) { + return DriverCursor::Software; + } + let xor = match number(4) { + level @ 0..=3 => level, + // The driver's default for a level it doesn't know: full support. + _ => 2, + }; + match (u32::try_from(number(2)), u32::try_from(number(3))) { + (Ok(max_x @ 1..), Ok(max_y @ 1..)) if switch(1) && matches!(xor, 2 | 3) => { + DriverCursor::Hardware(CursorCaps { + max_x, + max_y, + alpha: true, + xor, + }) + } + _ => DriverCursor::Unknown, + } +} + +/// `std::stoi` as the driver uses it: leading white space, a sign and +/// digits, ignoring what follows them; -1 (the driver's "unset") when +/// there are no digits or the number doesn't fit. +fn stoi(text: &str) -> i32 { + let text = text.trim_start(); + let (sign, digits) = match text.as_bytes().first() { + Some(b'-') => (-1, &text[1..]), + Some(b'+') => (1, &text[1..]), + _ => (1, text), + }; + let end = digits + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(digits.len()); + digits[..end] + .parse::() + .ok() + .and_then(|value| i32::try_from(sign * value).ok()) + .unwrap_or(-1) +} + +/// One start of the driver: the boot, and the device's last arrival +/// (`DEVPKEY_Device_LastArrivalDate`, a FILETIME). On the reference PC each +/// Enable changed the arrival and a driver turned off kept it; an arrival +/// only counts with the boot it came in. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct DriverInstance { + pub boot_id: u32, + pub arrival: u64, +} + +/// The driver as Configuration Manager reads it: one start running, none +/// running, or a read that failed or was ambiguous. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DriverRun { + Running(DriverInstance), + Off, + Unreadable, +} + +/// What a driver start read, kept on disk so a relaunched host knows it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CursorRecord { + pub instance: DriverInstance, + /// SHA-256 of the settings file it started with, for the logs. + pub settings_sha256: String, + pub cursor: DriverCursor, +} + +/// What the driver starts with when this host's Enable starts it: the +/// file's cursor (`cursor_set_by`), if the driver reads `` +/// (`reads_cursor`) and the registry leaves the file in charge +/// (`overridden` when it doesn't, see `file_in_charge`). The version that +/// says whether it reads `` is the installer's, which may not be +/// the running driver's, so an older one is not taken to mean no cursor. +pub fn starting_cursor(xml: Option<&str>, reads_cursor: bool, overridden: bool) -> DriverCursor { + match xml { + Some(xml) if reads_cursor && !overridden => cursor_set_by(xml), + _ => DriverCursor::Unknown, + } +} + +/// The record to keep after an Enable that succeeded: a driver running +/// after it (`after`) that was seen off before it (`before`) started in +/// this Enable, with the file this host just wrote (`xml`, read as +/// `starting_cursor`). An Enable that found it running (AlreadyDone, or a +/// relaunch with the driver left on), or couldn't tell, keeps no record: +/// the one on disk decides (`running_cursor`). +pub fn record_after_enable( + before: DriverRun, + after: DriverRun, + xml: Option<&str>, + cursor: DriverCursor, +) -> Option { + let (DriverRun::Off, DriverRun::Running(instance)) = (before, after) else { + return None; + }; + Some(CursorRecord { + instance, + settings_sha256: xml.map_or_else(String::new, |xml| { + let digest = ring::digest::digest(&ring::digest::SHA256, xml.as_bytes()); + digest.as_ref().iter().map(|b| format!("{b:02x}")).collect() + }), + cursor, + }) +} + +/// What the running driver does with the pointer: what the record says, +/// when it is of this very start of the driver (same boot, same arrival); +/// `Unknown` for any other, and when the driver is not running. +pub fn running_cursor(record: Option<&CursorRecord>, running: DriverRun) -> DriverCursor { + match (record, running) { + (Some(record), DriverRun::Running(running)) if record.instance == running => record.cursor, + _ => DriverCursor::Unknown, + } +} + +/// Where the record is kept, beside the app's other files. +pub const CURSOR_RECORD: &str = "vdd-cursor.json"; + +pub fn load_record(path: &Path) -> Option { + serde_json::from_str(&std::fs::read_to_string(path).ok()?).ok() +} + +pub fn save_record(path: &Path, record: &CursorRecord) -> io::Result<()> { + replace( + path, + &serde_json::to_string(record).map_err(io::Error::other)?, + ) +} + +/// Whether the installed driver reads `` at all: releases from +/// 24.10.27 on do. `version` is its installer's display version, such as +/// `25.05.03`; not one that doesn't read as a version. +pub fn driver_reads_cursor(version: &str) -> bool { + let parts: Option> = version + .trim() + .split('.') + .map(|part| part.parse().ok()) + .collect(); + match parts.as_deref() { + Some([year, month, day, ..]) => (*year, *month, *day) >= (24, 10, 27), + _ => false, + } +} + +/// A value under the driver's registry key, as read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RegValue { + /// The key or the value doesn't exist. + Absent, + /// It exists; the text, for a string read as one. + Present(String), + /// Any other error. + Failed, +} + +/// The settings folder the driver uses without `VDDPATH`, and the one this +/// host writes to. +const DRIVER_FOLDER: &str = r"C:\VirtualDisplayDriver"; + +/// The cursor values the driver reads from its registry key before the +/// file (`EnabledQuery`, `GetIntegerSetting`). +pub const REGISTRY_CURSOR_VALUES: [&str; 5] = [ + "HARDWARECURSOR", + "ALPHACURSORSUPPORT", + "CURSORMAXX", + "CURSORMAXY", + "XORCURSORSUPPORTLEVEL", +]; + +/// Whether the driver reads its cursor from the file this host writes: its +/// `VDDPATH` (read at `DriverEntry`) is unset or names +/// `C:\VirtualDisplayDriver`, ignoring case and a trailing `\`, as +/// `vdd_snapshot::collect` compares it, and no cursor value is set under +/// its key. A value that couldn't be read may be either. +pub fn file_in_charge(vddpath: &RegValue, cursor_values: &[RegValue]) -> bool { + let folder = match vddpath { + RegValue::Absent => true, + RegValue::Present(path) => path + .trim_end_matches('\\') + .eq_ignore_ascii_case(DRIVER_FOLDER), + RegValue::Failed => false, + }; + folder && cursor_values.iter().all(|value| *value == RegValue::Absent) +} + +/// Whether the driver's registry key may take the cursor away from the +/// file this host writes (`file_in_charge`, read now). +#[cfg(windows)] +pub fn registry_overrides_cursor() -> bool { + let vddpath = registry_value("VDDPATH", true); + let cursor: Vec = REGISTRY_CURSOR_VALUES + .iter() + .map(|name| registry_value(name, false)) + .collect(); + !file_in_charge(&vddpath, &cursor) +} + +/// One value under HKLM\SOFTWARE\MikeTheTech\VirtualDisplayDriver, with its +/// text when `text` (a REG_SZ; any other type fails). +#[cfg(windows)] +fn registry_value(name: &str, text: bool) -> RegValue { + use windows::core::{w, PCWSTR}; + use windows::Win32::Foundation::{ERROR_FILE_NOT_FOUND, ERROR_SUCCESS}; + use windows::Win32::System::Registry::{ + RegGetValueW, HKEY_LOCAL_MACHINE, RRF_RT_ANY, RRF_RT_REG_SZ, + }; + let key = w!(r"SOFTWARE\MikeTheTech\VirtualDisplayDriver"); + let name: Vec = name.encode_utf16().chain(Some(0)).collect(); + let mut buffer = [0u16; 1024]; + let mut bytes = std::mem::size_of_val(&buffer) as u32; + let result = unsafe { + if text { + RegGetValueW( + HKEY_LOCAL_MACHINE, + key, + PCWSTR(name.as_ptr()), + RRF_RT_REG_SZ, + None, + Some(buffer.as_mut_ptr().cast()), + Some(&mut bytes), + ) + } else { + RegGetValueW( + HKEY_LOCAL_MACHINE, + key, + PCWSTR(name.as_ptr()), + RRF_RT_ANY, + None, + None, + None, + ) + } + }; + match result { + ERROR_SUCCESS => { + let end = buffer + .iter() + .position(|&unit| unit == 0) + .unwrap_or(buffer.len()); + RegValue::Present(String::from_utf16_lossy(&buffer[..end])) + } + ERROR_FILE_NOT_FOUND => RegValue::Absent, + _ => RegValue::Failed, + } +} + fn escape(text: &str) -> String { text.replace('&', "&") .replace('<', "<") @@ -384,18 +732,19 @@ pub fn with_gpu(xml: &str, gpu: &str) -> Option { } /// Point the settings file the host does not otherwise write at `gpu`. -/// True when it changed. A missing file stays missing: without one the -/// driver offers its built-in modes, and a document holding only a GPU -/// would leave it none. A file that does not parse as expected is left -/// alone, and the error says why. +/// True when it changed: a file read from a code page is written back as +/// UTF-8 even when it already names `gpu`. A missing file stays missing: +/// without one the driver offers its built-in modes, and a document holding +/// only a GPU would leave it none. A file that does not parse as expected +/// is left alone, and the error says why. pub fn pin_gpu_in(path: &Path, gpu: &str) -> io::Result { - let current = std::fs::read_to_string(path)?; + let (current, legacy) = read_settings(path)?; let updated = gpu_edit(¤t, gpu) .map_err(|reason| io::Error::new(io::ErrorKind::InvalidData, reason))?; - if updated == current { + if updated == current && !legacy { return Ok(false); } - replace(path, &updated)?; + write_settings(path, legacy, || replace(path, &updated))?; Ok(true) } @@ -543,7 +892,9 @@ pub fn remember_pins(record: &Path, names: &[&str]) -> io::Result<()> { /// Make the settings file at `path` name `wanted`, changing only the card, /// under the rules of `pin_change`. `record` holds the names this host /// recorded writing. A missing file stays missing, and one that does not -/// parse as expected is left alone; the error says why. +/// parse as expected is left alone; the error says why. A file read from a +/// code page is written back as UTF-8 even when its card stays, so the +/// driver reads all of it. pub fn apply_pin( path: &Path, record: &Path, @@ -551,19 +902,27 @@ pub fn apply_pin( wanted: Option<&str>, ) -> io::Result { let invalid = |reason| io::Error::new(io::ErrorKind::InvalidData, reason); - let current = std::fs::read_to_string(path)?; + let (current, legacy) = read_settings(path)?; let named = named_gpu(¤t).map_err(invalid)?; let ours = recorded_pins(record); let ours: Vec<&str> = ours.iter().map(String::as_str).collect(); + let unchanged = |outcome| { + if legacy { + write_settings(path, legacy, || replace(path, ¤t))?; + } + Ok(outcome) + }; let (name, outcome) = match pin_change(named.as_deref(), wanted, owned, &ours) { - PinChange::Leave => return Ok(PinOutcome::Unchanged), - PinChange::Refuse(reason) => return Ok(PinOutcome::Kept(reason)), + PinChange::Leave => return unchanged(PinOutcome::Unchanged), + PinChange::Refuse(reason) => return unchanged(PinOutcome::Kept(reason)), PinChange::Write(card) => (card, PinOutcome::Named), PinChange::Reset => ("default".to_string(), PinOutcome::Reset), }; let updated = gpu_edit(¤t, &name).map_err(invalid)?; let new_record = (outcome == PinOutcome::Named).then_some(name.as_str()); - record_around(record, new_record, &ours, || replace(path, &updated))?; + record_around(record, new_record, &ours, || { + write_settings(path, legacy, || replace(path, &updated)) + })?; Ok(outcome) } @@ -622,13 +981,13 @@ pub fn write_with_pin( modes: &[VddMode], wanted: Option<&str>, ) -> io::Result { - let named = match std::fs::read_to_string(path) { - Ok(current) => match named_gpu(¤t) { - Ok(named) => named, - Err(_) if owned && wanted.is_some() => None, + let (named, legacy) = match read_settings(path) { + Ok((current, legacy)) => match named_gpu(¤t) { + Ok(named) => (named, legacy), + Err(_) if owned && wanted.is_some() => (None, legacy), Err(reason) => return Err(io::Error::new(io::ErrorKind::InvalidData, reason)), }, - Err(error) if error.kind() == io::ErrorKind::NotFound => None, + Err(error) if error.kind() == io::ErrorKind::NotFound => (None, false), Err(error) => return Err(error), }; let ours = recorded_pins(record); @@ -639,7 +998,7 @@ pub fn write_with_pin( PinChange::Write(card) => (Some(card), PinOutcome::Named), PinChange::Reset => (None, PinOutcome::Reset), }; - let write = || write_to(path, modes, name.as_deref()); + let write = || write_settings(path, legacy, || write_to(path, modes, name.as_deref())); match outcome { PinOutcome::Named => record_around(record, name.as_deref(), &ours, write)?, PinOutcome::Reset => record_around(record, None, &ours, write)?, @@ -662,6 +1021,253 @@ pub fn pin_result( } } +/// The settings file as text, and whether it was in a legacy code page. +/// Another tool may have saved it as UTF-16 (Windows PowerShell's +/// `Out-File` does), with a byte order mark or, as XML allows, without one; +/// or in a Windows code page, as Notepad's ANSI does, keeping the driver's +/// `encoding='utf-8'` declaration or naming the code page. It is read and, +/// when this host writes it, written back as UTF-8, the encoding the +/// driver's own file has and every file this host writes. The driver's XML +/// reader takes UTF-8 and UTF-16 but not double-byte code pages: declared, +/// it reads nothing; otherwise it stops at the first byte it cannot read, +/// keeping only the settings before it. A UTF-16 or code page +/// document's declaration names UTF-8 with it, so the declaration always +/// names what is on disk. UTF-8 keeps any byte order mark. Any other +/// encoding, UTF-32 included, is left alone, and the error says so. +fn read_settings(path: &Path) -> io::Result<(String, bool)> { + decode_settings(&std::fs::read(path)?, ANSI).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "the settings file is in no encoding this host reads", + ) + }) +} + +/// `CP_ACP`: the system's ANSI code page, which Notepad and other older +/// tools save text in. +const ANSI: u32 = 0; + +/// `bytes` as text, and whether they were in a code page; text that is not +/// UTF and declares no other encoding is in code page `ansi`. +fn decode_settings(bytes: &[u8], ansi: u32) -> Option<(String, bool)> { + let (text, legacy) = decode_text(bytes, ansi)?; + // XML never holds NUL: text with one was read in the wrong encoding, + // such as UTF-32 without a byte order mark, which reads as UTF-8. + (!text.contains('\0')).then_some((text, legacy)) +} + +fn decode_text(bytes: &[u8], ansi: u32) -> Option<(String, bool)> { + // UTF-32's byte order marks; the little-endian one starts like UTF-16's. + if bytes.starts_with(&[0xFF, 0xFE, 0, 0]) || bytes.starts_with(&[0, 0, 0xFE, 0xFF]) { + return None; + } + let (units, big_endian): (&[u8], bool) = match bytes { + [0xFF, 0xFE, rest @ ..] => (rest, false), + [0xFE, 0xFF, rest @ ..] => (rest, true), + // " (bytes, false), + [0, b'<', 0, b'?', ..] => (bytes, true), + _ => return decode_bytes(bytes, ansi), + }; + let (pairs, odd) = units.as_chunks::<2>(); + if !odd.is_empty() { + return None; + } + let units = pairs.iter().map(|&pair| { + if big_endian { + u16::from_be_bytes(pair) + } else { + u16::from_le_bytes(pair) + } + }); + let text = char::decode_utf16(units) + .collect::>() + .ok()?; + let label = encoding_label(text.as_bytes()).map(|label| text[label].to_ascii_lowercase()); + match label.as_deref() { + Some("utf-16" | "utf-16le" | "utf-16be") => Some((declared_utf8(text), false)), + _ => Some((text, false)), + } +} + +/// Bytes in an 8-bit encoding: UTF-8, or the code page the declaration +/// names. Without one, bytes that are not UTF-8 are in code page `ansi`; +/// so are bytes declared UTF-8 that are not, as Notepad leaves the driver's +/// file when it saves it as ANSI. A UTF-8 file with stray bytes is neither: +/// one with a byte order mark, or, in a single-byte code page, one that +/// still holds UTF-8 beyond ASCII, which that code page's text all but +/// never does. Read in the code page, its UTF-8 text would be garbled. +fn decode_bytes(bytes: &[u8], ansi: u32) -> Option<(String, bool)> { + let label = encoding_label(bytes).map(|label| bytes[label].to_ascii_lowercase()); + let code_page = match label.as_deref() { + None | Some(b"utf-8") => match String::from_utf8(bytes.to_vec()) { + Ok(text) => return Some((text, false)), + Err(_) if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) => return None, + Err(_) if single_byte(ansi) && holds_utf8(bytes) => return None, + Err(_) => ansi, + }, + Some(label) => match std::str::from_utf8(label).ok().and_then(code_page) { + Some(code_page) => code_page, + // The declaration is not one this host edits. + None => { + return String::from_utf8(bytes.to_vec()) + .ok() + .map(|text| (text, false)) + } + }, + }; + let text = decode_code_page(bytes, code_page)?; + Some((declared_utf8(text), true)) +} + +/// The Windows multi-byte code page an XML encoding name stands for. The +/// driver's reader takes none of these, declared or not; it does take the +/// single-byte ones, so a file declaring one stays as it is. +fn code_page(label: &str) -> Option { + let numbered = ["windows-", "x-windows-", "cp", "x-cp"] + .iter() + .find_map(|prefix| label.strip_prefix(prefix)?.parse().ok()); + let code_page = numbered.or(match label { + "shift_jis" | "shift-jis" | "sjis" | "ms_kanji" | "windows-31j" => Some(932), + "gb2312" | "gbk" | "x-gbk" => Some(936), + "euc-kr" | "ks_c_5601-1987" | "ks_c_5601-1989" | "ksc5601" | "uhc" => Some(949), + "big5" => Some(950), + "gb18030" => Some(54936), + _ => None, + })?; + [932, 936, 949, 950, 54936] + .contains(&code_page) + .then_some(code_page) +} + +/// `bytes` in Windows code page `code_page`; `None` when any byte is not +/// a character there, or off Windows. +#[cfg(windows)] +fn decode_code_page(bytes: &[u8], code_page: u32) -> Option { + use windows::Win32::Globalization::{MultiByteToWideChar, MB_ERR_INVALID_CHARS}; + if bytes.is_empty() { + return Some(String::new()); + } + let length = unsafe { MultiByteToWideChar(code_page, MB_ERR_INVALID_CHARS, bytes, None) }; + let mut wide = vec![0u16; usize::try_from(length).ok().filter(|&n| n > 0)?]; + let length = + unsafe { MultiByteToWideChar(code_page, MB_ERR_INVALID_CHARS, bytes, Some(&mut wide)) }; + String::from_utf16(wide.get(..usize::try_from(length).ok().filter(|&n| n > 0)?)?).ok() +} + +#[cfg(not(windows))] +fn decode_code_page(_bytes: &[u8], _code_page: u32) -> Option { + None +} + +/// Whether Windows code page `code_page` has one byte per character. +#[cfg(windows)] +fn single_byte(code_page: u32) -> bool { + use windows::Win32::Globalization::{GetCPInfo, CPINFO}; + let mut info = CPINFO::default(); + unsafe { GetCPInfo(code_page, &mut info) }.is_ok() && info.MaxCharSize == 1 +} + +#[cfg(not(windows))] +fn single_byte(_code_page: u32) -> bool { + false +} + +/// Whether `bytes` hold a UTF-8 character beyond ASCII anywhere. +fn holds_utf8(bytes: &[u8]) -> bool { + bytes.utf8_chunks().any(|chunk| !chunk.valid().is_ascii()) +} + +/// Where the XML declaration at the start of `xml` names its encoding. +fn encoding_label(xml: &[u8]) -> Option> { + let end = xml + .starts_with(b""))??; + let declaration = &xml[..end]; + let at = declaration.windows(9).position(|w| w == b"encoding=")? + 9; + let quote = *declaration.get(at).filter(|&&c| c == b'"' || c == b'\'')?; + let length = declaration[at + 1..].iter().position(|&c| c == quote)?; + Some(at + 1..at + 1 + length) +} + +/// `xml` with its declaration's encoding named as UTF-8, quote and all else +/// unchanged. +fn declared_utf8(xml: String) -> String { + match encoding_label(xml.as_bytes()) { + Some(label) => format!("{}utf-8{}", &xml[..label.start], &xml[label.end..]), + None => xml, + } +} + +/// Keep the file at `path` as it is, beside it, as +/// `vdd_settings.xml.bak-`, or `-2`, `-3`… when today's backups +/// hold something else; one that holds the same bytes is kept as it is. +/// The backup's file name. +fn back_up(path: &Path) -> io::Result { + back_up_dated(path, &utc_date(std::time::SystemTime::now())) +} + +/// `back_up` on `date`. +fn back_up_dated(path: &Path, date: &str) -> io::Result { + use std::io::Write; + let original = std::fs::read(path)?; + let name = path.file_name().unwrap_or_default().to_string_lossy(); + let mut n = 1; + loop { + let backup = match n { + 1 => format!("{name}.bak-{date}"), + n => format!("{name}.bak-{date}-{n}"), + }; + let at = path.with_file_name(&backup); + match std::fs::read(&at) { + Ok(held) if held == original => return Ok(backup), + Ok(_) => n += 1, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + let mut file = std::fs::File::create_new(&at)?; + file.write_all(&original)?; + file.sync_all()?; + return Ok(backup); + } + Err(error) => return Err(error), + } + } +} + +/// `time`'s UTC date as `YYYYMMDD` (Howard Hinnant's civil_from_days). +fn utc_date(time: std::time::SystemTime) -> String { + let seconds = time + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |since| since.as_secs()); + let days = (seconds / 86_400) as i64 + 719_468; + let era = days / 146_097; + let doe = days - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = yoe + era * 400 + i64::from(month <= 2); + format!("{year:04}{month:02}{day:02}") +} + +/// Replace the settings file with `write`. One read from a code page +/// (`legacy`) is backed up first, and its conversion logged once done. +fn write_settings( + path: &Path, + legacy: bool, + write: impl FnOnce() -> io::Result<()>, +) -> io::Result<()> { + let backup = legacy.then(|| back_up(path)).transpose()?; + write()?; + if let Some(backup) = backup { + tracing::info!( + backup, + "Wrote vdd_settings.xml as UTF-8; the original is kept beside it" + ); + } + Ok(()) +} + /// Write atomically (tmp + rename) so the driver never reads a torn file. /// Creates the directory if the driver hasn't made it yet. pub fn write_to(path: &Path, modes: &[VddMode], gpu: Option<&str>) -> io::Result<()> { @@ -792,6 +1398,364 @@ mod tests { 120" )); assert!(xml.ends_with("\n")); + assert_ne!( + cursor_set_by(&xml), + DriverCursor::Unknown, + "the second screen's pointer is reported apart" + ); + assert!(xml.contains( + " \n true\n \ + 128\n 128\n \ + true\n \ + 2\n \n" + )); + } + + /// A settings file with this cursor block. + fn with_cursor(block: &str) -> String { + format!("\n1{block}") + } + + fn hardware(max_x: u32, max_y: u32) -> DriverCursor { + DriverCursor::Hardware(CursorCaps { + max_x, + max_y, + alpha: true, + xor: 2, + }) + } + + /// The driver turns its hardware cursor on only when the file says so; + /// a file from before this version, or the vendor's with it off, keeps + /// the pointer in the second screen's picture. Read as the driver reads + /// it: comments don't count, only the text `true` does, and numbers as + /// `std::stoi` reads them. + #[test] + fn the_cursor_is_read_as_the_driver_reads_it() { + assert_eq!(cursor_set_by(VENDOR), DriverCursor::Software); + let v032 = render(&modes_for_client(1668, 2420, 60), None).replace(CURSOR, ""); + assert_eq!(cursor_set_by(&v032), DriverCursor::Software); + let caps = "128128true"; + for (text, on) in [ + ("true", true), + (" true", false), + ("1", false), + ("false", false), + ("TRUE", false), + ] { + let xml = with_cursor(&format!("{text}{caps}")); + assert_eq!(cursor_set_by(&xml) == hardware(128, 128), on, "{text:?}"); + } + let commented = format!("false{caps}"); + assert_eq!( + cursor_set_by(&commented), + DriverCursor::Software, + "a commented-out line is not a setting" + ); + let first_wins = format!("falsetrue{caps}"); + assert_eq!(cursor_set_by(&first_wins), DriverCursor::Software); + let blank_skipped = format!(" true{caps}"); + assert_eq!( + cursor_set_by(&blank_skipped), + hardware(128, 128), + "white space alone is not text to XmlLite" + ); + assert_eq!( + cursor_set_by("true"), + DriverCursor::Unknown, + "not well-formed" + ); + for (x, y, cursor) in [ + ("64", "64", hardware(64, 64)), + ("256", "256", hardware(256, 256)), + (" 96px", "+48", hardware(96, 48)), + ("0", "128", DriverCursor::Unknown), + ("-64", "128", DriverCursor::Unknown), + ("big", "128", DriverCursor::Unknown), + ("99999999999", "128", DriverCursor::Unknown), + ] { + let xml = with_cursor(&format!("true{x}{y}true")); + assert_eq!(cursor_set_by(&xml), cursor, "{x:?} x {y:?}"); + } + let missing = with_cursor("true64true"); + assert_eq!( + cursor_set_by(&missing), + DriverCursor::Unknown, + "a missing size reads as -1" + ); + let no_alpha = with_cursor("true6464false"); + assert_eq!( + cursor_set_by(&no_alpha), + DriverCursor::Unknown, + "not every pointer type" + ); + for (level, cursor) in [ + ("1", DriverCursor::Unknown), + ("0", DriverCursor::Unknown), + ( + "3", + DriverCursor::Hardware(CursorCaps { + max_x: 64, + max_y: 64, + alpha: true, + xor: 3, + }), + ), + ("7", hardware(64, 64)), + ] { + let xml = with_cursor(&format!("true6464true{level}")); + assert_eq!(cursor_set_by(&xml), cursor, "XOR level {level}"); + } + } + + #[test] + fn this_hosts_file_gives_a_128_px_cursor_that_takes_every_pointer_type() { + let xml = render(&modes_for_client(1668, 2420, 60), None); + assert_eq!(cursor_set_by(&xml), hardware(128, 128)); + assert_eq!(starting_cursor(Some(&xml), true, false), hardware(128, 128)); + assert_eq!( + starting_cursor(Some(&xml), true, true), + DriverCursor::Unknown, + "the driver's registry key overrides the file" + ); + assert_eq!( + starting_cursor(None, true, false), + DriverCursor::Unknown, + "no file read" + ); + } + + /// Review r5: whether the driver reads `` comes from the + /// installer's version, which may not be the running driver's. An old + /// or unreadable version is Unknown, never Software: the pointer is + /// then handled as in 0.3.4, which for a truly old driver gives the + /// same picture. + #[test] + fn a_driver_version_that_doesnt_read_the_cursor_is_unknown_not_software() { + let xml = render(&modes_for_client(1668, 2420, 60), None); + for version in ["24.10.16", "23.12.2", "24.10", "beta", ""] { + let reads = driver_reads_cursor(version); + assert!(!reads, "{version:?}"); + assert_eq!( + starting_cursor(Some(&xml), reads, false), + DriverCursor::Unknown, + "{version:?}" + ); + assert_eq!( + starting_cursor(Some(VENDOR), reads, false), + DriverCursor::Unknown, + "{version:?}" + ); + } + assert_eq!( + starting_cursor(Some(VENDOR), true, false), + DriverCursor::Software, + "only the file a driver that reads it started with says Software" + ); + } + + /// Review r5: XmlLite expands character and entity references into a + /// setting's text; quick-xml reports them apart, so a setting with one + /// is not known. + #[test] + fn a_reference_inside_a_cursor_setting_is_unknown() { + let caps = "6464true"; + for setting in [ + "true", + "true", + "true", + ] { + let xml = with_cursor(&format!("{setting}{caps}")); + assert_eq!(cursor_set_by(&xml), DriverCursor::Unknown, "{setting}"); + } + let size = with_cursor("true6464true"); + assert_eq!(cursor_set_by(&size), DriverCursor::Unknown); + let elsewhere = format!( + "A & Btrue{caps}" + ); + assert_eq!( + cursor_set_by(&elsewhere), + hardware(64, 64), + "a reference in another setting" + ); + } + + /// Review r5: the driver reads its settings folder from `VDDPATH`, and + /// its cursor values first from its registry key. Only a read that + /// finds nothing leaves the file this host writes in charge; a failed + /// read could be either. + #[test] + fn the_registry_leaves_the_file_in_charge_only_when_it_says_nothing() { + use RegValue::{Absent, Failed, Present}; + let none = [Absent, Absent, Absent, Absent, Absent]; + assert!(file_in_charge(&Absent, &none)); + for path in [ + r"C:\VirtualDisplayDriver", + r"C:\VirtualDisplayDriver\", + r"c:\virtualdisplaydriver", + r"C:\VIRTUALDISPLAYDRIVER\\", + ] { + assert!(file_in_charge(&Present(path.into()), &none), "{path:?}"); + } + for path in [ + r"D:\VirtualDisplayDriver", + r"C:\VirtualDisplayDriver\sub", + "", + r"C:\VirtualDisplayDriver2", + ] { + assert!(!file_in_charge(&Present(path.into()), &none), "{path:?}"); + } + assert!(!file_in_charge(&Failed, &none), "VDDPATH unreadable"); + for at in 0..none.len() { + for value in [Present(String::new()), Failed] { + let mut cursor = none.clone(); + cursor[at] = value.clone(); + assert!( + !file_in_charge(&Absent, &cursor), + "{} {value:?}", + REGISTRY_CURSOR_VALUES[at] + ); + } + } + } + + /// Review r4: what the running driver does with the pointer is tied to + /// its start (boot and device arrival), kept on disk. + #[test] + fn the_cursor_record_belongs_to_one_start_of_the_driver() { + let dir = std::env::temp_dir().join(format!("vdd-cursor-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(CURSOR_RECORD); + let start = |arrival| DriverInstance { + boot_id: 205, + arrival, + }; + let run = |arrival| DriverRun::Running(start(arrival)); + let ours = render(&modes_for_client(1668, 2420, 60), None); + let cursor = starting_cursor(Some(&ours), true, false); + + // Enable from off: the driver started with the file just written. + let record = record_after_enable(DriverRun::Off, run(10), Some(&ours), cursor).unwrap(); + assert_eq!(record.cursor, hardware(128, 128)); + assert_eq!(record.settings_sha256.len(), 64); + save_record(&path, &record).unwrap(); + assert_eq!( + running_cursor(load_record(&path).as_ref(), run(10)), + hardware(128, 128) + ); + + // AlreadyDone with a matching record: the driver kept what it read, + // whatever the file says now. + let vendor = starting_cursor(Some(VENDOR), true, false); + assert_eq!( + record_after_enable(run(10), run(10), Some(VENDOR), vendor), + None + ); + assert_eq!( + running_cursor(load_record(&path).as_ref(), run(10)), + hardware(128, 128) + ); + + // AlreadyDone with a changed arrival: someone else restarted it. + assert_eq!( + running_cursor(load_record(&path).as_ref(), run(11)), + DriverCursor::Unknown + ); + + // Relaunch: a new host reads the record from disk. Same start, same + // answer; after a reboot the same arrival counts for nothing; with + // no record, the driver off, or a failed read, nothing is known. + let relaunched = load_record(&path); + assert_eq!( + running_cursor(relaunched.as_ref(), run(10)), + hardware(128, 128) + ); + let rebooted = DriverInstance { + boot_id: 206, + arrival: 10, + }; + assert_eq!( + running_cursor(relaunched.as_ref(), DriverRun::Running(rebooted)), + DriverCursor::Unknown + ); + assert_eq!(running_cursor(None, run(10)), DriverCursor::Unknown); + for state in [DriverRun::Off, DriverRun::Unreadable] { + assert_eq!( + running_cursor(relaunched.as_ref(), state), + DriverCursor::Unknown + ); + } + + // Enables from off with the user's own caps of 64 and then 256: + // each start keeps its own. + for (arrival, max) in [(12, 64), (13, 256)] { + let users = with_cursor(&format!("true{max}{max}true")); + let cursor = starting_cursor(Some(&users), true, false); + let record = + record_after_enable(DriverRun::Off, run(arrival), Some(&users), cursor).unwrap(); + save_record(&path, &record).unwrap(); + let running = running_cursor(load_record(&path).as_ref(), run(arrival)); + assert_eq!(running, hardware(max, max)); + let DriverCursor::Hardware(caps) = running else { + unreachable!() + }; + assert!(caps.takes((max, max))); + assert!(!caps.takes((max + 1, 1))); + } + + // A start with a file that leaves the cursor off is kept as such. + let record = record_after_enable(DriverRun::Off, run(14), Some(VENDOR), vendor).unwrap(); + assert_eq!(record.cursor, DriverCursor::Software); + // A driver that is not running after the Enable keeps no record. + for after in [DriverRun::Off, DriverRun::Unreadable] { + assert_eq!( + record_after_enable(DriverRun::Off, after, Some(&ours), cursor), + None + ); + } + // A damaged record is no record. + std::fs::write(&path, "{").unwrap(); + assert_eq!(load_record(&path), None); + std::fs::remove_dir_all(&dir).unwrap(); + } + + /// Review r5: only a driver seen off before the Enable started in it. + /// A failed read before it, or one that was running (even another + /// start than after it), may have read another file. + #[test] + fn a_record_is_kept_only_for_a_driver_seen_off_before_the_enable() { + let run = |arrival| { + DriverRun::Running(DriverInstance { + boot_id: 205, + arrival, + }) + }; + let ours = render(&modes_for_client(1668, 2420, 60), None); + let cursor = starting_cursor(Some(&ours), true, false); + assert!(record_after_enable(DriverRun::Off, run(20), Some(&ours), cursor).is_some()); + for before in [DriverRun::Unreadable, run(20), run(19)] { + assert_eq!( + record_after_enable(before, run(20), Some(&ours), cursor), + None, + "{before:?}" + ); + } + } + + #[test] + fn only_drivers_that_read_the_cursor_block_are_trusted_with_it() { + for (version, reads) in [ + ("25.05.03", true), + ("24.10.27", true), + ("24.12.24", true), + ("24.10.16", false), + ("23.12.2", false), + ("24.10", false), + ("", false), + ("beta", false), + ] { + assert_eq!(driver_reads_cursor(version), reads, "{version:?}"); + } } #[test] @@ -1552,4 +2516,526 @@ mod tests { assert!(!path.with_extension("xml.tmp").exists(), "tmp cleaned up"); let _ = std::fs::remove_dir_all(&dir); } + + /// `text` as UTF-16, little or big endian, with or without a byte order + /// mark. + fn utf16(text: &str, big_endian: bool, bom: bool) -> Vec { + let units = bom.then_some(0xFEFF).into_iter().chain(text.encode_utf16()); + units + .flat_map(|unit| { + if big_endian { + unit.to_be_bytes() + } else { + unit.to_le_bytes() + } + }) + .collect() + } + + /// Field triage round 2: a settings file saved as UTF-16 was "not valid + /// UTF-8", so the second screen kept the driver's old modes and the card + /// was never named. + #[test] + fn a_utf16_or_bom_settings_file_is_read_and_written_back_as_utf8() { + let dir = std::env::temp_dir().join(format!("vdd-encoding-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let (path, record) = ( + dir.join("vdd_settings.xml"), + dir.join("app").join("vdd-pin.txt"), + ); + const INTEL: &str = "Intel(R) HD Graphics 630"; + let declared16 = VENDOR.replace("encoding='utf-8'", "encoding='UTF-16'"); + let fixtures: Vec<(&str, Vec)> = vec![ + ("utf-16 le, bom", utf16(&declared16, false, true)), + ("utf-16 be, bom", utf16(&declared16, true, true)), + ("utf-16 le, no bom", utf16(&declared16, false, false)), + ("utf-16 be, no bom", utf16(&declared16, true, false)), + ("utf-16 le, bom, utf-8 declared", utf16(VENDOR, false, true)), + ( + "utf-8, bom", + [&[0xEF, 0xBB, 0xBF][..], VENDOR.as_bytes()].concat(), + ), + ]; + for (name, bytes) in &fixtures { + // Only the card changes, as the driver will read it. + std::fs::write(&path, bytes).unwrap(); + let _ = std::fs::remove_file(&record); + let pinned = apply_pin(&path, &record, false, Some(INTEL)); + assert_eq!(pinned.ok(), Some(PinOutcome::Named), "{name}"); + let written = std::fs::read(&path).unwrap(); + let text = String::from_utf8(written.clone()).expect(name); + let body = text.strip_prefix('\u{feff}').unwrap_or(&text); + assert!( + body.starts_with("\r\n"), + "{name}: {body:?}" + ); + assert_eq!(named_gpu(&text), Ok(Some(INTEL.to_string())), "{name}"); + assert_eq!( + text.starts_with('\u{feff}'), + name.starts_with("utf-8"), + "{name}" + ); + assert_eq!( + body, + VENDOR.replace(">default<", &format!(">{INTEL}<")), + "{name}" + ); + + // Match the iPad's resolution writes the whole file, as UTF-8. + std::fs::write(&path, bytes).unwrap(); + let _ = std::fs::remove_file(&record); + let modes = modes_for_client(1640, 2360, 60); + let written = write_with_pin(&path, &record, false, &modes, Some(INTEL)); + assert_eq!(written.ok(), Some(PinOutcome::Named), "{name}"); + let text = std::fs::read_to_string(&path).expect(name); + assert_eq!(text, render(&modes, Some(INTEL)), "{name}"); + } + std::fs::remove_dir_all(&dir).unwrap(); + } + + /// Review r1: UTF-32's little-endian byte order mark starts like + /// UTF-16's. Read as UTF-16 it held NULs, did not parse, and a driver + /// Setup installed with a card to name had the whole file replaced. + #[test] + fn a_utf32_settings_file_is_left_byte_for_byte() { + let dir = std::env::temp_dir().join(format!("vdd-encoding-32-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let (path, record) = ( + dir.join("vdd_settings.xml"), + dir.join("app").join("vdd-pin.txt"), + ); + const INTEL: &str = "Intel(R) HD Graphics 630"; + let utf32 = |big_endian: bool, bom: bool| -> Vec { + bom.then_some('\u{feff}') + .into_iter() + .chain(VENDOR.chars()) + .flat_map(|c| { + let c = c as u32; + if big_endian { + c.to_be_bytes() + } else { + c.to_le_bytes() + } + }) + .collect() + }; + for (name, bytes) in [ + ("utf-32 le, bom", utf32(false, true)), + ("utf-32 be, bom", utf32(true, true)), + ("utf-32 le, no bom", utf32(false, false)), + ("utf-32 be, no bom", utf32(true, false)), + ] { + for owned in [false, true] { + std::fs::write(&path, &bytes).unwrap(); + let written = write_with_pin(&path, &record, owned, &default_modes(), Some(INTEL)); + assert!(written.is_err(), "{name} owned={owned}"); + assert!( + apply_pin(&path, &record, owned, Some(INTEL)).is_err(), + "{name}" + ); + assert!(pin_gpu_in(&path, INTEL).is_err(), "{name}"); + assert_eq!(std::fs::read(&path).unwrap(), bytes, "{name} owned={owned}"); + assert!(!record.exists(), "{name} owned={owned}"); + } + } + std::fs::remove_dir_all(&dir).unwrap(); + } + + /// A file in another encoding, or broken UTF-16, may hold anything: + /// it stays byte for byte, and the error says why. + #[test] + fn a_settings_file_in_another_encoding_is_left_alone() { + let dir = std::env::temp_dir().join(format!("vdd-encoding-other-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let (path, record) = ( + dir.join("vdd_settings.xml"), + dir.join("app").join("vdd-pin.txt"), + ); + // Not UTF-8, in an encoding this host does not know. + let unknown = stray_byte(&VENDOR.replace("encoding='utf-8'", "encoding='x-unknown'")); + // UTF-8 with a stray byte: after a byte order mark, or with UTF-8 + // beyond ASCII after it (the vendor file's comments) or before it. + let bom = [&[0xEF, 0xBB, 0xBF][..], &stray_byte(VENDOR)].concat(); + // A double-byte system code page may read these as its own text, so + // they are refused only in a single-byte one (and off Windows, where + // no code page is read); utf8_with_a_stray_byte_is_not_read_in_a_ + // single_byte_code_page tests both in 1252 on any system. + let mixed = if cfg!(not(windows)) || single_byte(ANSI) { + vec![ + ("utf-8 after a stray byte", mixed_after()), + ("utf-8 before a stray byte", mixed_before()), + ] + } else { + eprintln!("skipped the mixed UTF-8 cases: the system's code page is not single-byte"); + Vec::new() + }; + // A double-byte character cut short. + let mut broken = declared(CP949, "euc-kr"); + let at = broken.iter().position(|&b| b >= 0x80).unwrap(); + broken[at + 1] = b' '; + let mut odd = utf16(VENDOR, false, true); + odd.push(b' '); + let mut lone = utf16(VENDOR, false, true); + lone.extend_from_slice(&0xD800u16.to_le_bytes()); + // Off Windows no code page is read; on it, the system's code page + // reads this (a_file_in_the_systems_code_page_is_written_back_as_utf8). + let ansi = cfg!(not(windows)).then(|| ("ansi", stray_byte(VENDOR))); + for (name, bytes) in [ + ("unknown encoding", unknown), + ("utf-8 bom, stray byte", bom), + ("broken code page", broken), + ("odd utf-16", odd), + ("lone surrogate", lone), + ] + .into_iter() + .chain(mixed) + .chain(ansi) + { + std::fs::write(&path, &bytes).unwrap(); + let pinned = apply_pin(&path, &record, false, Some("Intel(R) HD Graphics 630")); + let error = pinned.expect_err(name); + assert_eq!(error.kind(), io::ErrorKind::InvalidData, "{name}"); + assert_eq!( + error.to_string(), + "the settings file is in no encoding this host reads" + ); + for owned in [false, true] { + let written = write_with_pin(&path, &record, owned, &default_modes(), Some("x")); + assert!(written.is_err(), "{name}"); + } + assert_eq!(std::fs::read(&path).unwrap(), bytes, "{name}"); + assert_eq!(backups(&dir), Vec::>::new(), "{name}"); + } + std::fs::remove_dir_all(&dir).unwrap(); + } + + /// `text` naming the card `Radeon` followed by the byte 0xAE, which is + /// `®` in code page 1252 and not UTF-8. + fn stray_byte(text: &str) -> Vec { + let mut bytes = text.replace(">default<", ">Radeon<").into_bytes(); + let at = bytes.windows(7).position(|w| w == b"Radeon<").unwrap() + 6; + bytes.insert(at, 0xAE); + bytes + } + + /// UTF-8 beyond ASCII after a stray byte: the vendor file's comments. + fn mixed_after() -> Vec { + stray_byte(VENDOR_FILE) + } + + /// UTF-8 beyond ASCII before a stray byte. + fn mixed_before() -> Vec { + stray_byte(&VENDOR.replace("", "")) + } + + /// The driver's file saved in code page 949 as Notepad's ANSI does: its + /// `encoding='utf-8'` declaration stays. A comment holds a folder path in + /// non-ASCII text, and so does the card's name. + const CP949: &[u8] = include_bytes!("../tests/fixtures/vdd_settings-25.5.2-cp949.xml"); + const VENDOR_DECLARATION: &str = "\r\n"; + + /// The code page 949 fixture and its code page 936 twin, each with its + /// code page, the text it holds, and the card it names. + #[cfg(windows)] + fn code_page_fixtures() -> [(&'static [u8], u32, String, &'static str); 2] { + let text = |comment: &str, card: &str| { + VENDOR_FILE + .replacen( + "\r\n", + &format!("\r\n \r\n"), + 1, + ) + .replace(">default<", &format!(">{card}<")) + }; + const CP949_CARD: &str = "Microsoft 기본 렌더링 드라이버"; + const CP936_CARD: &str = "Microsoft 基本呈现驱动程序"; + [ + ( + CP949, + 949, + text("가상 디스플레이 설정: D:\\도구\\디스플레이", CP949_CARD), + CP949_CARD, + ), + ( + include_bytes!("../tests/fixtures/vdd_settings-25.5.2-cp936.xml"), + 936, + text("虚拟显示器设置: D:\\工具\\显示器", CP936_CARD), + CP936_CARD, + ), + ] + } + + /// A code page fixture declaring `label` as its encoding instead. + fn declared(fixture: &[u8], label: &str) -> Vec { + let body = fixture.strip_prefix(VENDOR_DECLARATION.as_bytes()).unwrap(); + let declaration = format!("\r\n"); + [declaration.as_bytes(), body].concat() + } + + /// The backups beside the settings file in `dir`. + fn backups(dir: &Path) -> Vec> { + let mut found: Vec<_> = std::fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| { + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + name.starts_with("vdd_settings.xml.bak-") + }) + .collect(); + found.sort(); + found + .iter() + .map(|path| std::fs::read(path).unwrap()) + .collect() + } + + /// Field triage round 3: a settings file saved in a Windows code page + /// was "not valid UTF-8", so the second screen kept the driver's old + /// modes and the card was never named. Declared UTF-8 as Notepad leaves + /// it, or not declared, it is in the system's code page, given here. + #[cfg(windows)] + #[test] + fn a_code_page_settings_file_is_read_in_the_systems_code_page() { + const INTEL: &str = "Intel(R) HD Graphics 630"; + for (bytes, code_page, text, card) in code_page_fixtures() { + let undeclared = bytes.strip_prefix(VENDOR_DECLARATION.as_bytes()).unwrap(); + let body = text.strip_prefix(VENDOR_DECLARATION).unwrap(); + for (bytes, text) in [(bytes, text.as_str()), (undeclared, body)] { + assert_eq!( + decode_settings(bytes, code_page), + Some((text.to_string(), true)), + "{code_page}" + ); + assert_eq!(named_gpu(text), Ok(Some(card.to_string()))); + assert_eq!(with_gpu(text, INTEL), Some(text.replace(card, INTEL))); + } + // UTF-8 stays UTF-8 whatever the system's code page. + assert_eq!( + decode_settings(text.as_bytes(), code_page), + Some((text.clone(), false)) + ); + } + } + + /// A declaration may name the code page instead. Either way the file is + /// written back as UTF-8, declared so, which the driver's reader takes to + /// the end, and the file as it was is kept beside it. + #[cfg(windows)] + #[test] + fn a_code_page_settings_file_is_written_back_as_utf8_with_a_backup() { + let dir = std::env::temp_dir().join(format!("vdd-encoding-cp-{}", std::process::id())); + let (path, record) = ( + dir.join("vdd_settings.xml"), + dir.join("app").join("vdd-pin.txt"), + ); + const INTEL: &str = "Intel(R) HD Graphics 630"; + const AMD: &str = "AMD Radeon(TM) Graphics"; + let modes = modes_for_client(1640, 2360, 60); + let labels = [ + ( + 949, + &["euc-kr", "ks_c_5601-1987", "windows-949", "CP949"][..], + ), + (936, &["gb2312", "GBK", "windows-936", "cp936"][..]), + ]; + for (bytes, code_page, text, card) in code_page_fixtures() { + let utf8 = text.replacen( + VENDOR_DECLARATION, + "\r\n", + 1, + ); + let (_, labels) = labels.iter().find(|(page, _)| *page == code_page).unwrap(); + for label in *labels { + let declared = declared(bytes, label); + let fresh = || { + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(&path, &declared).unwrap(); + }; + + // Only the card changes, as the driver will read it. + fresh(); + let pinned = apply_pin(&path, &record, true, Some(INTEL)); + assert_eq!(pinned.ok(), Some(PinOutcome::Named), "{label}"); + let written = std::fs::read_to_string(&path).expect(label); + assert_eq!(written, utf8.replace(card, INTEL), "{label}"); + assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + // The file is UTF-8 now: the next write keeps the backup. + let pinned = apply_pin(&path, &record, true, Some(AMD)); + assert_eq!(pinned.ok(), Some(PinOutcome::Named), "{label}"); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + utf8.replace(card, AMD) + ); + assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + + // When the card stays, another app's or the driver's choice, + // the file is still written as UTF-8, so the driver reads it + // to the end. + for (wanted, outcome) in [ + (Some(INTEL), PinOutcome::Kept(NAMED_ELSEWHERE)), + (Some(card), PinOutcome::Unchanged), + (None, PinOutcome::Unchanged), + ] { + fresh(); + let pinned = apply_pin(&path, &record, false, wanted); + assert_eq!(pinned.ok(), Some(outcome), "{label} {wanted:?}"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), utf8, "{label}"); + assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + } + fresh(); + assert_eq!(pin_gpu_in(&path, card).ok(), Some(true), "{label}"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), utf8, "{label}"); + assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + + // Match the iPad's resolution writes the whole file, keeping + // the card another app named. + fresh(); + let written = write_with_pin(&path, &record, false, &modes, Some(INTEL)); + assert_eq!( + written.ok(), + Some(PinOutcome::Kept(NAMED_ELSEWHERE)), + "{label}" + ); + let text = std::fs::read_to_string(&path).expect(label); + assert_eq!(text, render(&modes, Some(card)), "{label}"); + assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + } + } + std::fs::remove_dir_all(&dir).unwrap(); + } + + /// Review r1: the system's code page, which reads a file declared UTF-8 + /// that is not or declaring nothing, had no test of its own. Here it is + /// the driver's file saved as ANSI on a system whose code page is 1252, + /// as on CI and the test PC, naming a card outside ASCII. Another code + /// page reads these bytes as other characters, so the test needs 1252. + #[cfg(windows)] + #[test] + fn a_file_in_the_systems_code_page_is_written_back_as_utf8() { + let ansi_code_page = unsafe { windows::Win32::Globalization::GetACP() }; + if ansi_code_page != 1252 { + eprintln!("skipped: the system's code page is {ansi_code_page}, not 1252"); + return; + } + let dir = std::env::temp_dir().join(format!("vdd-encoding-acp-{}", std::process::id())); + let (path, record) = ( + dir.join("vdd_settings.xml"), + dir.join("app").join("vdd-pin.txt"), + ); + const INTEL: &str = "Intel(R) HD Graphics 630"; + const CARD: &str = "Radeon\u{AE}"; + let text = VENDOR_FILE.replace(">default<", &format!(">{CARD}<")); + // As Notepad saves it: `®` is 0xAE and the comments' `’` 0x92. + let ansi: Vec = text + .chars() + .map(|c| match c { + '\u{AE}' => 0xAE, + '\u{2019}' => 0x92, + c => u8::try_from(c).ok().filter(u8::is_ascii).unwrap(), + }) + .collect(); + let fresh = || { + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(&path, &ansi).unwrap(); + }; + assert_eq!(decode_settings(&ansi, ANSI), Some((text.clone(), true))); + + // The card another app named stays, or this one, or the driver's + // choice; the file is written as UTF-8 all the same. + for (owned, wanted, outcome, card) in [ + (false, Some(INTEL), PinOutcome::Kept(NAMED_ELSEWHERE), CARD), + (false, Some(CARD), PinOutcome::Unchanged, CARD), + (true, Some(INTEL), PinOutcome::Named, INTEL), + ] { + fresh(); + let pinned = apply_pin(&path, &record, owned, wanted); + assert_eq!(pinned.ok(), Some(outcome), "{wanted:?}"); + let written = std::fs::read_to_string(&path).unwrap(); + assert_eq!(written, text.replace(CARD, card), "{wanted:?}"); + assert_eq!(backups(&dir), vec![ansi.clone()], "{wanted:?}"); + } + + // Match the iPad's resolution writes the whole file. + fresh(); + let modes = modes_for_client(1640, 2360, 60); + let written = write_with_pin(&path, &record, false, &modes, Some(INTEL)); + assert_eq!(written.ok(), Some(PinOutcome::Kept(NAMED_ELSEWHERE))); + let written = std::fs::read_to_string(&path).unwrap(); + assert_eq!(written, render(&modes, Some(CARD))); + assert_eq!(backups(&dir), vec![ansi.clone()]); + std::fs::remove_dir_all(&dir).unwrap(); + } + + /// Review r1: UTF-8 with a stray byte read in a single-byte code page + /// garbles its UTF-8 text. A double-byte code page's text may hold + /// what reads as UTF-8, so only single-byte code pages refuse it. + #[cfg(windows)] + #[test] + fn utf8_with_a_stray_byte_is_not_read_in_a_single_byte_code_page() { + let text = VENDOR.replace(">default<", ">Radeon\u{AE}<"); + assert_eq!( + decode_settings(&stray_byte(VENDOR), 1252), + Some((text, true)) + ); + let bom = [&[0xEF, 0xBB, 0xBF][..], &stray_byte(VENDOR)].concat(); + assert_eq!(decode_settings(&bom, 1252), None); + assert_eq!(decode_settings(&mixed_after(), 1252), None); + assert_eq!(decode_settings(&mixed_before(), 1252), None); + assert!(single_byte(1252) && !single_byte(949) && !single_byte(936)); + } + + /// Review r1: a second conversion the same day kept no copy of the file + /// it replaced. + #[test] + fn each_original_of_a_day_is_backed_up_once() { + let dir = std::env::temp_dir().join(format!("vdd-backup-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("vdd_settings.xml"); + let date = "20261108"; + let mut names = Vec::new(); + for original in ["one", "two", "one", "three", "two"] { + std::fs::write(&path, original).unwrap(); + names.push(back_up_dated(&path, date).unwrap()); + } + let name = |suffix: &str| format!("vdd_settings.xml.bak-{date}{suffix}"); + assert_eq!( + names, + [name(""), name("-2"), name(""), name("-3"), name("-2")] + ); + assert_eq!( + backups(&dir), + [b"one".to_vec(), b"two".to_vec(), b"three".to_vec()] + ); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn encoding_names_are_windows_code_pages() { + for (label, expected) in [ + ("euc-kr", Some(949)), + ("ks_c_5601-1987", Some(949)), + ("windows-949", Some(949)), + ("cp949", Some(949)), + ("gb2312", Some(936)), + ("gbk", Some(936)), + ("x-windows-936", Some(936)), + ("windows-1252", None), + ("iso-8859-1", None), + ("utf-16", None), + ("x-unknown", None), + ] { + assert_eq!(code_page(label), expected, "{label}"); + } + let at = |seconds| std::time::UNIX_EPOCH + std::time::Duration::from_secs(seconds); + assert_eq!(utc_date(at(0)), "19700101"); + assert_eq!(utc_date(at(951_868_800)), "20000301"); + assert_eq!(utc_date(at(1_709_251_199)), "20240229"); + assert_eq!(utc_date(at(1_794_096_000)), "20261108"); + } } diff --git a/host/src/vdd_task.ps1 b/host/src/vdd_task.ps1 index 49acd03d..f9fffb6a 100644 --- a/host/src/vdd_task.ps1 +++ b/host/src/vdd_task.ps1 @@ -19,10 +19,13 @@ try { if ($_.Exception.GetBaseException().HResult -ne -2147024894) { throw } } } + # A timeout says whether the instance was queued (Task Scheduler never + # started it) or in progress (the toggle itself did not finish). + $stateNames = @{ 2 = 'queued'; 4 = 'in progress' } $deadline = (Get-Date).AddSeconds(10) foreach ($name in $waitFor) { - while ($service.GetFolder('\').GetTask($name).State -in @(2,4)) { - if ((Get-Date) -gt $deadline) { throw "Previous VDD task timed out: $name" } + while (($state = $service.GetFolder('\').GetTask($name).State) -in @(2,4)) { + if ((Get-Date) -gt $deadline) { throw "Previous VDD task timed out: $name ($($stateNames[[int]$state]))" } Start-Sleep -Milliseconds 50 } } @@ -34,7 +37,7 @@ try { if (!$running) { throw "VDD task did not start: $Task" } $deadline = (Get-Date).AddSeconds(15) while ($true) { - if ((Get-Date) -gt $deadline) { throw "VDD task timed out: $Task" } + if ((Get-Date) -gt $deadline) { throw "VDD task timed out: $Task ($($stateNames[[int]$running.State]))" } try { $running.Refresh() } catch { # SCHED_E_TASK_NOT_RUNNING means the recorded instance completed. if ($_.Exception.GetBaseException().HResult -eq -2147216629) { break } diff --git a/host/tests/amf_encoder.rs b/host/tests/amf_encoder.rs index 6a35ea53..f93a0d77 100644 --- a/host/tests/amf_encoder.rs +++ b/host/tests/amf_encoder.rs @@ -12,21 +12,80 @@ //! desktop mode under a running stream. use std::sync::atomic::Ordering; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; use eternal_host::capture::synthetic::{decode_frame_counter_from_luma, render_synthetic_frame}; use eternal_host::capture::{frame_buffer, frame_slot, RawFrame}; use eternal_host::control::SharedControl; -use eternal_host::encoder::{run_encode_stage, NALUnit, NAL_CHANNEL_CAPACITY}; +use eternal_host::encoder::{ + forget_launch_failures, run_encode_stage, NALUnit, NAL_CHANNEL_CAPACITY, +}; use eternal_host::gpu::GpuInfo; use eternal_host::stats::PIPELINE_STATS; use eternal_wire::h264::{contains_nal_type, nal_type, parse_annex_b_units}; use ffmpeg_next::codec::Id; -/// The stage and PIPELINE_STATS are process-global, so stages run one at a time. +/// The encode stage's state is process-wide: PIPELINE_STATS, the encoders +/// that failed to open this launch, and the lane FFmpeg's hardware opens +/// share. So is an AMD driver's limit on concurrent AMF sessions. Each test +/// holds [`Serial`] from its first line, through its AMF probes and stages. static STAGE_LOCK: Mutex<()> = Mutex::new(()); +/// One test's turn at the encoders. A stage and an AMF probe take it by +/// reference, so neither runs beside another test's, and PIPELINE_STATS is +/// read only for the stage that just ran. +struct Serial(#[allow(dead_code)] MutexGuard<'static, ()>); + +/// Waits for this test's turn, then starts it from a clean launch: no +/// earlier test's encoder open is still running, and no earlier failure +/// keeps an encoder out of this test's stages. +fn serial() -> Serial { + let guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + wait_for_clear_lane(); + forget_launch_failures(); + Serial(guard) +} + +/// An open that went on in the background (see `encoder::Opening`) holds +/// the encoders' lane until it returns, and a stage's open in that lane is +/// refused meanwhile. +fn lane_busy() -> bool { + eternal_host::stall::abandoned(eternal_host::stall::Call::EncoderOpen) > 0 +} + +fn wait_for_clear_lane() { + // Past the 15 s limit of the slowest open in the lane. + let deadline = Instant::now() + Duration::from_secs(30); + while lane_busy() { + assert!( + Instant::now() < deadline, + "an earlier encoder open is still running" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + +/// A stage whose hardware open outlasted the first picture's wait starts on +/// libx264 while the open goes on in the background, as it should when a +/// driver is slow. On a busy test PC that says nothing about what a test +/// checks, so the stage runs once more. +fn started_on_the_fallback(_: &Serial) -> bool { + PIPELINE_STATS.lock().using_software_fallback && lane_busy() +} + +const STAGE_TRIES: u32 = 2; + +/// Before a stage runs again: its open in the background is released, and +/// what it remembered is forgotten, as for a new test. +fn before_another_try(encoder: &str) { + eprintln!( + "{encoder} was still opening when the first picture was due; running the stage again" + ); + wait_for_clear_lane(); + forget_launch_failures(); +} + const FRAME_INTERVAL: Duration = Duration::from_millis(20); const FRAME_COUNT: u64 = 90; /// Mid-GOP. The encoder's own IDRs land on frames 0, 30 and 60. @@ -71,11 +130,11 @@ fn amf_expectation() -> Option { /// Whether an AMF encoder opens on this machine. Cases that need a working /// AMF skip when it does not, except under `ETERNAL_AMF_TEST=hardware`. -fn amf_available(name: &str) -> bool { +fn amf_available(serial: &Serial, name: &str) -> bool { let Some(expectation) = amf_expectation() else { return false; }; - let opens = amf_opens(name); + let opens = amf_opens(serial, name); if expectation == AmfExpectation::Hardware { assert!( opens, @@ -88,7 +147,7 @@ fn amf_available(name: &str) -> bool { opens } -fn amf_opens(name: &str) -> bool { +fn amf_opens(_: &Serial, name: &str) -> bool { let Some(codec) = ffmpeg_next::encoder::find_by_name(name) else { return false; }; @@ -109,16 +168,39 @@ struct StageRun { packets: Vec, software_fallback: bool, codec_name: String, + /// Pictures the stage gave the encoder again (`keyframe_resubmits`). + resubmits: u64, + /// When each frame was published, by frame number. + published: Vec, } /// Feeds `FRAME_COUNT` frames through a fresh encode stage at /// `FRAME_INTERVAL`, sized by `size_of`, and raises a keyframe request just /// before frame `keyframe_at`. Panics if the stage fails or panics. fn run_stage( + serial: &Serial, encoder: &str, size_of: impl Fn(u64) -> (u32, u32), keyframe_at: Option, ) -> StageRun { + for _ in 1..STAGE_TRIES { + match run_stage_once(serial, encoder, &size_of, keyframe_at) { + Some(run) => return run, + None => before_another_try(encoder), + } + } + run_stage_once(serial, encoder, &size_of, keyframe_at) + .unwrap_or_else(|| panic!("{encoder} was still opening when the first picture was due")) +} + +/// [`run_stage`] once. None if the stage started on the fallback while its +/// encoder was still opening (see [`started_on_the_fallback`]). +fn run_stage_once( + serial: &Serial, + encoder: &str, + size_of: impl Fn(u64) -> (u32, u32), + keyframe_at: Option, +) -> Option { ffmpeg_next::init().unwrap(); let (producer, consumer) = frame_slot(); let (tx, mut rx) = tokio::sync::mpsc::channel(NAL_CHANNEL_CAPACITY); @@ -130,33 +212,55 @@ fn run_stage( ..GpuInfo::software_fallback() }; let generation = eternal_host::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let resubmits_before = PIPELINE_STATS.lock().keyframe_resubmits; let stage_shared = shared.clone(); let stage = std::thread::spawn(move || { run_encode_stage(consumer, tx, stage_shared, gpu, generation).map_err(|e| e.to_string()) }); // The output channel holds only a few frames, so drain it as we go. + let (first_tx, first) = std::sync::mpsc::channel(); let collector = std::thread::spawn(move || { let mut packets = Vec::new(); while let Some(nal) = rx.blocking_recv() { + if packets.is_empty() { + let _ = first_tx.send(()); + } packets.push(nal); } packets }); - for number in 0..FRAME_COUNT { - if keyframe_at == Some(number) { - shared.force_next_idr.store(true, Ordering::SeqCst); - } + let mut published = Vec::new(); + let mut publish = |number: u64| { let (width, height) = size_of(number); let mut data = frame_buffer((width * height * 4) as usize); render_synthetic_frame(&mut data, width, height, number); + published.push(Instant::now()); producer.publish(RawFrame { frame_number: number, timestamp: Instant::now(), data: Arc::new(data), width, height, + cached: false, }); + }; + // The stage opens its encoder on the first frame. The rest follow once + // it is out, however long the open took. + publish(0); + if first.recv_timeout(Duration::from_secs(20)).is_ok() && started_on_the_fallback(serial) { + producer.close(); + let _ = stage.join(); + let _ = collector.join(); + return None; + } + std::thread::sleep(FRAME_INTERVAL); + + for number in 1..FRAME_COUNT { + if keyframe_at == Some(number) { + shared.force_next_idr.store(true, Ordering::SeqCst); + } + publish(number); std::thread::sleep(FRAME_INTERVAL); } producer.close(); @@ -169,11 +273,62 @@ fn run_stage( let packets = collector.join().unwrap(); assert!(!packets.is_empty(), "{encoder} produced no packets"); let stats = PIPELINE_STATS.lock(); - StageRun { + Some(StageRun { packets, software_fallback: stats.using_software_fallback, codec_name: stats.codec_name.clone(), + resubmits: stats.keyframe_resubmits - resubmits_before, + published, + }) +} + +/// A stream whose frames come every `FRAME_INTERVAL` is never still, so no +/// picture may be given to the encoder again and none encoded twice. A +/// picture may repeat only where the feeder itself stalled right after it, +/// for a frame and a half (the encode loop's wait, see `keyframe_flush_wait`), +/// at most once per 20 ms of that stall, and no more often in all than the +/// stage reports giving pictures again. Everywhere else counters advance. +fn assert_no_duplicate_encodes(run: &StageRun, pictures: &[(u32, u32, u64)]) { + let counters: Vec = pictures.iter().map(|picture| picture.2).collect(); + assert!( + counters.windows(2).all(|pair| pair[0] <= pair[1]), + "frame counters go back: {counters:?}" + ); + let mut repeats = std::collections::BTreeMap::::new(); + for pair in counters.windows(2).filter(|pair| pair[0] == pair[1]) { + *repeats.entry(pair[0]).or_default() += 1; } + for (&counter, ×) in &repeats { + let stall = run + .published + .get(counter as usize + 1) + .zip(run.published.get(counter as usize)) + .map(|(next, this)| next.saturating_duration_since(*this)); + let allowed = stall + .filter(|stall| *stall >= FRAME_INTERVAL * 3 / 2) + .map_or(0, |stall| (stall.as_millis() / 20) as u32); + assert!( + times <= allowed, + "frame {counter} encoded {} times, though the next frame came {stall:?} \ + after it: {counters:?}", + times + 1 + ); + } + let stalled = run + .published + .windows(2) + .any(|pair| pair[1].saturating_duration_since(pair[0]) >= FRAME_INTERVAL * 3 / 2); + assert!( + stalled || run.resubmits == 0, + "{} pictures given again though no frame came later than {FRAME_INTERVAL:?}", + run.resubmits + ); + let total: u64 = repeats.values().map(|×| u64::from(times)).sum(); + assert!( + total <= run.resubmits, + "{total} pictures encoded again but {} given again: {counters:?}", + run.resubmits + ); } /// Decodes `packets` with a fresh, strict software decoder and returns each @@ -242,7 +397,8 @@ fn assert_hevc_keyframe(packet: &NALUnit, what: &str) { /// picture is at most `KEYFRAME_WINDOW` frames past the request, and a decoder /// that joins there must decode everything after it. fn assert_keyframe_on_request(run: &StageRun, codec: Id) { - decode(codec, &run.packets.iter().collect::>()); + let all = decode(codec, &run.packets.iter().collect::>()); + assert_no_duplicate_encodes(run, &all); let keyframes: Vec = run .packets .iter() @@ -274,10 +430,7 @@ fn assert_keyframe_on_request(run: &StageRun, codec: Id) { waited for the encoder's own IDR; keyframes at {keyframes:?}", counters[0] ); - assert!( - counters.windows(2).all(|pair| pair[0] < pair[1]), - "frame counters do not advance: {counters:?}" - ); + // Advancing counters across the whole stream: `assert_no_duplicate_encodes`. } /// Size schedule for the mode-change runs: small, then large, then small. @@ -292,7 +445,8 @@ fn mode_changes(number: u64) -> (u32, u32) { /// After each mode change the stream must restart on a keyframe at the new /// size that a fresh decoder can join. fn assert_follows_mode_changes(run: &StageRun, codec: Id) { - decode(codec, &run.packets.iter().collect::>()); + let all = decode(codec, &run.packets.iter().collect::>()); + assert_no_duplicate_encodes(run, &all); for (from, size) in [(30u64, LARGE), (60, SMALL)] { let start = run .packets @@ -320,17 +474,65 @@ fn assert_follows_mode_changes(run: &StageRun, codec: Id) { } } +#[test] +fn a_repeat_is_excused_only_by_the_stall_right_after_it() { + // Frames every 20 ms, except a 60 ms stall after frame 10. + let start = Instant::now(); + let published: Vec = (0..60u32) + .map(|n| { + start + + FRAME_INTERVAL * n + + if n > 10 { + FRAME_INTERVAL * 2 + } else { + Duration::ZERO + } + }) + .collect(); + let run = |resubmits| StageRun { + packets: Vec::new(), + software_fallback: false, + codec_name: String::new(), + resubmits, + published: published.clone(), + }; + let pictures = |repeated: u64| -> Vec<(u32, u32, u64)> { + (0..60u64) + .flat_map(|n| std::iter::repeat_n((0, 0, n), if n == repeated { 2 } else { 1 })) + .collect() + }; + assert_no_duplicate_encodes(&run(1), &pictures(10)); + for (resubmits, repeated, what) in [ + (1, 50, "a repeat far from the stall"), + (0, 10, "a repeat the stage did not give again"), + (1, 9, "a repeat just before the stall"), + ] { + let failed = std::panic::catch_unwind(|| { + assert_no_duplicate_encodes(&run(resubmits), &pictures(repeated)) + }) + .is_err(); + assert!(failed, "{what} was accepted"); + } + // Pictures given again with no stall at all. + let steady = StageRun { + published: (0..60u32).map(|n| start + FRAME_INTERVAL * n).collect(), + ..run(1) + }; + let counters: Vec<(u32, u32, u64)> = (0..60u64).map(|n| (0, 0, n)).collect(); + assert!(std::panic::catch_unwind(|| assert_no_duplicate_encodes(&steady, &counters)).is_err()); +} + #[test] fn software_encoder_follows_capture_mode_changes() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("libx264", mode_changes, None); + let serial = serial(); + let run = run_stage(&serial, "libx264", mode_changes, None); assert_follows_mode_changes(&run, Id::H264); } #[test] fn software_encoder_honours_keyframe_requests() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("libx264", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); + let serial = serial(); + let run = run_stage(&serial, "libx264", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); assert!( !run.software_fallback, "explicit CPU selection is not a fallback" @@ -344,8 +546,8 @@ fn software_encoder_honours_keyframe_requests() { #[test] #[ignore = "requires the NVIDIA/AMD reference PC without an Intel GPU"] fn unavailable_quick_sync_uses_another_gpu_and_decodes() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("h264_qsv", |_| LARGE, None); + let serial = serial(); + let run = run_stage(&serial, "h264_qsv", |_| LARGE, None); assert!(!run.software_fallback, "fell straight from QSV to CPU"); assert_eq!(run.codec_name, "H.264 (NVENC)"); assert_h264_keyframe(&run.packets[0], "the first packet"); @@ -371,11 +573,11 @@ fn unavailable_quick_sync_uses_another_gpu_and_decodes() { #[test] #[ignore = "native: an NVIDIA card, with this exe on another card"] fn nvenc_encodes_while_the_process_runs_on_another_card() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let serial = serial(); let facts = eternal_host::gpu_preference::facts().expect("graphics card facts"); let app_gpu = facts.running_on().expect("a hardware adapter").name.clone(); let started = Instant::now(); - let run = run_stage("h264_nvenc", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); + let run = run_stage(&serial, "h264_nvenc", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); assert!(!run.software_fallback, "NVENC fell back to the CPU"); assert!(run.codec_name.contains("NVENC"), "{}", run.codec_name); assert_keyframe_on_request(&run, Id::H264); @@ -388,8 +590,8 @@ fn nvenc_encodes_while_the_process_runs_on_another_card() { #[test] fn strict_decoder_rejects_broken_access_units() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("libx264", |_| SMALL, None); + let serial = serial(); + let run = run_stage(&serial, "libx264", |_| SMALL, None); let predicted = run .packets .iter() @@ -413,11 +615,11 @@ fn strict_decoder_rejects_broken_access_units() { #[test] fn amf_honours_keyframe_requests() { + let serial = serial(); let Some(expectation) = amf_expectation() else { return; }; - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("h264_amf", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); + let run = run_stage(&serial, "h264_amf", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); match expectation { AmfExpectation::Hardware => assert!( !run.software_fallback, @@ -460,45 +662,287 @@ fn amf_honours_keyframe_requests() { #[test] fn amf_follows_capture_mode_changes() { - if !amf_available("h264_amf") { + let serial = serial(); + if !amf_available(&serial, "h264_amf") { return; } - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("h264_amf", mode_changes, None); + let run = run_stage(&serial, "h264_amf", mode_changes, None); assert!(!run.software_fallback, "AMF fell back to libx264"); assert_follows_mode_changes(&run, Id::H264); } #[test] fn amf_hevc_follows_capture_mode_changes() { - if !amf_available("hevc_amf") { + let serial = serial(); + if !amf_available(&serial, "hevc_amf") { return; } - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("hevc_amf", mode_changes, None); + let run = run_stage(&serial, "hevc_amf", mode_changes, None); assert!(!run.software_fallback, "AMF fell back to libx264"); assert_follows_mode_changes(&run, Id::HEVC); } #[test] fn amf_hevc_honours_keyframe_requests() { - if !amf_available("hevc_amf") { + let serial = serial(); + if !amf_available(&serial, "hevc_amf") { return; } - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("hevc_amf", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); + let run = run_stage(&serial, "hevc_amf", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); assert_hevc_keyframe(&run.packets[0], "the first packet"); assert_keyframe_on_request(&run, Id::HEVC); } +/// How long a keyframe requested on a still desktop may take to come out. +/// Capture resends the last picture once for the request, and then nothing +/// until the 750 ms keepalive, so an encoder that holds a frame until the +/// next one arrives kept a new viewer waiting that long. Nothing is published +/// meanwhile: a keyframe that comes out at all came out without a next frame. +/// How fast is printed, not checked, because an AMD driver's own reopen took +/// from 50 to over 400 ms in a fresh test process. +const STILL_WINDOW: Duration = Duration::from_millis(1500); +/// How long the encoder may take to open and encode the warm-up frames. +const WARM_UP_LIMIT: Duration = Duration::from_secs(10); +/// The warm-up is over once no packet has come for this long. +const WARM_UP_QUIET: Duration = Duration::from_millis(300); + +/// Streams a few frames and waits for the encoder to encode them, goes still, +/// then requests a keyframe with one resent picture and nothing after it, as +/// for a reconnecting viewer. +fn keyframe_on_a_still_desktop(serial: &Serial, encoder: &str) -> StillRun { + for _ in 1..STAGE_TRIES { + match keyframe_on_a_still_desktop_once(serial, encoder) { + Some(result) => return result, + None => before_another_try(encoder), + } + } + keyframe_on_a_still_desktop_once(serial, encoder) + .unwrap_or_else(|| panic!("{encoder} was still opening when the first picture was due")) +} + +/// [`keyframe_on_a_still_desktop`] once, or None as [`run_stage_once`]. +fn keyframe_on_a_still_desktop_once(serial: &Serial, encoder: &str) -> Option { + ffmpeg_next::init().unwrap(); + let (producer, consumer) = frame_slot(); + let (tx, mut rx) = tokio::sync::mpsc::channel(NAL_CHANNEL_CAPACITY); + let shared = SharedControl::new(0, 8_000_000); + shared.running.store(true, Ordering::SeqCst); + let gpu = GpuInfo { + encoder_name: encoder.to_string(), + codec_display_name: encoder.to_string(), + ..GpuInfo::software_fallback() + }; + let generation = eternal_host::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let resubmits_before = PIPELINE_STATS.lock().keyframe_resubmits; + let stage_shared = shared.clone(); + let stage = std::thread::spawn(move || { + run_encode_stage(consumer, tx, stage_shared, gpu, generation).map_err(|e| e.to_string()) + }); + let (packets_tx, packets) = std::sync::mpsc::channel(); + let collector = std::thread::spawn(move || { + while let Some(nal) = rx.blocking_recv() { + if packets_tx.send((Instant::now(), nal)).is_err() { + break; + } + } + }); + let publish = |number: u64| { + let (width, height) = LARGE; + let mut data = frame_buffer((width * height * 4) as usize); + render_synthetic_frame(&mut data, width, height, number); + producer.publish(RawFrame { + frame_number: number, + timestamp: Instant::now(), + data: Arc::new(data), + width, + height, + // The warm-up's frames are new; the resent one is the cache. + cached: number == RESENT, + }); + }; + + for number in 0..10 { + publish(number); + std::thread::sleep(FRAME_INTERVAL); + } + // The request must find a session that has encoded frames, so that it + // takes the reopen a reconnect takes, whatever the encoder's open cost. + let warm_up_until = Instant::now() + WARM_UP_LIMIT; + let mut seen = Vec::new(); + loop { + match packets.recv_timeout(WARM_UP_QUIET) { + Ok(packet) => { + if seen.is_empty() && started_on_the_fallback(serial) { + producer.close(); + let _ = stage.join(); + let _ = collector.join(); + return None; + } + seen.push(packet); + } + Err(std::sync::mpsc::RecvTimeoutError::Timeout) if !seen.is_empty() => break, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => assert!( + Instant::now() < warm_up_until, + "{encoder} encoded nothing within {WARM_UP_LIMIT:?}" + ), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + panic!( + "{encoder} stage ended during the warm-up: {:?}", + stage.join() + ) + } + } + } + + // A viewer connects to the still desktop. + let requested = Instant::now(); + shared.force_next_idr.store(true, Ordering::SeqCst); + publish(RESENT); + let mut keyframe = None; + while keyframe.is_none() { + let left = STILL_WINDOW.saturating_sub(requested.elapsed()); + match packets.recv_timeout(left) { + Ok((at, nal)) => { + if nal.is_keyframe && nal.sequence == RESENT { + keyframe = Some(at.duration_since(requested)); + } + seen.push((at, nal)); + } + Err(_) => break, + } + } + producer.close(); + + match stage.join() { + Ok(Ok(())) => {} + Ok(Err(error)) => panic!("{encoder} encode stage failed: {error}"), + Err(_) => panic!("{encoder} encode stage panicked"), + } + collector.join().unwrap(); + let (software_fallback, resubmits) = { + let stats = PIPELINE_STATS.lock(); + ( + stats.using_software_fallback, + stats.keyframe_resubmits - resubmits_before, + ) + }; + eprintln!( + "{encoder}: still-desktop keyframe after {keyframe:?}; packets {:?}", + seen.iter() + .map(|(at, nal)| ( + nal.sequence, + nal.is_keyframe, + at.saturating_duration_since(requested).as_millis() as i64 + - requested.saturating_duration_since(*at).as_millis() as i64 + )) + .collect::>() + ); + Some(StillRun { + keyframe, + software_fallback, + resubmits, + sequences: seen.iter().map(|(_, nal)| nal.sequence).collect(), + }) +} + +/// The picture resent for the viewer, after the warm-up's frames 0 to 9. +const RESENT: u64 = 100; + +struct StillRun { + /// How long the resent picture's keyframe took to leave the stage, or + /// `None` if it did not within [`STILL_WINDOW`]. + keyframe: Option, + software_fallback: bool, + resubmits: u64, + /// The capture number of each packet, in order. + sequences: Vec, +} + +fn assert_still_keyframe(serial: &Serial, encoder: &str) { + let run = keyframe_on_a_still_desktop(serial, encoder); + assert!( + encoder == "libx264" || !run.software_fallback, + "{encoder} fell back to libx264" + ); + assert!( + run.keyframe.is_some(), + "{encoder} held the requested keyframe until the next captured frame" + ); + // Only the still picture may be encoded twice, and only by being given + // again: the warm-up's frames came every 20 ms. + let warm_up: Vec = run + .sequences + .iter() + .copied() + .filter(|&sequence| sequence != RESENT) + .collect(); + assert!( + warm_up.windows(2).all(|pair| pair[0] < pair[1]), + "{encoder} encoded a moving picture twice: {:?}", + run.sequences + ); + let still = run + .sequences + .iter() + .filter(|&&sequence| sequence == RESENT) + .count() as u64; + assert!( + still.saturating_sub(1) <= run.resubmits, + "{encoder}: the still picture came out {still} times, given again {}", + run.resubmits + ); + if eternal_host::encoder::holds_last_frame(encoder) { + assert!( + run.resubmits >= 1, + "{encoder} brought out the keyframe without being given the picture again" + ); + } else { + assert_eq!( + run.resubmits, 0, + "{encoder} returns each frame, so nothing is given again" + ); + } +} + +#[test] +fn software_encoder_sends_a_requested_keyframe_on_a_still_desktop() { + let serial = serial(); + assert_still_keyframe(&serial, "libx264"); +} + +#[test] +fn amf_sends_a_requested_keyframe_on_a_still_desktop() { + let serial = serial(); + for name in ["h264_amf", "hevc_amf"] { + if !amf_available(&serial, name) { + continue; + } + assert_still_keyframe(&serial, name); + } +} + +#[cfg(windows)] +#[test] +#[ignore = "native: an NVIDIA card"] +fn nvenc_sends_a_requested_keyframe_on_a_still_desktop() { + let serial = serial(); + for name in ["h264_nvenc", "hevc_nvenc"] { + assert_still_keyframe(&serial, name); + } +} + #[test] fn amf_fails_cleanly_without_an_amd_driver() { + let serial = serial(); if amf_expectation() != Some(AmfExpectation::Fallback) { return; } // Without an AMD driver, both AMF encoders must fail to open rather than // hang or crash. The HEVC one is chosen at runtime for HEVC clients. for name in ["h264_amf", "hevc_amf"] { - assert!(!amf_opens(name), "{name} opened without an AMD driver"); + assert!( + !amf_opens(&serial, name), + "{name} opened without an AMD driver" + ); } } diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index 4a889c10..e8a2a0a0 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -26,7 +26,8 @@ use eternal_wire::h264::contains_nal_type; use eternal_wire::reassembly::{AddOutcome, Reassembler, MAX_STALL_ALLOWANCE}; use eternal_wire::v2::control::{ encode_control, parse_control, ByeReason, ControlMessage, Hello2, HelloAck, HelloStatus, - KeyframeReason, KeyframeRequest, ReceiverReport, CAP_DECODE_H264, + KeyframeReason, KeyframeRequest, Ping, ReceiverReport, CAP_DECODE_H264, + HOSTCAP_KEYFRAME_PER_LOSS, }; use eternal_wire::v2::media::MediaHeader; use eternal_wire::v2::{classify, Classified}; @@ -707,6 +708,7 @@ fn run_slow_usb(bytes_per_sec: f64, run_for: Duration) -> SlowUsbRun { stream_epoch: epoch, last_complete_seq: seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, }); write_datagram(&mut stream, &encode_control(session_id, msg_seq, &request)) .await @@ -789,6 +791,407 @@ fn slow_usb_link_measurement() { assert!(run.tunnel_open, "the tunnel must stay open"); } +/// Measures recovery from lossy WiFi through the real encoder and transport: +/// how long the picture stands still and how many keyframes reach the fake +/// iPad whole. `ETERNAL_TEST_LOSS` (default 0.10) is the loss each way. +/// The host's fault injection drops first transmissions only, so this +/// receiver also loses that share of the repairs and of its own requests, +/// as WiFi does. `ETERNAL_TEST_ENCODER` picks the encoder (default libx264), +/// `ETERNAL_TEST_SECONDS` the length (default 30), `ETERNAL_TEST_FPS` the +/// capture rate (default 60; the encoder's own IDR comes every 30 frames, so +/// a slower desktop waits longer for it). `ETERNAL_TEST_BURST_MS` +/// adds a total outage each way of that length every +/// `ETERNAL_TEST_BURST_EVERY_MS` (default 4000), after a 2 s start. +/// +/// It also reports each recovery: from the first sign of a broken picture +/// (a dropped frame, an undecodable one, or a burst's start) to the next +/// decodable frame, with the requests sent and the keyframes that began +/// meanwhile, and the size and send spread of keyframes and delta frames. +/// `ETERNAL_TEST_TRACE=1` prints each of those events with its time. +/// `ETERNAL_TEST_OLD_CLIENT=1` makes the receiver ask as iPads before 0.3.5 +/// did, without the request for a new loss after a keyframe. +/// Timing on a busy machine changes the numbers, so this only reports them; +/// `wifi_repair_sim.rs` holds the assertions. +#[test] +#[ignore = "measurement; run with --ignored --nocapture"] +fn lossy_wifi_recovery_measurement() { + let _env = test_env(); + init_test_tracing(); + let _ = ffmpeg_next::init(); + let env = |name: &str| std::env::var(name).ok().filter(|v| !v.is_empty()); + let loss: f64 = env("ETERNAL_TEST_LOSS").map_or(0.10, |v| v.parse().unwrap()); + let seconds: u64 = env("ETERNAL_TEST_SECONDS").map_or(30, |v| v.parse().unwrap()); + let encoder = env("ETERNAL_TEST_ENCODER").unwrap_or_else(|| "libx264".into()); + let fps: u32 = env("ETERNAL_TEST_FPS").map_or(60, |v| v.parse().unwrap()); + let burst = + Duration::from_millis(env("ETERNAL_TEST_BURST_MS").map_or(0, |v| v.parse().unwrap())); + let burst_every = Duration::from_millis( + env("ETERNAL_TEST_BURST_EVERY_MS").map_or(4000, |v| v.parse().unwrap()), + ); + if env("ETERNAL_SYNTH_SIZE").is_none() { + std::env::set_var("ETERNAL_SYNTH_SIZE", "1920x1080"); + } + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_DROP", loss.to_string()); + if env("ETERNAL_REORDER").is_none() { + std::env::set_var("ETERNAL_REORDER", "0.02"); + } + if env("ETERNAL_JITTER_MS").is_none() { + std::env::set_var("ETERNAL_JITTER_MS", "4"); + } + let port = free_udp_port(); + let shared = SharedControl::new(port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some(encoder.clone()); + shared + .target_fps + .store(fps, std::sync::atomic::Ordering::SeqCst); + let (tx, rx) = mpsc::channel(); + let pipeline_shared = shared.clone(); + let pipeline_tx = tx.clone(); + let gpu = if encoder == "libx264" { + GpuInfo::software_fallback() + } else { + GpuInfo::detect() + }; + let supervisor = std::thread::spawn(move || { + eternal_host::supervisor::run(port, pipeline_shared, gpu, pipeline_tx, rx); + }); + let mut receiver = FakeReceiver::connect_full( + port, + CAP_DECODE_H264, + eternal_wire::v2::control::FEATURE_SUPPORTS_NACK, + ); + receiver + .socket + .set_read_timeout(Some(Duration::from_millis(1))) + .unwrap(); + let mut assembler = Reassembler::new(); + assembler.configure_repair(true, Duration::from_secs(1) / fps, Duration::from_millis(5)); + // The iPad's clock, as in `lossy_stream_recovers_and_adapts`: a pass + // counts at most MAX_PASS, so a descheduled test thread expires nothing. + const MAX_PASS: Duration = Duration::from_millis(20); + let mut ipad_now = Instant::now(); + let mut pass_started = ipad_now; + let mut rng = 0x2545_F491_4F6C_DD1Du64; + let mut lost = move || { + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + ((rng >> 11) as f64 / (1u64 << 53) as f64) < loss + }; + // The app's requests: the assembler asks when it drops a frame, then + // the decoder when a frame cannot be decoded, and again while it waits + // for a keyframe, at most every 600 ms (`VideoDecoder`); only the + // assembler's name the last frame. The control channel sends one + // request per 500 ms and drops the others, but to a host with + // `HOSTCAP_KEYFRAME_PER_LOSS` the first GapLoss after a keyframe goes at + // once without moving that spacing (`ControlChannel.sendKeyframeRequest`). + let old_client = env("ETERNAL_TEST_OLD_CLIENT").is_some(); + let mut last_request: Option = None; + let mut keyframe_since_request = false; + let mut early_requests = 0u64; + let mut decoder_waiting: Option = None; + let mut decoder_asked: Option = None; + let mut requests_dropped = 0u64; + let mut keyframes_seen = std::collections::HashSet::new(); + let mut keyframes_complete = 0u64; + let mut previous: Option<(u32, u32, bool)> = None; + let mut last_decodable: Option = None; + let mut freezes = Vec::new(); + let mut decodable = 0u64; + let mut delivered = 0u64; + let mut requests = 0u64; + let mut bitrate = 0; + let mut bytes = [0; 2048]; + // A broken picture until the next decodable frame. + #[derive(Debug)] + struct Episode { + since: Instant, + requests: u32, + delivered_requests: u32, + /// Requests sent while a newer keyframe was already arriving. + behind_keyframe: u32, + keyframes_begun: u32, + } + let begin = |since| Episode { + since, + requests: 0, + delivered_requests: 0, + behind_keyframe: 0, + keyframes_begun: 0, + }; + let mut episode: Option = None; + let mut episodes: Vec<(Duration, Episode)> = Vec::new(); + // The newest keyframe whose fragments began to arrive, until it is delivered. + let mut arriving_keyframe: Option<(u32, u32)> = None; + // First and last first-transmission arrival per frame, and keyframe flag. + let mut spread: HashMap<(u32, u32), (Instant, Instant, bool)> = HashMap::new(); + let mut keyframe_bytes = Vec::new(); + let mut delta_bytes = Vec::new(); + let mut was_in_burst = false; + // From a delivered request to the first fragment of the next keyframe. + let mut asked_at: Option = None; + let mut answer_us = Vec::new(); + let started = Instant::now(); + let tracing = env("ETERNAL_TEST_TRACE").is_some(); + let trace = |what: std::fmt::Arguments| { + if tracing { + eprintln!("LOSSY_TRACE {:>6} {what}", started.elapsed().as_millis()); + } + }; + while started.elapsed() < Duration::from_secs(seconds) { + let woke = Instant::now(); + ipad_now += (woke - pass_started).min(MAX_PASS); + pass_started = woke; + let in_burst = !burst.is_zero() + && started.elapsed() >= Duration::from_secs(2) + && ((started.elapsed() - Duration::from_secs(2)).as_millis() % burst_every.as_millis()) + < burst.as_millis(); + if in_burst && !was_in_burst && episode.is_none() { + trace(format_args!("begin burst")); + episode = Some(begin(ipad_now)); + } + was_in_burst = in_burst; + if let Ok((len, _)) = receiver.socket.recv_from(&mut bytes) { + if in_burst { + // Everything is lost each way during a burst. + } else if let Ok((header, payload)) = MediaHeader::decode(&bytes[..len]) { + if header.session_id == receiver.session_id && !(header.is_retransmit && lost()) { + let id = (header.stream_epoch, header.frame_seq); + if header.is_keyframe && keyframes_seen.insert(id) { + if let Some(episode) = episode.as_mut() { + episode.keyframes_begun += 1; + } + trace(format_args!("keyframe {id:?} first fragment")); + arriving_keyframe = Some(id); + if let Some(at) = asked_at.take() { + answer_us.push((woke - at).as_micros()); + } + } + if !header.is_retransmit { + let entry = spread.entry(id).or_insert((woke, woke, header.is_keyframe)); + entry.1 = woke; + } + assembler.add_media(header, payload, ipad_now); + } + } else if let Ok((_, ControlMessage::Heartbeat(hb))) = parse_control(&bytes[..len]) { + bitrate = hb.stream_config.bitrate_bps; + } + } + assembler.tick(ipad_now); + for nack in assembler.take_nacks() { + if !lost() && !in_burst { + receiver.send(&ControlMessage::Nack(nack)); + } + } + let mut asks = Vec::new(); + if assembler.take_keyframe_request().is_some() { + asks.push(KeyframeReason::GapLoss); + if episode.is_none() { + trace(format_args!("begin dropped frame")); + } + episode.get_or_insert(begin(ipad_now)); + } + for frame in assembler.take_ready() { + delivered += 1; + keyframes_complete += u64::from(frame.is_keyframe); + let epoch = assembler.counters().stream_epoch; + if frame.is_keyframe { + keyframe_since_request = true; + keyframe_bytes.push(frame.payload.len()); + if arriving_keyframe.is_some_and(|(e, seq)| e == epoch && seq <= frame.seq) { + arriving_keyframe = None; + } + } else { + delta_bytes.push(frame.payload.len()); + } + // The host numbers sent frames contiguously per epoch, so a + // delta decodes only right after the frame before it decoded. + let decodes = frame.is_keyframe + || previous.is_some_and(|(e, seq, ok)| ok && e == epoch && seq + 1 == frame.seq); + previous = Some((epoch, frame.seq, decodes)); + if episode.is_some() || frame.is_keyframe { + trace(format_args!( + "deliver {} key={} decodes={decodes}", + frame.seq, frame.is_keyframe + )); + } + if decodes { + decoder_waiting = None; + decodable += 1; + if let Some(at) = last_decodable { + freezes.push(ipad_now - at); + } + last_decodable = Some(ipad_now); + if let Some(episode) = episode.take() { + trace(format_args!("end {:?}", ipad_now - episode.since)); + episodes.push((ipad_now - episode.since, episode)); + } + } else if last_decodable.is_some() { + if episode.is_none() { + trace(format_args!("begin undecodable frame")); + } + episode.get_or_insert(begin(ipad_now)); + if decoder_waiting.is_none() { + decoder_waiting = Some(ipad_now); + } + } + } + if let Some(since) = decoder_waiting { + if decoder_asked.is_none_or(|at| ipad_now - at >= Duration::from_millis(600)) { + decoder_asked = Some(ipad_now); + asks.push(if ipad_now - since >= Duration::from_secs(2) { + KeyframeReason::Resume + } else { + KeyframeReason::DecodeError + }); + } + } + for reason in asks { + let spaced = last_request.is_none_or(|at| ipad_now - at >= Duration::from_millis(500)); + let new_loss = !old_client + && receiver.host_caps & HOSTCAP_KEYFRAME_PER_LOSS != 0 + && reason == KeyframeReason::GapLoss + && keyframe_since_request; + if !spaced && !new_loss { + requests_dropped += 1; + } else { + trace(format_args!("request {reason:?} early={}", !spaced)); + if spaced { + last_request = Some(ipad_now); + } + keyframe_since_request = false; + early_requests += u64::from(!spaced); + requests += 1; + let c = assembler.counters(); + let delivered_request = !lost() && !in_burst; + if let Some(episode) = episode.as_mut() { + episode.requests += 1; + episode.delivered_requests += u32::from(delivered_request); + episode.behind_keyframe += u32::from(arriving_keyframe.is_some()); + } + if delivered_request { + asked_at.get_or_insert(woke); + let gap = reason == KeyframeReason::GapLoss; + receiver.send(&ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: if gap { c.stream_epoch } else { 0 }, + last_complete_seq: if gap { + assembler.latest_completed_seq() + } else { + 0 + }, + reason, + max_media_dgram: 0, + })); + } + } + } + if in_burst { + continue; + } + if receiver.last_report.elapsed() >= Duration::from_millis(500) { + receiver.last_report = Instant::now(); + let c = assembler.counters(); + receiver.send(&ControlMessage::ReceiverReport(ReceiverReport { + stream_epoch: c.stream_epoch, + highest_seq: c.highest_seq, + frames_complete: c.frames_complete as u32, + frames_dropped: c.frames_dropped as u32, + frags_received: c.frags_received as u32, + frags_lost: c.frags_lost as u32, + frags_repaired: c.frags_repaired as u32, + nacks_sent: c.nacks_sent as u32, + assembler_depth: c.assembler_depth, + jitter_us: c.jitter_us, + ..Default::default() + })); + } + } + let (sent, retransmits) = { + let stats = eternal_host::stats::PIPELINE_STATS.lock(); + (stats.transport_packets_sent, stats.transport_retransmits) + }; + shared.stop(); + tx.send(SupervisorCommand::Shutdown).unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = supervisor.join(); + let _ = done_tx.send(()); + }); + let _ = done_rx.recv_timeout(Duration::from_secs(5)); + freezes.sort(); + let over = |ms: u64| { + freezes + .iter() + .filter(|f| **f > Duration::from_millis(ms)) + .count() + }; + let longest = freezes.last().copied().unwrap_or_default(); + let percentile = |values: &mut Vec, p: usize| { + values.sort_unstable(); + values + .get((values.len() * p / 100).min(values.len().saturating_sub(1))) + .copied() + .unwrap_or(0) + }; + let mut recovery_ms: Vec = episodes.iter().map(|(d, _)| d.as_millis()).collect(); + let storms = episodes + .iter() + .filter(|(_, e)| e.delivered_requests > 1) + .count(); + let extra_keyframes: u32 = episodes + .iter() + .map(|(_, e)| e.keyframes_begun.saturating_sub(1)) + .sum(); + let behind_keyframe: u32 = episodes.iter().map(|(_, e)| e.behind_keyframe).sum(); + let episode_requests: u32 = episodes.iter().map(|(_, e)| e.delivered_requests).sum(); + let mut key_sizes: Vec = keyframe_bytes.iter().map(|b| *b as u128).collect(); + let mut delta_sizes: Vec = delta_bytes.iter().map(|b| *b as u128).collect(); + let key_p50 = percentile(&mut key_sizes, 50); + let delta_p50 = percentile(&mut delta_sizes, 50); + let mut key_spread_us: Vec = spread + .values() + .filter(|(_, _, key)| *key) + .map(|(first, last, _)| (*last - *first).as_micros()) + .collect(); + let mut delta_spread_us: Vec = spread + .values() + .filter(|(_, _, key)| !*key) + .map(|(first, last, _)| (*last - *first).as_micros()) + .collect(); + eprintln!( + "LOSSY_E2E encoder={encoder} fps={fps} loss={loss} burst_ms={} seconds={seconds} keyframes={keyframes_complete}/{} \ + decodable={decodable}/{delivered} sent={sent} retransmits={retransmits} requests={requests} \ + requests_dropped={requests_dropped} early_requests={early_requests} old_client={old_client} \ + longest_freeze_ms={} freezes_over_250ms={} freezes_over_1s={} end_bitrate={bitrate} \ + episodes={} recovery_p50_ms={} recovery_p90_ms={} recovery_max_ms={} \ + episode_requests={episode_requests} storm_episodes={storms} requests_behind_keyframe={behind_keyframe} \ + extra_keyframes={extra_keyframes} keyframe_p50_bytes={key_p50} keyframe_max_bytes={} \ + delta_p50_bytes={delta_p50} keyframe_spread_p50_us={} keyframe_spread_max_us={} \ + delta_spread_p50_us={} answers={} answer_p50_us={} answer_p90_us={} answer_max_us={} counters={:?}", + burst.as_millis(), + keyframes_seen.len(), + longest.as_millis(), + over(250), + over(1000), + episodes.len(), + percentile(&mut recovery_ms, 50), + percentile(&mut recovery_ms, 90), + recovery_ms.last().copied().unwrap_or(0), + key_sizes.last().copied().unwrap_or(0), + percentile(&mut key_spread_us, 50), + key_spread_us.last().copied().unwrap_or(0), + percentile(&mut delta_spread_us, 50), + answer_us.len(), + percentile(&mut answer_us, 50), + percentile(&mut answer_us, 90), + answer_us.last().copied().unwrap_or(0), + assembler.counters() + ); + assert!(delivered > 0, "the stream must deliver frames"); +} + /// Accepts the host's USB tunnel on `listener` within `within`, past its /// EMLINK preamble, as `read_preamble` reads it. Other programs dial a /// test's loopback listener too: a development tool that probes each new @@ -885,6 +1288,8 @@ fn free_udp_port() -> u16 { struct H264TestDecoder { decoder: ffmpeg_next::codec::decoder::Video, frame: ffmpeg_next::frame::Video, + /// Width and height of the last picture that came out. + size: (u32, u32), } impl H264TestDecoder { @@ -899,6 +1304,7 @@ impl H264TestDecoder { Self { decoder, frame: ffmpeg_next::frame::Video::empty(), + size: (0, 0), } } @@ -911,6 +1317,7 @@ impl H264TestDecoder { } let mut counters = Vec::new(); while self.decoder.receive_frame(&mut self.frame).is_ok() { + self.size = (self.frame.width(), self.frame.height()); let stride = self.frame.stride(0); let luma = self.frame.data(0); counters.push(decode_frame_counter_from_luma(luma, stride, SYNTH_W)); @@ -1011,12 +1418,34 @@ impl FakeReceiver { ) } + /// The handshake, then a ping, as every iPad since 0.3.0 sends the + /// moment its HELLO_ACK lands. fn connect_as( host_port: u16, decoder_caps: u16, feature_caps: u16, preferred_fps: u8, client_nonce: u32, + ) -> Self { + let mut receiver = Self::handshake( + host_port, + decoder_caps, + feature_caps, + preferred_fps, + client_nonce, + ); + receiver.send(&ControlMessage::Ping(Ping { t1_us: 1 })); + receiver + } + + /// The handshake alone: the first report, 400 ms later, is the first + /// thing the PC hears from this receiver. + fn handshake( + host_port: u16, + decoder_caps: u16, + feature_caps: u16, + preferred_fps: u8, + client_nonce: u32, ) -> Self { let socket = UdpSocket::bind("127.0.0.1:0").expect("receiver socket"); socket @@ -1363,6 +1792,224 @@ fn audio_stream_end_to_end() { } } +/// What one fake iPad saw for `window`: the largest media datagram, whether +/// a keyframe came, and the latest heartbeat frame count. +fn media_seen(receiver: &mut FakeReceiver, window: Duration) -> (usize, bool, Option) { + let started = Instant::now(); + let mut buffer = [0; 2048]; + let (mut largest, mut keyframe, mut frames_sent) = (0, false, None); + while started.elapsed() < window { + receiver.maybe_report(0, 0); + let Ok((len, _)) = receiver.socket.recv_from(&mut buffer) else { + continue; + }; + match classify(&buffer[..len]) { + Classified::Media { flags } => { + largest = largest.max(len); + keyframe |= flags & 1 != 0; + } + Classified::Control(_) => { + if let Ok((_, ControlMessage::Heartbeat(heartbeat))) = parse_control(&buffer[..len]) + { + frames_sent = heartbeat.frames_sent; + } + } + _ => {} + } + } + (largest, keyframe, frames_sent) +} + +/// An iPad that said it may ask for smaller media datagrams gets them for +/// the rest of its session, starting with a keyframe, and heartbeats that +/// count the frames sent. An iPad that didn't say so gets neither, and its +/// request is the keyframe request it always was. +#[test] +fn an_ipad_gets_the_smaller_media_datagrams_it_asks_for() { + use eternal_wire::v2::control::{FEATURE_MEDIA_SIZE, HOSTCAP_MEDIA_SIZE}; + let _env = test_env(); + init_test_tracing(); + ffmpeg_next::init().unwrap(); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + for name in ["ETERNAL_DROP", "ETERNAL_REORDER", "ETERNAL_JITTER_MS"] { + std::env::remove_var(name); + } + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".into()); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor_shared = shared.clone(); + let supervisor_sender = supervisor_tx.clone(); + let supervisor = std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + supervisor_shared, + GpuInfo::software_fallback(), + supervisor_sender, + supervisor_rx, + ) + }); + + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let sized = |max_media_dgram| { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 0, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram, + }) + }; + for offered in [true, false] { + let features = if offered { FEATURE_MEDIA_SIZE } else { 0 }; + let mut receiver = FakeReceiver::connect_full(listen_port, CAP_DECODE_H264, features); + assert_eq!(receiver.host_caps & HOSTCAP_MEDIA_SIZE != 0, offered); + let (largest, _, frames_sent) = media_seen(&mut receiver, Duration::from_secs(2)); + assert!( + largest > 1000, + "{offered}: the keyframe goes in full-size datagrams ({largest})" + ); + assert_eq!( + frames_sent.is_some_and(|frames| frames > 0), + offered, + "{frames_sent:?}" + ); + if !offered { + assert_eq!(frames_sent, None); + } + + receiver.send(&sized(576)); + // Frames already queued may still be large; what follows isn't. + let (_, queued_keyframe, _) = media_seen(&mut receiver, Duration::from_millis(500)); + let (largest, keyframe, _) = media_seen(&mut receiver, Duration::from_secs(2)); + if offered { + assert!( + queued_keyframe || keyframe, + "the smaller size starts with a keyframe" + ); + assert!(largest > 0 && largest <= 576, "{largest}"); + } else { + assert!(largest > 576, "{largest}"); + } + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + std::thread::sleep(Duration::from_millis(200)); + } + // The size was this session's: the next one starts at the PC's. + let mut receiver = + FakeReceiver::connect_full(listen_port, CAP_DECODE_H264, FEATURE_MEDIA_SIZE); + let (largest, _, _) = media_seen(&mut receiver, Duration::from_secs(2)); + assert!(largest > 1000, "{largest}"); + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + })); + let _ = supervisor_tx.send(SupervisorCommand::Shutdown); + supervisor.join().unwrap(); + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } +} + +/// The first keyframe `receiver` gets after frame `after`, arriving before +/// `until`: its stream epoch and frame number. +fn keyframe_after(receiver: &mut FakeReceiver, after: u32, until: Instant) -> Option<(u32, u32)> { + let mut buffer = [0; 2048]; + while Instant::now() < until { + receiver.maybe_report(0, 0); + let Ok((len, _)) = receiver.socket.recv_from(&mut buffer) else { + continue; + }; + if let Ok((header, _)) = MediaHeader::decode(&buffer[..len]) { + if header.is_keyframe && !header.is_retransmit && header.frame_seq > after { + return Some((header.stream_epoch, header.frame_seq)); + } + } + } + None +} + +/// Under Wi-Fi loss the next frame is often lost soon after a recovery IDR +/// lands. The iPad then asks again at once, naming a frame from that +/// keyframe on, and the PC sends that loss its own IDR inside its 500 ms +/// request window rather than after it. Each round asks for a keyframe, +/// waits for it, and asks again naming it; the second IDR must arrive less +/// than 500 ms after the first request, which no IDR granted on the window +/// can. At 10 fps the encoder's own IDR comes 3 s after the last one, so +/// none falls in a round. +#[test] +fn a_loss_soon_after_a_recovery_gets_its_own_keyframe() { + let _env = test_env(); + init_test_tracing(); + ffmpeg_next::init().unwrap(); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + for name in ["ETERNAL_DROP", "ETERNAL_REORDER", "ETERNAL_JITTER_MS"] { + std::env::remove_var(name); + } + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".into()); + shared + .target_fps + .store(10, std::sync::atomic::Ordering::SeqCst); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor_shared = shared.clone(); + let supervisor_sender = supervisor_tx.clone(); + let supervisor = std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + supervisor_shared, + GpuInfo::software_fallback(), + supervisor_sender, + supervisor_rx, + ) + }); + + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut receiver = FakeReceiver::connect(listen_port); + let gap_loss = |stream_epoch, last_complete_seq| { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch, + last_complete_seq, + reason: KeyframeReason::GapLoss, + max_media_dgram: 0, + }) + }; + let (epoch, mut last) = + keyframe_after(&mut receiver, 0, Instant::now() + Duration::from_secs(10)) + .expect("the opening keyframe"); + let mut in_window = Vec::new(); + for _ in 0..6 { + // Past the window of the round before. + let _ = keyframe_after( + &mut receiver, + u32::MAX, + Instant::now() + Duration::from_millis(700), + ); + let asked = Instant::now(); + receiver.send(&gap_loss(epoch, last)); + let (_, recovery) = keyframe_after(&mut receiver, last, asked + Duration::from_secs(2)) + .expect("the first loss gets an IDR"); + receiver.send(&gap_loss(epoch, recovery)); + let answer = + keyframe_after(&mut receiver, recovery, asked + Duration::from_millis(500)); + in_window.push(answer.is_some()); + last = answer.map_or(recovery, |(_, seq)| seq); + } + let answered = in_window.iter().filter(|answered| **answered).count(); + assert!( + answered >= 4, + "a new loss is answered inside the window: {in_window:?}" + ); + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + })); + let _ = supervisor_tx.send(SupervisorCommand::Shutdown); + supervisor.join().unwrap(); + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } +} + /// Native first-second audio measurement across repeated connects with the /// PC's real WASAPI loopback (inaudible: it only reads the mix). #[cfg(windows)] @@ -1568,6 +2215,7 @@ fn synthetic_stream_end_to_end_v2() { stream_epoch: header.stream_epoch, last_complete_seq: header.frame_seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } if let Some(at) = keyframe_requested_at { @@ -1737,6 +2385,147 @@ fn a_session_ends_with_one_summary_naming_the_clients_nonce() { } } +/// A hybrid laptop where Windows won't share the screen from the graphics +/// card EternalMonitor runs on (capture fails with DXGI_ERROR_UNSUPPORTED, +/// faked here). The iPad got 10 s of black, then "no picture", while only +/// the PC's window said why. Now the session's heartbeats tell it after the +/// refusal lasts 2 s, the summary names a capture failure, and once Windows +/// lets capture work, the next session streams with no notice. +#[cfg(debug_assertions)] +#[test] +fn a_screen_refused_from_this_graphics_card_is_told_to_the_ipad() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_UNSUPPORTED_GPU}; + let _env = test_env(); + init_test_tracing(); + let _ = ffmpeg_next::init(); + + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_REFUSE_GPU", "1"); + std::env::remove_var("ETERNAL_DROP"); + + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".to_string()); + let gpu_info = GpuInfo::software_fallback(); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor = { + let (shared, supervisor_tx) = (shared.clone(), supervisor_tx.clone()); + std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + shared, + gpu_info, + supervisor_tx, + supervisor_rx, + ); + }) + }; + + // The session the iPad gives up on: heartbeats, no video. + const REFUSED: u32 = 0x6B0_0001; + let mut receiver = FakeReceiver::connect_as(listen_port, CAP_DECODE_H264, 0, 0, REFUSED); + let accepted = Instant::now(); + let mut datagram = [0u8; 2048]; + let mut told_after = None; + while told_after.is_none() { + assert!( + accepted.elapsed() < Duration::from_secs(8), + "no notice within 8 s of the session" + ); + receiver.maybe_report(0, 0); + let Ok((len, _)) = receiver.socket.recv_from(&mut datagram) else { + continue; + }; + match classify(&datagram[..len]) { + Classified::Media { .. } => panic!("video went out from a refused screen"), + Classified::Control(_) => { + if let Ok((_, ControlMessage::Heartbeat(hb))) = parse_control(&datagram[..len]) { + if hb.notice == NOTICE_UNSUPPORTED_GPU { + told_after = Some(accepted.elapsed()); + } else { + assert_eq!(hb.notice, NOTICE_NONE); + } + } + } + _ => {} + } + } + let told_after = told_after.unwrap(); + assert!( + told_after >= Duration::from_millis(1500), + "told after {told_after:?}: a refusal that lasts a moment says nothing" + ); + // The iPad's first-picture wait runs out. + receiver.send(&ControlMessage::Bye(ByeReason::Error)); + receiver.send(&ControlMessage::Bye(ByeReason::Error)); + let deadline = Instant::now() + Duration::from_secs(3); + while summaries_with_nonce(REFUSED).is_empty() { + assert!( + Instant::now() < deadline, + "no summary within 3 s of the goodbye" + ); + std::thread::sleep(Duration::from_millis(20)); + } + let line = summaries_with_nonce(REFUSED)[0].clone(); + assert!( + line.contains(" stage=accepted outcome=no_picture failure=capture_failed "), + "{line}" + ); + + // Windows lets capture work again (the user switched the card). + std::env::remove_var("ETERNAL_SYNTH_REFUSE_GPU"); + const WORKS: u32 = 0x6B0_0002; + let mut receiver = FakeReceiver::connect_as(listen_port, CAP_DECODE_H264, 0, 0, WORKS); + let deadline = Instant::now() + DEADLINE; + let mut video = false; + let mut notices_after_video = Vec::new(); + while notices_after_video.len() < 2 { + assert!( + Instant::now() < deadline, + "no video and heartbeats within {DEADLINE:?}" + ); + receiver.maybe_report(0, 0); + let Ok((len, _)) = receiver.socket.recv_from(&mut datagram) else { + continue; + }; + match classify(&datagram[..len]) { + Classified::Media { .. } => video = true, + Classified::Control(_) if video => { + if let Ok((_, ControlMessage::Heartbeat(hb))) = parse_control(&datagram[..len]) { + notices_after_video.push(hb.notice); + } + } + _ => {} + } + } + assert_eq!(notices_after_video, [NOTICE_NONE, NOTICE_NONE]); + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + + shared.stop(); + supervisor_tx + .send(SupervisorCommand::Shutdown) + .expect("send shutdown"); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = supervisor.join(); + let _ = done_tx.send(()); + }); + done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("supervisor must shut down within 5s"); + if let Some(path) = std::env::var_os("ETERNAL_SUMMARY_DUMP") { + use std::io::Write; + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .unwrap(); + file.write_all(format!("{line}\n").as_bytes()).unwrap(); + } +} + /// Repair is negotiated and exercised through the real encoder and transport. #[test] fn lossy_stream_recovers_and_adapts() { @@ -1989,6 +2778,7 @@ fn lossy_stream_recovers_and_adapts() { stream_epoch: epoch, last_complete_seq: seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } for frame in assembler.take_ready() { @@ -2158,6 +2948,7 @@ fn lossy_legacy_stream_uses_keyframe_recovery() { stream_epoch: current_epoch, last_complete_seq: highest_seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } @@ -2700,6 +3491,184 @@ fn input_relay_maps_touches_end_to_end() { .expect("supervisor must shut down within 5s"); } +/// A screen with an odd size streams at the even size below it: libx264 +/// refused 1680x945 and the pipeline gave up after six restarts. The iPad +/// is told the size it decodes, the picture is intact, and a touch on its +/// last pixel lands there while its far edge reaches the row and column left +/// out. Off Windows only: the input recorder stands in for real injection. +#[cfg(not(windows))] +#[test] +fn an_odd_screen_streams_cropped_and_input_still_reaches_its_edge() { + use eternal_host::input::{ + desktop_pixel_to_abs, recorder, Injection, VirtualScreen, KIND_TOUCH, + }; + use eternal_wire::v2::control::{InputEvent, FEATURE_WANTS_INPUT}; + const ODD_W: u32 = SYNTH_W + 1; + const ODD_H: u32 = SYNTH_H + 1; + + let _env = test_env(); + init_test_tracing(); + let _ = ffmpeg_next::init(); + + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{ODD_W}x{ODD_H}")); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::remove_var("ETERNAL_DROP"); + + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".to_string()); + let gpu_info = GpuInfo::software_fallback(); + + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor_shared = shared.clone(); + let supervisor_tx_clone = supervisor_tx.clone(); + let supervisor = std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + supervisor_shared, + gpu_info, + supervisor_tx_clone, + supervisor_rx, + ); + }); + + let mut receiver = FakeReceiver::connect_with_caps(listen_port, FEATURE_WANTS_INPUT); + let deadline = Instant::now() + DEADLINE; + let mut reassembler = Reassembler::new(); + let mut decoder = H264TestDecoder::new(); + let mut datagram = [0u8; 2048]; + let mut decoded_counters: Vec = Vec::new(); + let mut heartbeats_seen = 0u32; + let mut highest_seq = 0u32; + while decoded_counters.len() < WANT_DECODED_FRAMES || heartbeats_seen == 0 { + assert!( + Instant::now() < deadline, + "timed out: {} decoded frames, heartbeats={heartbeats_seen}", + decoded_counters.len() + ); + receiver.maybe_report(highest_seq, decoded_counters.len() as u32); + let Ok((len, _)) = receiver.socket.recv_from(&mut datagram) else { + continue; + }; + let bytes = &datagram[..len]; + match classify(bytes) { + Classified::Control(_) => { + // Heartbeats before the first capture carry no size. + if let Ok((_, ControlMessage::Heartbeat(hb))) = parse_control(bytes) { + let size = (hb.stream_config.width, hb.stream_config.height); + if size != (0, 0) { + heartbeats_seen += 1; + assert_eq!(size, (SYNTH_W as u16, SYNTH_H as u16)); + } + } + } + Classified::Media { .. } => { + let (header, payload) = MediaHeader::decode(bytes).expect("valid media datagram"); + highest_seq = highest_seq.max(header.frame_seq); + let AddOutcome::Completed(frame_bytes) = reassembler.add_fragment( + header.frame_seq, + header.frag_index, + header.frag_count, + header.stream_epoch, + payload, + Instant::now(), + ) else { + continue; + }; + let counters = decoder.decode(&frame_bytes); + if !counters.is_empty() { + assert_eq!(decoder.size, (SYNTH_W, SYNTH_H)); + } + decoded_counters.extend(counters); + } + _ => {} + } + } + // Real pictures: the counter in the top-left corner reads back and rises. + for pair in decoded_counters.windows(2) { + assert!(pair[1] > pair[0], "decoded counters must rise: {pair:?}"); + } + + let _ = recorder::take(); + let screen = VirtualScreen { + left: 0, + top: 0, + width: ODD_W, + height: ODD_H, + }; + // TouchRelay.swift: the touch's fraction of the 640x360 picture. + let centre = |pixel: u32, pixels: u32| { + ((f64::from(pixel) + 0.5) / f64::from(pixels) * 65_535.0).round() as u16 + }; + let tap = InputEvent { + input_ver: 1, + kind: KIND_TOUCH, + phase: 0, + buttons: 1, + event_id: 1, + x_norm: centre(SYNTH_W - 1, SYNTH_W), + y_norm: centre(SYNTH_H - 1, SYNTH_H), + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: 0, + modifiers: 0, + client_time_us: 0, + tilt_x: 0, + tilt_y: 0, + }; + for event in [ + tap, + InputEvent { + phase: 1, + event_id: 2, + x_norm: 65_535, + y_norm: 65_535, + ..tap + }, + InputEvent { + phase: 2, + event_id: 3, + x_norm: 65_535, + y_norm: 65_535, + ..tap + }, + ] { + receiver.send(&ControlMessage::InputEvent(event)); + } + let inject_deadline = Instant::now() + Duration::from_secs(5); + while recorder::peek().len() < 5 && Instant::now() < inject_deadline { + std::thread::sleep(Duration::from_millis(50)); + } + std::thread::sleep(Duration::from_millis(150)); + let (lx, ly) = desktop_pixel_to_abs(SYNTH_W as i32 - 1, SYNTH_H as i32 - 1, screen); + let (ex, ey) = desktop_pixel_to_abs(ODD_W as i32 - 1, ODD_H as i32 - 1, screen); + assert_eq!( + recorder::take(), + [ + Injection::MoveAbs { x: lx, y: ly }, + Injection::LeftDown { x: lx, y: ly }, + Injection::MoveAbs { x: ex, y: ey }, + Injection::MoveAbs { x: ex, y: ey }, + Injection::LeftUp { x: ex, y: ey }, + ] + ); + + shared.stop(); + supervisor_tx + .send(SupervisorCommand::Shutdown) + .expect("send shutdown"); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = supervisor.join(); + let _ = done_tx.send(()); + }); + done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("supervisor must shut down within 5s"); +} + /// An encoder crash mid-stream must be contained by the supervisor: automatic /// restart with backoff, a fresh stream epoch, and the SAME session resuming /// (the client never re-handshakes; its assembler resets on the epoch bump). @@ -3308,3 +4277,311 @@ fn quiet_clients_keep_their_session_until_they_go_silent() { let _ = supervisor_tx.send(SupervisorCommand::Shutdown); let _ = supervisor.join(); } + +/// #183: an iPad that draws the pointer gets it apart from the video. Over a +/// still picture the pointer moves every frame while the host sends only an +/// IDR and its idle keepalive; an older iPad on the same host gets a frame +/// for every move and no pointer messages, as before. +#[test] +fn the_pointer_moves_without_video_frames_for_an_ipad_that_draws_it() { + use eternal_host::capture::synthetic::pointer_path; + use eternal_wire::v2::control::{FEATURE_DRAWS_POINTER, HOSTCAP_POINTER, POINTER_VISIBLE}; + let _env = test_env(); + init_test_tracing(); + let _ = ffmpeg_next::init(); + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_POINTER", "1"); + for name in ["ETERNAL_DROP", "ETERNAL_REORDER", "ETERNAL_JITTER_MS"] { + std::env::remove_var(name); + } + + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".to_string()); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor = { + let shared = shared.clone(); + let tx = supervisor_tx.clone(); + std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + shared, + GpuInfo::software_fallback(), + tx, + supervisor_rx, + ); + }) + }; + // Positions on the synthetic path, to check every state against. + let path: std::collections::HashSet<(i32, i32)> = (0..20_000) + .map(|step| pointer_path(step, SYNTH_W, SYNTH_H)) + .collect(); + const WINDOW: Duration = Duration::from_secs(3); + + for (feature_caps, nonce) in [(FEATURE_DRAWS_POINTER, 0xE2E0_0183u32), (0, 0xE2E0_0184)] { + let draws = feature_caps != 0; + let mut receiver = + FakeReceiver::connect_as(listen_port, CAP_DECODE_H264, feature_caps, 60, nonce); + assert_ne!( + receiver.host_caps & HOSTCAP_POINTER, + 0, + "this host sends the pointer" + ); + let mut reassembler = Reassembler::new(); + let mut datagram = [0u8; 2048]; + let (mut frames, mut highest, mut states) = (0u32, 0u32, Vec::new()); + let mut shape: Option<(u32, Vec, usize)> = None; + let mut held_shape = 0; + let mut hidden_at_start = 0u32; + let mut window_start: Option = None; + let deadline = Instant::now() + DEADLINE; + while window_start.is_none_or(|at| at.elapsed() < WINDOW) { + assert!(Instant::now() < deadline, "no first frame (draws={draws})"); + if receiver.last_report.elapsed() >= Duration::from_millis(400) { + receiver.last_report = Instant::now(); + receiver.send(&ControlMessage::ReceiverReport(ReceiverReport { + highest_seq: highest, + frames_complete: frames, + pointer_shape_id: held_shape, + ..Default::default() + })); + } + let Ok((len, _)) = receiver.socket.recv_from(&mut datagram) else { + continue; + }; + let bytes = &datagram[..len]; + match classify(bytes) { + Classified::Media { .. } => { + let (header, payload) = MediaHeader::decode(bytes).unwrap(); + highest = highest.max(header.frame_seq); + let done = reassembler.add_fragment( + header.frame_seq, + header.frag_index, + header.frag_count, + header.stream_epoch, + payload, + Instant::now(), + ); + if matches!(done, AddOutcome::Completed(_)) { + // The window starts after the session's first frame, + // or, for an iPad that draws the pointer, its first + // shown position: until its report names the image, + // the pointer stays in the video. + if window_start.is_some() { + frames += 1; + } else if !draws { + window_start = Some(Instant::now()); + } + } + } + Classified::Control(_) => match parse_control(bytes).map(|(_, m)| m) { + Ok( + message @ (ControlMessage::PointerState(_) + | ControlMessage::PointerShape(_)), + ) if !draws => { + panic!("an older iPad got a pointer message: {message:?}") + } + Ok(ControlMessage::PointerState(state)) + if state.flags & POINTER_VISIBLE != 0 => + { + assert!(held_shape != 0, "shown before the iPad held the image"); + if window_start.is_some() { + states.push((state.x, state.y)); + } + window_start.get_or_insert_with(Instant::now); + } + // Hidden only while the session starts: until the iPad's + // report names the image, the pointer stays in the video. + // The synthetic pointer never hides. + Ok(ControlMessage::PointerState(state)) => { + assert!( + window_start.is_none(), + "a hidden state after the pointer went apart: {state:?}" + ); + hidden_at_start += 1; + } + Ok(ControlMessage::PointerShape(chunk)) => { + let (id, bytes, got) = shape.get_or_insert_with(|| { + (chunk.shape_id, vec![0; chunk.total_len as usize], 0) + }); + if *id == chunk.shape_id && held_shape != chunk.shape_id { + let at = chunk.offset as usize; + bytes[at..at + chunk.data.len()].copy_from_slice(&chunk.data); + *got += chunk.data.len(); + if *got >= bytes.len() { + held_shape = chunk.shape_id; + assert_eq!((chunk.width, chunk.height), (12, 19), "the arrow"); + } + } + } + _ => {} + }, + other => panic!("unexpected datagram from host: {other:?}"), + } + } + let distinct: std::collections::HashSet<_> = states.iter().collect(); + let load = std::fs::read_to_string("/proc/loadavg").unwrap_or_default(); + eprintln!( + "draws={draws}: {frames} frames, {} pointer states ({} distinct) in {WINDOW:?}, \ + {hidden_at_start} hidden before {load}", + states.len(), + distinct.len() + ); + if draws { + assert_ne!(held_shape, 0, "the pointer image arrived whole"); + // 60 fps for 3 s is 180 moves; leave room for a loaded runner. + assert!( + distinct.len() >= 60, + "{} distinct positions", + distinct.len() + ); + assert!( + states.iter().all(|p| path.contains(p)), + "states off the path" + ); + // The 750 ms keepalive, and nothing for the moves. + assert!( + frames <= 6, + "{frames} video frames while only the pointer moved" + ); + } else { + assert!( + states.is_empty() && shape.is_none(), + "an older iPad gets no pointer messages" + ); + assert!( + frames >= 60, + "only {frames} frames: each move must be a frame" + ); + } + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + let gone = Instant::now() + Duration::from_secs(5); + while shared.client_connected() { + assert!(Instant::now() < gone, "the session did not end after BYE"); + std::thread::sleep(Duration::from_millis(20)); + } + } + + shared.stop(); + supervisor_tx.send(SupervisorCommand::Shutdown).unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = supervisor.join(); + let _ = done_tx.send(()); + }); + done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("supervisor must shut down within 5s"); +} + +/// Review r2: a HELLO2 that only claims `FEATURE_DRAWS_POINTER` must not get +/// pointer messages (an image can be hundreds of kilobytes) before it has +/// answered, as audio waits for it; the pointer stays in its video until +/// then. Once it answers, pointer messages start. +#[test] +fn pointer_messages_start_once_the_ipad_has_answered() { + use eternal_wire::v2::control::FEATURE_DRAWS_POINTER; + let _env = test_env(); + init_test_tracing(); + let _ = ffmpeg_next::init(); + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_POINTER", "1"); + for name in ["ETERNAL_DROP", "ETERNAL_REORDER", "ETERNAL_JITTER_MS"] { + std::env::remove_var(name); + } + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".to_string()); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor = { + let shared = shared.clone(); + let tx = supervisor_tx.clone(); + std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + shared, + GpuInfo::software_fallback(), + tx, + supervisor_rx, + ); + }) + }; + + // The handshake alone, no ping: nothing confirms it got its HELLO_ACK. + let mut receiver = FakeReceiver::handshake( + listen_port, + CAP_DECODE_H264, + FEATURE_DRAWS_POINTER, + 60, + 0xE2E0_0185, + ); + let mut datagram = [0u8; 2048]; + let count = |receiver: &mut FakeReceiver, datagram: &mut [u8; 2048], for_: Duration| { + let (mut pointer, mut media) = (0, 0); + let until = Instant::now() + for_; + while Instant::now() < until { + let Ok((len, _)) = receiver.socket.recv_from(datagram) else { + continue; + }; + match classify(&datagram[..len]) { + Classified::Media { .. } => media += 1, + Classified::Control(_) => { + if matches!( + parse_control(&datagram[..len]).map(|(_, m)| m), + Ok(ControlMessage::PointerShape(_) | ControlMessage::PointerState(_)) + ) { + pointer += 1; + } + } + _ => {} + } + } + (pointer, media) + }; + // Silent after its HELLO2: the pointer moves every frame meanwhile. + let (pointer, media) = count(&mut receiver, &mut datagram, Duration::from_millis(1500)); + assert_eq!(pointer, 0, "pointer messages before the iPad answered"); + assert!(media > 0, "its first picture still came"); + assert!( + !shared.client_draws_pointer(), + "capture keeps the pointer in its video" + ); + + // Answered, without the image yet: messages start, and the pointer + // stays in the video until a report names the image. + receiver.send(&ControlMessage::ReceiverReport(ReceiverReport::default())); + let (pointer, _) = count(&mut receiver, &mut datagram, Duration::from_millis(1500)); + assert!(pointer > 0, "no pointer messages after it answered"); + assert!( + !shared.client_draws_pointer(), + "not before it holds the image" + ); + let shape_id = shared.pointer.latest().0.unwrap().id; + receiver.send(&ControlMessage::ReceiverReport(ReceiverReport { + pointer_shape_id: shape_id, + ..Default::default() + })); + let (pointer, _) = count(&mut receiver, &mut datagram, Duration::from_millis(1500)); + assert!( + pointer > 10, + "only {pointer} pointer messages once it held the image" + ); + assert!(shared.client_draws_pointer()); + + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + shared.stop(); + supervisor_tx.send(SupervisorCommand::Shutdown).unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = supervisor.join(); + let _ = done_tx.send(()); + }); + done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("supervisor must shut down within 5s"); +} diff --git a/host/tests/e2e_synthetic/second_screen.rs b/host/tests/e2e_synthetic/second_screen.rs index eac94743..02879b61 100644 --- a/host/tests/e2e_synthetic/second_screen.rs +++ b/host/tests/e2e_synthetic/second_screen.rs @@ -71,8 +71,11 @@ impl Host { } } + /// Unanswered until its first report: an answer runs the second screen's + /// next step at once, and with an instant attach these tests would see + /// no mirror before the switch. fn connect(&self, nonce: u32) -> FakeReceiver { - FakeReceiver::connect_as(self.port, CAP_DECODE_H264, 0, 0, nonce) + FakeReceiver::handshake(self.port, CAP_DECODE_H264, 0, 0, nonce) } fn status(&self) -> VddStatus { @@ -562,7 +565,10 @@ fn a_screen_moved_to_the_other_card_never_traps_the_ipad_on_a_blocked_mirror() { let _env = test_env(); synthetic_env(); let memory = memory_dir("moved-screen"); - let display = FakeDisplay::new(Some(Duration::ZERO), Duration::from_secs(10)); + // It attaches only once the mirror has shown, as below: an instant + // attach could switch the stream before the iPad decoded a mirror + // picture, and then no picture ever came after the first epoch. + let display = FakeDisplay::new(Some(Duration::from_secs(25)), Duration::from_secs(30)); let host = Host::start(&display, &memory, SystemTime::now()); // The fake clock skips the waits between display changes and the // attach wait; none of it saves a failure at the moved clock. @@ -574,8 +580,15 @@ fn a_screen_moved_to_the_other_card_never_traps_the_ipad_on_a_blocked_mirror() { // The second screen works, with the screen on the app's card. let mut receiver = host.connect(0x5707_2001); + let mut advanced = false; let pictures = watch(&mut receiver, Duration::from_secs(40), |pictures| { - skip_waits(); + if !advanced && !pictures.is_empty() && !display.toggles().is_empty() { + // Mirroring while the display attaches. Now its 25 s pass. + assert_eq!(epochs(pictures).len(), 1); + assert_eq!(host.status(), VddStatus::Attaching); + display.advance(Duration::from_secs(25)); + advanced = true; + } after_first_epoch(pictures) >= 10 }); assert_eq!(epochs(&pictures).len(), 2, "mirror, then the second screen"); diff --git a/host/tests/fixtures/README.md b/host/tests/fixtures/README.md index 767c2c01..028b7d6b 100644 --- a/host/tests/fixtures/README.md +++ b/host/tests/fixtures/README.md @@ -5,3 +5,9 @@ 25.5.2 (`Virtual Display Driver (HDR)/vdd_settings.xml`), byte for byte. It is MIT licensed by its authors. `host/src/vdd_settings.rs` tests that the host edits this file in place. + +`vdd_settings-25.5.2-cp949.xml` and `vdd_settings-25.5.2-cp936.xml` are that +file with a comment holding a folder path and a card name in non-ASCII text, +saved in Windows code page 949 and 936 as Notepad's ANSI does: the +`encoding='utf-8'` declaration stays. They were made with Python's `cp949` and +`gbk` codecs, so they do not depend on the Windows conversion under test. diff --git a/host/tests/fixtures/vdd_settings-25.5.2-cp936.xml b/host/tests/fixtures/vdd_settings-25.5.2-cp936.xml new file mode 100644 index 00000000..c42a794c --- /dev/null +++ b/host/tests/fixtures/vdd_settings-25.5.2-cp936.xml @@ -0,0 +1,98 @@ + + + + + 1 + + + Microsoft + + + + 60 + 90 + 120 + 144 + 165 + 244 + + + + 800 + 600 + 30 + + + 1366 + 768 + 30 + + + 1920 + 1080 + 30 + + + 2560 + 1440 + 30 + + + 3840 + 2160 + 30 + + + + + + true + false + + false + + + + + + + false + false + RGB + + + + true + + 128 + + 128 + + true + + 2 + + + + + false + + false + + false + + + \ No newline at end of file diff --git a/host/tests/fixtures/vdd_settings-25.5.2-cp949.xml b/host/tests/fixtures/vdd_settings-25.5.2-cp949.xml new file mode 100644 index 00000000..a31024c7 --- /dev/null +++ b/host/tests/fixtures/vdd_settings-25.5.2-cp949.xml @@ -0,0 +1,98 @@ + + + + + 1 + + + Microsoft ⺻ ̹ + + + + 60 + 90 + 120 + 144 + 165 + 244 + + + + 800 + 600 + 30 + + + 1366 + 768 + 30 + + + 1920 + 1080 + 30 + + + 2560 + 1440 + 30 + + + 3840 + 2160 + 30 + + + + + + true + false + + false + + + + + + + false + false + RGB + + + + true + + 128 + + 128 + + true + + 2 + + + + + false + + false + + false + + + \ No newline at end of file diff --git a/host/tests/frame_numbering.rs b/host/tests/frame_numbering.rs index abe5d7cc..9a1ec519 100644 --- a/host/tests/frame_numbering.rs +++ b/host/tests/frame_numbering.rs @@ -47,6 +47,7 @@ fn skipped_capture_frames_leave_no_gap_on_the_wire() { port, sender_shared, supervisor_tx, + 1, )) .unwrap(); }); diff --git a/host/tests/goodbye.rs b/host/tests/goodbye.rs index b669a9de..546fab36 100644 --- a/host/tests/goodbye.rs +++ b/host/tests/goodbye.rs @@ -1,13 +1,13 @@ -//! When Stop and update ends a stream, the iPad must hear the host's goodbye -//! however the transport stops, over UDP and over the USB tunnel. Drives the -//! real transport loop with a fake iPad on loopback. +//! When Stop and update ends a stream, or the app quits, the iPad must hear +//! the host's goodbye however the transport stops, over UDP and over the USB +//! tunnel. Drives the real transport loop with a fake iPad on loopback. use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; use eternal_host::control::SharedControl; use eternal_host::transport::link::{read_datagram, read_preamble, write_datagram, PeerId}; -use eternal_host::transport::session::ConfigSource; +use eternal_host::transport::session::{ConfigSource, StageFailure, StreamStop}; use eternal_host::transport::{self}; use eternal_wire::v2::control::{ encode_control, parse_control, ByeReason, ControlMessage, Hello2, StreamConfig, CAP_DECODE_H264, @@ -58,31 +58,45 @@ fn hello() -> ControlMessage { }) } -fn is_goodbye(bytes: &[u8]) -> bool { - matches!( - parse_control(bytes), - Ok((_, ControlMessage::Bye(ByeReason::HostShuttingDown))) - ) +fn is_bye(bytes: &[u8], reason: ByeReason) -> bool { + matches!(parse_control(bytes), Ok((_, ControlMessage::Bye(said))) if said == reason) } +/// The transport binds every address, so the port must be free on all. fn free_udp_port() -> u16 { - std::net::UdpSocket::bind("127.0.0.1:0") + std::net::UdpSocket::bind("0.0.0.0:0") .and_then(|s| s.local_addr()) .map(|a| a.port()) .unwrap() } -/// How the transport stops after Stop and update ended the stream. +/// How the stream ends. +#[derive(Clone, Copy)] +enum End { + /// Stop and update. + Update, + /// The app quits. + Quit, + /// The app quits for "Restart EternalMonitor". + Restart, + /// The encoder kept failing: it closed its channel as it exited, and the + /// supervisor, on hearing why, gave up (the storm brake). + GaveUp, +} + +/// How the transport stops after the stream ended. enum Stop { /// The host is closing: `running` goes false. Shutdown, - /// The encoder stopped first and closed its channel. + /// The encoder stopped first and closed its channel, then the host + /// closes. EncoderEof, } -/// Runs the transport with a UDP iPad, stops the stream for an update, stops -/// the transport the given way and counts the goodbyes the iPad received. -fn udp_goodbyes(stop: Stop) -> usize { +/// Runs the transport with a UDP iPad, ends the stream the given way, stops +/// the transport the given way and counts the goodbyes the iPad received +/// with `reason`. +fn udp_goodbyes(end: End, stop: Stop, reason: ByeReason) -> usize { let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); eternal_host::clock::init(); // Never reach a real iPad: this unreachable endpoint stands in for USB. @@ -105,20 +119,45 @@ fn udp_goodbyes(stop: Stop) -> usize { .handle_control(peer, 0, hello(), &Config, Instant::now()); assert_eq!(admitted.new_target, Some(peer)); let (nal_tx, nal_rx) = mpsc::channel(4); + let mut nal_tx = Some(nal_tx); let (_audio_tx, audio_rx) = mpsc::channel(4); let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); let control = shared.clone(); let transport = tokio::spawn(async move { - transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx) + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx, 1) .await .unwrap() }); // Let the loop take its first ticks. tokio::time::sleep(Duration::from_millis(40)).await; - assert!(shared.session.lock().begin_update(true)); + match end { + End::GaveUp => { + // The encoder thread drops its sender before its exit report + // reaches the supervisor. Longer than a liveness tick: the + // transport used to end here, before the goodbye existed. + drop(nal_tx.take()); + tokio::time::sleep(Duration::from_millis(400)).await; + eternal_host::diagnostics::stop_session( + &shared, + StreamStop::GaveUp(StageFailure::Encoder), + ); + } + End::Update => assert!(shared.session.lock().begin_update(true)), + End::Quit => eternal_host::diagnostics::log_session_end_at_quit(&shared), + End::Restart => { + shared + .handoff + .claim(eternal_host::handoff::Handoff::Restart) + .unwrap(); + eternal_host::diagnostics::log_session_end_at_quit(&shared); + } + } match stop { Stop::Shutdown => shared.stop(), - Stop::EncoderEof => drop(nal_tx), + Stop::EncoderEof => { + drop(nal_tx.take()); + shared.stop(); + } } // Only a backstop for a transport that never stops. It stops at its // next liveness tick, 250 ms away, but a loaded machine can leave @@ -136,7 +175,7 @@ fn udp_goodbyes(stop: Stop) -> usize { while let Ok(Ok((len, _))) = tokio::time::timeout(Duration::from_millis(100), ipad.recv_from(&mut bytes)).await { - goodbyes += usize::from(is_goodbye(&bytes[..len])); + goodbyes += usize::from(is_bye(&bytes[..len], reason)); } goodbyes }) @@ -145,14 +184,108 @@ fn udp_goodbyes(stop: Stop) -> usize { /// One lost datagram must not turn a planned update into a lost connection. #[test] fn udp_goodbye_is_a_short_burst() { - assert_eq!(udp_goodbyes(Stop::Shutdown), 3); + assert_eq!( + udp_goodbyes(End::Update, Stop::Shutdown, ByeReason::HostShuttingDown), + 3 + ); } /// The encoder can close its channel before the next liveness tick while the /// host shuts down; the goodbye must still go out. #[test] fn encoder_eof_still_says_goodbye() { - assert!(udp_goodbyes(Stop::EncoderEof) >= 1); + assert!(udp_goodbyes(End::Update, Stop::EncoderEof, ByeReason::HostShuttingDown) >= 1); +} + +/// Quitting used to end the session without a word: the iPad waited out a +/// lost signal and then tried to reconnect to a PC that had closed. It now +/// hears that the PC ended the session, the goodbye that keeps it from +/// reconnecting by itself. +#[test] +fn quitting_tells_the_ipad_the_pc_ended_the_session() { + assert_eq!( + udp_goodbyes(End::Quit, Stop::Shutdown, ByeReason::UserDisconnect), + 3 + ); +} + +/// The storm brake gives up after the encoder kept failing. The encoder +/// closes its channel before the supervisor hears why, so the transport must +/// still be there to send the goodbye the supervisor leaves. +#[test] +fn giving_up_on_a_failing_encoder_tells_the_ipad_to_connect_again() { + assert_eq!( + udp_goodbyes(End::GaveUp, Stop::Shutdown, ByeReason::Error), + 3 + ); +} + +/// Stop lands while a pipeline generation is starting: after the supervisor +/// re-armed `running`, before the new transport reopened to clients. The +/// stopped session's goodbye is current, and the transport going up must +/// send it on its way out. Startup used to take it for one left while no +/// transport ran and drop it, and the iPad timed out instead of showing the +/// card. +#[test] +fn a_stop_during_startup_still_says_goodbye() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + eternal_host::clock::init(); + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let goodbyes = runtime.block_on(async { + let ipad = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let port = free_udp_port(); + let shared = SharedControl::new(port, 10_000_000); + shared.pairing.lock().required = false; + // The new generation's `running`, re-armed before its threads start. + shared.running.store(true, Ordering::SeqCst); + let peer = PeerId::udp(ipad.local_addr().unwrap()); + // The session carried over from the previous generation. + shared + .session + .lock() + .handle_control(peer, 0, hello(), &Config, Instant::now()); + // Stop, as the supervisor's stop_pipeline does it. + eternal_host::diagnostics::stop_session(&shared, StreamStop::User); + shared.stop(); + let (_nal_tx, nal_rx) = mpsc::channel(4); + let (_audio_tx, audio_rx) = mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + let control = shared.clone(); + tokio::time::timeout( + Duration::from_secs(30), + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx, 1), + ) + .await + .expect("the transport stops") + .unwrap(); + assert!( + shared.session.lock().take_goodbye().is_none(), + "the goodbye was not left behind" + ); + let mut bytes = [0; 2048]; + let mut goodbyes = 0; + while let Ok(Ok((len, _))) = + tokio::time::timeout(Duration::from_millis(100), ipad.recv_from(&mut bytes)).await + { + goodbyes += usize::from(is_bye(&bytes[..len], ByeReason::UserDisconnect)); + } + goodbyes + }); + assert_eq!(goodbyes, 3); +} + +/// A restart hands over to a new copy that is back in seconds: the iPad is +/// told to connect again, at once. +#[test] +fn a_restart_tells_the_ipad_to_connect_again() { + assert_eq!( + udp_goodbyes(End::Restart, Stop::Shutdown, ByeReason::Error), + 3 + ); } /// Over USB the goodbye is queued for a writer task; the transport must wait @@ -161,6 +294,17 @@ fn encoder_eof_still_says_goodbye() { /// one goodbye arrives on a link that stays open. #[test] fn usb_goodbye_arrives_before_the_link_closes() { + usb_goodbye(End::Update, ByeReason::HostShuttingDown); +} + +/// Quitting over the cable: the iPad used to see the tunnel close and showed +/// an unplugged cable. +#[test] +fn quitting_over_usb_says_goodbye_before_the_link_closes() { + usb_goodbye(End::Quit, ByeReason::UserDisconnect); +} + +fn usb_goodbye(end: End, reason: ByeReason) { let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); eternal_host::clock::init(); let runtime = tokio::runtime::Builder::new_current_thread() @@ -182,7 +326,7 @@ fn usb_goodbye_arrives_before_the_link_closes() { let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); let control = shared.clone(); let transport = tokio::spawn(async move { - transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx) + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx, 1) .await .unwrap() }); @@ -204,7 +348,12 @@ fn usb_goodbye_arrives_before_the_link_closes() { } } tokio::time::sleep(Duration::from_millis(40)).await; - assert!(shared.session.lock().begin_update(true)); + match end { + End::Update => assert!(shared.session.lock().begin_update(true)), + End::Quit | End::Restart | End::GaveUp => { + eternal_host::diagnostics::log_session_end_at_quit(&shared) + } + } shared.stop(); let stopping = Instant::now(); tokio::time::timeout(Duration::from_secs(2), transport) @@ -217,7 +366,7 @@ fn usb_goodbye_arrives_before_the_link_closes() { let mut goodbyes = 0; loop { match tokio::time::timeout(Duration::from_millis(500), read_datagram(&mut ipad)).await { - Ok(Ok(bytes)) => goodbyes += usize::from(is_goodbye(&bytes)), + Ok(Ok(bytes)) => goodbyes += usize::from(is_bye(&bytes, reason)), Ok(Err(error)) => panic!("the USB hub closed the tunnel: {error}"), Err(_) => break, } diff --git a/host/tests/gpu_detect_fallback.rs b/host/tests/gpu_detect_fallback.rs new file mode 100644 index 00000000..97fd72cc --- /dev/null +++ b/host/tests/gpu_detect_fallback.rs @@ -0,0 +1,76 @@ +//! The real encode stage on the software facts startup uses when GPU +//! detection did not return in time: the processor encodes, and the window +//! says why. + +use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use eternal_host::capture::synthetic::render_synthetic_frame; +use eternal_host::capture::{frame_buffer, frame_slot, RawFrame}; +use eternal_host::control::SharedControl; +use eternal_host::encoder::{run_encode_stage, NAL_CHANNEL_CAPACITY}; +use eternal_host::gpu::GpuInfo; +use eternal_host::stats::PIPELINE_STATS; + +/// The stage and PIPELINE_STATS are process-global, so stages run one at a time. +static STAGE_LOCK: Mutex<()> = Mutex::new(()); + +/// Runs a few frames through a fresh encode stage with `gpu` and the +/// encoder override `chosen`, and returns the fallback notice's two flags. +fn notice_after_stage(gpu: GpuInfo, chosen: Option<&str>) -> (bool, bool) { + let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + ffmpeg_next::init().unwrap(); + let (producer, consumer) = frame_slot(); + let (tx, mut rx) = tokio::sync::mpsc::channel(NAL_CHANNEL_CAPACITY); + let shared = SharedControl::new(0, 8_000_000); + shared.running.store(true, Ordering::SeqCst); + *shared.encoder_override.lock() = chosen.map(str::to_string); + let generation = eternal_host::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let stage = std::thread::spawn(move || { + run_encode_stage(consumer, tx, shared, gpu, generation).map_err(|e| e.to_string()) + }); + let collector = std::thread::spawn(move || { + let mut packets = 0; + while rx.blocking_recv().is_some() { + packets += 1; + } + packets + }); + let (width, height) = (640, 360); + for number in 0..10 { + let mut data = frame_buffer((width * height * 4) as usize); + render_synthetic_frame(&mut data, width, height, number); + producer.publish(RawFrame { + frame_number: number, + timestamp: Instant::now(), + data: Arc::new(data), + width, + height, + cached: false, + }); + std::thread::sleep(Duration::from_millis(20)); + } + producer.close(); + stage.join().unwrap().expect("the encode stage failed"); + assert!(collector.join().unwrap() > 0, "no packets"); + let stats = PIPELINE_STATS.lock(); + (stats.using_software_fallback, stats.gpu_detection_stalled) +} + +#[test] +fn a_stalled_detection_encodes_on_the_processor_with_the_notice() { + let stalled = GpuInfo { + detection_stalled: true, + ..GpuInfo::software_fallback() + }; + assert_eq!(notice_after_stage(stalled.clone(), None), (true, true)); + // libx264 chosen in Settings is the user's choice, not a fallback. + assert_eq!(notice_after_stage(stalled, Some("libx264")), (false, false)); + // A detection that answered and found no graphics card: no notice, as + // before. + assert_eq!( + notice_after_stage(GpuInfo::software_fallback(), None), + (false, false) + ); +} diff --git a/host/tests/gui_snapshots.rs b/host/tests/gui_snapshots.rs index 2e547602..e5dcc433 100644 --- a/host/tests/gui_snapshots.rs +++ b/host/tests/gui_snapshots.rs @@ -247,6 +247,7 @@ fn connect_app( nacks_sent: 10, audio_packets_lost: 0, audio_buffer_ms: 60, + pointer_shape_id: 0, }), &Fixture, now, diff --git a/host/tests/hello_standby.rs b/host/tests/hello_standby.rs new file mode 100644 index 00000000..f671e9eb --- /dev/null +++ b/host/tests/hello_standby.rs @@ -0,0 +1,1132 @@ +//! The PC answers an iPad's HELLO2 while it isn't streaming: in the pause +//! before an automatic restart, across a restart, after Stop, and after the +//! storm brake. A stopped PC also proves it is the saved PC to an iPad that +//! found it at a new address. Drives the REAL supervisor with synthetic +//! capture and libx264, talked to by fake iPads over loopback UDP on +//! ephemeral ports. + +use std::collections::BTreeMap; +use std::net::UdpSocket; +use std::sync::atomic::Ordering; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +use eternal_host::control::{SharedControl, SupervisorCommand}; +use eternal_host::gpu::GpuInfo; +use eternal_host::pipeline; +use eternal_host::stats::PIPELINE_STATS; +use eternal_host::transport::session::NotStreaming; +use eternal_wire::v2::control::{ + encode_control, parse_control, ControlMessage, Hello2, HelloAck, HelloStatus, CAP_DECODE_H264, + FEATURE_STANDBY_STATUS, +}; + +/// Tests share process-global env (ETERNAL_*) and pipeline state; run them +/// one at a time. +static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// An iPad on 0.3.5 or later, and one on 0.3.4 or older. +const NEW: u16 = FEATURE_STANDBY_STATUS; +const OLD: u16 = 0; + +struct Host { + shared: SharedControl, + port: u16, + commands: mpsc::Sender, + supervisor: Option>, + _env: std::sync::MutexGuard<'static, ()>, +} + +impl Host { + fn start(gpu: GpuInfo, pairing: bool) -> Self { + Self::start_with_backoff(gpu, pairing, 500) + } + + /// With the first automatic restart `backoff_ms` after a failure. + fn start_with_backoff(gpu: GpuInfo, pairing: bool, backoff_ms: u64) -> Self { + Self::start_with(gpu, backoff_ms, |shared| { + shared.pairing.lock().required = pairing + }) + } + + /// With `configure` applied before the supervisor starts. + fn start_with(gpu: GpuInfo, backoff_ms: u64, configure: impl FnOnce(&SharedControl)) -> Self { + let env = ENV_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + eternal_host::supervisor::TEST_BACKOFF_BASE_MS.store(backoff_ms, Ordering::SeqCst); + eternal_host::clock::init(); + // Never claim a physically connected iPad. + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_SIZE", "640x360"); + for name in [ + "ETERNAL_DROP", + "ETERNAL_REORDER", + "ETERNAL_JITTER_MS", + "ETERNAL_FAULT_ENCODER_AFTER", + ] { + std::env::remove_var(name); + } + let _ = ffmpeg_next::init(); + let port = UdpSocket::bind("0.0.0.0:0") + .unwrap() + .local_addr() + .unwrap() + .port(); + let shared = SharedControl::new(port, pipeline::DEFAULT_BITRATE_BPS); + configure(&shared); + let (commands, command_rx) = mpsc::channel(); + let supervisor = { + let (shared, commands) = (shared.clone(), commands.clone()); + std::thread::spawn(move || { + eternal_host::supervisor::run(port, shared, gpu, commands, command_rx); + }) + }; + Self { + shared, + port, + commands, + supervisor: Some(supervisor), + _env: env, + } + } + + fn send(&self, command: SupervisorCommand) { + self.commands.send(command).unwrap(); + } + + fn streaming(&self) -> bool { + PIPELINE_STATS.lock().pipeline_running + } + + /// Waits for the supervisor to stop the stream after Stop. + fn wait_until_stopped(&self) { + let deadline = Instant::now() + Duration::from_secs(20); + while self.shared.session.lock().not_streaming() != Some(NotStreaming::Stopped) { + assert!(Instant::now() < deadline, "the stream never stopped"); + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +impl Drop for Host { + fn drop(&mut self) { + eternal_host::supervisor::TEST_BACKOFF_BASE_MS.store(500, Ordering::SeqCst); + eternal_host::transport::session::TEST_RESTART_HOLD_MS.store( + eternal_host::transport::session::RESTART_HOLD_LIMIT.as_millis() as u64, + Ordering::SeqCst, + ); + #[cfg(debug_assertions)] + eternal_host::network::set_test_adapters(None); + self.shared.stop(); + let _ = self.commands.send(SupervisorCommand::Shutdown); + if let Some(supervisor) = self.supervisor.take() { + let _ = supervisor.join(); + } + } +} + +/// The supervisor's log lines since [`capture_supervisor_log`], from every +/// thread. +#[cfg(debug_assertions)] +static SUPERVISOR_LOG: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + +#[cfg(debug_assertions)] +struct SupervisorLog; + +#[cfg(debug_assertions)] +impl tracing_subscriber::Layer for SupervisorLog { + fn on_event(&self, event: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { + struct Message(Option); + impl tracing::field::Visit for Message { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + if field.name() == "message" { + self.0 = Some(format!("{value:?}")); + } + } + } + let mut message = Message(None); + event.record(&mut message); + if let Some(line) = message.0 { + SUPERVISOR_LOG.lock().unwrap().push(line); + } + } +} + +/// Starts capturing the supervisor's log lines afresh. The capture is the +/// process's subscriber, and sees nothing else; `ENV_LOCK` runs one test at +/// a time. +#[cfg(debug_assertions)] +fn capture_supervisor_log() { + use tracing_subscriber::prelude::*; + static INSTALLED: std::sync::Once = std::sync::Once::new(); + INSTALLED.call_once(|| { + let only_the_supervisor = tracing_subscriber::filter::filter_fn(|metadata| { + metadata.target() == "eternal_host::supervisor" + }); + let _ = tracing_subscriber::registry() + .with(SupervisorLog.with_filter(only_the_supervisor)) + .try_init(); + }); + SUPERVISOR_LOG.lock().unwrap().clear(); +} + +/// Waits for the supervisor to log `line`. +#[cfg(debug_assertions)] +fn wait_for_supervisor_line(line: &str) { + let deadline = Instant::now() + Duration::from_secs(20); + loop { + let logged = SUPERVISOR_LOG.lock().unwrap().clone(); + if logged.iter().any(|logged| logged == line) { + return; + } + assert!( + Instant::now() < deadline, + "the supervisor never logged {line:?}: {logged:?}" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + +/// A GPU whose encoder FFmpeg doesn't have: every generation fails as it +/// starts, as a broken driver does. +fn broken_gpu() -> GpuInfo { + GpuInfo { + encoder_name: "no_such_encoder".into(), + ..GpuInfo::software_fallback() + } +} + +struct FakeIpad { + socket: UdpSocket, + host: String, + device_id: u64, + feature_caps: u16, + auth_token: [u8; 16], + /// The bytes of the last HELLO2 sent and of the last ACK received. + sizes: std::cell::Cell<(usize, usize)>, +} + +impl FakeIpad { + fn new(host: &Host, device_id: u64, feature_caps: u16) -> Self { + Self::at(std::net::Ipv4Addr::LOCALHOST, host, device_id, feature_caps) + } + + /// An iPad that reaches the PC at `ip`. + fn at(ip: std::net::Ipv4Addr, host: &Host, device_id: u64, feature_caps: u16) -> Self { + let socket = UdpSocket::bind((ip, 0)).unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let _ = socket2::SockRef::from(&socket).set_recv_buffer_size(4 * 1024 * 1024); + Self { + socket, + host: format!("{ip}:{}", host.port), + device_id, + feature_caps, + auth_token: [0; 16], + sizes: Default::default(), + } + } + + fn hello(&self, nonce: u32) { + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: nonce, + listen_port: self.socket.local_addr().unwrap().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: self.feature_caps, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 640, + screen_pt_h: 360, + refresh_hz: 60, + // A long name, so a 0.3.4 iPad's HELLO2 is larger than an + // answer carrying any PC name a test machine has. + device_name: "standby fake iPad named like a real one".to_string(), + device_id: self.device_id, + preferred_fps: 0, + auth_token: self.auth_token, + pairing_code: 0, + }); + self.send(&encode_control(0, nonce, &hello)); + } + + fn send(&self, datagram: &[u8]) { + self.socket.send_to(datagram, &self.host).unwrap(); + self.sizes.set((datagram.len(), 0)); + } + + /// The next HELLO_ACK for `nonce` within `wait`, skipping video, + /// heartbeats and older answers. + fn ack(&self, nonce: u32, wait: Duration) -> Option { + let deadline = Instant::now() + wait; + let mut buffer = [0; 2048]; + while Instant::now() < deadline { + if let Ok(len) = self.socket.recv(&mut buffer) { + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&buffer[..len]) { + if ack.client_nonce == nonce { + self.sizes.set((self.sizes.get().0, len)); + return Some(ack); + } + } + } + } + None + } + + /// Asks once, as an iPad's first HELLO2 does. The answer must be prompt. + fn ask(&self, nonce: u32) -> Option { + self.hello(nonce); + self.ack(nonce, Duration::from_secs(2)) + } + + /// Keeps asking, as a waiting iPad does, until the PC answers at all. + fn ask_until_answered(&self, nonce: u32, within: Duration) -> Option { + let deadline = Instant::now() + within; + while Instant::now() < deadline { + self.hello(nonce); + if let Some(ack) = self.ack(nonce, Duration::from_millis(500)) { + return Some(ack); + } + } + None + } + + /// Keeps asking, as an iPad does, until the PC streams to it. + fn connect(&self, nonce: u32) -> HelloAck { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + assert!(Instant::now() < deadline, "never connected"); + self.hello(nonce); + if let Some(ack) = self.ack(nonce, Duration::from_millis(250)) { + if ack.status == HelloStatus::Ok { + return ack; + } + } + } + } +} + +/// The smallest HELLO2 there is: no name, no device id or token, no +/// feature bits: 38 bytes. +fn minimal_hello(nonce: u32) -> Vec { + encode_control( + 0, + nonce, + &ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: nonce, + listen_port: 0, + decoder_caps: CAP_DECODE_H264, + feature_caps: 0, + screen_px_w: 1, + screen_px_h: 1, + screen_pt_w: 1, + screen_pt_h: 1, + refresh_hz: 60, + device_name: String::new(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }), + ) +} + +/// While no stream runs, the PC never answers with more bytes than it was +/// sent. A small HELLO2 from an unpaired sender hears nothing; an iPad that +/// asks for standby answers pads its HELLO2 and hears every answer. +#[test] +fn a_stopped_pc_never_answers_with_more_than_it_was_sent() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + + let small_sender = FakeIpad::new(&host, 1, OLD); + let small = minimal_hello(1); + assert_eq!(small.len(), 38); + small_sender.send(&small); + assert!( + small_sender.ack(1, Duration::from_secs(2)).is_none(), + "an answer larger than a 38-byte HELLO2 went out" + ); + + let unpaired = FakeIpad::new(&host, 2, NEW); + let ack = unpaired.ask(2).expect("a padded HELLO2 is answered"); + assert_eq!(ack.status, HelloStatus::Unauthorized); + let (sent, received) = unpaired.sizes.get(); + assert!(received <= sent, "{received} > {sent}"); + + let mut paired = FakeIpad::new(&host, 3, NEW); + paired.auth_token = host.shared.pairing.lock().token(); + let ack = paired.ask(3).expect("a padded HELLO2 is answered"); + assert_eq!(ack.status, HelloStatus::Stopped); + let (sent, received) = paired.sizes.get(); + assert!(received <= sent, "{received} > {sent}"); + + // A paired 0.3.4 iPad: an answer that fits goes out, one that doesn't + // is today's silence. + let mut old = FakeIpad::new(&host, 4, OLD); + old.auth_token = paired.auth_token; + if let Some(ack) = old.ask(4) { + assert_eq!(ack.status, HelloStatus::Error); + let (sent, received) = old.sizes.get(); + assert!(received <= sent, "{received} > {sent}"); + } +} + +/// After Stop, a new iPad hears that the PC isn't streaming, an old one is +/// turned down at once instead of timing out, and nothing starts by itself. +/// After Start, the waiting iPad connects. +#[test] +fn a_stopped_pc_says_so_and_streams_again_only_after_start() { + let host = Host::start(GpuInfo::software_fallback(), false); + let ipad = FakeIpad::new(&host, 1, NEW); + assert_eq!(ipad.connect(1).status, HelloStatus::Ok); + + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let ack = ipad.ask(2).expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Stopped); + assert_eq!(ack.session_id, 0); + let old = FakeIpad::new(&host, 2, OLD); + let ack = old.ask(3).expect("a stopped PC must answer an old iPad"); + assert_eq!(ack.status, HelloStatus::Error); + + // Asking again and again never starts the stream: only Start does. + for nonce in 4..10 { + let ack = ipad.ask(nonce).expect("still answered"); + assert_eq!(ack.status, HelloStatus::Stopped); + std::thread::sleep(Duration::from_millis(200)); + } + assert!(!host.streaming(), "a HELLO2 started the stream after Stop"); + + host.send(SupervisorCommand::Start); + let ack = ipad.connect(10); + assert_ne!(ack.session_id, 0); +} + +/// An unpaired iPad learns nothing new from a stopped PC: it is asked for +/// the pairing code, as when it streams. A paired one hears it's stopped. +#[test] +fn a_stopped_pc_still_asks_an_unpaired_ipad_to_pair() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let unpaired = FakeIpad::new(&host, 1, NEW); + let ack = unpaired.ask(1).expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Unauthorized); + assert!(ack.reason.is_empty()); + let mut paired = FakeIpad::new(&host, 2, NEW); + paired.auth_token = host.shared.pairing.lock().token(); + let ack = paired.ask(2).expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Stopped); + assert_eq!(ack.auth_token, [0; 16], "no token while stopped"); +} + +/// While the supervisor waits out the pause before an automatic restart, a +/// new iPad hears "restarting", and an old iPad is never turned down: it +/// hears nothing and keeps retrying, as before. +#[test] +fn a_pc_restarting_after_a_failure_says_so() { + let host = Host::start(broken_gpu(), false); + let ipad = FakeIpad::new(&host, 1, NEW); + let old = FakeIpad::new(&host, 2, OLD); + let deadline = Instant::now() + Duration::from_secs(20); + let mut restarting = 0; + let mut old_answers = Vec::new(); + for nonce in 1.. { + assert!(Instant::now() < deadline, "never heard Restarting"); + if let Some(ack) = ipad.ask(nonce) { + restarting += usize::from(ack.status == HelloStatus::Restarting); + } + old.hello(nonce); + if let Some(ack) = old.ack(nonce, Duration::from_millis(50)) { + old_answers.push(ack.status); + } + if restarting >= 3 { + break; + } + } + assert!( + old_answers.iter().all(|status| *status == HelloStatus::Ok), + "an old iPad must not be turned down mid-restart: {old_answers:?}" + ); +} + +/// Every HELLO2 sent across a Restart is answered: the port stays bound +/// while it changes hands, so none falls into the gap. +#[test] +fn no_hello_is_lost_across_a_restart() { + let host = Host::start(GpuInfo::software_fallback(), false); + let ipad = FakeIpad::new(&host, 1, NEW); + assert_eq!(ipad.connect(1).status, HelloStatus::Ok); + + const SENT: u32 = 300; + let mut answers = BTreeMap::new(); + let mut buffer = [0; 2048]; + let mut collect = |ipad: &FakeIpad, answers: &mut BTreeMap| { + while let Ok(len) = ipad.socket.recv(&mut buffer) { + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&buffer[..len]) { + answers.insert(ack.client_nonce, ack.status); + } + } + }; + for nonce in 100..100 + SENT { + if nonce == 150 { + host.send(SupervisorCommand::Restart); + } + ipad.hello(nonce); + std::thread::sleep(Duration::from_millis(5)); + if nonce % 10 == 0 { + collect(&ipad, &mut answers); + } + } + let deadline = Instant::now() + Duration::from_secs(10); + while answers.len() < SENT as usize && Instant::now() < deadline { + collect(&ipad, &mut answers); + } + let missing: Vec<_> = (100..100 + SENT) + .filter(|nonce| !answers.contains_key(nonce)) + .collect(); + assert!(missing.is_empty(), "unanswered HELLO2 nonces: {missing:?}"); + assert!( + answers + .values() + .all(|status| matches!(status, HelloStatus::Ok | HelloStatus::Restarting)), + "{answers:?}" + ); +} + +/// After repeated failures the storm brake stops the stream until someone +/// clicks Start. A new iPad hears why, an old one is turned down, and the +/// stream doesn't start by itself. +#[test] +fn the_storm_brake_says_why_and_waits_for_start() { + let host = Host::start_with_backoff(broken_gpu(), false, 1); + let ipad = FakeIpad::new(&host, 1, NEW); + let deadline = Instant::now() + Duration::from_secs(30); + let failed = (1..) + .find_map(|nonce| { + assert!(Instant::now() < deadline, "never heard Failed"); + ipad.ask(nonce) + .filter(|ack| ack.status == HelloStatus::Failed) + }) + .unwrap(); + assert_eq!(failed.reason, "encoder"); + assert_eq!(failed.session_id, 0); + let (sent, received) = ipad.sizes.get(); + assert!(received <= sent, "{received} > {sent}"); + let old = FakeIpad::new(&host, 2, OLD); + assert_eq!( + old.ask(1) + .expect("a failed PC must answer an old iPad") + .status, + HelloStatus::Error + ); + std::thread::sleep(Duration::from_secs(1)); + assert!(!host.streaming()); + let ack = ipad.ask(1000).expect("still answered"); + assert_eq!(ack.status, HelloStatus::Failed); +} + +/// A stream that finds its chosen network changed as Stop stops it asks to +/// start again on the new address. That request comes from a stream the +/// user already stopped: the PC stays stopped, and only Start streams. +/// +/// The session lock holds the stream between its bind and its first look +/// at the network, so it always looks after Stop, on the new address. +#[cfg(debug_assertions)] +#[test] +fn a_stream_stopping_as_its_network_changes_stays_stopped() { + use eternal_host::network::{set_test_adapters, Adapter, Kind, ListenOn}; + let wifi = |up| { + vec![Adapter { + name: "Wi-Fi".into(), + kind: Kind::Wifi, + up, + gateway: true, + ipv4: vec![std::net::Ipv4Addr::new(192, 168, 1, 20)], + }] + }; + let host = Host::start_with(GpuInfo::software_fallback(), 500, |shared| { + shared.pairing.lock().required = false; + *shared.listen_on.lock() = ListenOn::Adapter("Wi-Fi".into()); + set_test_adapters(Some(wifi(false))); + std::mem::forget(shared.session.lock()); + }); + capture_supervisor_log(); + // The stream binds this PC only, the network being down, then waits + // for the session. + let shown = format!("unknown:{}", host.port); + let deadline = Instant::now() + Duration::from_secs(20); + while PIPELINE_STATS.lock().listen_addr != shown { + assert!(Instant::now() < deadline, "the stream never bound"); + std::thread::sleep(Duration::from_millis(10)); + } + host.send(SupervisorCommand::Stop); + set_test_adapters(Some(wifi(true))); + // SAFETY: the guard forgotten above was taken on this lock, and nothing + // else holds it. + unsafe { host.shared.session.force_unlock() }; + + host.wait_until_stopped(); + // The stream asked after Stop, and the supervisor turned it down. + wait_for_supervisor_line("A stream that no longer runs asked to start again: left as it is"); + assert_eq!( + host.shared.session.lock().not_streaming(), + Some(NotStreaming::Stopped), + "the stopped stream restarted itself" + ); + assert!(!host.streaming(), "the stopped stream restarted itself"); +} + +/// A stopped PC listens where iPads can reach it, as the stream does: on +/// this PC only while the chosen network is down, on the network's address +/// once it is back, on this PC again when it goes, and on every network +/// once Listen on says so. It never starts the stream to do so. +#[cfg(debug_assertions)] +#[test] +fn a_stopped_pc_follows_the_chosen_network() { + use eternal_host::network::{default_route, set_test_adapters, Adapter, Kind, ListenOn}; + use std::net::IpAddr; + // Binding the network's address needs a real one. + let Some(IpAddr::V4(ip)) = default_route().filter(|ip| !ip.is_loopback()) else { + eprintln!("SKIP: no IPv4 default route to stand in for the chosen network"); + return; + }; + let wifi = move |up| { + vec![Adapter { + name: "Wi-Fi".into(), + kind: Kind::Wifi, + up, + gateway: true, + ipv4: vec![ip], + }] + }; + let host = Host::start_with(GpuInfo::software_fallback(), 500, |shared| { + shared.pairing.lock().required = false; + *shared.listen_on.lock() = ListenOn::Adapter("Wi-Fi".into()); + set_test_adapters(Some(wifi(false))); + }); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let here = FakeIpad::new(&host, 1, NEW); + let lan = FakeIpad::at(ip, &host, 2, NEW); + let ack = here + .ask(1) + .expect("answered on this PC while the network is down"); + assert_eq!(ack.status, HelloStatus::Stopped); + + set_test_adapters(Some(wifi(true))); + let ack = lan + .ask_until_answered(2, Duration::from_secs(20)) + .expect("never answered on the network's address once it came back"); + assert_eq!(ack.status, HelloStatus::Stopped); + + set_test_adapters(Some(wifi(false))); + let ack = here + .ask_until_answered(3, Duration::from_secs(20)) + .expect("never answered on this PC once the network went"); + assert_eq!(ack.status, HelloStatus::Stopped); + + // Listen on goes back to every network while stopped: the network's + // address answers again, though the adapter is still down. + *host.shared.listen_on.lock() = ListenOn::All; + let ack = lan + .ask_until_answered(4, Duration::from_secs(20)) + .expect("never answered on every network after Listen on changed to All"); + assert_eq!(ack.status, HelloStatus::Stopped); + assert!( + !host.streaming(), + "following the network started the stream" + ); + assert_eq!( + host.shared.session.lock().not_streaming(), + Some(NotStreaming::Stopped) + ); +} + +/// A stopped PC still proves it is the saved PC to an iPad that found it +/// at a new address, within the same bounds as while it streams. The PC's +/// home network address is a test listing, so debug builds only. +#[cfg(debug_assertions)] +mod identify { + use super::*; + use eternal_host::network::{ + home_addresses, listed_home_addresses, set_test_adapters, Adapter, Kind, + }; + use eternal_wire::v2::control::{identity_message, Identify, Identity, IDENTIFY_VERSION}; + + impl FakeIpad { + /// Asks the PC to prove it is the saved PC, as an iPad that found it at a + /// new address does. + fn identify(&self, nonce: [u8; 16]) { + let identify = ControlMessage::Identify(Identify { + version: IDENTIFY_VERSION, + nonce, + }); + self.send(&encode_control(0, 1, &identify)); + } + + /// The next IDENTITY for `nonce` within `wait`, skipping anything else. + fn identity(&self, nonce: [u8; 16], wait: Duration) -> Option { + let deadline = Instant::now() + wait; + let mut buffer = [0; 2048]; + while Instant::now() < deadline { + if let Ok(len) = self.socket.recv(&mut buffer) { + if let Ok((_, ControlMessage::Identity(identity))) = + parse_control(&buffer[..len]) + { + if identity.nonce == nonce { + self.sizes.set((self.sizes.get().0, len)); + return Some(identity); + } + } + } + } + None + } + + /// Asks up to 8 times, 250 ms apart, as an iPad's check does, and + /// returns the first answer. + fn check(&self, nonce: [u8; 16]) -> Option { + (0..8).find_map(|_| { + self.identify(nonce); + self.identity(nonce, Duration::from_millis(250)) + }) + } + } + + /// The address the test PC lists as its home network's. It listens on + /// every address, so an iPad that found it there reaches it on loopback. + const HOME: std::net::Ipv4Addr = std::net::Ipv4Addr::new(192, 168, 1, 20); + + /// Makes [`HOME`] the PC's only home network address. + fn set_home_adapter() { + set_test_adapters(Some(vec![Adapter { + name: "Wi-Fi".into(), + kind: Kind::Wifi, + up: true, + gateway: true, + ipv4: vec![HOME], + }])); + } + + /// Makes [`HOME`] the PC's only home network address, and waits until a + /// fresh listing has it. + fn list_home_address() { + set_home_adapter(); + let deadline = Instant::now() + Duration::from_secs(10); + while home_addresses() != [HOME] { + assert!( + Instant::now() < deadline, + "the home address was never listed" + ); + std::thread::sleep(Duration::from_millis(5)); + } + } + + /// What the iPad checks before it moves to `asked`: the proof is the + /// pairing token's over its nonce and the port and addresses the answer + /// names, and those are the port and one of the addresses it asked at. + fn ipad_accepts( + identity: &Identity, + token: [u8; 16], + nonce: [u8; 16], + asked: std::net::SocketAddrV4, + ) -> bool { + let key = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, &token); + let signed = identity_message(&nonce, identity.port, &identity.addresses); + ring::hmac::verify(&key, &signed, &identity.proof).is_ok() + && identity.port == asked.port() + && identity.addresses.contains(&asked.ip().octets()) + } + + /// A stopped PC that changed address proves it is the saved PC, so the iPad + /// moves to it and hears that it's stopped. The answer is no larger than + /// the request, and nothing starts. + #[test] + fn a_stopped_pc_proves_it_is_the_saved_pc() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let mut ipad = FakeIpad::new(&host, 1, NEW); + let token = host.shared.pairing.lock().token(); + + list_home_address(); + let identity = ipad + .check([1; 16]) + .expect("a stopped PC must answer IDENTIFY"); + let asked = std::net::SocketAddrV4::new(HOME, host.port); + assert!( + ipad_accepts(&identity, token, [1; 16], asked), + "{identity:?}" + ); + assert!(!ipad_accepts(&identity, [9; 16], [1; 16], asked)); + assert_eq!(ipad.sizes.get(), (84, 84)); + + assert!(!host.streaming(), "an IDENTIFY started the stream"); + assert_eq!( + host.shared.session.lock().not_streaming(), + Some(NotStreaming::Stopped) + ); + ipad.auth_token = token; + let ack = ipad.ask(2).expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Stopped); + } + + /// Something at another address that passes the iPad's IDENTIFY on to the + /// stopped PC gets the PC's real answer back, and the iPad turns it down: + /// it names the PC's own port, and no rewrite of the port or addresses + /// keeps the proof. + #[test] + fn a_forwarded_identify_fails_the_ipads_check_while_stopped() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let token = host.shared.pairing.lock().token(); + let forwarder = UdpSocket::bind("127.0.0.1:0").unwrap(); + forwarder + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let std::net::SocketAddr::V4(forwarder_addr) = forwarder.local_addr().unwrap() else { + unreachable!() + }; + let mut ipad = FakeIpad::new(&host, 1, NEW); + ipad.host = forwarder_addr.to_string(); + // Passes what the iPad sends on to the PC, and the PC's answers back. + let relaying = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + let relay = { + let relaying = relaying.clone(); + let pc = std::net::SocketAddr::from(([127, 0, 0, 1], host.port)); + std::thread::spawn(move || { + let mut buffer = [0; 2048]; + let mut asker = None; + while relaying.load(Ordering::SeqCst) { + let Ok((len, from)) = forwarder.recv_from(&mut buffer) else { + continue; + }; + let to = if from == pc { + asker + } else { + asker = Some(from); + Some(pc) + }; + if let Some(to) = to { + let _ = forwarder.send_to(&buffer[..len], to); + } + } + }) + }; + + list_home_address(); + let identity = ipad.check([1; 16]); + relaying.store(false, Ordering::SeqCst); + relay.join().unwrap(); + let identity = identity.expect("a stopped PC must answer a forwarded IDENTIFY"); + + // It is the PC's real answer, for the PC's own address... + let pc = std::net::SocketAddrV4::new(HOME, host.port); + assert!(ipad_accepts(&identity, token, [1; 16], pc)); + // ...so not for the address the iPad asked at. + assert!(!ipad_accepts(&identity, token, [1; 16], forwarder_addr)); + let mut rewritten = identity.clone(); + rewritten.port = forwarder_addr.port(); + rewritten.addresses = vec![forwarder_addr.ip().octets()]; + assert!(!ipad_accepts(&rewritten, token, [1; 16], forwarder_addr)); + } + + /// A stopped PC answers no more IDENTIFYs than a streaming one: a burst + /// from one address gets the per-address limit, and the PC still answers + /// HELLO2. + #[test] + fn identify_answers_stay_bounded_while_stopped() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let mut ipad = FakeIpad::new(&host, 1, NEW); + + list_home_address(); + for nonce in 0..20u8 { + ipad.identify([nonce; 16]); + } + let mut answers = 0; + let mut buffer = [0; 2048]; + let deadline = Instant::now() + Duration::from_millis(1500); + while Instant::now() < deadline { + if let Ok(len) = ipad.socket.recv(&mut buffer) { + answers += usize::from(matches!( + parse_control(&buffer[..len]), + Ok((_, ControlMessage::Identity(_))) + )); + } + } + // Per asking address and 10 s window (`IDENTIFY_PER_SOURCE`). + assert_eq!(answers, 8); + + ipad.auth_token = host.shared.pairing.lock().token(); + let ack = ipad.ask(1).expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Stopped); + } + + /// A stopped PC signs no address listing older than its 2 s freshness: the + /// IDENTIFY that finds it expired goes unanswered, the next one is answered + /// from the fresh listing it started. + #[test] + fn a_stopped_pc_signs_no_expired_listing() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let ipad = FakeIpad::new(&host, 1, NEW); + + list_home_address(); + std::thread::sleep(Duration::from_millis(2200)); + ipad.identify([1; 16]); + assert!( + ipad.identity([1; 16], Duration::from_secs(1)).is_none(), + "signed an expired listing" + ); + ipad.identify([2; 16]); + let identity = ipad + .identity([2; 16], Duration::from_secs(2)) + .expect("never answered from a fresh listing"); + assert_eq!(identity.addresses, [HOME.octets()]); + } + + /// The standby lists the home network addresses as it starts on every + /// address, as the stream does, so the first IDENTIFY after Stop is + /// answered though the stream's listing expired long ago. + #[test] + fn the_first_identify_after_stop_is_answered() { + let host = Host::start_with(GpuInfo::software_fallback(), 500, |shared| { + shared.pairing.lock().required = true; + set_home_adapter(); + }); + let deadline = Instant::now() + Duration::from_secs(20); + while !host.streaming() { + assert!(Instant::now() < deadline, "never streamed"); + std::thread::sleep(Duration::from_millis(10)); + } + std::thread::sleep(Duration::from_millis(2200)); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let mut ipad = FakeIpad::new(&host, 1, NEW); + ipad.auth_token = host.shared.pairing.lock().token(); + // Answered: the standby holds the port. + let ack = ipad + .ask_until_answered(1, Duration::from_secs(10)) + .expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Stopped); + // Its listing runs on a thread of its own. Waiting for it starts + // none, so only the standby's can come. + let deadline = Instant::now() + Duration::from_secs(10); + while listed_home_addresses().is_none() { + assert!( + Instant::now() < deadline, + "the standby never listed the home network addresses" + ); + std::thread::sleep(Duration::from_millis(5)); + } + ipad.identify([1; 16]); + assert!( + ipad.identity([1; 16], Duration::from_secs(2)).is_some(), + "the first IDENTIFY after Stop went unanswered" + ); + } +} + +/// What a streaming iPad heard over a stretch of time. +#[derive(Default, Debug)] +struct Heard { + /// When each heartbeat for the session came. + heartbeats: Vec, + /// Each heartbeat's notice, with when it came. + notices: Vec<(Instant, u8)>, + /// Each media datagram for the session: when, its epoch, and whether + /// it was part of a keyframe. + media: Vec<(Instant, u32, bool)>, + /// Goodbyes for the session. + goodbyes: Vec, +} + +impl FakeIpad { + /// Streams as an iPad does for `within`, or until `done` says so: + /// reports every 250 ms for `session_id`, and notes what comes back. + fn stream(&self, session_id: u32, within: Duration, done: impl Fn(&Heard) -> bool) -> Heard { + use eternal_wire::v2::control::ReceiverReport; + use eternal_wire::v2::media::MediaHeader; + let mut heard = Heard::default(); + let deadline = Instant::now() + within; + let mut next_report = Instant::now(); + let mut seq = 100; + let mut buffer = [0; 2048]; + while Instant::now() < deadline && !done(&heard) { + if Instant::now() >= next_report { + seq += 1; + let report = ControlMessage::ReceiverReport(ReceiverReport { + frames_complete: 1, + ..ReceiverReport::default() + }); + self.send(&encode_control(session_id, seq, &report)); + next_report += Duration::from_millis(250); + } + let Ok(len) = self.socket.recv(&mut buffer) else { + continue; + }; + let bytes = &buffer[..len]; + let at = Instant::now(); + match eternal_wire::v2::classify(bytes) { + eternal_wire::v2::Classified::Control(_) => match parse_control(bytes) { + Ok((header, ControlMessage::Heartbeat(hb))) + if header.session_id == session_id => + { + heard.heartbeats.push(at); + heard.notices.push((at, hb.notice)); + } + Ok((header, ControlMessage::Bye(reason))) + if header.session_id == session_id => + { + heard.goodbyes.push(reason) + } + _ => {} + }, + eternal_wire::v2::Classified::Media { .. } => { + if let Ok((header, _)) = MediaHeader::decode(bytes) { + if header.session_id == session_id { + heard + .media + .push((at, header.stream_epoch, header.is_keyframe)); + } + } + } + _ => {} + } + } + heard + } +} + +/// The encoder fails once, after this many frames: a capture or encoder +/// error that the supervisor restarts the stream for. +fn fail_the_encoder_once(after: u32) -> impl FnOnce(&SharedControl) { + move |shared| { + shared.pairing.lock().required = false; + std::env::set_var("ETERNAL_FAULT_ENCODER_AFTER", after.to_string()); + } +} + +/// A stream restart longer than the iPad's liveness (3 s) used to drop it: +/// nothing sent it heartbeats while the PC waited to restart, and it lost +/// the signal. The PC now keeps the session alive with heartbeats, and the +/// picture comes back on the same session with a keyframe. +#[test] +fn a_restart_longer_than_the_ipads_liveness_keeps_its_session() { + let host = Host::start_with( + GpuInfo::software_fallback(), + 4_000, + fail_the_encoder_once(240), + ); + let ipad = FakeIpad::new(&host, 1, NEW); + let ack = ipad.connect(1); + let id = ack.session_id; + // Until the stream has restarted: media from a second epoch. + let heard = ipad.stream(id, Duration::from_secs(60), |heard| { + heard + .media + .first() + .is_some_and(|(_, first, _)| heard.media.iter().any(|(_, epoch, _)| epoch != first)) + }); + assert!( + heard.goodbyes.is_empty(), + "the iPad was told to go: {:?}", + heard.goodbyes + ); + let first_epoch = heard.media.first().expect("a picture before the restart").1; + let last_before = heard + .media + .iter() + .filter(|(_, epoch, _)| *epoch == first_epoch) + .map(|(at, _, _)| *at) + .next_back() + .unwrap(); + let resumed = heard + .media + .iter() + .find(|(_, epoch, _)| *epoch != first_epoch) + .expect("the picture came back"); + let gap = resumed.0 - last_before; + assert!( + gap > Duration::from_secs(3), + "the restart took only {gap:?}" + ); + assert!(resumed.2, "the picture comes back with a keyframe"); + // Heartbeats kept coming through the gap, as the iPad's liveness needs. + let mut beats: Vec<_> = heard + .heartbeats + .iter() + .copied() + .filter(|at| *at > last_before && *at < resumed.0) + .collect(); + beats.insert(0, last_before); + beats.push(resumed.0); + let longest = beats + .windows(2) + .map(|pair| pair[1] - pair[0]) + .max() + .unwrap(); + assert!( + longest < Duration::from_millis(2_500), + "{longest:?} without a heartbeat in a {gap:?} restart" + ); + assert_eq!(host.shared.session.lock().session_id(), Some(id)); + // This iPad understands it: the heartbeats said the PC was restarting + // until the picture came back. + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_RESTARTING}; + assert!( + heard.notices.iter().any(|(at, notice)| *at > last_before + && *at < resumed.0 + && *notice == NOTICE_RESTARTING), + "{:?}", + heard.notices + ); + let after = ipad.stream(id, Duration::from_secs(5), |heard| heard.notices.len() >= 2); + assert!( + after + .notices + .iter() + .all(|(_, notice)| *notice == NOTICE_NONE), + "still restarting after the picture came back: {:?}", + after.notices + ); +} + +/// A restart that outlasts the hold limit ends the session with the goodbye +/// that tells the iPad to connect again; its HELLO2 then hears that the PC +/// is restarting. +#[test] +fn a_restart_past_the_hold_limit_tells_the_ipad_to_connect_again() { + // Set under the lock Host takes: another test's Host resets it on drop. + let fail = fail_the_encoder_once(240); + let host = Host::start_with(GpuInfo::software_fallback(), 10_000, |shared| { + eternal_host::transport::session::TEST_RESTART_HOLD_MS.store(2_000, Ordering::SeqCst); + fail(shared); + }); + let ipad = FakeIpad::new(&host, 1, NEW); + let id = ipad.connect(1).session_id; + let heard = ipad.stream(id, Duration::from_secs(60), |heard| { + !heard.goodbyes.is_empty() + }); + assert_eq!( + heard.goodbyes.first(), + Some(&eternal_wire::v2::control::ByeReason::Error) + ); + assert!(heard.heartbeats.len() >= 2, "kept alive until the limit"); + assert_eq!(host.shared.session.lock().session_id(), None); + let ack = ipad.ask(2).expect("a restarting PC answers"); + assert_eq!(ack.status, HelloStatus::Restarting); +} diff --git a/host/tests/input_restart.rs b/host/tests/input_restart.rs new file mode 100644 index 00000000..5e44a3d0 --- /dev/null +++ b/host/tests/input_restart.rs @@ -0,0 +1,234 @@ +//! A stream restarts while Windows holds the old generation's input. The +//! iPad keeps its session and presses a key again on the new generation. +//! Each generation used to inject on a thread of its own: the new one's +//! press landed at once, and when Windows let the old thread go, its +//! cleanup released that key, which then read as up while the iPad held +//! it. Now the old generation's release lands first. +//! +//! The real transport loop, twice on one port, with the off-Windows input +//! recorder. Only the old generation's stuck injection is held. +#![cfg(not(windows))] + +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use eternal_host::control::SharedControl; +use eternal_host::input::{hold, recorder, CaptureGeometry, Injection, KIND_KEY, PHASE_BEGAN}; +use eternal_host::transport; +use eternal_wire::v2::control::{ + encode_control, parse_control, ControlMessage, Hello2, HelloStatus, InputEvent, + CAP_DECODE_H264, FEATURE_WANTS_INPUT, +}; +use tokio::sync::mpsc; + +const KEY_A: u16 = 0x04; +const KEY_B: u16 = 0x05; +/// The scan code Windows gets for A. +const SCAN_A: u16 = 0x1e; + +/// One stream generation: the real transport loop on its own thread. +struct Generation { + nal_tx: mpsc::Sender, + thread: std::thread::JoinHandle<()>, +} + +fn start(shared: &SharedControl, port: u16) -> Generation { + shared.running.store(true, Ordering::SeqCst); + let (nal_tx, nal_rx) = mpsc::channel(4); + let control = shared.clone(); + let thread = std::thread::spawn(move || { + let (_audio_tx, audio_rx) = mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(transport::start_sender( + nal_rx, + audio_rx, + port, + control, + supervisor_tx, + 1, + )) + .unwrap(); + }); + Generation { nal_tx, thread } +} + +fn stop(shared: &SharedControl, generation: Generation) { + shared.stop(); + drop(generation.nal_tx); + let stopping = Instant::now(); + while !generation.thread.is_finished() { + assert!( + stopping.elapsed() < Duration::from_secs(3), + "the stopped loop waits for Windows" + ); + std::thread::sleep(Duration::from_millis(10)); + } + generation.thread.join().unwrap(); +} + +struct Ipad { + socket: UdpSocket, + host: SocketAddr, + msg_seq: u32, + event_id: u32, + session_id: u32, +} + +impl Ipad { + fn send(&mut self, message: &ControlMessage) { + self.msg_seq += 1; + self.socket + .send_to( + &encode_control(self.session_id, self.msg_seq, message), + self.host, + ) + .unwrap(); + } + + fn connect(&mut self) { + let started = Instant::now(); + let mut bytes = [0u8; 2048]; + loop { + assert!( + started.elapsed() < Duration::from_secs(10), + "no HELLO answer" + ); + self.send(&ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 1, + listen_port: self.socket.local_addr().unwrap().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_WANTS_INPUT, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "Input restart iPad".into(), + device_id: 1, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + })); + let until = Instant::now() + Duration::from_millis(300); + while Instant::now() < until { + if let Ok((len, _)) = self.socket.recv_from(&mut bytes) { + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&bytes[..len]) { + if ack.status == HelloStatus::Ok { + self.session_id = ack.session_id; + return; + } + } + } + } + } + } + + fn press(&mut self, keycode: u16) { + self.event_id += 1; + let event = InputEvent { + input_ver: 1, + kind: KIND_KEY, + phase: PHASE_BEGAN, + buttons: 0, + event_id: self.event_id, + x_norm: 0, + y_norm: 0, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode, + modifiers: 0, + client_time_us: 0, + tilt_x: 0, + tilt_y: 0, + }; + self.send(&ControlMessage::InputEvent(event)); + } +} + +fn wait_until(what: &str, mut done: impl FnMut() -> bool) { + let started = Instant::now(); + while !done() { + assert!(started.elapsed() < Duration::from_secs(5), "{what}"); + std::thread::sleep(Duration::from_millis(5)); + } +} + +/// Whether Windows ends up holding A: its last press or release. +fn a_is_down(injected: &[Injection]) -> Option { + injected.iter().rev().find_map(|injection| match injection { + Injection::KeyDown { scan: SCAN_A, .. } => Some(true), + Injection::KeyUp { scan: SCAN_A, .. } => Some(false), + _ => None, + }) +} + +#[test] +fn a_restart_while_windows_holds_input_never_releases_the_new_generations_keys() { + eternal_host::clock::init(); + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + // Bound first, so its port is never the one the PC takes. + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let port = UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap(); + let shared = SharedControl::new(port, 10_000_000); + shared.pairing.lock().required = false; + *shared.capture_geometry.lock() = Some(CaptureGeometry { + left: 0, + top: 0, + width: 1280, + height: 720, + }); + recorder::take(); + let old = start(&shared, port); + let mut ipad = Ipad { + socket, + host: SocketAddr::from(([127, 0, 0, 1], port)), + msg_seq: 0, + event_id: 0, + session_id: 0, + }; + ipad.connect(); + ipad.press(KEY_A); + wait_until("A goes down", || a_is_down(&recorder::peek()) == Some(true)); + + // Windows stops taking the old generation's input, and the stream + // restarts on the same port. + hold::hold_next(); + ipad.press(KEY_B); + wait_until("B reaches Windows", || hold::waiting() == 1); + stop(&shared, old); + let new = start(&shared, port); + + // The iPad, still in its session, presses A again on the new stream, + // and the press is queued while Windows still holds the old one's. + wait_until("the new loop starts", || { + shared.transport_beat.read().is_some() + }); + let queued = shared.input.relays_queued(); + ipad.press(KEY_A); + wait_until("the new press is queued", || { + shared.input.relays_queued() > queued + }); + assert_eq!(hold::waiting(), 1, "Windows still holds the old input"); + + // Windows answers again. Once everything queued has landed, A is down. + hold::release(); + assert!(shared.input.wait_idle(Duration::from_secs(5))); + let injected = recorder::take(); + assert_eq!(a_is_down(&injected), Some(true), "{injected:?}"); + + stop(&shared, new); +} diff --git a/host/tests/input_stall.rs b/host/tests/input_stall.rs new file mode 100644 index 00000000..0e795f94 --- /dev/null +++ b/host/tests/input_stall.rs @@ -0,0 +1,248 @@ +//! Windows' input calls can wait for seconds while it reconfigures displays. +//! In the field the transport loop sat in one for about 53 s during a +//! virtual display change: its iPad heard nothing, new HELLOs went +//! unanswered, and the stopped stream kept its port from the next one. +//! +//! The real transport loop on loopback, with every injection held where +//! Windows' calls are. The input it takes is a key with no Windows scan +//! code, so nothing reaches the desktop on Windows either. + +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use eternal_host::control::SharedControl; +use eternal_host::input::{hold, CaptureGeometry, KIND_KEY, PHASE_BEGAN}; +use eternal_host::transport; +use eternal_host::transport::beat::Watch; +use eternal_wire::v2::control::{ + encode_control, parse_control, ControlMessage, Hello2, HelloStatus, InputEvent, Ping, + CAP_DECODE_H264, FEATURE_WANTS_INPUT, +}; +use tokio::sync::mpsc; + +/// A HID usage Windows has no scan code for: the relay injects nothing. +const NO_KEY: u16 = 0x01; + +struct Ipad { + socket: UdpSocket, + host: SocketAddr, + device_id: u64, + msg_seq: u32, +} + +impl Ipad { + fn new(host: SocketAddr, device_id: u64) -> Self { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + Self { + socket, + host, + device_id, + msg_seq: 0, + } + } + + fn send(&mut self, session_id: u32, message: &ControlMessage) { + self.msg_seq += 1; + self.socket + .send_to( + &encode_control(session_id, self.msg_seq, message), + self.host, + ) + .unwrap(); + } + + fn hello(&mut self) { + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: self.device_id as u32, + listen_port: self.socket.local_addr().unwrap().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_WANTS_INPUT, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "Input stall iPad".into(), + device_id: self.device_id, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + self.send(0, &hello); + } + + /// Control messages heard until `until`. + fn listen(&mut self, until: Instant) -> Vec { + let mut heard = Vec::new(); + let mut bytes = [0u8; 2048]; + while Instant::now() < until { + if let Ok((len, _)) = self.socket.recv_from(&mut bytes) { + if let Ok((_, message)) = parse_control(&bytes[..len]) { + heard.push(message); + } + } + } + heard + } + + fn accepted(&mut self) -> u32 { + let started = Instant::now(); + loop { + assert!( + started.elapsed() < Duration::from_secs(10), + "no HELLO answer" + ); + self.hello(); + let heard = self.listen(Instant::now() + Duration::from_millis(500)); + if let Some(id) = heard.iter().find_map(|message| match message { + ControlMessage::HelloAck(ack) if ack.status == HelloStatus::Ok => { + Some(ack.session_id) + } + _ => None, + }) { + return id; + } + } + } +} + +fn now_ms() -> u64 { + eternal_host::clock::host_now_us() / 1000 +} + +#[test] +fn input_windows_holds_never_stops_the_transport_loop() { + eternal_host::clock::init(); + // Never reach a real iPad over USB. + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + // Bound first, so their ports are never the one the PC takes. + let unbound = SocketAddr::from(([127, 0, 0, 1], 0)); + let mut ipad = Ipad::new(unbound, 1); + let mut other = Ipad::new(unbound, 2); + let port = UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap(); + let shared = SharedControl::new(port, 10_000_000); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + *shared.capture_geometry.lock() = Some(CaptureGeometry { + left: 0, + top: 0, + width: 1280, + height: 720, + }); + let (nal_tx, nal_rx) = mpsc::channel(4); + let (_audio_tx, audio_rx) = mpsc::channel(4); + let control = shared.clone(); + let transport = std::thread::spawn(move || { + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(transport::start_sender( + nal_rx, + audio_rx, + port, + control, + supervisor_tx, + 1, + )) + .unwrap(); + }); + let host = SocketAddr::from(([127, 0, 0, 1], port)); + ipad.host = host; + other.host = host; + let session_id = ipad.accepted(); + + // Windows stops taking input, and the iPad presses a key. + hold::hold(); + ipad.send( + session_id, + &ControlMessage::InputEvent(InputEvent { + input_ver: 1, + kind: KIND_KEY, + phase: PHASE_BEGAN, + buttons: 0, + event_id: 1, + x_norm: 0, + y_norm: 0, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: NO_KEY, + modifiers: 0, + client_time_us: 0, + tilt_x: 0, + tilt_y: 0, + }), + ); + let started = Instant::now(); + while hold::waiting() == 0 { + assert!( + started.elapsed() < Duration::from_secs(5), + "the key reaches Windows" + ); + std::thread::sleep(Duration::from_millis(5)); + } + + // While Windows holds it, the iPad still hears its heartbeats, another + // iPad's HELLO is answered, and the watch sees the loop moving. + let mut watch = Watch::default(); + let mut verdicts = Vec::new(); + other.hello(); + let mut heartbeats = 0; + let mut answered = false; + let until = Instant::now() + Duration::from_secs(3); + while Instant::now() < until { + ipad.send(session_id, &ControlMessage::Ping(Ping { t1_us: 1 })); + let step = Instant::now() + Duration::from_millis(100); + heartbeats += ipad + .listen(step) + .iter() + .filter(|message| matches!(message, ControlMessage::Heartbeat(_))) + .count(); + answered |= other + .listen(Instant::now() + Duration::from_millis(5)) + .iter() + .any(|message| matches!(message, ControlMessage::HelloAck(_))); + verdicts.extend(watch.check(shared.transport_beat.read(), now_ms())); + } + assert_eq!(hold::waiting(), 1, "Windows still holds the key"); + assert!( + heartbeats >= 2, + "heartbeats while Windows held input: {heartbeats}" + ); + assert!(answered, "the other iPad's HELLO went unanswered"); + assert_eq!(verdicts, []); + + // A stream stopped meanwhile ends at once, and hands its port on: the + // socket stays bound for the next generation or the responder that + // answers while nothing streams, not with a loop waiting for Windows. + shared.stop(); + drop(nal_tx); + let stopping = Instant::now(); + while !transport.is_finished() { + assert!( + stopping.elapsed() < Duration::from_secs(3), + "the stopped loop waits for Windows" + ); + std::thread::sleep(Duration::from_millis(10)); + } + transport.join().unwrap(); + let parked = shared + .stream_socket + .lock() + .as_ref() + .and_then(|socket| socket.local_addr().ok()) + .map(|addr| addr.port()); + assert_eq!(parked, Some(port), "the stopped loop handed its port on"); + hold::release(); +} diff --git a/host/tests/listen_on.rs b/host/tests/listen_on.rs index d7a40034..af594008 100644 --- a/host/tests/listen_on.rs +++ b/host/tests/listen_on.rs @@ -29,6 +29,9 @@ fn wifi(up: bool, ip: Ipv4Addr) -> Adapter { } } +/// The pipeline generation of the streams this test starts. +const GENERATION: u64 = 7; + /// Runs the transport until it stops, on its own thread. fn start( shared: &SharedControl, @@ -55,6 +58,7 @@ fn start( port, shared, supervisor_tx, + GENERATION, )) .unwrap(); }); @@ -91,7 +95,12 @@ fn the_address_row_shows_a_chosen_network_that_came_back() { let command = supervisor_rx .recv_timeout(Duration::from_secs(30)) .expect("the network check asks for a restart"); - assert!(matches!(command, SupervisorCommand::Restart)); + assert_eq!( + command, + SupervisorCommand::RestartStream { + generation: GENERATION + } + ); transport.join().unwrap(); // The supervisor starts the stream again, now on the network's address. diff --git a/host/tests/liveness_hold.rs b/host/tests/liveness_hold.rs index d92e078b..28a519d9 100644 --- a/host/tests/liveness_hold.rs +++ b/host/tests/liveness_hold.rs @@ -381,7 +381,7 @@ impl Sim { let control = shared.clone(); let transport = tokio::spawn(async move { let _audio = audio_tx; - transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx).await + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx, 1).await }); // The transport binds before its first wait. for _ in 0..10 { @@ -777,3 +777,253 @@ fn another_ipad_takes_the_slot_of_a_silent_one() { sim.stop().await; }); } + +/// The same iPad over the cable: the tunnel the host's USB worker opened to +/// the test's listener, read and written as the app's link does. Its pings +/// wait for their pong without moving the clock, so the host hears them +/// when they are sent. +struct UsbIpad { + stream: std::net::TcpStream, + buffer: Vec, + msg_seq: u32, + nonce: u32, + session_id: Option, + received: Vec<(Duration, Heard)>, + pongs: usize, + /// When the host's end of the tunnel closed, if it did. + closed: Option, + next_report: Duration, + next_ping: Duration, +} + +impl UsbIpad { + async fn accept(listener: &std::net::TcpListener) -> Self { + let mut accepted = None; + settle("the host's USB tunnel", || { + accepted = listener.accept().ok(); + accepted.is_some() + }) + .await; + let (stream, _) = accepted.unwrap(); + stream.set_nonblocking(true).unwrap(); + stream.set_nodelay(true).unwrap(); + let mut ipad = Self { + stream, + buffer: Vec::new(), + msg_seq: 0, + nonce: 100, + session_id: None, + received: Vec::new(), + pongs: 0, + closed: None, + next_report: Duration::MAX, + next_ping: Duration::MAX, + }; + let preamble = eternal_host::transport::link::PREAMBLE; + settle("the EMLINK preamble", || { + ipad.fill(Duration::ZERO); + ipad.buffer.len() >= preamble.len() || ipad.closed.is_some() + }) + .await; + assert_eq!(&ipad.buffer[..preamble.len()], preamble); + ipad.buffer.drain(..preamble.len()); + ipad + } + + fn fill(&mut self, now: Duration) { + use std::io::Read; + let mut bytes = [0; 64 * 1024]; + loop { + match self.stream.read(&mut bytes) { + Ok(0) => { + self.closed.get_or_insert(now); + return; + } + Ok(len) => self.buffer.extend_from_slice(&bytes[..len]), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => return, + Err(_) => { + self.closed.get_or_insert(now); + return; + } + } + } + } + + fn drain(&mut self, now: Duration) { + self.fill(now); + while self.buffer.len() >= 2 { + let len = usize::from(u16::from_le_bytes([self.buffer[0], self.buffer[1]])); + if self.buffer.len() < 2 + len { + break; + } + let packet: Vec = self.buffer.drain(..2 + len).skip(2).collect(); + if matches!(parse_control(&packet), Ok((_, ControlMessage::Pong(_)))) { + self.pongs += 1; + } + let heard = heard(&packet); + if let (None, Heard::Ack(HelloStatus::Ok, id)) = (self.session_id, heard) { + self.session_id = Some(id); + self.next_report = now + Duration::from_millis(500); + self.next_ping = now + Duration::from_secs(2); + } + self.received.push((now, heard)); + } + } + + fn send(&mut self, message: &ControlMessage) { + use std::io::Write; + self.msg_seq += 1; + let bytes = encode_control(self.session_id.unwrap_or(0), self.msg_seq, message); + let mut framed = (bytes.len() as u16).to_le_bytes().to_vec(); + framed.extend_from_slice(&bytes); + self.stream.write_all(&framed).unwrap(); + } + + /// HELLO2 as the iPad that streamed over Wi-Fi, and its ACK. + async fn hello(&mut self, device_id: u64, now: Duration) -> u32 { + self.nonce += 1; + self.session_id = None; + self.send(&ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: self.nonce, + listen_port: 0, + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_WANTS_INPUT, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "Stall test iPad".into(), + device_id, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + })); + settle("the ACK over USB", || { + self.drain(now); + self.session_id.is_some() || self.closed.is_some() + }) + .await; + self.session_id.expect("an ACK over USB") + } + + async fn step(&mut self, now: Duration) { + self.drain(now); + let Some(id) = self.session_id else { + return; + }; + if self.closed.is_some() { + return; + } + if now >= self.next_report { + self.next_report += Duration::from_millis(500); + let frames = self + .received + .iter() + .filter(|(_, heard)| *heard == Heard::Media(id)) + .count() as u32; + self.send(&ControlMessage::ReceiverReport(ReceiverReport { + stream_epoch: 1, + highest_seq: frames, + frames_complete: frames, + ..ReceiverReport::default() + })); + } + if now >= self.next_ping { + self.next_ping += Duration::from_secs(2); + let pongs = self.pongs; + self.send(&ControlMessage::Ping(Ping { t1_us: 1 })); + // A session that ended answers with a goodbye instead. + settle("a pong over USB", || { + self.drain(now); + self.pongs > pongs || self.closed.is_some() || !byes(&self.received).is_empty() + }) + .await; + } + } +} + +/// The iPad moves from Wi-Fi to the cable: it says goodbye on Wi-Fi +/// (`superseded`) and sends HELLO2 over USB, in either order. In the field +/// some of these tunnels closed on both sides 9.5-9.8 s later, with no +/// reason on either side. Nothing in the host times that: for 30 s the +/// session and the tunnel stay up and video flows over the cable, and +/// neither link hears a goodbye. A deadline left from the Wi-Fi session or +/// the takeover wait (1 s), the liveness hold (10 s) or the tunnel would +/// show here. +#[test] +fn a_usb_takeover_from_wifi_keeps_its_tunnel_and_session() { + for bye_first in [true, false] { + run(async { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + std::env::set_var( + "ETERNAL_USB_DIRECT", + listener.local_addr().unwrap().to_string(), + ); + let mut sim = Sim::start().await; + let wifi = sim.add_ipad(77).await; + sim.ipads[wifi].hello(); + sim.run_until(secs(2.0)).await; + let (Some(wifi_id), None) = sim.session() else { + panic!("no session: {:?}", sim.session()) + }; + assert!(sim.ipads[wifi].video_after(wifi_id, secs(1.5))); + let mut usb = UsbIpad::accept(&listener).await; + + let took_over = sim.start.elapsed(); + let bye = ControlMessage::Bye(ByeReason::Superseded); + if bye_first { + sim.ipads[wifi].send_now(wifi_id, &bye).await; + } + let usb_id = usb.hello(77, took_over).await; + if !bye_first { + // The Wi-Fi goodbye arrives late, on a session already gone. + sim.ipads[wifi].send_now(wifi_id, &bye).await; + } + let wifi = sim.ipads.remove(wifi); + assert_ne!(usb_id, wifi_id); + + let end = took_over + secs(30.0); + let mut next = took_over; + while next < end { + next += Duration::from_millis(50); + sim.run_until(next).await; + usb.step(sim.start.elapsed()).await; + } + + eprintln!( + "USB_TAKEOVER_SIM bye_first={bye_first} took_over={took_over:?} closed={:?} \ + session={:?} usb_byes={:?}", + usb.closed, + sim.session(), + byes(&usb.received) + ); + assert_eq!(usb.closed, None, "the host closed the tunnel"); + assert_eq!( + sim.session(), + (Some(usb_id), Some((wifi_id, EndReason::Superseded))) + ); + assert!(byes(&usb.received).is_empty(), "{:?}", byes(&usb.received)); + assert!( + byes(&wifi.received()).is_empty(), + "{:?}", + byes(&wifi.received()) + ); + let last_video = usb + .received + .iter() + .filter(|(_, heard)| *heard == Heard::Media(usb_id)) + .map(|(at, _)| *at) + .max(); + assert!( + last_video.is_some_and(|at| at >= end - secs(0.5)), + "{last_video:?}" + ); + sim.stop().await; + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + }); + } +} diff --git a/host/tests/power_goodbye.rs b/host/tests/power_goodbye.rs new file mode 100644 index 00000000..8e3eb5e4 --- /dev/null +++ b/host/tests/power_goodbye.rs @@ -0,0 +1,448 @@ +//! When Windows puts the PC to sleep, shuts it down or signs the user out +//! while an iPad streams, the iPad that understands it hears why, an older +//! one hears nothing as before, and the session's summary is written first. +//! Drives the real transport loop with a fake iPad on loopback and gives it +//! the messages Windows would; nothing here sleeps or shuts down a PC. + +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use eternal_host::control::SharedControl; +use eternal_host::power::{self, Message, SLEEP_WAIT, SUMMARY_WAIT}; +use eternal_host::transport; +use eternal_wire::v2::control::{ + encode_control, parse_control, ByeReason, ControlMessage, Hello2, HelloStatus, CAP_DECODE_H264, + FEATURE_POWER_BYE, FEATURE_SUPPORTS_NACK, +}; +use tokio::sync::mpsc; + +/// The tests share the environment and the session log; one at a time. +static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Keeps the session log, which a summary written before a shutdown goes +/// to directly, out of the real app data folder. +fn private_app_data() -> std::path::PathBuf { + static DIR: std::sync::OnceLock = std::sync::OnceLock::new(); + DIR.get_or_init(|| { + let dir = std::env::temp_dir().join(format!("em-power-goodbye-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::env::set_var("APPDATA", &dir); + dir + }) + .clone() +} + +fn free_udp_port() -> u16 { + // The transport binds every address. + std::net::UdpSocket::bind("0.0.0.0:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap() +} + +fn hello(nonce: u32, listen_port: u16, feature_caps: u16) -> Vec { + encode_control( + 0, + nonce, + &ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: nonce, + listen_port, + decoder_caps: CAP_DECODE_H264, + feature_caps, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "Power test iPad".into(), + device_id: 7, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }), + ) +} + +/// A fake iPad streaming from a real transport on loopback. +struct Rig { + shared: SharedControl, + ipad: UdpSocket, + host: SocketAddr, + caps: u16, + nal_tx: Option>, + transport: Option>, +} + +impl Rig { + fn start(caps: u16) -> Self { + eternal_host::clock::init(); + private_app_data(); + // Never reach a real iPad over USB. + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + let port = free_udp_port(); + let shared = SharedControl::new(port, 10_000_000); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + let (nal_tx, nal_rx) = mpsc::channel(4); + let control = shared.clone(); + let transport = std::thread::spawn(move || { + let (_audio_tx, audio_rx) = mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(transport::start_sender( + nal_rx, + audio_rx, + port, + control, + supervisor_tx, + 1, + )) + .unwrap(); + }); + let ipad = UdpSocket::bind("127.0.0.1:0").unwrap(); + ipad.set_read_timeout(Some(Duration::from_millis(50))) + .unwrap(); + let mut rig = Self { + shared, + ipad, + host: SocketAddr::from(([127, 0, 0, 1], port)), + caps, + nal_tx: Some(nal_tx), + transport: Some(transport), + }; + assert_eq!(rig.connect(1), Some(HelloStatus::Ok), "the iPad streams"); + rig + } + + /// Sends HELLO2 `nonce` until an answer comes, for at most `patience`. + fn ask(&self, nonce: u32, patience: Duration) -> Option { + let port = self.ipad.local_addr().unwrap().port(); + let until = Instant::now() + patience; + let mut bytes = [0; 2048]; + while Instant::now() < until { + assert!( + !self.transport.as_ref().unwrap().is_finished(), + "the transport ended early" + ); + self.ipad + .send_to(&hello(nonce, port, self.caps), self.host) + .unwrap(); + let asked = Instant::now(); + while asked.elapsed() < Duration::from_millis(200) { + if let Ok(len) = self.ipad.recv(&mut bytes) { + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&bytes[..len]) { + if ack.client_nonce == nonce { + return Some(ack.status); + } + } + } + } + } + None + } + + fn connect(&mut self, nonce: u32) -> Option { + // The transport binds its socket first. + self.ask(nonce, Duration::from_secs(30)) + } + + /// The goodbyes that arrive within `window`. + fn goodbyes(&self, window: Duration) -> Vec { + let until = Instant::now() + window; + let mut bytes = [0; 2048]; + let mut said = Vec::new(); + while Instant::now() < until { + if let Ok(len) = self.ipad.recv(&mut bytes) { + if let Ok((_, ControlMessage::Bye(reason))) = parse_control(&bytes[..len]) { + said.push(reason); + } + } + } + said + } + + /// Gives the host `message` as its window would, and how long that took. + fn tell(&self, message: Message) -> (isize, Duration) { + let started = Instant::now(); + let answer = power::handle(&self.shared, message); + (answer, started.elapsed()) + } +} + +impl Drop for Rig { + fn drop(&mut self) { + self.shared.stop(); + drop(self.nal_tx.take()); + if let Some(transport) = self.transport.take() { + let _ = transport.join(); + } + } +} + +fn session_log() -> String { + std::fs::read_to_string(eternal_host::logging::session_log_path()).unwrap_or_default() +} + +/// Sleep: the iPad that understands it hears that the PC went to sleep, in +/// the usual burst of three, before the message returns; then the PC answers +/// no HELLO2 until it wakes, and streams again after. +#[test] +fn sleep_tells_the_ipad_and_waking_lets_it_back() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let mut rig = Rig::start(FEATURE_POWER_BYE | FEATURE_SUPPORTS_NACK); + let (answer, took) = rig.tell(Message::Suspend); + assert_eq!(answer, 1, "never asks Windows to wait"); + assert!(took < SLEEP_WAIT + Duration::from_millis(100), "{took:?}"); + assert_eq!( + rig.goodbyes(Duration::from_millis(500)), + [ByeReason::HostSleeping; 3] + ); + assert!(!rig.shared.client_connected()); + + assert_eq!( + rig.ask(2, Duration::from_millis(600)), + None, + "no answer while it sleeps" + ); + assert!(!rig.shared.client_connected()); + assert_eq!(rig.tell(Message::Resume).0, 1); + assert_eq!( + rig.connect(3), + Some(HelloStatus::Ok), + "streams again after waking" + ); +} + +/// An iPad from before these goodbyes gets none, as before: it notices the +/// silence. Nothing waits for a goodbye that isn't coming. +#[test] +fn sleep_says_nothing_to_an_older_ipad() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let rig = Rig::start(FEATURE_SUPPORTS_NACK); + let (answer, took) = rig.tell(Message::Suspend); + assert_eq!(answer, 1); + assert!(took < SLEEP_WAIT, "{took:?}"); + assert!( + !rig.shared.client_connected(), + "the session ended all the same" + ); + let heard = rig.goodbyes(Duration::from_millis(500)); + assert!(heard.is_empty(), "{heard:?}"); +} + +/// Shutdown and sign-out: the iPad hears that the PC is powering off, and +/// the session's summary is in the session log on disk before the message +/// returns, ready for the next launch to upload. +#[test] +fn shutting_down_or_signing_out_says_so_and_saves_the_summary() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + for (nonce, message) in [ + ( + 11, + Message::EndSession { + ending: true, + logoff: false, + close_app: false, + }, + ), + ( + 12, + Message::EndSession { + ending: true, + logoff: true, + close_app: false, + }, + ), + (13, Message::SessionLogoff), + ] { + let mut rig = Rig::start(FEATURE_POWER_BYE); + // Asking is not going: another app may still call it off. + assert_eq!(rig.tell(Message::QueryEndSession).0, 1); + assert!(rig.shared.client_connected()); + assert_eq!( + rig.tell(Message::EndSession { + ending: false, + logoff: false, + close_app: false, + }) + .0, + 0 + ); + assert!( + rig.shared.client_connected(), + "a cancelled shutdown changes nothing" + ); + // The session to end, under its own nonce. + assert_eq!(rig.connect(nonce), Some(HelloStatus::Ok)); + + let (answer, took) = rig.tell(message); + assert_eq!(answer, 0, "{message:?}"); + assert!(took < SUMMARY_WAIT + Duration::from_millis(100), "{took:?}"); + // A sign-out's session change follows its end of session: the + // repeat ends nothing, tells nothing and writes nothing again. + let (_, repeat_took) = rig.tell(Message::SessionLogoff); + assert!(repeat_took < Duration::from_millis(100), "{repeat_took:?}"); + let summary = format!(" nonce={nonce:016x} "); + let summaries = session_log() + .lines() + .filter(|line| { + line.contains("session_summary v=1 role=pc") + && line.contains(&summary) + && line.contains(" end=shutdown ") + }) + .count(); + assert_eq!(summaries, 1, "{message:?}: one summary in the log"); + assert_eq!( + rig.goodbyes(Duration::from_millis(500)), + [ByeReason::HostPoweringOff; 3], + "{message:?}" + ); + } +} + +/// Restart Manager closes the app for an installer, as Setup does on an +/// update, while the PC stays up: the stream ends as on a quit, with the +/// goodbye every iPad reads, not as the PC powering off. An update that +/// hands over to its new copy says connect again. +#[test] +fn an_installer_closing_the_app_ends_the_stream_as_a_quit() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + for (nonce, handoff, bye) in [ + (21, false, ByeReason::UserDisconnect), + (22, true, ByeReason::Error), + ] { + let mut rig = Rig::start(FEATURE_POWER_BYE); + assert_eq!(rig.connect(nonce), Some(HelloStatus::Ok)); + if handoff { + rig.shared + .handoff + .claim(eternal_host::handoff::Handoff::Update) + .unwrap(); + } + let (answer, took) = rig.tell(Message::EndSession { + ending: true, + logoff: false, + close_app: true, + }); + assert_eq!(answer, 0); + assert!(took < SUMMARY_WAIT + Duration::from_millis(100), "{took:?}"); + assert_eq!(rig.goodbyes(Duration::from_millis(500)), [bye; 3]); + assert!( + !rig.shared.running.load(Ordering::SeqCst), + "the stream stopped" + ); + let summary = format!(" nonce={nonce:016x} "); + assert!( + session_log() + .lines() + .any(|line| line.contains("session_summary v=1 role=pc") && line.contains(&summary)), + "no summary in the log" + ); + } +} + +/// The real window, given the message Windows sends before sleep, then +/// those Restart Manager sends to close the app for an installer. Windows +/// only: it checks the window gets Windows' messages and answers them, and +/// that closing the app takes the app's quit path once while the window +/// stays. +#[cfg(windows)] +#[test] +fn the_hidden_window_turns_windows_messages_into_goodbyes() { + use windows::Win32::Foundation::{HWND, LPARAM, WPARAM}; + use windows::Win32::UI::WindowsAndMessaging::{ + IsWindow, SendMessageW, ENDSESSION_CLOSEAPP, PBT_APMRESUMEAUTOMATIC, PBT_APMSUSPEND, + WM_CLOSE, WM_ENDSESSION, WM_POWERBROADCAST, + }; + + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let rig = Rig::start(FEATURE_POWER_BYE); + let (quit_tx, quits) = std::sync::mpsc::channel(); + power::watch( + rig.shared.clone(), + Box::new(move || quit_tx.send(()).unwrap()), + ); + let started = Instant::now(); + let hwnd = loop { + if let Some(hwnd) = power::window() { + break HWND(hwnd as _); + } + assert!( + started.elapsed() < Duration::from_secs(10), + "the window opens" + ); + std::thread::sleep(Duration::from_millis(10)); + }; + let answer = unsafe { + SendMessageW( + hwnd, + WM_POWERBROADCAST, + WPARAM(PBT_APMSUSPEND as usize), + LPARAM(0), + ) + }; + assert_eq!(answer.0, 1); + assert_eq!( + rig.goodbyes(Duration::from_millis(500)), + [ByeReason::HostSleeping; 3] + ); + let answer = unsafe { + SendMessageW( + hwnd, + WM_POWERBROADCAST, + WPARAM(PBT_APMRESUMEAUTOMATIC as usize), + LPARAM(0), + ) + }; + assert_eq!(answer.0, 1); + assert_eq!(rig.ask(2, Duration::from_secs(10)), Some(HelloStatus::Ok)); + + // Restart Manager closes the app: the quit goodbye, then the quit path. + let answer = unsafe { + SendMessageW( + hwnd, + WM_ENDSESSION, + WPARAM(1), + LPARAM(ENDSESSION_CLOSEAPP as isize), + ) + }; + assert_eq!(answer.0, 0); + assert_eq!( + rig.goodbyes(Duration::from_millis(500)), + [ByeReason::UserDisconnect; 3] + ); + assert_eq!( + quits.recv_timeout(Duration::from_secs(5)), + Ok(()), + "the app quits" + ); + // Its WM_CLOSE that follows neither quits again nor destroys the window, + // which still hears Windows. + let answer = unsafe { SendMessageW(hwnd, WM_CLOSE, WPARAM(0), LPARAM(0)) }; + assert_eq!(answer.0, 0); + assert!(unsafe { IsWindow(hwnd) }.as_bool(), "the window stays"); + assert!(quits.recv_timeout(Duration::from_millis(500)).is_err()); + let answer = unsafe { + SendMessageW( + hwnd, + WM_POWERBROADCAST, + WPARAM(PBT_APMSUSPEND as usize), + LPARAM(0), + ) + }; + assert_eq!(answer.0, 1, "still answers Windows"); + unsafe { + SendMessageW( + hwnd, + WM_POWERBROADCAST, + WPARAM(PBT_APMRESUMEAUTOMATIC as usize), + LPARAM(0), + ) + }; +} diff --git a/host/tests/send_budget.rs b/host/tests/send_budget.rs new file mode 100644 index 00000000..842d7723 --- /dev/null +++ b/host/tests/send_budget.rs @@ -0,0 +1,599 @@ +//! The PC sends the first picture, then waits to hear back from the iPad +//! before it streams more. +//! +//! Paused-clock simulations of the real transport loop, with a fake encoder +//! and fake clients on loopback. The tokio clock only moves when every task +//! waits, and nothing waits while a datagram is on the loopback, so a busy +//! machine cannot change the result. Frames stay at 16 fragments or fewer, +//! which go out unpaced: pacing waits on a real-time thread. + +use std::net::SocketAddr; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; + +use eternal_host::control::SharedControl; +use eternal_host::encoder::NALUnit; +use eternal_host::transport; +use eternal_host::transport::probe; +use eternal_host::transport::session::{EndReason, UNANSWERED_MEDIA_BUDGET}; +use eternal_wire::v2::control::{ + encode_control, parse_control, ControlMessage, Hello2, HelloStatus, Ping, ReceiverReport, + CAP_DECODE_H264, FEATURE_SUPPORTS_NACK, +}; +use eternal_wire::v2::media::MediaHeader; +use eternal_wire::v2::{classify, Classified}; +use parking_lot::Mutex; +use tokio::sync::mpsc; +use tokio::time::Instant; + +/// The tests share ETERNAL_USB_DIRECT and the transport probe; run them one +/// at a time. +static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +const FRAME_INTERVAL: Duration = Duration::from_micros(16_667); +const STEP: Duration = Duration::from_millis(10); +/// 16 fragments: the largest frame that goes out unpaced. +const KEYFRAME_BYTES: usize = 21_000; +const DELTA_BYTES: usize = 8_000; +/// Real time a datagram may take across the loopback before a test fails. +const LOOPBACK_LIMIT: Duration = Duration::from_secs(10); + +/// Waits until `done`, without moving the paused clock: a yielding task +/// keeps the runtime busy, so tokio polls the sockets but advances no time. +async fn settle(what: &str, mut done: impl FnMut() -> bool) { + let limit = std::time::Instant::now() + LOOPBACK_LIMIT; + while !done() { + assert!( + std::time::Instant::now() < limit, + "{what} still in flight after {LOOPBACK_LIMIT:?}" + ); + tokio::task::yield_now().await; + } +} + +fn secs(seconds: f64) -> Duration { + Duration::from_secs_f64(seconds) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Heard { + /// Session, frame number, keyframe. + Media(u32, u32, bool), + Ack(HelloStatus, u32), + Heartbeat, + Bye, + Other, +} + +/// How a fake client answers its session. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Answers { + /// As the app does since 0.3.0: a ping the moment the HELLO_ACK lands, + /// four more 200 ms apart, then one every 2 s, and a report every + /// 500 ms. + AtOnce, + /// Its first report, this long after the HELLO_ACK. + After(Duration), + Never, +} + +struct Client { + socket: std::net::UdpSocket, + host: SocketAddr, + answers: Answers, + nonce: u32, + msg_seq: u32, + session: Option<(u32, Duration)>, + /// What arrived, when, and its size. + received: Vec<(Duration, Heard, usize)>, + /// When each packet reached the PC's socket. + delivered: Vec, + sent_before: u64, + taken_before: u64, + next_ping: Option<(Duration, u32)>, + next_report: Option, + start: Instant, +} + +impl Client { + fn new(host: SocketAddr, answers: Answers, start: Instant) -> Self { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.set_nonblocking(true).unwrap(); + let address = socket.local_addr().unwrap(); + // A port an earlier test used may come round again. + let (sent_before, taken_before) = (probe::sent_to(address), probe::taken_from(address)); + Self { + socket, + host, + answers, + nonce: 0, + msg_seq: 0, + session: None, + received: Vec::new(), + delivered: Vec::new(), + sent_before, + taken_before, + next_ping: None, + next_report: None, + start, + } + } + + fn address(&self) -> SocketAddr { + self.socket.local_addr().unwrap() + } + + /// Sends, and waits until the PC took it. + async fn send(&mut self, bytes: Vec) { + self.socket.send_to(&bytes, self.host).unwrap(); + self.delivered.push(self.start.elapsed()); + let (address, sent) = ( + self.address(), + self.taken_before + self.delivered.len() as u64, + ); + settle("a packet to the PC", || probe::taken_from(address) >= sent).await; + } + + async fn send_message(&mut self, message: &ControlMessage) { + self.msg_seq += 1; + let session_id = self.session.map_or(0, |(id, _)| id); + let bytes = encode_control(session_id, self.msg_seq, message); + self.send(bytes).await; + } + + /// The shipped iPad's HELLO2: the generic name, a device id, and + /// `listen_port` its own. + async fn hello(&mut self) { + self.nonce += 1; + self.session = None; + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: self.nonce, + listen_port: self.address().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_SUPPORTS_NACK, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "iPad".into(), + device_id: 77, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + self.send_message(&hello).await; + } + + /// The smallest HELLO2 there is: no name and no device id. 38 bytes. + async fn bare_hello(&mut self) { + self.nonce += 1; + let bytes = encode_control( + 0, + 1, + &ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: self.nonce, + listen_port: self.address().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: 0, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: String::new(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }), + ); + assert_eq!(bytes.len(), 38); + self.send(bytes).await; + } + + /// Takes everything the PC has sent so far, as received `now`. + async fn drain(&mut self, now: Duration) { + let address = self.address(); + let sent_before = self.sent_before; + let mut bytes = [0; 2048]; + loop { + while let Ok((len, _)) = self.socket.recv_from(&mut bytes) { + let datagram = &bytes[..len]; + self.received.push((now, heard(datagram), len)); + } + if self.received.len() as u64 >= probe::sent_to(address) - sent_before { + break; + } + tokio::task::yield_now().await; + } + } + + /// What the client does every step: take the session from an ACK, then + /// answer as it does. + async fn step(&mut self, now: Duration) { + self.drain(now).await; + if self.session.is_none() { + let accepted = self + .received + .iter() + .rev() + .find_map(|(_, heard, _)| match heard { + Heard::Ack(HelloStatus::Ok, id) => Some(*id), + _ => None, + }); + if let Some(id) = accepted { + self.session = Some((id, now)); + match self.answers { + Answers::AtOnce => { + self.next_ping = Some((now, 5)); + self.next_report = Some(now + Duration::from_millis(500)); + } + Answers::After(delay) => self.next_report = Some(now + delay), + Answers::Never => {} + } + } + } + if let Some((at, burst)) = self.next_ping.filter(|(at, _)| now >= *at) { + self.next_ping = Some(if burst > 1 { + (at + Duration::from_millis(200), burst - 1) + } else { + (at + Duration::from_secs(2), 1) + }); + self.send_message(&ControlMessage::Ping(Ping { t1_us: 1 })) + .await; + } + if let Some(at) = self.next_report.filter(|at| now >= *at) { + self.next_report = Some(at + Duration::from_millis(500)); + let frames = self.frames().len() as u32; + self.send_message(&ControlMessage::ReceiverReport(ReceiverReport { + stream_epoch: 1, + highest_seq: frames, + frames_complete: frames, + ..ReceiverReport::default() + })) + .await; + } + } + + /// Each video frame's first fragment: when, its number, keyframe. + fn frames(&self) -> Vec<(Duration, u32, bool)> { + let mut seen = std::collections::BTreeSet::new(); + self.received + .iter() + .filter_map(|(at, heard, _)| match heard { + Heard::Media(_, seq, key) if seen.insert(*seq) => Some((*at, *seq, *key)), + _ => None, + }) + .collect() + } + + fn media_bytes(&self) -> usize { + self.received + .iter() + .filter(|(_, heard, _)| matches!(heard, Heard::Media(..))) + .map(|(_, _, len)| len) + .sum() + } + + fn count(&self, kind: fn(&Heard) -> bool) -> usize { + self.received + .iter() + .filter(|(_, heard, _)| kind(heard)) + .count() + } +} + +fn heard(bytes: &[u8]) -> Heard { + match classify(bytes) { + Classified::Media { .. } => { + let header = MediaHeader::decode(bytes).unwrap().0; + Heard::Media(header.session_id, header.frame_seq, header.is_keyframe) + } + Classified::Control(_) => match parse_control(bytes) { + Ok((_, ControlMessage::HelloAck(ack))) => Heard::Ack(ack.status, ack.session_id), + Ok((_, ControlMessage::Heartbeat(_))) => Heard::Heartbeat, + Ok((_, ControlMessage::Bye(_))) => Heard::Bye, + _ => Heard::Other, + }, + _ => Heard::Other, + } +} + +/// The real transport loop and a fake encoder at 60 fps that makes a +/// keyframe whenever one is asked for. +struct Sim { + start: Instant, + shared: SharedControl, + host: SocketAddr, + transport: tokio::task::JoinHandle>>, + encoder: tokio::task::JoinHandle<()>, + /// When the encoder found an IDR asked for. + forced: Arc>>, + clients: Vec, +} + +impl Sim { + async fn start() -> Self { + let start = Instant::now(); + // Another process can take the free port before the transport binds it. + let (port, shared, nal_tx, transport) = loop { + let port = std::net::UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap(); + let shared = SharedControl::new(port, 10_000_000); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + let (nal_tx, nal_rx) = mpsc::channel(4); + let (audio_tx, audio_rx) = mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + let control = shared.clone(); + let transport = tokio::spawn(async move { + let _audio = audio_tx; + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx, 1).await + }); + // The transport binds before its first wait. + for _ in 0..10 { + tokio::task::yield_now().await; + } + if !transport.is_finished() { + break (port, shared, nal_tx, transport); + } + let error = transport.await.unwrap().unwrap_err(); + assert!( + error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::AddrInUse), + "{error}" + ); + }; + let forced = Arc::new(Mutex::new(Vec::new())); + let encoder = { + let (force, forced) = (shared.force_next_idr.clone(), forced.clone()); + tokio::spawn(async move { + let mut next = Instant::now(); + for sequence in 0.. { + tokio::time::sleep_until(next).await; + next += FRAME_INTERVAL; + let is_keyframe = force.swap(false, Ordering::SeqCst); + if is_keyframe { + forced.lock().push(start.elapsed()); + } + let mut data = vec![ + 0xAA; + if is_keyframe { + KEYFRAME_BYTES + } else { + DELTA_BYTES + } + ]; + data[..5].copy_from_slice(&[0, 0, 0, 1, if is_keyframe { 0x65 } else { 0x41 }]); + let nal = NALUnit { + data, + sequence, + timestamp: Instant::now().into_std(), + encode_duration_us: 0, + is_keyframe, + }; + if nal_tx.send(nal).await.is_err() { + break; + } + } + }) + }; + Self { + start, + shared, + host: SocketAddr::from(([127, 0, 0, 1], port)), + transport, + encoder, + forced, + clients: Vec::new(), + } + } + + fn add_client(&mut self, answers: Answers) -> usize { + self.clients + .push(Client::new(self.host, answers, self.start)); + self.clients.len() - 1 + } + + fn now(&self) -> Duration { + self.start.elapsed() + } + + async fn run_until(&mut self, at: Duration) { + while self.start.elapsed() < at { + let now = self.start.elapsed(); + for client in &mut self.clients { + client.step(now).await; + } + tokio::time::sleep(STEP).await; + let now = self.start.elapsed(); + for client in &mut self.clients { + client.drain(now).await; + } + } + } + + fn last_end(&self) -> Option<(u32, EndReason)> { + self.shared.session.lock().last_end() + } + + async fn stop(self) { + self.shared.stop(); + tokio::time::timeout(Duration::from_secs(5), self.transport) + .await + .expect("the transport stops") + .unwrap() + .unwrap(); + self.encoder.abort(); + } +} + +fn run(test: impl std::future::Future) { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + eternal_host::clock::init(); + // Never reach a real iPad: this unreachable endpoint stands in for USB. + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .unwrap() + .block_on(test); +} + +/// A HELLO2 from an address that never answers gets the first picture, at +/// most [`UNANSWERED_MEDIA_BUDGET`] of media, and nothing more however +/// often it asks again: a new HELLO2 there brings no new budget for a +/// minute. Its session ends at the liveness timeout, as before. +#[test] +fn an_address_that_never_answers_gets_only_the_first_picture() { + run(async { + let mut sim = Sim::start().await; + let silent = sim.add_client(Answers::Never); + sim.clients[silent].hello().await; + sim.run_until(secs(2.0)).await; + // It asks again and again, from the same address. + for _ in 0..10 { + sim.clients[silent].hello().await; + let next = sim.now() + secs(0.1); + sim.run_until(next).await; + } + sim.run_until(secs(8.0)).await; + let (ended, reason) = sim.last_end().expect("the session ended"); + assert_eq!(reason, EndReason::Liveness, "session {ended}"); + sim.clients[silent].bare_hello().await; + sim.run_until(secs(12.0)).await; + + let client = &sim.clients[silent]; + let frames = client.frames(); + eprintln!( + "UNANSWERED_SIM media_bytes={} frames={} first={:?} last={:?} acks={} heartbeats={}", + client.media_bytes(), + frames.len(), + frames.first(), + frames.last(), + client.count(|h| matches!(h, Heard::Ack(..))), + client.count(|h| *h == Heard::Heartbeat), + ); + assert!( + client.media_bytes() <= UNANSWERED_MEDIA_BUDGET, + "{} bytes of media", + client.media_bytes() + ); + let (_, first_seq, first_key) = frames[0]; + assert!( + first_key && first_seq == 1, + "the first picture: {:?}", + frames[0] + ); + let (last_at, _, _) = *frames.last().unwrap(); + assert!(last_at < secs(1.2), "media at {last_at:?}"); + sim.stop().await; + }); +} + +/// An iPad that answers as the app does streams exactly as before: one +/// keyframe for its session, and every frame from the first on, with no +/// pause while the PC waits to hear from it. +#[test] +fn an_ipad_that_answers_at_once_streams_as_before() { + run(async { + let mut sim = Sim::start().await; + let ipad = sim.add_client(Answers::AtOnce); + sim.clients[ipad].hello().await; + sim.run_until(secs(4.0)).await; + + let client = &sim.clients[ipad]; + let (id, accepted) = client.session.expect("a session"); + let frames = client.frames(); + eprintln!( + "ANSWERED_SIM accepted={accepted:?} media_bytes={} frames={} first={:?} forced={:?}", + client.media_bytes(), + frames.len(), + frames.first(), + sim.forced.lock() + ); + assert_eq!(sim.shared.session.lock().session_id(), Some(id)); + assert_eq!(sim.forced.lock().len(), 1, "one keyframe"); + assert!(frames[0].2 && frames[0].1 == 1); + assert!( + frames.windows(2).all(|pair| pair[1].1 == pair[0].1 + 1), + "frames missing" + ); + let gap = frames + .windows(2) + .map(|pair| pair[1].0 - pair[0].0) + .max() + .unwrap(); + assert!(gap <= FRAME_INTERVAL + STEP, "video stopped for {gap:?}"); + assert!(frames.last().unwrap().0 >= secs(3.9)); + assert!(client.media_bytes() > 3 * UNANSWERED_MEDIA_BUDGET); + sim.stop().await; + }); +} + +/// An iPad first heard after its budget ran out gets nothing more until +/// then, and then a keyframe and the stream, numbered on from the last +/// frame it got. +#[test] +fn an_ipad_heard_late_gets_a_keyframe_then_the_stream() { + run(async { + let mut sim = Sim::start().await; + let ipad = sim.add_client(Answers::After(secs(2.0))); + sim.clients[ipad].hello().await; + sim.run_until(secs(4.0)).await; + + let client = &sim.clients[ipad]; + let (id, accepted) = client.session.expect("a session"); + let answered = accepted + secs(2.0); + let frames = client.frames(); + let before: Vec<_> = frames.iter().filter(|(at, _, _)| *at < answered).collect(); + let after: Vec<_> = frames.iter().filter(|(at, _, _)| *at >= answered).collect(); + let held_bytes: usize = client + .received + .iter() + .filter(|(at, heard, _)| *at < answered && matches!(heard, Heard::Media(..))) + .map(|(_, _, len)| len) + .sum(); + eprintln!( + "LATE_SIM accepted={accepted:?} before={} ({held_bytes} bytes, last {:?}) \ + after_first={:?} forced={:?}", + before.len(), + before.last(), + after.first(), + sim.forced.lock() + ); + assert_eq!(sim.shared.session.lock().session_id(), Some(id)); + assert!(held_bytes <= UNANSWERED_MEDIA_BUDGET, "{held_bytes}"); + assert!(before.last().unwrap().0 < secs(1.2), "{:?}", before.last()); + let (resumed, seq, key) = *after[0]; + assert!(key, "the stream starts again with a keyframe"); + assert_eq!(seq, before.last().unwrap().1 + 1, "numbered on"); + assert!( + resumed - answered <= FRAME_INTERVAL + 2 * STEP, + "{resumed:?}" + ); + assert_eq!( + sim.forced + .lock() + .iter() + .filter(|at| **at >= answered) + .count(), + 1 + ); + assert!( + frames.windows(2).all(|pair| pair[1].1 == pair[0].1 + 1), + "frames missing" + ); + sim.stop().await; + }); +} diff --git a/host/tests/send_congestion.rs b/host/tests/send_congestion.rs new file mode 100644 index 00000000..985eb2c2 --- /dev/null +++ b/host/tests/send_congestion.rs @@ -0,0 +1,486 @@ +//! When the system has no buffer space for a datagram (WSAENOBUFS 10055 on +//! Windows, ENOBUFS elsewhere), the real transport loop backs off, steps the +//! bitrate down at most once per episode, gives up a frame it can't send +//! rather than wait on it, retries a control reply once, and logs one line +//! per episode plus its end. The sends are refused through the test-hooks +//! seam, which also records each video send and pacing wait, so the order +//! and deadlines are checked where the transport sent, not where the fake +//! iPad happened to read. + +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::Ordering; +use std::sync::mpsc as std_mpsc; +use std::time::{Duration, Instant}; + +use eternal_host::control::SharedControl; +use eternal_host::encoder::NALUnit; +use eternal_host::transport::congestion::{BACKOFF, EPISODE_QUIET}; +use eternal_host::transport::probe::{self, Attempt, Fail, Kind}; +use eternal_wire::v2::control::{ + encode_control, parse_control, ControlMessage, Hello2, Nack, Ping, CAP_DECODE_H264, + FEATURE_SUPPORTS_NACK, +}; +use eternal_wire::v2::media::{media_payload_size, MediaHeader}; +use parking_lot::Mutex; + +static LOG: Mutex> = Mutex::new(Vec::new()); + +struct Sink; + +impl std::io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + LOG.lock().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn log_lines(containing: &str) -> Vec { + String::from_utf8_lossy(&LOG.lock()) + .lines() + .filter(|line| line.contains(containing)) + .map(|line| line.trim().to_owned()) + .collect() +} + +fn free_udp_port() -> u16 { + UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap() +} + +struct Ipad { + socket: UdpSocket, + host: String, + session_id: u32, + msg_seq: u32, + pinged: Instant, +} + +impl Ipad { + fn addr(&self) -> SocketAddr { + self.socket.local_addr().unwrap() + } + + fn send(&mut self, message: &ControlMessage) { + self.msg_seq += 1; + self.socket + .send_to( + &encode_control(self.session_id, self.msg_seq, message), + &self.host, + ) + .unwrap(); + } + + /// Keeps the session alive while the test waits. + fn keep_alive(&mut self) { + if self.pinged.elapsed() >= Duration::from_millis(250) { + self.pinged = Instant::now(); + self.send(&ControlMessage::Ping(Ping { t1_us: 1 })); + } + } + + /// Every datagram for `wait`, with when it came. + fn receive(&mut self, wait: Duration) -> Vec<(Instant, Vec)> { + let mut buffer = [0; 2048]; + let mut received = Vec::new(); + let until = Instant::now() + wait; + while Instant::now() < until { + self.keep_alive(); + if let Ok(len) = self.socket.recv(&mut buffer) { + received.push((Instant::now(), buffer[..len].to_vec())); + } + } + received + } + + /// Frame `frame_seq`'s original fragments among `received`, in order. + fn fragments(received: &[(Instant, Vec)], frame_seq: u32) -> Vec { + received + .iter() + .filter_map(|(_, bytes)| { + let (header, _) = MediaHeader::decode(bytes).ok()?; + (header.frame_seq == frame_seq && !header.is_retransmit) + .then_some(header.frag_index) + }) + .collect() + } + + /// Waits until the transport has tried at least `count` sends of frame + /// `frame_seq`'s originals, and returns them. + fn attempts(&mut self, frame_seq: u32, count: usize) -> Vec { + self.attempted(frame_seq, false, count) + } + + /// As [`Self::attempts`], for the frame's repairs. + fn retransmits(&mut self, frame_seq: u32, count: usize) -> Vec { + self.attempted(frame_seq, true, count) + } + + fn attempted(&mut self, frame_seq: u32, retransmit: bool, count: usize) -> Vec { + let until = Instant::now() + Duration::from_secs(10); + loop { + let attempts = attempts_of(frame_seq, retransmit); + if attempts.len() >= count { + return attempts; + } + assert!( + Instant::now() < until, + "only {} sends: {attempts:?}", + attempts.len() + ); + self.keep_alive(); + std::thread::sleep(Duration::from_millis(1)); + } + } + + /// Waits for a heartbeat, so the next second has none. + fn after_heartbeat(&mut self) { + let mut buffer = [0; 2048]; + let until = Instant::now() + Duration::from_secs(3); + while Instant::now() < until { + self.keep_alive(); + if let Ok(len) = self.socket.recv(&mut buffer) { + if let Ok((_, ControlMessage::Heartbeat(_))) = parse_control(&buffer[..len]) { + return; + } + } + } + panic!("no heartbeat"); + } +} + +fn attempts_of(frame_seq: u32, retransmit: bool) -> Vec { + probe::attempts() + .into_iter() + .filter(|a| a.frame_seq == frame_seq && a.retransmit == retransmit) + .collect() +} + +fn indexes(attempts: &[Attempt]) -> Vec { + attempts.iter().map(|a| a.frag_index).collect() +} + +/// Which attempts failed, by position, and how. +fn failures(attempts: &[Attempt]) -> Vec<(usize, Fail)> { + attempts + .iter() + .enumerate() + .filter_map(|(n, a)| a.failed.map(|fail| (n, fail))) + .collect() +} + +/// The pacing waits asked for after `from` and before `to`. +fn waits_between(from: Instant, to: Instant) -> Vec<(Instant, Instant)> { + probe::waits() + .into_iter() + .filter(|&(asked, _)| asked > from && asked < to) + .collect() +} + +/// A keyframe of exactly `fragments` datagrams. +fn keyframe(shared: &SharedControl, fragments: usize) -> NALUnit { + let payload = media_payload_size(shared.max_dgram.load(Ordering::SeqCst) as usize).unwrap(); + let mut data = vec![0x41; fragments * payload]; + data[..5].copy_from_slice(&[0, 0, 0, 1, 0x65]); + NALUnit { + data, + sequence: 0, + timestamp: Instant::now(), + encode_duration_us: 0, + is_keyframe: true, + } +} + +#[test] +fn refused_sends_back_off_step_down_once_and_log_one_episode() { + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_target(false) + .with_max_level(tracing::Level::INFO) + .with_writer(|| Sink) + .finish(); + tracing::subscriber::set_global_default(subscriber).unwrap(); + + let port = free_udp_port(); + let shared = SharedControl::new(port, 15_000_000); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + let (nal_tx, nal_rx) = tokio::sync::mpsc::channel(16); + let (_audio_tx, audio_rx) = tokio::sync::mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + let sender_shared = shared.clone(); + let sender = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(eternal_host::transport::start_sender( + nal_rx, + audio_rx, + port, + sender_shared, + supervisor_tx, + 1, + )) + .unwrap(); + }); + + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(5))) + .unwrap(); + let mut ipad = Ipad { + host: format!("127.0.0.1:{port}"), + session_id: 0, + msg_seq: 0, + pinged: Instant::now(), + socket, + }; + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 9, + listen_port: ipad.addr().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_SUPPORTS_NACK, + screen_px_w: 640, + screen_px_h: 360, + screen_pt_w: 640, + screen_pt_h: 360, + refresh_hz: 60, + device_name: "Congested iPad".into(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + let mut buffer = [0; 2048]; + let deadline = Instant::now() + Duration::from_secs(10); + ipad.session_id = loop { + assert!(Instant::now() < deadline, "no HELLO_ACK"); + ipad.send(&hello); + if let Ok(len) = ipad.socket.recv(&mut buffer) { + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&buffer[..len]) { + break ack.session_id; + } + } + }; + let addr = ipad.addr(); + assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 15_000_000); + probe::record(addr); + + // Frame 1: fragment 10 is refused once. The frame asks the pacing clock + // for one back-off, sends it again once that deadline has passed, and + // the bitrate steps down without waiting for a loss report. + probe::refuse_sends(addr, Kind::Media, 10, 1); + nal_tx.blocking_send(keyframe(&shared, 40)).unwrap(); + let frame = ipad.attempts(1, 41); + let expected: Vec = (0..=10).chain(10..40).collect(); + assert_eq!( + indexes(&frame), + expected, + "fragment 10 is tried twice, in order" + ); + assert_eq!(failures(&frame), [(10, Fail::NoBufferSpace)]); + let (refused, retried) = (frame[10].at, frame[11].at); + let backoffs = waits_between(refused, retried); + assert_eq!(backoffs.len(), 1, "one back-off: {backoffs:?}"); + let (_, deadline) = backoffs[0]; + assert!( + deadline >= refused + BACKOFF, + "the back-off lasts {BACKOFF:?}" + ); + assert!(retried >= deadline, "the retry waits for the deadline"); + let stepped = Instant::now(); + assert_eq!( + shared.abr_current_bps.load(Ordering::SeqCst), + 12_000_000, + "one rung down at once" + ); + let received = ipad.receive(Duration::from_millis(100)); + assert_eq!( + Ipad::fragments(&received, 1), + (0..40).collect::>(), + "the refused fragment is sent again, not lost" + ); + + // Frame 2, in the same episode: fragment 5 is refused, and so is its + // retry. The frame gives up its rest, which repair recovers, and the + // bitrate stays: one step per episode. + probe::refuse_sends(addr, Kind::Media, 5, 2); + nal_tx.blocking_send(keyframe(&shared, 40)).unwrap(); + let frame = ipad.attempts(2, 7); + let received = ipad.receive(Duration::from_millis(100)); + let frame_after = attempts_of(2, false); + assert_eq!( + frame_after.len(), + frame.len(), + "nothing after the second refusal" + ); + assert_eq!(indexes(&frame), [0, 1, 2, 3, 4, 5, 5]); + assert_eq!( + failures(&frame), + [(5, Fail::NoBufferSpace), (6, Fail::NoBufferSpace)] + ); + assert_eq!(waits_between(frame[5].at, frame[6].at).len(), 1); + assert_eq!( + Ipad::fragments(&received, 2), + (0..5).collect::>(), + "the rest is given up" + ); + assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 12_000_000); + + // Repairs go through the same sends: five of them are refused, count in + // the episode, and don't arrive. + probe::refuse_sends(addr, Kind::Media, 10, 5); + let epoch = received + .iter() + .find_map(|(_, bytes)| MediaHeader::decode(bytes).ok()) + .unwrap() + .0 + .stream_epoch; + ipad.send(&ControlMessage::Nack(Nack { + stream_epoch: epoch, + frame_seq: 2, + frag_count: 40, + missing: (5..40).collect(), + })); + let repairs = ipad.retransmits(2, 35); + assert_eq!( + failures(&repairs), + (10..15) + .map(|n| (n, Fail::NoBufferSpace)) + .collect::>() + ); + let repaired = ipad + .receive(Duration::from_millis(100)) + .iter() + .filter(|(_, bytes)| { + MediaHeader::decode(bytes) + .is_ok_and(|(header, _)| header.frame_seq == 2 && header.is_retransmit) + }) + .count(); + assert_eq!( + repaired, 30, + "repair recovers what was given up, less what was refused" + ); + + // Frame 3: fragment 3 is refused, and its retry fails another way. That + // is logged as any failed send is, and the frame goes on. + probe::fail_sends(addr, Kind::Media, 3, &[Fail::NoBufferSpace, Fail::Other]); + nal_tx.blocking_send(keyframe(&shared, 40)).unwrap(); + let frame = ipad.attempts(3, 41); + let expected: Vec = (0..=3).chain(3..40).collect(); + assert_eq!(indexes(&frame), expected); + assert_eq!( + failures(&frame), + [(3, Fail::NoBufferSpace), (4, Fail::Other)] + ); + let failed = log_lines("UDP send failed link="); + assert_eq!( + failed, + ["WARN UDP send failed link=wifi error=connection reset"] + ); + + // Five frames of 100 fragments while every send is refused: each frame + // asks for one back-off, no longer, and gives up after its retry, so no + // frame holds the loop for more than one back-off. + probe::refuse_sends(addr, Kind::Media, 0, 1000); + for _ in 0..5 { + nal_tx.blocking_send(keyframe(&shared, 100)).unwrap(); + } + let last = ipad.attempts(8, 2); + ipad.receive(Duration::from_millis(100)); + assert_eq!(probe::refusals_left(addr), 990, "two refusals per frame"); + let first = attempts_of(4, false)[0].at; + let backoffs = waits_between(first, last[1].at); + assert_eq!(backoffs.len(), 5, "{backoffs:?}"); + for (asked, until) in &backoffs { + assert!( + *until <= *asked + BACKOFF, + "a pause of {:?}", + *until - *asked + ); + } + for frame_seq in 4..=8 { + let frame = attempts_of(frame_seq, false); + assert_eq!(indexes(&frame), [0, 0], "frame {frame_seq}"); + assert_eq!( + failures(&frame), + [(0, Fail::NoBufferSpace), (1, Fail::NoBufferSpace)] + ); + } + probe::refuse_sends(addr, Kind::Media, 0, 0); + assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 12_000_000); + + // The episode ends after a quiet second, with one summary line. + let until = Instant::now() + EPISODE_QUIET + Duration::from_secs(2); + while log_lines("buffer space again").is_empty() && Instant::now() < until { + ipad.receive(Duration::from_millis(50)); + } + let started = log_lines("no buffer space"); + assert_eq!(started.len(), 1, "{started:#?}"); + assert!( + started[0].starts_with("WARN UDP send failed: no buffer space, backing off link=wifi"), + "{started:#?}" + ); + let ended = log_lines("buffer space again"); + assert_eq!(ended.len(), 1, "{ended:#?}"); + assert!( + ended[0] + .starts_with("INFO UDP sends have buffer space again link=wifi refused=19 took_ms="), + "{ended:#?}" + ); + let steps = log_lines("ABR stepping bitrate down"); + assert_eq!(steps.len(), 1, "{steps:#?}"); + assert!( + steps[0].contains("reason=\"no_buffer_space\""), + "{steps:#?}" + ); + + // A refused control reply is sent again once. This is a new episode, + // and the downgrade cooldown (3 s) since the first step is over, so it + // steps down again. + while stepped.elapsed() < Duration::from_millis(3_200) { + ipad.receive(Duration::from_millis(50)); + } + ipad.after_heartbeat(); + // No keep-alive Pong still on its way to take the refusal. + ipad.pinged = Instant::now(); + ipad.receive(Duration::from_millis(30)); + probe::refuse_sends(addr, Kind::Control, 0, 1); + ipad.send(&ControlMessage::Ping(Ping { t1_us: 0xC0FFEE })); + let ponged = ipad + .receive(Duration::from_millis(200)) + .iter() + .any(|(_, bytes)| { + matches!( + parse_control(bytes), + Ok((_, ControlMessage::Pong(pong))) if pong.t1_us == 0xC0FFEE + ) + }); + assert!(ponged, "the reply went on its retry"); + assert_eq!(probe::refusals_left(addr), 0); + // Stepped on the next liveness tick. + let until = Instant::now() + Duration::from_secs(1); + while shared.abr_current_bps.load(Ordering::SeqCst) != 10_000_000 && Instant::now() < until { + ipad.receive(Duration::from_millis(20)); + } + assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 10_000_000); + assert_eq!(log_lines("no buffer space").len(), 2); + assert!(log_lines("Failed to send control reply").is_empty()); + assert_eq!( + log_lines("UDP send failed link=").len(), + 1, + "no line per send" + ); + + drop(nal_tx); + shared.stop(); + sender.join().unwrap(); +} diff --git a/host/tests/transport_stall.rs b/host/tests/transport_stall.rs new file mode 100644 index 00000000..20d75212 --- /dev/null +++ b/host/tests/transport_stall.rs @@ -0,0 +1,137 @@ +//! A transport loop stuck in one call is named by the supervisor's watch, with +//! the step it is stuck in, and again when it moves. Drives the real loop and +//! blocks it the way a stuck call would: the session lock it needs at its +//! next tick, or at the second screen's step after it, is held elsewhere. + +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use eternal_host::control::SharedControl; +use eternal_host::transport::beat::{Phase, Verdict, Watch}; +use eternal_host::transport::{self}; +use tokio::sync::mpsc; + +/// The transport binds every address, so the port must be free on all. +fn free_udp_port() -> u16 { + std::net::UdpSocket::bind("0.0.0.0:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap() +} + +fn now_ms() -> u64 { + eternal_host::clock::host_now_us() / 1000 +} + +/// Checks the beat as the supervisor does, every 100 ms, until `until`. +/// Fails at once if the transport ended (it could not bind, say), which +/// would otherwise read as a loop that never marks. +fn watch_until( + watch: &mut Watch, + shared: &SharedControl, + transport: &std::thread::JoinHandle<()>, + until: Instant, +) -> Vec { + let mut verdicts = Vec::new(); + while Instant::now() < until { + assert!(!transport.is_finished(), "the transport ended early"); + verdicts.extend(watch.check(shared.transport_beat.read(), now_ms())); + std::thread::sleep(Duration::from_millis(100)); + } + verdicts +} + +#[test] +fn a_stuck_transport_loop_is_named_with_its_step() { + eternal_host::clock::init(); + // Never reach a real iPad over USB. + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + let port = free_udp_port(); + let shared = SharedControl::new(port, 10_000_000); + shared.running.store(true, Ordering::SeqCst); + let (nal_tx, nal_rx) = mpsc::channel(4); + let (_audio_tx, audio_rx) = mpsc::channel(4); + let control = shared.clone(); + let transport = std::thread::spawn(move || { + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(transport::start_sender( + nal_rx, + audio_rx, + port, + control, + supervisor_tx, + 1, + )) + .unwrap(); + }); + let mut watch = Watch::default(); + let started = Instant::now(); + while shared.transport_beat.read().is_none() { + assert!( + started.elapsed() < Duration::from_secs(30), + "the loop starts" + ); + assert!(!transport.is_finished(), "the transport ended early"); + std::thread::sleep(Duration::from_millis(10)); + } + // Running freely, it ticks every 250 ms. + assert_eq!( + watch_until( + &mut watch, + &shared, + &transport, + Instant::now() + Duration::from_millis(1500) + ), + [] + ); + + let held = shared.session.lock(); + let stuck = watch_until( + &mut watch, + &shared, + &transport, + Instant::now() + Duration::from_secs(4), + ); + drop(held); + // The loop takes the session lock at its tick and again, after each + // tick's replies, for the second screen's next step. Whichever it + // reaches first while the lock is held is where it is stuck. + let [Verdict::Stalled { + phase: stuck_in, + silent_ms, + }] = stuck.as_slice() + else { + panic!("{stuck:?}"); + }; + assert!( + matches!(stuck_in, Phase::Tick | Phase::SecondScreen), + "{stuck:?}" + ); + assert!(*silent_ms >= 2_000, "{stuck:?}"); + let moved = watch_until( + &mut watch, + &shared, + &transport, + Instant::now() + Duration::from_secs(1), + ); + assert!( + matches!( + moved.as_slice(), + [Verdict::Moving { phase, stalled_ms }] if phase == stuck_in && *stalled_ms >= 2_000 + ), + "{moved:?}" + ); + + shared.stop(); + drop(nal_tx); + transport.join().unwrap(); + assert_eq!( + shared.transport_beat.read(), + None, + "an ended loop is not stuck" + ); +} diff --git a/host/tests/window_device_loss.rs b/host/tests/window_device_loss.rs new file mode 100644 index 00000000..ad0db156 --- /dev/null +++ b/host/tests/window_device_loss.rs @@ -0,0 +1,407 @@ +//! The window survives the loss of its graphics device, as in a driver +//! reset. egui-wgpu 0.36 panicked on the next frame ("Failed to create +//! staging buffer for index data"), which ended the app and the stream; the +//! patched copy in `vendor/egui-wgpu` skips the frame and replaces the device +//! on a schedule (`Replacement`), giving up after repeated failures. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use eframe::egui; +use eframe::egui_wgpu::winit::{watch, Answer, Asked, Replacement, REPLACEMENT_TIMEOUT}; +use eframe::egui_wgpu::{ + replacement_candidates, AdapterKey, RenderState, Renderer, RendererOptions, ScreenDescriptor, + WgpuConfiguration, +}; +use eframe::wgpu; + +fn block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(future) +} + +fn device() -> (wgpu::Device, wgpu::Queue) { + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); + let adapter = block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())) + .expect("a graphics adapter"); + block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).expect("a device") +} + +/// One pass of a window with text and a button, tessellated. +fn pass(ctx: &egui::Context) -> (Vec, egui::TexturesDelta) { + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + ui.heading("EternalMonitor"); + let _ = ui.button("Start streaming"); + }); + let primitives = ctx.tessellate(output.shapes, output.pixels_per_point); + (primitives, output.textures_delta) +} + +#[test] +fn a_lost_device_skips_the_frame_instead_of_panicking() { + let (device, queue) = device(); + let mut renderer = Renderer::new( + &device, + wgpu::TextureFormat::Rgba8Unorm, + RendererOptions::default(), + ); + let ctx = egui::Context::default(); + let screen = ScreenDescriptor { + size_in_pixels: [640, 480], + pixels_per_point: 1.0, + }; + + let (primitives, mut textures) = pass(&ctx); + assert!(!primitives.is_empty()); + for (id, deltas) in &textures.set { + for delta in deltas { + renderer.update_texture(&device, &queue, *id, delta); + } + } + textures.clear(); + let mut encoder = device.create_command_encoder(&Default::default()); + renderer.update_buffers(&device, &queue, &mut encoder, &primitives, &screen); + queue.submit([encoder.finish()]); + assert!(!renderer.upload_failed()); + + // wgpu now refuses every staging buffer, as after a driver reset. + device.destroy(); + let (primitives, mut textures) = pass(&ctx); + textures.clear(); + let mut encoder = device.create_command_encoder(&Default::default()); + let callbacks = renderer.update_buffers(&device, &queue, &mut encoder, &primitives, &screen); + assert!(callbacks.is_empty()); + assert!(renderer.upload_failed()); +} + +const SECOND: Duration = Duration::from_secs(1); + +/// The waits between tries until the painter gives up, when every +/// replacement fails at once. +fn waits_until_given_up(replacement: &mut Replacement, mut now: Instant) -> Vec { + let mut waits = Vec::new(); + while let Some(wait) = replacement.wait(now) { + waits.push(wait); + now += wait; + assert!(waits.len() <= 20, "never gives up: {waits:?}"); + replacement.failed(now); + } + waits +} + +#[test] +fn a_lost_device_is_replaced_at_once_then_backs_off_then_gives_up() { + let start = Instant::now(); + let mut replacement = Replacement::default(); + assert_eq!(replacement.wait(start), Some(Duration::ZERO)); + replacement.drew(start); + assert!(!replacement.lost(start + SECOND)); + let ms = Duration::from_millis; + assert_eq!( + waits_until_given_up(&mut replacement, start + SECOND), + [ + ms(0), + ms(500), + ms(1000), + ms(2000), + ms(4000), + ms(8000), + ms(16000), + ms(30000), + ms(30000), + ms(30000), + ] + ); + assert_eq!(replacement.wait(start + Duration::from_secs(3600)), None); +} + +#[test] +fn a_new_device_that_fails_at_once_every_time_ends_in_giving_up() { + // As with a wgpu error on every frame: each new device is lost again + // right after it is made. + let mut now = Instant::now(); + let mut replacement = Replacement::default(); + let mut gave_up = 0; + for _ in 0..30 { + let Some(wait) = replacement.wait(now) else { + break; + }; + now += wait; + replacement.drew(now); + now += Duration::from_millis(20); + gave_up += usize::from(replacement.lost(now)); + } + assert_eq!(replacement.wait(now), None); + assert_eq!(gave_up, 1, "said once"); +} + +#[test] +fn a_device_that_drew_for_a_while_starts_the_count_again() { + let mut now = Instant::now(); + let mut replacement = Replacement::default(); + replacement.lost(now); + for _ in 0..(Replacement::MAX_FAILURES - 1) { + replacement.failed(now); + } + // One failure short of giving up, a new device works for a while. + now += replacement.wait(now).unwrap(); + replacement.drew(now); + now += Replacement::STABLE_AFTER; + assert!(!replacement.lost(now)); + assert_eq!(replacement.wait(now), Some(Duration::ZERO)); + replacement.failed(now); + assert_eq!(replacement.wait(now), Some(Duration::from_millis(500))); +} + +#[test] +fn a_stuck_replacement_times_out_without_blocking_and_its_late_device_is_taken() { + let (release, released) = std::sync::mpsc::channel::<()>(); + let start = Instant::now(); + // As a driver call that stalls while the replacement device is made. + let mut answer = Answer::ask("test replacement", start, REPLACEMENT_TIMEOUT, move || { + let _ = released.recv(); + "device" + }) + .unwrap(); + assert!(matches!(answer.poll(start), Asked::Waiting)); + assert!(matches!( + answer.poll(start + REPLACEMENT_TIMEOUT), + Asked::TimedOut + )); + // One timeout per period, each a failed try for the backoff. + assert!(matches!( + answer.poll(start + REPLACEMENT_TIMEOUT), + Asked::Waiting + )); + assert!(matches!( + answer.poll(start + REPLACEMENT_TIMEOUT * 2), + Asked::TimedOut + )); + + release.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + match answer.poll(start) { + Asked::Answered(device) => break assert_eq!(device, "device"), + Asked::Waiting => std::thread::sleep(Duration::from_millis(10)), + other => panic!("{other:?}"), + } + assert!(Instant::now() < deadline); + } +} + +#[test] +fn a_replacement_thread_that_dies_ends_the_wait() { + let start = Instant::now(); + let mut answer = Answer::<()>::ask("test replacement", start, REPLACEMENT_TIMEOUT, || { + panic!("simulated driver panic") + }) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !matches!(answer.poll(start), Asked::Ended) { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +fn a_replacement_that_never_answers_ends_in_giving_up() { + // The painter counts each timeout of a stuck replacement as a failed + // try, and starts no other while it is out. + let (_never, never) = std::sync::mpsc::channel::<()>(); + let mut now = Instant::now(); + let mut replacement = Replacement::default(); + replacement.lost(now); + let mut answer = Answer::ask("test replacement", now, REPLACEMENT_TIMEOUT, move || { + let _ = never.recv(); + }) + .unwrap(); + let mut timeouts = 0; + for _ in 0..100 { + if replacement.wait(now).is_none() { + break; + } + now += REPLACEMENT_TIMEOUT; + if let Asked::TimedOut = answer.poll(now) { + timeouts += 1; + replacement.failed(now); + } + assert!(timeouts <= Replacement::MAX_FAILURES); + } + assert_eq!(timeouts, Replacement::MAX_FAILURES); +} + +#[test] +fn a_replacement_device_is_made_without_any_window_or_surface() { + // What the replacement thread runs gets the configuration, the instance and + // adapter identities only, so a thread stuck in a driver can't hold the + // window or touch it after it is gone. + let config = WgpuConfiguration::default(); + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); + let drew_on = block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())) + .expect("a graphics adapter"); + let preferred = AdapterKey::of(&drew_on); + let lost = Arc::new(AtomicBool::new(false)); + let new = block_on(RenderState::new_device( + &config, + &instance, + Some(&preferred), + &[], + |device| watch(device, &lost), + )) + .expect("a device"); + assert_eq!(new.adapter_key(), preferred); + // The loss handlers are on the device: a wgpu error marks it lost, where + // wgpu's default handler would panic. + assert!(!lost.load(Ordering::SeqCst)); + let _misaligned = new.device().create_buffer(&wgpu::BufferDescriptor { + label: None, + size: 3, + usage: wgpu::BufferUsages::COPY_DST, + mapped_at_creation: true, + }); + assert!(lost.load(Ordering::SeqCst)); + let buffer = new.device().create_buffer(&wgpu::BufferDescriptor { + label: None, + size: 256, + usage: wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + assert_eq!(buffer.size(), 256); +} + +fn adapter(name: &str, backend: wgpu::Backend, vendor: u32) -> AdapterKey { + AdapterKey { + luid: None, + backend, + vendor, + device: 1, + device_type: wgpu::DeviceType::DiscreteGpu, + name: name.into(), + pci_bus: String::new(), + } +} + +#[test] +fn a_replacement_prefers_the_adapter_that_drew_and_moves_past_ones_that_cannot() { + use wgpu::Backend::{Dx12, Vulkan}; + // A hybrid PC: the window started on the dGPU with DX12. + let igpu = adapter("iGPU", Dx12, 0x1002); + let dgpu = adapter("dGPU", Dx12, 0x10de); + let dgpu_vulkan = adapter("dGPU", Vulkan, 0x10de); + let listed = [igpu.clone(), dgpu.clone(), dgpu_vulkan.clone()]; + + // The adapter that drew is the only choice while it is listed. + assert_eq!(replacement_candidates(&listed, Some(&dgpu), &[]), [1]); + // Gone from the listing, as across a driver reset: any of the others. + assert_eq!( + replacement_candidates(&[igpu.clone(), dgpu_vulkan.clone()], Some(&dgpu), &[]), + [0, 1] + ); + // Nothing left to try. + assert!(replacement_candidates(&listed, Some(&dgpu), &listed).is_empty()); + + // Each one the window can't draw on is left out, and the next try moves on, + // with the selector taking the first candidate. + let can_draw = |key: &AdapterKey| *key == dgpu_vulkan; + let mut excluded = Vec::new(); + let mut tried = Vec::new(); + let chosen = loop { + let candidates = replacement_candidates(&listed, Some(&dgpu), &excluded); + let key = listed[candidates[0]].clone(); + tried.push(key.clone()); + if can_draw(&key) { + break key; + } + excluded.push(key); + assert!(tried.len() <= listed.len()); + }; + assert_eq!(chosen, dgpu_vulkan); + assert_eq!(tried, [dgpu, igpu, dgpu_vulkan]); +} + +#[test] +fn two_identical_cards_are_told_apart_by_their_luid() { + use wgpu::Backend::Dx12; + // Two of the same card on different buses: wgpu's DX12 bus comes from the + // PCI ids, so everything but the LUID matches. + let card = |luid| AdapterKey { + luid: Some(luid), + pci_bus: "0000:01:00.0".into(), + ..adapter("RTX 5080", Dx12, 0x10de) + }; + let (first, second) = (card(0x0000_0000_0001_a2b3), card(0x0000_0000_0001_c4d5)); + let listed = [first.clone(), second.clone()]; + + // The one that drew is chosen, not its twin listed first. + assert_eq!(replacement_candidates(&listed, Some(&second), &[]), [1]); + // Leaving one out leaves the other. + assert_eq!( + replacement_candidates(&listed, Some(&first), std::slice::from_ref(&first)), + [1] + ); + assert_eq!(replacement_candidates(&listed, None, &[second]), [0]); +} + +#[cfg(windows)] +#[test] +fn every_dx12_adapter_has_its_own_luid() { + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { + backends: wgpu::Backends::DX12, + ..wgpu::InstanceDescriptor::new_without_display_handle() + }); + let adapters = block_on(instance.enumerate_adapters(wgpu::Backends::DX12)); + assert!(!adapters.is_empty()); + let luids: Vec = adapters + .iter() + .map(|adapter| AdapterKey::of(adapter).luid.expect("a DX12 LUID")) + .collect(); + let mut unique = luids.clone(); + unique.sort_unstable(); + unique.dedup(); + assert_eq!(unique.len(), luids.len(), "{luids:x?}"); +} + +#[test] +fn an_adapter_with_a_new_luid_after_a_driver_update_is_still_itself() { + use wgpu::Backend::Vulkan; + // Two of the same card; Vulkan reports each one's own PCI bus. + let card = |luid, bus: &str| AdapterKey { + luid: Some(luid), + pci_bus: bus.into(), + ..adapter("RTX 5080", Vulkan, 0x10de) + }; + let drew = card(1, "0000:01:00.0"); + let twin = card(2, "0000:02:00.0"); + // A driver update gives the card that drew a new LUID; its twin is listed first. + let listed = [twin.clone(), card(3, "0000:01:00.0")]; + assert_eq!(replacement_candidates(&listed, Some(&drew), &[]), [1]); + // An adapter left out stays left out with its new LUID. + assert_eq!( + replacement_candidates(&listed, None, std::slice::from_ref(&drew)), + [0] + ); + assert_eq!( + replacement_candidates(&listed, Some(&twin), std::slice::from_ref(&drew)), + [0] + ); +} + +#[test] +fn two_cards_alike_but_for_their_new_luids_are_no_preference() { + use wgpu::Backend::Dx12; + // On DX12 identical cards share the bus too: only the LUID tells them apart, + // and after an update both have new ones. + let card = |luid| AdapterKey { + luid: Some(luid), + pci_bus: "0000:01:00.0".into(), + ..adapter("RTX 5080", Dx12, 0x10de) + }; + let listed = [card(3), card(4)]; + assert_eq!(replacement_candidates(&listed, Some(&card(1)), &[]), [0, 1]); + // Nor is either left out for an old LUID. + assert_eq!(replacement_candidates(&listed, None, &[card(2)]), [0, 1]); +} diff --git a/installer/EternalMonitor.iss b/installer/EternalMonitor.iss index b29b673b..e66679f4 100644 --- a/installer/EternalMonitor.iss +++ b/installer/EternalMonitor.iss @@ -206,18 +206,24 @@ Name: "{autodesktop}\EternalMonitor"; Filename: "{app}\EternalMonitor-host. ; listens only on UDP; the TCP rule older versions added is removed and not re-added. Filename: "{sys}\netsh.exe"; Parameters: "advfirewall firewall delete rule name=""EternalMonitor Host UDP"""; Flags: runhidden waituntilterminated Filename: "{sys}\netsh.exe"; Parameters: "advfirewall firewall delete rule name=""EternalMonitor Host TCP"""; Flags: runhidden waituntilterminated +; Every other inbound rule for the host goes too. Windows adds Block rules for it when its +; "allow access" prompt is dismissed, and a Block rule beats the Allow rule below: iPads then +; get no answer. Running Setup again is the fix the PC and the iPad offer for that. +Filename: "{sys}\netsh.exe"; Parameters: "advfirewall firewall delete rule name=all dir=in program=""{app}\EternalMonitor-host.exe"""; Flags: runhidden waituntilterminated Filename: "{sys}\netsh.exe"; Parameters: "advfirewall firewall add rule name=""EternalMonitor Host UDP"" dir=in action=allow program=""{app}\EternalMonitor-host.exe"" protocol=UDP profile=domain,private,public enable=yes"; StatusMsg: "{cm:FirewallStatus}"; Flags: runhidden waituntilterminated #ifdef IncludeDriver ; Save the screens before the driver steps: starting the virtual display can leave a PC with no ; screen at all. The check after them puts the saved screens back only then. Both always exit 0. Filename: "powershell.exe"; Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\scripts\display-guard.ps1"" -Action save -Path ""{tmp}\display-layout.json"""; Flags: runhidden waituntilterminated logoutput ; The vendor setup prompts to uninstall an existing installation even with silent -; switches. Preserve an existing driver and install only when it is absent. +; switches. Preserve an existing driver and install only when it is absent, and only on +; Windows 11: Second screen needs it (see WindowsRunsVdd). Filename: "{app}\driver\vdd-setup-x64.exe"; Parameters: "/VERYSILENT /SUPPRESSMSGBOXES /NORESTART"; StatusMsg: "{cm:DriverStatus}"; Flags: waituntilterminated; Check: NeedsVddInstall; AfterInstall: RecordVddOwnership ; Register the enable/disable scheduled tasks and leave the virtual display OFF by default — ; EternalMonitor turns it on only while streaming to it, so there's no phantom monitor. -; A preserved driver whose device was removed or broke is repaired first. -Filename: "powershell.exe"; Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\scripts\vdd-tasks-setup.ps1"""; StatusMsg: "{cm:DisplayStatus}"; Flags: runhidden waituntilterminated; Check: VddInstallSucceeded; BeforeInstall: RepairVddDriver; AfterInstall: VerifyVddDriver +; A preserved driver whose device was removed or broke is repaired first. On Windows 10 a +; driver an earlier version installed keeps its tasks, which keep it off. +Filename: "powershell.exe"; Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\scripts\vdd-tasks-setup.ps1"""; StatusMsg: "{cm:DisplayStatus}"; Flags: runhidden waituntilterminated; Check: VddTasksWanted; BeforeInstall: RepairVddDriver; AfterInstall: VerifyVddDriver ; After the Disable above, and also when a failed driver step skipped it. Filename: "powershell.exe"; Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\scripts\display-guard.ps1"" -Action check -Path ""{tmp}\display-layout.json"""; Flags: runhidden waituntilterminated logoutput #endif @@ -302,19 +308,50 @@ begin Result := Result and (Directory <> '') and (Version <> ''); end; +const + FirstWindows11Build = 22000; + +{ The bundled driver needs Windows 11's display driver framework. On Windows 10 + it installs and starts, but Windows never adds its screen, and the app shows + the mirror there without turning it on (second_screen::driver_supports_windows + in the host). } +function WindowsRunsVdd: Boolean; +var + Windows: TWindowsVersion; +begin + GetWindowsVersionEx(Windows); + Result := Windows.Build >= FirstWindows11Build; +end; + function NeedsVddInstall: Boolean; var Directory, Version: String; begin Result := not ReadVddRegistration(Directory, Version); if not Result then - Log('Preserving existing Virtual Display Driver package ' + Version); + Log('Preserving existing Virtual Display Driver package ' + Version) + else if not WindowsRunsVdd then begin + Log('Not installing the Virtual Display Driver: Second screen needs Windows 11'); + Result := False; + end; +end; + +{ The driver's tasks: wherever a driver is installed, so one an earlier version + installed on Windows 10 stays off. None on Windows 10 without one. } +function VddTasksWanted: Boolean; +var + Directory, Version: String; +begin + Result := VddInstallSucceeded and (WindowsRunsVdd or ReadVddRegistration(Directory, Version)); end; procedure VerifyVddDriver; var ExitCode: Integer; begin + { Windows 10 doesn't use the driver: a kept one isn't checked. } + if not WindowsRunsVdd then + exit; { [Run] ignores a child process's exit code. Check the actual PnP binding explicitly, including upgrades that preserve an existing registration. } if not Exec(ExpandConstant('{sys}\WindowsPowerShell\v1.0\powershell.exe'), @@ -332,7 +369,12 @@ var begin { Setup keeps an existing driver instead of running its setup again. When Setup installed it and no device works, put one back (exit codes in the - script). A driver another app installed is left alone. } + script). A driver another app installed is left alone. Windows 10 doesn't + use the driver: a kept one is left as it is, with no device changes. } + if not WindowsRunsVdd then begin + Log('Leaving the Virtual Display Driver as it is: Second screen needs Windows 11'); + exit; + end; if not Exec(ExpandConstant('{sys}\WindowsPowerShell\v1.0\powershell.exe'), '-NoProfile -ExecutionPolicy Bypass -File "' + ExpandConstant('{app}\scripts\vdd-driver-repair.ps1') + '"', '', SW_HIDE, ewWaitUntilTerminated, ExitCode) then diff --git a/installer/tests/Test-InstallerWindows10.ps1 b/installer/tests/Test-InstallerWindows10.ps1 new file mode 100644 index 00000000..165004d7 --- /dev/null +++ b/installer/tests/Test-InstallerWindows10.ps1 @@ -0,0 +1,75 @@ +# Static checks for Setup on Windows 10, where Second screen needs Windows 11: a fresh install +# skips the Virtual Display Driver and its tasks, and an upgrade keeps a driver an earlier +# version installed, with its tasks, without repairing or verifying it. Windows 11 is +# unchanged. Nothing is compiled or installed. The checks also run against broken copies of +# the script, which they must reject. +param([string]$IssPath) +$ErrorActionPreference = 'Stop' +$source = if ($IssPath) { $IssPath } else { Join-Path $PSScriptRoot '..\EternalMonitor.iss' } +$iss = Get-Content -Raw -Encoding UTF8 $source + +# One [Code] routine, from its header to its closing "end;". +function Get-Routine([string]$Text, [string]$Header) { + [regex]::Match($Text, '(?s)\n' + [regex]::Escape($Header) + '.*?\nend;').Value +} + +function Get-Failures([string]$Text) { + $failures = @() + if ($Text -notmatch '(?m)^\s*FirstWindows11Build = 22000;') { + $failures += 'The first Windows 11 build is not 22000' + } + $runs = Get-Routine $Text 'function WindowsRunsVdd: Boolean;' + if ($runs -notmatch 'GetWindowsVersionEx\((\w+)\);' -or + $runs -notmatch "Result := $($Matches[1])\.Build >= FirstWindows11Build;") { + $failures += 'WindowsRunsVdd does not compare the Windows build with the first Windows 11 build' + } + + # Fresh install: a registered driver is kept on every Windows; with none, only Windows 11 + # installs one. + $needs = Get-Routine $Text 'function NeedsVddInstall: Boolean;' + if ($needs -notmatch '(?s)Result := not ReadVddRegistration\(Directory, Version\);.*else if not WindowsRunsVdd then begin.*Result := False;') { + $failures += 'NeedsVddInstall installs the driver on Windows 10' + } + if ($Text -notmatch '(?m)^Filename: "\{app\}\\driver\\vdd-setup-x64\.exe";.*Check: NeedsVddInstall;') { + $failures += 'The vendor setup does not run under NeedsVddInstall' + } + + # The tasks: wherever a driver is installed, and on Windows 11. + $wanted = Get-Routine $Text 'function VddTasksWanted: Boolean;' + if ($wanted -notmatch 'Result := VddInstallSucceeded and \(WindowsRunsVdd or ReadVddRegistration\(Directory, Version\)\);') { + $failures += 'VddTasksWanted does not keep the tasks for an installed driver only' + } + $tasks = $Text -split "\r?\n" | Where-Object { $_ -match '^Filename: "powershell\.exe";.*vdd-tasks-setup\.ps1' } + if (@($tasks).Count -ne 1 -or $tasks -notmatch 'Check: VddTasksWanted;') { + $failures += 'The driver tasks are set up without VddTasksWanted' + } + + # A driver kept on Windows 10 is neither repaired nor checked: no device changes, and no + # failed Setup over a driver Windows 10 doesn't use. + foreach ($routine in 'procedure RepairVddDriver;', 'procedure VerifyVddDriver;') { + $body = Get-Routine $Text $routine + $first = [regex]::Match($body, '(?s)\nbegin\s*(?:\{[^}]*\}\s*)*(?if not WindowsRunsVdd then(?: begin)?\s*(?:Log\([^;]*\);\s*)?exit;)') + if (!$first.Success) { $failures += "$($routine.Split(' ')[1].TrimEnd(';')) runs on Windows 10" } + } + $failures +} + +$failures = @(Get-Failures $iss) +if ($failures) { throw "Windows 10 checks failed:`n " + ($failures -join "`n ") } + +# Each broken copy must fail. +$broken = [ordered]@{ + 'Windows 10 build' = $iss.Replace('FirstWindows11Build = 22000;', 'FirstWindows11Build = 19041;') + 'build not read' = $iss.Replace('Result := Windows.Build >= FirstWindows11Build;', 'Result := True;') + 'fresh install on Windows 10' = $iss.Replace(" else if not WindowsRunsVdd then begin`r`n", " else if False then begin`r`n").Replace(" else if not WindowsRunsVdd then begin`n", " else if False then begin`n") + 'vendor setup unchecked' = $iss.Replace('Check: NeedsVddInstall;', 'Check: VddInstallSucceeded;') + 'tasks without a driver' = $iss.Replace('(WindowsRunsVdd or ReadVddRegistration(Directory, Version))', '(True or ReadVddRegistration(Directory, Version))') + 'tasks unchecked' = $iss.Replace('Check: VddTasksWanted;', 'Check: VddInstallSucceeded;') + 'repair on Windows 10' = [regex]::Replace($iss, '(?s)(procedure RepairVddDriver;.*?\nbegin.*?)if not WindowsRunsVdd then begin', '$1if False then begin') + 'verify on Windows 10' = [regex]::Replace($iss, '(?s)(procedure VerifyVddDriver;.*?\nbegin.*?)if not WindowsRunsVdd then', '$1if False then') +} +foreach ($case in $broken.Keys) { + if ($broken[$case] -ceq $iss) { throw "The '$case' copy is the real script; update this test" } + if (!@(Get-Failures $broken[$case])) { throw "The checks pass a broken script: $case" } +} +Write-Output "PASS: Windows 10 skips a fresh driver install and its tasks, keeps an installed driver unrepaired and unchecked; $($broken.Count) broken copies rejected" diff --git a/ios/EternalMonitor/App/CableHold.swift b/ios/EternalMonitor/App/CableHold.swift new file mode 100644 index 00000000..aa2d2b81 --- /dev/null +++ b/ios/EternalMonitor/App/CableHold.swift @@ -0,0 +1,28 @@ +import Foundation + +/// A PC that turns this iPad down over the cable dials it again about a +/// second after each refusal, so the refusal would repeat for as long as the +/// cable stays in. Until something changes, the iPad stops answering the +/// cable: after Busy for a wait that grows with each Busy in a row, and after +/// a version refusal until the user taps Try again or Connect USB. +struct CableHold: Equatable { + enum Reason: Equatable { + case busy + case version + } + + let reason: Reason + /// When the iPad answers the cable again; nil waits for the user. + let until: Date? + + /// The wait after `count` Busy refusals in a row: 30 s, doubling each + /// time, at most 5 minutes. + static func busySeconds(_ count: Int) -> Double { + min(30 * pow(2, Double(min(max(count, 1), 16) - 1)), 300) + } + + /// Whether the cable still goes unanswered at `now`. + func holds(at now: Date) -> Bool { + until.map { now < $0 } ?? true + } +} diff --git a/ios/EternalMonitor/App/ConnectView.swift b/ios/EternalMonitor/App/ConnectView.swift index daaaa718..3de3b88c 100644 --- a/ios/EternalMonitor/App/ConnectView.swift +++ b/ios/EternalMonitor/App/ConnectView.swift @@ -9,6 +9,7 @@ struct ConnectView: View { @ObservedObject private var pairings = PairingStore.shared @ObservedObject private var language = LanguageStore.shared @ObservedObject private var updates = UpdateChecker.shared + @ObservedObject private var oldPC = OldPCPrompt.shared @State private var hostIP: String = "" @State private var port: String = "9876" @@ -96,6 +97,10 @@ struct ConnectView: View { ) } + if let card = oldPC.card { + OldPCCard(card: card, close: { withAnimation(Motion.fade) { oldPC.close() } }) + } + if isConnecting { connectingCard .transition(.opacity.combined(with: .scale(scale: 0.97))) @@ -229,6 +234,12 @@ struct ConnectView: View { .onChange(of: scenePhase) { _, phase in if phase == .active && scanner.searchProblem == .localNetworkOff { scanner.startScan() } } + // The old-PC card shows only here, never over a stream. + .onAppear(perform: offerOldPC) + .onChange(of: connectionManager.state) { _, _ in offerOldPC() } + .onChange(of: scanner.hosts) { _, _ in offerOldPC() } + .onChange(of: showSetup) { _, _ in offerOldPC() } + .onDisappear { oldPC.close() } .animation(.easeInOut(duration: 0.2), value: isConnecting) .animation(.easeInOut(duration: 0.2), value: connectionManager.connectionError) .animation(.easeInOut(duration: 0.2), value: connectionManager.problem) @@ -275,7 +286,8 @@ struct ConnectView: View { case .connect(let host, let port): hostIP = host self.port = String(port) - connectionManager.connect(host: host, port: port) + // Setup offers only the PCs Bonjour found. + connectionManager.connect(host: host, port: port, found: true) return case .scanQR: showQRScanner = true @@ -340,6 +352,14 @@ struct ConnectView: View { .foregroundStyle(Theme.textMuted) .fixedSize(horizontal: false, vertical: true) .accessibilityIdentifier("connect.problemDetail") + if connectionManager.watchingForPC { + // Problem cards stay English. + Text(verbatim: "Waiting for your PC: this iPad connects by itself when EternalMonitor opens there.") + .font(.app(14, .medium, relativeTo: .callout)) + .foregroundStyle(Theme.text) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("connect.problemWaiting") + } if !problem.actions.isEmpty { HStack(spacing: 10) { ForEach(problem.actions, id: \.self) { action in @@ -559,10 +579,19 @@ struct ConnectView: View { Text(L("Connecting…")) .font(.app(18, .semibold, relativeTo: .headline)) .foregroundStyle(Theme.text) - Text(normalizedHostIP.isEmpty ? L("Waiting for your PC") : L("Reaching \(normalizedHostIP)")) - .font(.app(14, relativeTo: .callout)) - .foregroundStyle(Theme.textMuted) - .lineLimit(1) + // The PC answered and said why no picture comes yet. + if let card = connectionManager.pcGraphicsCard { + Text(card.short) + .font(.app(14, relativeTo: .callout)) + .foregroundStyle(Theme.text) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("connect.pcGraphicsCard") + } else { + Text(normalizedHostIP.isEmpty ? L("Waiting for your PC") : L("Reaching \(normalizedHostIP)")) + .font(.app(14, relativeTo: .callout)) + .foregroundStyle(Theme.textMuted) + .lineLimit(1) + } } Spacer(minLength: 0) } @@ -596,7 +625,8 @@ struct ConnectView: View { Text(L("USB cable")) .font(.app(15, .medium, relativeTo: .subheadline)) .foregroundStyle(Theme.text) - Text(USBStatusText.describe(connectionManager.usbStatus)) + Text(USBStatusText.describe(connectionManager.usbStatus, + appleDevicesUnreachable: connectionManager.appleDevicesUnreachable != nil)) .font(.app(13.5, relativeTo: .footnote)) .foregroundStyle(Theme.textMuted) .fixedSize(horizontal: false, vertical: true) @@ -699,14 +729,17 @@ struct ConnectView: View { guard let address = host.address else { return } hostIP = address port = "\(host.port)" - connectionManager.connect(host: address, port: host.port) + oldPC.connecting(service: host.name, address: address) + connectionManager.connect(host: address, port: host.port, found: true) } label: { hostRow( icon: "desktopcomputer", title: host.name, subtitle: Self.discoveredSubtitle(host), badge: nil, - paired: false + paired: false, + note: updates.pcNeedsUpdate(PCVersions.version(address: host.address, name: host.name)) + ? Self.pcUpdateNote : nil ) } .buttonStyle(RowButtonStyle()) @@ -720,6 +753,10 @@ struct ConnectView: View { .transition(.opacity.combined(with: .move(edge: .bottom))) } + /// Under a found PC whose EternalMonitor is older than the newest for + /// Windows. Update text stays English, like the other update text. + static let pcUpdateNote = "Update EternalMonitor on your PC" + static func discoveredSubtitle(_ host: DiscoveredHost) -> String { if let address = host.address { return L("Tap to connect · \(address)") } return host.finding @@ -727,6 +764,14 @@ struct ConnectView: View { : "Couldn't get its address. Type the Address shown on the PC." } + /// Notes the PCs the search lists, each with its own TXT version, and + /// shows a waiting old-PC card while nothing is streaming and setup is + /// closed. + private func offerOldPC() { + oldPC.discovered(scanner.hosts) + if connectionManager.state == .disconnected && !showSetup { oldPC.show() } + } + private var unlistedRecents: [RecentConnection] { RecentConnection.unlisted(recentStore.connections, discovered: scanner.hosts) } @@ -772,7 +817,8 @@ struct ConnectView: View { .padding(.leading, 70) } - private func hostRow(icon: String, title: String, subtitle: String, badge: String?, paired: Bool) -> some View { + private func hostRow(icon: String, title: String, subtitle: String, badge: String?, paired: Bool, + note: String? = nil) -> some View { HStack(spacing: 14) { Image(systemName: icon) .font(.system(size: 16, weight: .medium)) @@ -791,6 +837,13 @@ struct ConnectView: View { .font(.app(13, relativeTo: .footnote)) .foregroundStyle(Theme.textMuted) .lineLimit(1) + if let note { + Label(note, systemImage: "arrow.up.circle") + .font(.app(12.5, relativeTo: .caption)) + .foregroundStyle(Theme.textMuted) + .lineLimit(2) + .accessibilityIdentifier("discovered.pcUpdate") + } } Spacer(minLength: 8) if paired { diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 54023198..60cfb12f 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -86,7 +86,12 @@ final class ConnectionManager: ObservableObject { @Published var connectionError: String? /// A connection problem in plain words, with its next action. Shown in /// place of `connectionError` when set. - @Published var problem: ConnectionProblem? + @Published var problem: ConnectionProblem? { + didSet { problemChanged() } + } + /// The "didn't answer" card is up and this iPad connects by itself when + /// the PC shows up on the network (see `PCReturn`). + @Published private(set) var watchingForPC = false /// The link failure classified for the current attempt. A link's close /// callback that follows its own failure reports this, not a removed cable. private var linkFailure: ConnectionProblem? @@ -95,14 +100,20 @@ final class ConnectionManager: ObservableObject { @Published var pairingPrompt: PairingSheetModel? /// HELLO2 feature bits. Every build says it checks for its own updates, - /// so the PC tells only older iPad apps to update. + /// so the PC tells only older iPad apps to update, and that it + /// understands why a PC isn't streaming and why it went down. It draws + /// the PC's pointer itself, so pointer motion needs no video frame. nonisolated static func featureCaps(wantsInput: Bool, wantsAudio: Bool, isUSB: Bool) -> UInt16 { (wantsInput ? Hello2.featureWantsInput : 0) | (wantsAudio ? Hello2.featureWantsAudio : 0) - | (isUSB ? 0 : Hello2.featureSupportsNack) | Hello2.featureChecksUpdates + | (isUSB ? 0 : Hello2.featureSupportsNack | Hello2.featureMediaSize) | Hello2.featureChecksUpdates + | Hello2.featureStandbyStatus | Hello2.featurePowerBye + | (E2E.drawsPointer ? Hello2.featureDrawsPointer : 0) } private var awaitingPairing = false + /// The PC answered that it isn't streaming; the attempt waits for it. + private var waitingForStream = false private var explicitPairingToken: Data? - private let pairingStore = PairingStore.shared + private let pairingStore: PairingStore @Published private(set) var diagnostics: [DiagnosticEntry] = [] /// Live stream health for the HUD, recomputed at the 4 Hz flush. @Published private(set) var stats = StreamStats() @@ -111,6 +122,28 @@ final class ConnectionManager: ObservableObject { /// Frames still arrive, but the picture has been held for over a second /// waiting for the keyframe that replaces a lost frame. @Published var pictureRestoring = false + /// The PC is locked or shows a permission prompt, so it can't capture and + /// the picture is held. Its heartbeats say so until capture works again. + @Published var pcLocked = false + /// The PC has no display turned on, so there is nothing to capture and + /// the picture is held. Its heartbeats say so until one comes back. + @Published var pcNoDisplay = false + /// The PC's heartbeats say Windows won't let it capture from its + /// graphics card, and which screen. + @Published var pcGraphicsCard: PCGraphicsCard? + /// The PC's heartbeats counted video frames sent this attempt. + private var pcSentVideo = false + /// The PC is restarting its stream and kept this session, and no new + /// frame has come for `Timing.restartNoticeSeconds`. The first new frame + /// clears it. + @Published private(set) var pcRestarting = false + /// When the last frame was decoded, by the system uptime clock. + private var lastFrameAt: TimeInterval? + /// Shows the restarting pill once the picture has been held long enough. + private var restartNoticeTask: Task? + /// Second screen needs Windows 11, so the PC mirrors its main screen. + /// Its heartbeats say so for a few seconds after the session starts. + @Published var secondScreenNeedsWindows11 = false /// Nonzero while an automatic reconnect cycle is running. @Published private(set) var reconnectAttempt = 0 @@ -140,6 +173,25 @@ final class ConnectionManager: ObservableObject { @Published private(set) var hostStreamConfig: StreamConfig? @Published private(set) var usbStatus = "USB: waiting for a cable" + /// A PC left the USB capability out of its HELLO_ACK: when it was + /// sampled, that PC couldn't reach Apple's USB service. Kept only for + /// the PC at that target, and only until `until`; see `USBStatusText`. + struct AppleDevicesSighting: Equatable { + static let lifetime: TimeInterval = 30 * 60 + /// "host:port" of the PC over Wi-Fi, or "USB". + let target: String + let at: Date + let until: Date + + func applies(now: Date) -> Bool { now >= at && now < until } + } + /// Cleared when it expires, so an idle Connect screen drops the hint too. + @Published private(set) var appleDevicesUnreachable: AppleDevicesSighting? + private var appleDevicesExpiry: Task? + /// The wall clock for the Apple Devices sighting. Tests replace it. + var now: () -> Date = { Date() } + /// Hears, from each accepted HELLO_ACK, whether the PC is older than 0.3.4. + var oldPCPrompt = OldPCPrompt.shared private var mediaLink: (any MediaLink)? private var usbListener: USBListener? private var isForeground = true @@ -148,6 +200,11 @@ final class ConnectionManager: ObservableObject { /// Disconnect, coming back to the app keeps it stopped until Try again /// or Connect USB. private var usbStoppedForNoPicture = false + /// The PC turned the cable down: it isn't answered until the hold ends + /// (see `CableHold`). Busy refusals in a row set the wait. + private var cableHold: CableHold? + private var cableBusyInARow = 0 + private var cableHoldTask: Task? private let currentAttempt = OSAllocatedUnfairLock(initialState: nil) private var frameAssembler: FrameAssembler? private var controlChannel: ControlChannel? @@ -181,6 +238,8 @@ final class ConnectionManager: ObservableObject { private var summarizedNonces: [UInt32] = [] let frameSlot = FrameSlot() + /// The PC's pointer, drawn by `MetalView` when the PC sends it apart. + let pointerOverlay = PointerOverlay() /// Lets hot-path closures reach the CURRENT control channel without /// retaining self (rebuilt each connect). private let controlChannelBox = ControlChannelBox() @@ -213,15 +272,108 @@ final class ConnectionManager: ObservableObject { ?? Int(ConnectionManager.connectionTimeoutSeconds)) var reconnectGraceSeconds = 2.0 var reconnectBackoffSeconds: (Int) -> Double = { attempt in Double(min(1 << (attempt - 1), 15)) } + var appleDevicesNoteSeconds = AppleDevicesSighting.lifetime + /// How long video the PC says it sent may go missing before the + /// next no-video step (see `NoVideoFallback`). + var noVideoStepSeconds = 1.5 + /// How long the cable goes unanswered after the given number of + /// Busy refusals in a row (see `CableHold`). + var cableBusyHoldSeconds: (Int) -> Double = CableHold.busySeconds + /// Waits out a cable hold. Tests replace it to end the wait when + /// they choose. + var cableHoldSleep: (Double) async -> Void = { try? await Task.sleep(for: .seconds($0)) } + /// How long the picture is held through a PC's stream restart before + /// the iPad says so. A quick restart shows nothing. + var restartNoticeSeconds = 1.5 + /// How long after the PC that didn't answer shows up on the network + /// it is asked, so a few Bonjour updates count as one. + var pcReturnDebounceSeconds = 1.0 + /// How often a listing of the PC that hasn't proved the pairing yet + /// is asked again while the card is up. + var pcReturnProbeSeconds = 5.0 + /// How long the "didn't answer" card watches for the PC. + var pcReturnLimitSeconds: TimeInterval = 180 } var timing: Timing /// The Wi-Fi link to a host's port. Tests pass a fake one, or replace it /// to fail a lookup without asking DNS. var makeUDPLink: (UInt16) -> any MediaLink - - init(timing: Timing = Timing(), udpReceiver: @escaping (UInt16) -> any MediaLink = { UDPReceiver(port: $0) }) { + /// Looks for PCs with Bonjour, given the service name wanted. Tests + /// replace it. + var findPCs: (String) async -> [DiscoveredHost] = { await MovedPC.find($0) } + /// Which addresses a saved PC may move from and to. Tests on loopback + /// widen it. + var homeAddress: (String) -> Bool = MovedPC.isHomeAddress + /// How this iPad reaches an address. Tests replace it. + var localRoute: (String, UInt16) async -> MovedPC.Route? = { await MovedPC.route(to: $0, port: $1) } + /// Where the iPad sends to reach a PC address it was given. Tests on + /// loopback point the addresses a fake signs at other fakes. + var dialAddress: (String, UInt16) -> (host: String, port: UInt16) = { ($0, $1) } + /// "host:port" of each address picked from Bonjour's list in this + /// launch. Saved entries keep the mark once they show a picture. + private var foundAddresses = Set() + /// A saved PC that didn't answer at its address and showed it is the + /// same PC at another. + private struct Move { + let from: (host: String, port: UInt16) + let name: String + /// Its pairing token, which the PC proved it holds. + let token: Data + /// The PC's video didn't arrive at `from`; it still answers there. + var noVideo = false + } + /// The move the next attempt makes, and the one the current attempt made. + private var pendingMove: Move? + private var attemptMove: Move? + /// The PC's video didn't arrive at the address asked for before, and did + /// at another: the next attempt goes there first, and back to the address + /// asked for (kept here) if the PC doesn't answer there. + private var pendingRedirect: (host: String, port: UInt16)? + private var attemptRedirect: (host: String, port: UInt16)? + /// The proof the PC at a remembered address is checked for, before any + /// attempt starts (see `connectViaRemembered`). + private var redirectCheck: UUID? + /// The current session's no-video steps, with a PC that takes them, and + /// the step whose result isn't known yet, as its log line names it. + private var noVideoFallback: NoVideoFallback? + private var noVideoStep: String? + /// The media size the current session runs at, when smaller than the + /// PC's own; whether a step this session asked for it, rather than the + /// size kept for this path; and where it is kept (see `NoVideoSizes`). + private var sessionMediaSize: UInt16? + private var mediaSizeFromStep = false + private var mediaSizeKey: NoVideoSizes.Key? + /// Watches Bonjour for the PC that didn't answer. Tests pass a fake. + var makePCBrowser: () -> any PCBrowser = { BonjourPCBrowser() } + /// The PC watched for, its search (stopped in the background), the + /// task asking its listings, and the time limit. + private var pcReturn: PCReturn? + private var pcReturnBrowser: (any PCBrowser)? + private var pcReturnPaused = false + private var pcReturnTry: Task? + private var pcReturnLimit: Task? + /// The PC an automatic try's session must be, for the next attempt and + /// the current one. The PC proved the pairing just before; a PC gives + /// the pairing back only when it is new, so the session must give the + /// saved PC's name, when this iPad knows it, and no other pairing. + private struct ExpectedPC { + let pairing: Data + let name: String? + + func accepts(_ info: ControlChannel.SessionInfo) -> Bool { + (!PairingStore.valid(info.authToken) || info.authToken == pairing) && (name == nil || info.hostName == name) + } + } + private var pendingPC: ExpectedPC? + private var attemptPC: ExpectedPC? + /// The automatic try is starting: the attempt it starts doesn't end the watch. + private var startingPCReturnTry = false + + init(timing: Timing = Timing(), udpReceiver: @escaping (UInt16) -> any MediaLink = { UDPReceiver(port: $0) }, + pairingStore: PairingStore? = nil) { self.timing = timing self.makeUDPLink = udpReceiver + self.pairingStore = pairingStore ?? .shared } func refreshUSBAvailability() { @@ -271,10 +423,47 @@ final class ConnectionManager: ObservableObject { func resumeUSBConnections() { noPictureStreak.restartCount() usbStoppedForNoPicture = false + endCableHold() + cableBusyInARow = 0 usbPaused = false refreshUSBAvailability() } + /// The PC turned this iPad down over the cable. Stop answering the cable + /// for the hold's wait, or until the user acts after a version refusal, + /// instead of taking the PC's next dial a second later. + private func holdCable(_ reason: CableHold.Reason) { + var wait: Double? + if reason == .busy { + cableBusyInARow += 1 + wait = timing.cableBusyHoldSeconds(cableBusyInARow) + } + let hold = CableHold(reason: reason, until: wait.map { now().addingTimeInterval($0) }) + endCableHold() + cableHold = hold + record(.info, "usb", "The PC turned the cable down (\(reason)); " + (wait.map { "answering it again in \(Int($0.rounded())) s" } + ?? "answering it again after Try again or Connect USB")) + usbPaused = true + usbListener?.stop() + usbListener = nil + usbStatus = "USB: disconnected" + guard let wait else { return } + let sleep = timing.cableHoldSleep + cableHoldTask = Task { @MainActor [weak self] in + await sleep(wait) + guard let self, !Task.isCancelled, self.cableHold == hold else { return } + self.endCableHold() + self.usbPaused = self.usbStoppedForNoPicture + self.refreshUSBAvailability() + } + } + + private func endCableHold() { + cableHoldTask?.cancel() + cableHoldTask = nil + cableHold = nil + } + // MARK: - Frame slot (thread-safe single-slot) var latestFrame: CVPixelBuffer? { @@ -301,16 +490,93 @@ final class ConnectionManager: ObservableObject { // MARK: - Connect / Disconnect - func connect(host: String, port: UInt16, token: Data? = nil) { + /// `found`: the address is the one Bonjour listed for the PC, so the PC + /// may be looked up by name again if the address changes. + func connect(host: String, port: UInt16, token: Data? = nil, found: Bool = false) { guard state == .disconnected else { return } + if !startingPCReturnTry { endPCReturn() } explicitPairingToken = token + if found { foundAddresses.insert(PairingStore.address(host: host, port: port)) } if let token { do { try pairingStore.save(token: token, hostName: "", address: PairingStore.address(host: host, port: port)) } catch { record(.warning, "pairing", "Could not save the scanned pairing on this iPad.") } + } else if let remembered = rememberedAddress(host: host, port: port) { + connectViaRemembered(from: (host.trimmingCharacters(in: .whitespacesAndNewlines), port), remembered) + return } connect(target: .udp(host: host, port: port)) } + /// Where the PC paired at `host:port` showed video after none arrived + /// there, kept for this pairing, with the pairing and the PC's name. + private func rememberedAddress(host: String, port: UInt16) -> (to: NoVideoPaths.Address, token: Data, name: String)? { + let host = host.trimmingCharacters(in: .whitespacesAndNewlines) + let address = PairingStore.address(host: host, port: port) + guard let token = (try? pairingStore.token(address: address)) ?? nil, + let to = NoVideoPaths.better(than: host, port: port, pairing: token), + let name = pairingStore.pcName(address: address) ?? RecentConnectionStore.shared.entry(host: host, port: port)?.name + else { return nil } + return (to, token, name) + } + + /// Connecting to `from`, where the PC's video didn't arrive before, goes + /// to the address where it did, once that address is on this iPad's own + /// network and the PC there shows it holds this pairing and receives + /// there: the address may belong to another device now. Only then does a + /// HELLO2 with the token go there, as a move, with the same-PC checks of + /// one. Otherwise the entry is forgotten and the attempt goes to `from`, + /// as it does if that move fails (see `leaveRememberedAddress`). + private func connectViaRemembered(from: (host: String, port: UInt16), + _ remembered: (to: NoVideoPaths.Address, token: Data, name: String)) { + endReconnectCycle() + // Try again and a resume go to the address tapped. + lastTarget = from + let check = UUID() + redirectCheck = check + connectionError = nil + problem = nil + state = .connecting + Task { @MainActor [weak self] in + guard let self else { return } + let to = remembered.to + let toFrom = { [weak self] (result: String) in + guard let self else { return } + self.redirectCheck = nil + self.state = .disconnected + NoVideoPaths.forget(from: .init(host: from.host, port: from.port)) + self.startAttempt(.udp(host: from.host, port: from.port)) + self.record(.info, "connection", "no_video_fallback step=remembered result=\(result)") + } + let route = await self.localRoute(to.host, to.port) + guard self.redirectCheck == check else { return } + guard let route, MovedPC.isLocal(to.host, over: route) else { return toFrom("not_local") } + let same = await MovedPC.confirm(host: to.host, port: to.port, token: remembered.token, + link: self.makeUDPLink(to.port)) + guard self.redirectCheck == check else { return } + guard same else { return toFrom("not_confirmed") } + self.redirectCheck = nil + self.state = .disconnected + self.pendingMove = Move(from: from, name: remembered.name, token: remembered.token, noVideo: true) + self.pendingRedirect = from + self.explicitPairingToken = remembered.token + self.startAttempt(.udp(host: to.host, port: to.port)) + self.pendingRedirect = nil + } + } + + /// The attempt at a remembered address (see `connectViaRemembered`) + /// passed its proof and failed after it: it ends with `end`, the entry is + /// forgotten, and a new attempt goes to the address tapped. False when + /// this attempt didn't go to a remembered address. + private func leaveRememberedAddress(_ end: AttemptEnd, result: String) -> Bool { + guard let from = attemptRedirect else { return false } + NoVideoPaths.forget(from: .init(host: from.host, port: from.port)) + record(.info, "connection", "no_video_fallback step=remembered result=\(result)") + disconnect(end) + startAttempt(.udp(host: from.host, port: from.port)) + return true + } + func submitPairing() { guard awaitingPairing, let code = pairingPrompt?.submit() else { return } // The code goes in a new HELLO2. The refused one ends here, as on the @@ -327,6 +593,8 @@ final class ConnectionManager: ObservableObject { func forgetPairedHosts() { do { try pairingStore.forgetAll() } catch { connectionError = "Could not remove saved pairings from this iPad." } + NoVideoPaths.forgetAll() + NoVideoSizes.forgetAll() } /// A connection the user or a cable starts. It replaces any automatic @@ -335,6 +603,8 @@ final class ConnectionManager: ObservableObject { guard state == .disconnected else { return } if case .udp(let host, let port) = target, host.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || port == 0 { return } + // The user picked a PC, or a cable took over. + if !startingPCReturnTry { endPCReturn() } endReconnectCycle() if case .udp(let host, _) = target, let typed = ConnectionProblem.typedAddress(host) { connectionError = nil @@ -359,7 +629,9 @@ final class ConnectionManager: ObservableObject { SupportDiagnostics.noteTypedAddress(normalizedHost) port = targetPort receiver = makeUDPLink(port) - lastTarget = (normalizedHost, port) + // A moved attempt becomes the target to retry only once the PC + // there is the saved one (see onSessionEstablished). + if pendingMove == nil { lastTarget = (normalizedHost, port) } case .usb(let link): normalizedHost = "USB" port = 9877 @@ -371,8 +643,22 @@ final class ConnectionManager: ObservableObject { } let isUSB = receiver.isUSB let pairingAddress = isUSB ? "USB" : PairingStore.address(host: normalizedHost, port: port) + let pcTarget = isUSB ? "USB" : "\(normalizedHost):\(port)" + // What one PC said about Apple Devices says nothing about another. + if appleDevicesUnreachable?.target != pcTarget { keepAppleDevicesSighting(nil) } let pairingToken = explicitPairingToken ?? ((try? pairingStore.token(address: pairingAddress)) ?? Data(repeating: 0, count: 16)) explicitPairingToken = nil + attemptMove = isUSB ? nil : pendingMove + pendingMove = nil + attemptRedirect = isUSB ? nil : pendingRedirect + pendingRedirect = nil + attemptPC = isUSB ? nil : pendingPC + pendingPC = nil + noVideoFallback = nil + noVideoStep = nil + sessionMediaSize = nil + mediaSizeFromStep = false + mediaSizeKey = nil let connectionID = UUID() currentAttempt.withLock { $0 = connectionID } transportMode = isUSB ? "USB" : "WiFi" @@ -385,10 +671,17 @@ final class ConnectionManager: ObservableObject { linkFailure = nil versionMismatch = nil signalLost = false + pcLocked = false + pcNoDisplay = false + pcGraphicsCard = nil + pcSentVideo = false + clearRestartNotice() + secondScreenNeedsWindows11 = false degradedSinceUs = nil prevCounters = FrameAssembler.Counters() diagnostics.removeAll() didExtendTimeout = false + waitingForStream = false attemptStartedAt = ProcessInfo.processInfo.systemUptime sessionAcceptedAt = nil firstFrameAt = nil @@ -402,7 +695,18 @@ final class ConnectionManager: ObservableObject { sessionWantsAudio = wantsAudio audioStats = AudioStats() debugState = ConnectionDebugState(host: normalizedHost, port: port) + // Watched for from the start: a PC listed by the time the card goes + // up is asked then. + if !isUSB, attemptMove == nil, attemptRedirect == nil, reconnectCycleID == nil { + watchForPC(host: normalizedHost, port: port) + } record(.info, "connection", "Starting connection attempt=\(connectionID) transport=\(transportMode) to \(normalizedHost):\(port) drawing=\(drawing.enabled) input=\(wantsInput) audio=\(wantsAudio)") + if attemptMove != nil { + record(.info, "connection", attemptMove?.noVideo == true + ? "Trying the PC at another of its addresses: its video didn't arrive at the first" + : "Trying the saved PC at the new address Bonjour gave") + } + if attemptRedirect != nil { record(.info, "connection", "no_video_fallback step=remembered result=tried") } let decoder = VideoDecoder() let metrics = ReceiverMetrics() @@ -449,11 +753,18 @@ final class ConnectionManager: ObservableObject { guard self.currentAttempt.withLock({ $0 == connectionID }) else { return } self.debugState.decodedFrames += 1 self.debugState.lastDecodedTimestampUs = ts + self.lastFrameAt = ProcessInfo.processInfo.systemUptime + if self.pcRestarting { self.record(.info, "ctrl", "Picture back after the PC's stream restart") } + self.clearRestartNotice() if self.debugState.decodedFrames == 1 { self.firstFrameAt = ProcessInfo.processInfo.systemUptime self.record(.info, "decode", "First frame decoded successfully" + self.attemptTimings) + self.endNoVideoStep("video") + self.noVideoFallback = nil + self.settleMediaSize() self.timeoutTask?.cancel() self.noPictureStreak = NoPictureStreak() + self.endPCReturn() self.state = .connected self.streamingSince = Date() // A reconnect attempt that streams ends its cycle. @@ -464,9 +775,15 @@ final class ConnectionManager: ObservableObject { UIApplication.shared.isIdleTimerDisabled = UserDefaults.standard.flag(forKey: "keepScreenAwake") E2E.firstFrame(width: frameWidth, height: frameHeight, link: isUSB ? "usb" : "udp") - RecentConnectionStore.shared.add(host: isUSB ? (self.hostInfo?.hostName ?? "USB host") : normalizedHost, - port: port, isUSB: isUSB) - if !isUSB { + var saved = true + if let move = self.attemptMove { + saved = self.savedMove(move, to: (normalizedHost, port), pairingAddress: pairingAddress) + } else { + RecentConnectionStore.shared.add(host: isUSB ? (self.hostInfo?.hostName ?? "USB host") : normalizedHost, + port: port, isUSB: isUSB, name: self.hostInfo?.hostName, + found: self.foundAddresses.contains(pairingAddress)) + } + if !isUSB && saved { UserDefaults.standard.set(normalizedHost, forKey: "lastHost") UserDefaults.standard.set(Int(port), forKey: "lastPort") } @@ -531,8 +848,9 @@ final class ConnectionManager: ObservableObject { } } - assembler.onFrameAssembled = { [weak self, weak decoder, weak assembler] data, seq, captureTsUs, isKeyframe in + assembler.onFrameAssembled = { [weak self, weak decoder, weak assembler, weak controlChannelBox] data, seq, captureTsUs, isKeyframe in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } + if isKeyframe { controlChannelBox?.value?.noteKeyframeAssembled() } self.streamCounters.recordAssembled(bytes: data.count) self.streamCounters.recordParsed() decoder?.decode(packet: FramePacket( @@ -553,11 +871,23 @@ final class ConnectionManager: ObservableObject { receiver?.send(data) }) controlChannelBox.value = channel + pointerOverlay.reset() + channel.pointer = pointerOverlay assembler.rttUs = { [weak channel] in channel?.clockSnapshot.withLock { $0.rttUs } ?? 10_000 } assembler.onNack = { [weak channel] nack in channel?.sendNack(nack) } assembler.onNeedsKeyframe = { [weak channel] epoch, seq in channel?.sendKeyframeRequest(streamEpoch: epoch, lastCompleteSeq: seq, reason: .gapLoss) } + // Video arrives but the first frame never completes: ask for a fresh + // keyframe. The PC's packet size is its own setting; no message + // asks it for smaller datagrams. + assembler.onNoFrameYet = { [weak self, weak channel] epoch, missing in + channel?.sendKeyframeRequest(streamEpoch: epoch, lastCompleteSeq: 0, reason: .startup) + Task { @MainActor in + guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } + self.record(.warning, "assembly", "Video arrives but no frame completed in 2 s; asked the PC for a fresh keyframe: \(missing)") + } + } channel.reportProvider = { [weak assembler, weak channel, weak decoder, weak audio] in var report = metrics.makeReport(assembler: assembler?.counters.withLock { $0 } ?? .init(), decodeDepth: decoder?.decodeDepth ?? 0, @@ -566,6 +896,7 @@ final class ConnectionManager: ObservableObject { let stats = audio?.stats ?? AudioStats() report.audioPacketsLost = UInt32(clamping: stats.lost) report.audioBufferMs = UInt16(clamping: stats.bufferMs) + report.pointerShapeId = channel?.pointer?.completeShapeId ?? 0 return report } channel.onHelloAttempt = { [weak self] attempt, total in @@ -589,12 +920,49 @@ final class ConnectionManager: ObservableObject { let acceptedAt = ProcessInfo.processInfo.systemUptime Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } + // An automatic try streams only from the PC it expects, + // whatever answered at that address. + if let expected = self.attemptPC, !expected.accepts(info) { + self.record(.warning, "connection", "pc_return result=other_pc") + self.disconnect(.otherPC) + self.problem = .noReply(usb: false) + return + } + if let move = self.attemptMove { + guard Self.isSamePC(move, info) else { + self.record(.warning, "connection", "The PC at the new address isn't the saved one") + if self.leaveRememberedAddress(.otherPC, result: "other_pc") { return } + self.disconnect(.otherPC) + self.problem = .noReply(usb: false) + return + } + self.lastTarget = (normalizedHost, port) + } self.sessionAcceptedAt = acceptedAt + self.noVideoFallback = !isUSB && info.hostCaps & HelloAck.hostCapMediaSize != 0 + ? NoVideoFallback(wait: self.timing.noVideoStepSeconds) : nil + let sessionToken = PairingStore.valid(info.authToken) ? info.authToken : pairingToken + if self.noVideoFallback != nil, PairingStore.valid(sessionToken) { + self.startWithRememberedSize(connectionID, address: .init(host: normalizedHost, port: port), + pairing: sessionToken) + } self.acceptedAddresses.insert("\(normalizedHost):\(port)") self.noPictureStreak.answered(by: info.hostName, pairingToken: info.authToken, at: isUSB ? nil : "\(normalizedHost):\(port)") self.livenessTimeoutUs = UInt64(info.livenessTimeoutMs) * 1000 self.hostInfo = info + self.oldPCPrompt.answered(hostName: info.hostName, pairingToken: info.authToken, + hostCaps: info.hostCaps, address: isUSB ? nil : normalizedHost) + if self.waitingForStream { + self.waitingForStream = false + if case .pcNotStreaming = self.problem { self.problem = nil } + self.armConnectTimeout(seconds: self.timing.connectTimeoutSeconds) + } + let seen = self.now() + self.keepAppleDevicesSighting(info.hostCaps & HelloAck.hostCapUSB == 0 + ? AppleDevicesSighting(target: pcTarget, at: seen, + until: seen.addingTimeInterval(self.timing.appleDevicesNoteSeconds)) + : nil) if self.awaitingPairing { self.awaitingPairing = false self.pairingPrompt = nil @@ -604,7 +972,10 @@ final class ConnectionManager: ObservableObject { do { try self.pairingStore.save(token: info.authToken, hostName: info.hostName, address: pairingAddress) } catch { self.record(.warning, "pairing", "Pairing was not saved on this iPad. Pair again next time.") } } - if isUSB { self.usbStatus = "USB: connected to \(info.hostName)" } + if isUSB { + self.usbStatus = "USB: connected to \(info.hostName)" + self.cableBusyInARow = 0 + } self.hostStreamConfig = info.streamConfig self.record( .info, "ctrl", @@ -617,6 +988,23 @@ final class ConnectionManager: ObservableObject { let mismatch = channel?.lastMismatch Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } + // Nor did the PC an automatic try reached: no pairing prompt, + // and the saved pairing stays. + if self.attemptPC != nil, self.attemptMove == nil, status == .unauthorized || status == .rateLimited { + self.record(.warning, "connection", "pc_return result=other_pc") + self.disconnect(.otherPC) + self.problem = .noReply(usb: false) + return + } + // A PC that wants a code didn't take the saved PC's pairing: + // it is another PC, and the saved pairing stays. + if self.attemptMove != nil, status == .unauthorized || status == .rateLimited { + self.record(.warning, "connection", "The PC at the new address isn't the saved one: it asked for pairing") + if self.leaveRememberedAddress(.otherPC, result: "other_pc") { return } + self.disconnect(.otherPC) + self.problem = .noReply(usb: false) + return + } // Busy passes: the PC is streaming elsewhere, often to this // iPad's own cable session that hasn't closed yet. An // automatic cycle keeps retrying and shows it if it runs out. @@ -646,7 +1034,30 @@ final class ConnectionManager: ObservableObject { self.connectionError = "Your PC turned down the connection." } self.record(.error, "ctrl", self.connectionError ?? self.problem?.title ?? "rejected") - self.disconnect(.pcRefused) + switch status { + case .busy: self.disconnect(.pcBusy) + case .versionUnsupported where mismatch == .pcOlder: self.disconnect(.oldPC) + default: self.disconnect(.pcRefused) + } + // Over the cable the PC dials again a second later, and would + // be turned down again: wait for a change instead. + if isUSB, status == .busy || status == .versionUnsupported { + self.holdCable(status == .busy ? .busy : .version) + } + } + } + channel.onPCNotStreaming = { [weak self] status, reason in + Task { @MainActor in + guard let self, self.currentAttempt.withLock({ $0 == connectionID }), + let wait = ConnectionProblem.StreamWait(status, reason: reason) else { return } + if self.problem != .pcNotStreaming(wait) { + self.record(.info, "ctrl", "PC isn't streaming (\(status)\(reason.isEmpty ? "" : " \(reason)")); waiting for it") + } + // No reconnect attempt is spent while the PC says why it + // isn't streaming. If it stops answering, the timeout says so. + self.waitingForStream = true + self.problem = .pcNotStreaming(wait) + self.armConnectTimeout(seconds: self.timing.connectTimeoutSeconds) } } channel.onHandshakeTimeout = { [weak self, weak receiver] in @@ -662,8 +1073,14 @@ final class ConnectionManager: ObservableObject { self.connectionError = VersionMismatch.pcLegacy.message self.record(.error, "ctrl", "Legacy v1 host detected") self.disconnect(.legacyPC) + if isUSB { self.holdCable(.version) } } else { self.record(.warning, "ctrl", "HELLO2 handshake got no reply yet") + if self.leaveRememberedAddress(.pcMoved, result: "no_reply") { + // Back to the address tapped. + } else if !isUSB { + self.lookForMovedPC(connectionID, host: normalizedHost, port: port) + } } } } @@ -683,9 +1100,10 @@ final class ConnectionManager: ObservableObject { } else { self.beginReconnect(.pcStoppedResponding, failure: failure, end: .pcGoodbye) } - case .userDisconnect, .superseded: - // Disconnected from the PC, or another device took over: - // stay away until the user connects again. + case .userDisconnect, .superseded, .hostSleeping, .hostPoweringOff: + // Disconnected from the PC, another device took over, or + // the PC is going down and won't answer: stay away until + // the user connects again. self.endReconnectCycle() self.disconnect(.pcGoodbye) self.problem = failure @@ -707,6 +1125,36 @@ final class ConnectionManager: ObservableObject { Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } self.hostStreamConfig = heartbeat.streamConfig + if let frames = heartbeat.framesSent { + self.noVideoFallback?.heard(framesSent: frames, at: ProcessInfo.processInfo.systemUptime) + if frames > 0 { self.pcSentVideo = true } + } + let locked = heartbeat.notice == Heartbeat.noticePCLocked + if locked != self.pcLocked { + self.pcLocked = locked + self.record(.info, "ctrl", locked ? "PC locked: holding the picture" : "PC unlocked") + } + if heartbeat.notice == Heartbeat.noticeRestarting { + self.noteRestartNotice() + } else { + self.clearRestartNotice() + } + let noDisplay = heartbeat.notice == Heartbeat.noticeNoDisplay + if noDisplay != self.pcNoDisplay { + self.pcNoDisplay = noDisplay + self.record(.info, "ctrl", noDisplay ? "PC has no active display: holding the picture" : "PC display back") + } + let windows10 = heartbeat.notice == Heartbeat.noticeSecondScreenNeedsWindows11 + if windows10 != self.secondScreenNeedsWindows11 { + self.secondScreenNeedsWindows11 = windows10 + if windows10 { self.record(.info, "ctrl", "PC mirrors: Second screen needs Windows 11") } + } + let card = PCGraphicsCard(notice: heartbeat.notice) + if card != self.pcGraphicsCard { + self.pcGraphicsCard = card + self.record(.info, "ctrl", card.map { "PC can't capture from its graphics card screen=\($0)" } + ?? "PC captures from its graphics card again") + } } } channel.onStreamConfig = { [weak self, weak assembler] config in @@ -778,7 +1226,7 @@ final class ConnectionManager: ObservableObject { return } guard isUSB else { - self.disconnect(.linkFailed) + self.disconnect(failure == .wifiOff ? .noNetwork : .linkFailed) self.problem = failure return } @@ -864,10 +1312,14 @@ final class ConnectionManager: ObservableObject { } } + self.advanceNoVideoFallback() // Runs on EVERY tick, including silent ones: a dead host sends // neither media nor heartbeats, so skipping this when nothing // arrived is exactly when the liveness watchdog is needed. self.refreshStatsAndWatchdog() + if E2E.enabled, self.state == .connected { + E2E.pointer(self.pointerOverlay.counters, decoded: self.debugState.decodedFrames) + } if ProcessInfo.processInfo.systemUptime - lastSummary >= 10 { lastSummary = ProcessInfo.processInfo.systemUptime self.recordStreamSummary(reason: "periodic") @@ -940,6 +1392,29 @@ final class ConnectionManager: ObservableObject { } } + /// The PC said it is restarting its stream and holds this session. Once + /// the picture has been held `restartNoticeSeconds` since its last + /// frame, say so; a frame first means a quick restart, and nothing shows. + private func noteRestartNotice() { + guard state == .connected, !pcRestarting, restartNoticeTask == nil else { return } + let heldFor = ProcessInfo.processInfo.systemUptime - (lastFrameAt ?? ProcessInfo.processInfo.systemUptime) + let wait = max(0, timing.restartNoticeSeconds - heldFor) + let frameAt = lastFrameAt + restartNoticeTask = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(wait)) + guard !Task.isCancelled, let self, self.state == .connected, self.lastFrameAt == frameAt else { return } + self.restartNoticeTask = nil + self.pcRestarting = true + self.record(.info, "ctrl", "PC is restarting its stream: holding the picture") + } + } + + private func clearRestartNotice() { + restartNoticeTask?.cancel() + restartNoticeTask = nil + pcRestarting = false + } + /// Ends the session that `reason` interrupted (`end` when it was not /// the link: the PC said goodbye) and reconnects if that is on. private func beginReconnect(_ reason: ConnectionProblem.LostReason, failure: ConnectionProblem? = nil, @@ -1000,6 +1475,199 @@ final class ConnectionManager: ObservableObject { } } + /// The attempt's HELLO2 got no answer at `host:port`. A paired PC that + /// this iPad found with Bonjour is looked for by name. Where Bonjour has + /// it at another home network address that this iPad reaches directly, + /// the iPad checks that it is the same PC, at that address, and only + /// then moves the attempt there with its pairing. + /// Otherwise the attempt goes on as before. + private func lookForMovedPC(_ connectionID: UUID, host: String, port: UInt16) { + let address = PairingStore.address(host: host, port: port) + let saved = RecentConnectionStore.shared.entry(host: host, port: port) + let name = saved?.name ?? pairingStore.pcName(address: address) + let token = (try? pairingStore.token(address: address)) ?? nil + guard attemptMove == nil, let name, let token, + MovedPC.canLookUp(host: host, pcName: name, found: foundAddresses.contains(address) || saved?.found == true, + paired: PairingStore.valid(token), homeAddress: homeAddress) + else { return } + record(.info, "connection", "Looking for the saved PC by name: no answer at its address") + let current = { [weak self] in + // The PC may have answered meanwhile, or the attempt ended. + guard let self else { return false } + return self.currentAttempt.withLock { $0 == connectionID } && self.state == .connecting + && self.hostInfo == nil && !self.awaitingPairing + } + Task { @MainActor [weak self] in + guard let self else { return } + let found = await self.findPCs(MovedPC.serviceName(pcName: name)) + guard current() else { return } + guard let moved = MovedPC.newAddress(pcName: name, host: host, port: port, found: found, + homeAddress: self.homeAddress) else { + self.record(.info, "connection", "The saved PC wasn't found at another address") + return + } + let route = await self.localRoute(moved.host, moved.port) + guard current() else { return } + guard let route, MovedPC.isLocal(moved.host, over: route) else { + self.record(.info, "connection", "The saved PC was found, but not on this iPad's local network: staying at the saved address") + return + } + self.record(.info, "connection", "Checking that the PC at \(moved.host):\(moved.port) is the saved one") + let same = await MovedPC.confirm(host: moved.host, port: moved.port, token: token, + link: self.makeUDPLink(moved.port)) + guard current() else { return } + guard same else { + self.record(.info, "connection", "The PC there didn't show it is the saved one: staying at the saved address") + return + } + self.record(.info, "connection", "It is the saved PC: connecting there") + self.disconnect(.pcMoved) + self.pendingMove = Move(from: (host, port), name: name, token: token) + self.explicitPairingToken = token + self.startAttempt(.udp(host: moved.host, port: moved.port)) + } + } + + /// The PC that accepted a moved attempt is the saved one: it has the + /// saved name, and gives back the saved pairing token, or none if its + /// pairing is off now. + private static func isSamePC(_ move: Move, _ info: ControlChannel.SessionInfo) -> Bool { + info.hostName == move.name && (!PairingStore.valid(info.authToken) || info.authToken == move.token) + } + + /// The moved PC showed a picture: its pairing is saved at the new + /// address, then the new entry replaces the old one. If the pairing + /// couldn't be saved, the old entry and pairing stay as they were. + private func savedMove(_ move: Move, to new: (host: String, port: UInt16), pairingAddress: String) -> Bool { + do { + try pairingStore.save(token: move.token, hostName: hostInfo?.hostName ?? move.name, address: pairingAddress) + guard try pairingStore.token(address: pairingAddress) == move.token else { throw CocoaError(.fileWriteUnknown) } + } catch { + record(.warning, "pairing", "The PC's new address wasn't saved on this iPad; keeping the saved one") + return false + } + RecentConnectionStore.shared.moved(from: move.from, to: new, name: hostInfo?.hostName ?? move.name, + replacesOld: move.noVideo || homeAddress(move.from.host)) + if move.noVideo { + // The PC still answers at the old address: its pairing stays, + // and connecting there goes here first. + NoVideoPaths.remember(from: .init(host: move.from.host, port: move.from.port), + to: .init(host: new.host, port: new.port), pairing: move.token) + } else if homeAddress(move.from.host) { + try? pairingStore.remove(address: PairingStore.address(host: move.from.host, port: move.from.port)) + } + return true + } + + /// Takes the next no-video step once it is due: the PC said it sent + /// video and no picture came of it (see `NoVideoFallback`). A path that + /// drops large datagrams often lets a frame's small last fragment by. + private func advanceNoVideoFallback() { + guard noVideoFallback != nil, state == .connecting, hostInfo != nil, !awaitingPairing, !waitingForStream, + let step = noVideoFallback?.next(at: ProcessInfo.processInfo.systemUptime) else { return } + endNoVideoStep("no_video") + switch step { + case .smallerPackets(let size): + // The PC is sending and the iPad is working on it: one fresh + // window, as when data starts to arrive. + if !didExtendTimeout { + didExtendTimeout = true + armConnectTimeout(seconds: timing.connectTimeoutSeconds) + } + noVideoStep = "step=smaller_packets size=\(size)" + sessionMediaSize = size + mediaSizeFromStep = true + controlChannel?.sendKeyframeRequest(streamEpoch: 0, lastCompleteSeq: 0, reason: .startup, maxMediaDgram: size) + case .otherAddress: + tryOtherAddress() + } + } + + /// A session at an address where the PC's video needed a smaller media + /// size before, over the same network, asks for it at once instead of + /// waiting through the steps again (see `NoVideoSizes`). + private func startWithRememberedSize(_ connectionID: UUID, address: NoVideoPaths.Address, pairing: Data) { + Task { @MainActor [weak self] in + guard let self else { return } + let route = await self.localRoute(address.host, address.port) + guard self.currentAttempt.withLock({ $0 == connectionID }) else { return } + let key = NoVideoSizes.Key(address: address, pairing: pairing, network: NoVideoSizes.network(route)) + self.mediaSizeKey = key + // A picture came first, at the PC's size or after a step. + guard self.state == .connecting else { return self.settleMediaSize() } + guard self.sessionMediaSize == nil, let size = NoVideoSizes.size(for: key, now: self.now()) else { return } + self.sessionMediaSize = size + self.noVideoFallback?.skipSizes(downTo: size) + self.controlChannel?.sendKeyframeRequest(streamEpoch: 0, lastCompleteSeq: 0, reason: .startup, maxMediaDgram: size) + self.record(.info, "connection", "no_video_fallback step=remembered_size size=\(size) result=asked") + } + } + + /// The session shows a picture: keep the smaller size a step found, or + /// forget the size kept for this address once the PC's own size works, + /// as after a change of network. + private func settleMediaSize() { + guard let key = mediaSizeKey else { return } + if let size = sessionMediaSize { + if mediaSizeFromStep { NoVideoSizes.remember(size, for: key, now: now()) } + } else if NoVideoSizes.forget(key) { + record(.info, "connection", "no_video_fallback step=remembered_size result=forgotten") + } + } + + /// Logs the pending no-video step's `result`. + private func endNoVideoStep(_ result: String) { + guard let step = noVideoStep else { return } + noVideoStep = nil + record(.info, "connection", "no_video_fallback \(step) result=\(result)") + } + + /// No picture came of the PC's video at this address, even in small + /// datagrams. Asked here, the PC signs its home network addresses with + /// this iPad's pairing; the first other one on this iPad's own network, + /// where the PC shows again that it holds the pairing and receives + /// there, gets a new attempt. A forwarded or spoofed + /// address can't make either answer. + private func tryOtherAddress() { + let host = debugState.host, port = debugState.port + let done = { [weak self] (result: String) -> Void in + self?.record(.info, "connection", "no_video_fallback step=other_address result=\(result)") + } + guard attemptMove == nil else { return done("moved_before") } + guard let connectionID = currentAttempt.withLock({ $0 }), let name = hostInfo?.hostName, + let token = (try? pairingStore.token(address: PairingStore.address(host: host, port: port))) ?? nil + else { return done("unpaired") } + let current = { [weak self] in + guard let self else { return false } + return self.currentAttempt.withLock { $0 == connectionID } && self.state == .connecting + } + Task { @MainActor [weak self] in + guard let self else { return } + let identity = await MovedPC.identify(host: host, port: port, token: token, link: self.makeUDPLink(port)) + guard current() else { return } + guard let identity else { return done("no_answer") } + for other in MovedPC.otherAddresses(identity, asked: host, homeAddress: self.homeAddress) { + let target = self.dialAddress(other, identity.port) + let route = await self.localRoute(target.host, target.port) + guard current() else { return } + // As for a PC Bonjour moved: only an address on this iPad's + // own network, so nothing relays the stream elsewhere. + guard let route, MovedPC.isLocal(target.host, over: route) else { continue } + let same = await MovedPC.confirm(host: target.host, port: target.port, token: token, + link: self.makeUDPLink(target.port)) + guard current() else { return } + guard same else { continue } + done("moved") + self.disconnect(.noVideoPath) + self.pendingMove = Move(from: (host, port), name: name, token: token, noVideo: true) + self.explicitPairingToken = token + self.startAttempt(.udp(host: target.host, port: target.port)) + return + } + done("none") + } + } + /// Retires the automatic reconnect cycle, if one is running: its task /// stops at its next check and can no longer write state. private func endReconnectCycle() { @@ -1010,6 +1678,184 @@ final class ConnectionManager: ObservableObject { reconnectAttempt = 0 } + // MARK: - The PC that didn't answer comes back + + private var showsNoReplyOverWiFi: Bool { + switch problem { + case .noReply(usb: false)?, .noReplyAtNewAddress?: return true + default: return false + } + } + + /// The pairing saved for the PC at `host:port`, if a valid one. + private func savedPairing(host: String, port: UInt16) -> Data? { + let token = (try? pairingStore.token(address: PairingStore.address(host: host, port: port))) ?? nil + return token.flatMap { PairingStore.valid($0) ? $0 : nil } + } + + /// Starts watching Bonjour for the paired PC at `host:port`, by its + /// service when this iPad knows its name, as an attempt to it starts. An + /// automatic try must prove it is the same PC with the pairing, so an + /// unpaired PC isn't watched for. A watch for that PC goes on as it was. + private func watchForPC(host: String, port: UInt16) { + if let watch = pcReturn, watch.host == host, watch.port == port { return } + endPCReturn() + guard savedPairing(host: host, port: port) != nil else { return } + let name = RecentConnectionStore.shared.entry(host: host, port: port)?.name + ?? pairingStore.pcName(address: PairingStore.address(host: host, port: port)) + pcReturn = PCReturn(name: name, host: host, port: port) + let browser = makePCBrowser() + browser.onHosts = { [weak self] hosts in self?.pcReturnHeard(hosts) } + pcReturnBrowser = browser + pcReturnPaused = !isForeground + if isForeground { browser.start() } + } + + /// The "didn't answer" card is up for the PC watched for: its listings + /// are asked until one proves the pairing, until the time limit. + private func armPCReturn() { + guard pcReturn != nil else { return } + if pcReturn?.armedAt == nil { + pcReturn?.arm(at: now()) + watchingForPC = true + record(.info, "connection", "pc_return result=watching") + let limit = timing.pcReturnLimitSeconds + pcReturnLimit = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(limit)) + guard !Task.isCancelled, let self, self.pcReturn != nil else { return } + self.record(.info, "connection", "pc_return result=expired") + self.endPCReturn() + } + } + askPCReturn(after: timing.pcReturnDebounceSeconds) + } + + private func pcReturnHeard(_ hosts: [DiscoveredHost]) { + guard pcReturn != nil else { return } + if pcReturn?.heard(hosts) == true { askPCReturn(after: timing.pcReturnDebounceSeconds) } + } + + /// The card is up, in the foreground, within the time limit. + private var pcReturnMayTry: Bool { + guard let watch = pcReturn, watch.armedAt != nil else { return false } + return isForeground && state == .disconnected && showsNoReplyOverWiFi + && !watch.expired(at: now(), limit: timing.pcReturnLimitSeconds) + } + + /// Asks each of the PC's listings in turn after `delay`, so a few + /// Bonjour updates count as one, then again every few seconds, until one + /// proves the pairing and has its try or none is left to ask. A newer + /// call starts over. + private func askPCReturn(after delay: Double) { + guard let watch = pcReturn, watch.armedAt != nil else { return } + guard !watch.expired(at: now(), limit: timing.pcReturnLimitSeconds) else { + record(.info, "connection", "pc_return result=expired") + return endPCReturn() + } + pcReturnTry?.cancel() + pcReturnTry = Task { @MainActor [weak self] in + var wait = delay + while true { + try? await Task.sleep(for: .seconds(wait)) + guard !Task.isCancelled, let self, self.pcReturnMayTry, + let round = self.pcReturn?.candidates, !round.isEmpty else { return } + for pc in round where self.pcReturn?.awaits(pc) == true { + let proved = await self.provePCReturn(pc) + guard !Task.isCancelled, self.pcReturnMayTry else { return } + if proved { + self.pcReturnTry = nil + self.pcReturn?.didTry(pc) + return self.tryPCAgain(at: pc) + } + } + wait = self.timing.pcReturnProbeSeconds + } + } + } + + /// Whether the PC at `pc`'s address shows it holds the saved pairing, + /// as a moved PC does; nothing with the pairing goes to it before. An + /// address other than the one tried must be on this iPad's own network + /// first: one that isn't never gets a try. + private func provePCReturn(_ pc: DiscoveredHost) async -> Bool { + guard let watch = pcReturn, let host = pc.address, + let token = savedPairing(host: watch.host, port: watch.port) else { return false } + if host != watch.host { + let route = homeAddress(host) ? await localRoute(host, pc.port) : nil + guard let route, MovedPC.isLocal(host, over: route) else { + if !Task.isCancelled, pcReturn != nil { + pcReturn?.didTry(pc) + record(.info, "connection", "pc_return result=not_local") + } + return false + } + } + return await MovedPC.confirm(host: host, port: pc.port, token: token, link: makeUDPLink(pc.port)) + } + + /// The one try at where Bonjour has the PC now, which proved the + /// pairing. The attempt carries the pairing, and only a session from + /// the PC it expects counts (see `ExpectedPC`). A PC at a new address or port moves the saved + /// entry once it shows a picture. If the PC still doesn't answer, the + /// card comes back and waits again. + private func tryPCAgain(at found: DiscoveredHost) { + guard let watch = pcReturn, let host = found.address, + let token = savedPairing(host: watch.host, port: watch.port) else { return } + record(.info, "connection", "pc_return result=tried") + startingPCReturnTry = true + defer { startingPCReturnTry = false } + problem = nil + connectionError = nil + if host != watch.host || found.port != watch.port, let name = watch.name { + pendingMove = Move(from: (watch.host, watch.port), name: name, token: token) + } + pendingPC = ExpectedPC(pairing: token, name: watch.name) + explicitPairingToken = token + connect(target: .udp(host: host, port: found.port)) + pendingMove = nil + pendingPC = nil + } + + /// Dismissed, or replaced by another card: the watch ends. An attempt + /// that starts clears the card too; a user's own ends the watch where + /// it starts. + private func problemChanged() { + guard pcReturn != nil, !startingPCReturnTry, !showsNoReplyOverWiFi else { return } + if problem == nil && state != .disconnected { return } + endPCReturn() + } + + /// The app left the foreground: the search stops and nothing is asked. + private func pausePCReturn() { + pcReturnTry?.cancel() + pcReturnTry = nil + guard let browser = pcReturnBrowser, !pcReturnPaused else { return } + pcReturnPaused = true + browser.stop() + } + + /// Back in the foreground: the search starts again, and the PC's + /// listings are asked again. + private func resumePCReturn() { + guard let browser = pcReturnBrowser, pcReturnPaused else { return } + pcReturnPaused = false + browser.start() + askPCReturn(after: timing.pcReturnDebounceSeconds) + } + + private func endPCReturn() { + pcReturnTry?.cancel() + pcReturnTry = nil + pcReturnLimit?.cancel() + pcReturnLimit = nil + pcReturnBrowser?.onHosts = nil + pcReturnBrowser?.stop() + pcReturnBrowser = nil + pcReturnPaused = false + pcReturn = nil + if watchingForPC { watchingForPC = false } + } + /// The session and how long the current attempt has taken, appended to /// the support log's first-frame and timeout lines. private var attemptTimings: String { @@ -1034,15 +1880,25 @@ final class ConnectionManager: ObservableObject { let level = debug.timeoutLevel(handshakeDone: handshakeDone, answeredHereBefore: self.acceptedAddresses.contains("\(debug.host):\(debug.port)")) self.record(level, "timeout", debug.timeoutSummary(handshakeDone: handshakeDone) + self.attemptTimings) + if self.leaveRememberedAddress(.connectTimeout, result: "no_picture") { return } // Only an attempt the PC answered counts. if handshakeDone { self.noPictureStreak.endedWithoutPicture() } - self.problem = self.noPictureStreak.shown(.timeout( + // A PC that said it isn't streaming and then went quiet (it + // closed, or slept) didn't answer: its answers were no picture. + // A PC that said why (its graphics card, which only the PC + // can switch) gets that card instead of "no picture". + let card = handshakeDone ? self.pcGraphicsCard : nil + let problem: ConnectionProblem = self.waitingForStream ? .noReply(usb: isUSB) + : card.map { .pcGraphicsCard($0) } ?? self.noPictureStreak.shown(.timeout( listenerReady: debug.listenerReadyPort != nil, helloAttempts: debug.helloAttempts, handshakeDone: handshakeDone, datagramsReceived: debug.datagramsReceived, isUSB: isUSB, host: debug.host), at: isUSB ? nil : "\(debug.host):\(debug.port)") + self.problem = problem == .noReply(usb: false) && self.attemptMove != nil + ? .noReplyAtNewAddress(debug.host) : problem // The app gave up, not the user: the PC's log tells them apart. self.disconnect(.connectTimeout) if handshakeDone && self.noPictureStreak.stopsReconnecting { self.stopAutomaticAttempts(isUSB: isUSB) } + if self.showsNoReplyOverWiFi && self.reconnectCycleID == nil { self.armPCReturn() } } } } @@ -1088,6 +1944,8 @@ final class ConnectionManager: ObservableObject { timeoutTask?.cancel() endReconnectCycle() disconnect() + // The user stopped waiting for the PC to stream. + if case .pcNotStreaming = problem { problem = nil } } /// Try the last connection again: the cable when that is what failed, @@ -1160,6 +2018,7 @@ final class ConnectionManager: ObservableObject { isForeground = false usbListener?.stop() usbListener = nil + pausePCReturn() // A suspended app can't reconnect. Backgrounding ends an automatic // cycle where running out would, and foregrounding resumes it as an // interrupted session. @@ -1182,12 +2041,40 @@ final class ConnectionManager: ObservableObject { } } + /// Shows `sighting` until it expires, then clears it, as long as no + /// newer one has replaced it. + private func keepAppleDevicesSighting(_ sighting: AppleDevicesSighting?) { + appleDevicesExpiry?.cancel() + appleDevicesExpiry = nil + guard let sighting, sighting.applies(now: now()) else { + appleDevicesUnreachable = nil + return + } + appleDevicesUnreachable = sighting + let wait = sighting.until.timeIntervalSince(now()) + appleDevicesExpiry = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(wait)) + guard !Task.isCancelled, let self, self.appleDevicesUnreachable == sighting else { return } + self.appleDevicesUnreachable = nil + } + } + + /// A suspended app's timer may not have fired on time: check the + /// sighting against the clock again, and wait out what is left of it. + func recheckAppleDevicesSighting() { + if let sighting = appleDevicesUnreachable { keepAppleDevicesSighting(sighting) } + } + /// Resume the session that backgrounding interrupted (opt-out toggle). func handleAppForegrounded() { endBackgroundGoodbye() isForeground = true - usbPaused = usbStoppedForNoPicture + recheckAppleDevicesSighting() + // A hold whose wait ran out in the background ends here. + if let hold = cableHold, !hold.holds(at: now()) { endCableHold() } + usbPaused = usbStoppedForNoPicture || cableHold != nil refreshUSBAvailability() + resumePCReturn() guard resumeOnForeground else { return } resumeOnForeground = false guard UserDefaults.standard.flag(forKey: "autoResumeOnForeground"), @@ -1214,12 +2101,23 @@ final class ConnectionManager: ObservableObject { /// Ends the attempt. `end` says why; only the user's own action is /// `.userDisconnect`. func disconnect(_ end: AttemptEnd = .userDisconnect) { + // Still checking the remembered address: nothing went out yet. + if redirectCheck != nil { + redirectCheck = nil + state = .disconnected + return + } + // Only an attempt that got no answer leaves the PC watched for. + if ![.connectTimeout, .pcMoved, .otherPC].contains(end) { endPCReturn() } if state != .disconnected { + endNoVideoStep("no_video") recordStreamSummary(reason: "disconnect=\(end.name)") record(.info, "connection", "Disconnect state=\(state) reason=\(end.name)") recordSummary(end) } awaitingPairing = false + waitingForStream = false + noVideoFallback = nil pairingPrompt = nil currentAttempt.withLock { $0 = nil } timeoutTask?.cancel() @@ -1258,6 +2156,11 @@ final class ConnectionManager: ObservableObject { stats = StreamStats() signalLost = false pictureRestoring = false + pcLocked = false + pcNoDisplay = false + pcGraphicsCard = nil + clearRestartNotice() + secondScreenNeedsWindows11 = false degradedSinceUs = nil prevCounters = FrameAssembler.Counters() hostInfo = nil @@ -1283,7 +2186,9 @@ final class ConnectionManager: ObservableObject { return SessionSummary( end: end, isUSB: mediaLink?.isUSB ?? (transportMode == "USB"), sessionId: hostInfo?.sessionId, nonce: controlChannel?.helloNonce ?? 0, awaitingPairing: awaitingPairing, - datagramsReceived: debugState.datagramsReceived, firstFrame: firstFrameAt.map { $0 - summaryStartedAt }, + datagramsReceived: debugState.datagramsReceived, captureRefused: pcGraphicsCard != nil, + pcSentVideo: pcSentVideo, + firstFrame: firstFrameAt.map { $0 - summaryStartedAt }, duration: now - summaryStartedAt) } @@ -1545,6 +2450,11 @@ struct RecentConnection: Codable, Identifiable { let port: UInt16 let lastUsed: Date let isUSB: Bool + /// The name the PC gave. Entries saved before 0.3.5 have none. + var name: String? = nil + /// The address is the one Bonjour listed for the PC. Only such an + /// entry may move to a new address by itself; nil is unknown. + var found: Bool? = nil } extension RecentConnection { @@ -1568,16 +2478,36 @@ final class RecentConnectionStore: ObservableObject { load() } - func add(host: String, port: UInt16, isUSB: Bool) { + /// Saves a connection that showed a picture. A Wi-Fi entry keeps its + /// name and found mark when this connection gives none. + func add(host: String, port: UInt16, isUSB: Bool, name: String? = nil, found: Bool = false) { + let previous = isUSB ? nil : entry(host: host, port: port) connections.removeAll { $0.host == host && $0.port == port && $0.isUSB == isUSB } connections.insert( - RecentConnection(host: host, port: port, lastUsed: Date(), isUSB: isUSB), + RecentConnection(host: host, port: port, lastUsed: Date(), isUSB: isUSB, + name: isUSB ? nil : name.flatMap { $0.isEmpty ? nil : $0 } ?? previous?.name, + found: isUSB ? nil : (found || previous?.found == true)), at: 0 ) if connections.count > 10 { connections = Array(connections.prefix(10)) } save() } + /// The saved Wi-Fi entry for `host:port`. + func entry(host: String, port: UInt16) -> RecentConnection? { + connections.first { !$0.isUSB && $0.host == host && $0.port == port } + } + + /// The PC moved from `old` to `new`, an address Bonjour gave. A home + /// network entry gives way to it (`replacesOld`). + func moved(from old: (host: String, port: UInt16), to new: (host: String, port: UInt16), name: String, + replacesOld: Bool) { + if replacesOld { + connections.removeAll { !$0.isUSB && $0.host == old.host && $0.port == old.port } + } + add(host: new.host, port: new.port, isUSB: false, name: name, found: true) + } + private func load() { guard let data = UserDefaults.standard.data(forKey: key), let decoded = try? JSONDecoder().decode([RecentConnection].self, from: data) else { return } diff --git a/ios/EternalMonitor/App/ConnectionProblem.swift b/ios/EternalMonitor/App/ConnectionProblem.swift index d476ad3c..0ef36c91 100644 --- a/ios/EternalMonitor/App/ConnectionProblem.swift +++ b/ios/EternalMonitor/App/ConnectionProblem.swift @@ -8,6 +8,9 @@ enum ConnectionProblem: Equatable { /// Nothing came back from the address (PC closed, asleep, other network, /// blocked by a firewall or the router). case noReply(usb: Bool) + /// The saved address didn't answer, and Bonjour found the PC at this + /// new one, which didn't answer either. + case noReplyAtNewAddress(String) /// The PC answered, but no picture could be shown. case noPicture /// This iPad could not open its side of the connection. @@ -19,6 +22,9 @@ enum ConnectionProblem: Equatable { /// usually the router. case routerAddress case noNetwork + /// iPadOS called the path unsatisfied for want of any usable network: + /// Wi-Fi is off, or joined to no network. + case wifiOff case localNetworkBlocked case couldNotConnect /// The cable connection failed after it had opened. @@ -31,10 +37,57 @@ enum ConnectionProblem: Equatable { /// The PC ended the session on an error: it stopped hearing this iPad (a /// Wi-Fi stall) or hit a problem of its own. Nobody stopped the stream. case pcEndedSessionOnError + /// The PC said it is going to sleep. + case pcAsleep + /// The PC said it is shutting down, restarting or signing out. + case pcPoweringOff /// The PC is streaming to another device, or still to this iPad's cable. case pcBusy /// The PC stopped the stream to install an update and restarts on it. case pcUpdating(UpdateWait) + /// The PC answered but isn't streaming. This iPad keeps asking and + /// connects as soon as it is. + case pcNotStreaming(StreamWait) + /// The PC answered, but Windows won't let it capture its screen from the + /// graphics card EternalMonitor runs on, as on some laptops with two. + case pcGraphicsCard(PCGraphicsCard) + + enum StreamWait: Equatable { + /// Someone clicked Stop on the PC. + case stopped + case restarting + /// The PC stopped after repeated errors, for the reason it gave (a + /// token; one this iPad doesn't know is no reason). + case failed(String) + + /// What the PC's answer means, if it is one of these. + init?(_ status: HelloStatus, reason: String) { + switch status { + case .stopped: self = .stopped + case .restarting: self = .restarting + case .failed: self = .failed(reason) + default: return nil + } + } + + /// The PC's reason for `.failed`, in words, if this iPad knows it. + /// The PC's Stream page says the same. + static func failedReason(_ token: String) -> String? { + switch token { + case "capture": return "Capturing your screen kept failing" + case "encoder": return "Encoding the video kept failing" + case "transport": return "Sending the video kept failing" + case "secure_desktop": return "Your PC is showing the lock screen or a permission prompt" + case "display_changing": return "Windows is changing the display" + case "no_display": return "No display is on" + case "unsupported_gpu": return "Windows won't share this screen from the graphics card EternalMonitor uses" + case "too_many_capture_apps": return "Another app is recording or sharing your screen" + case "session_disconnected": return "This Windows session is disconnected" + case "driver_not_responding": return "The graphics driver isn't responding" + default: return nil + } + } + } /// What this iPad does while the PC updates. enum UpdateWait: Equatable { @@ -61,12 +114,14 @@ enum ConnectionProblem: Equatable { switch self { case .noReply(usb: false): return "Your PC didn't answer" case .noReply(usb: true): return "Your PC didn't answer over the cable" + case .noReplyAtNewAddress: return "Your PC didn't answer at its new address" case .noPicture: return "Your PC answered, but no picture arrived" case .receiverNotReady: return "This iPad couldn't start the connection" case .badAddress: return "That address doesn't look right" case .pairingCodeAsAddress: return "That looks like the pairing code, not the PC's address" case .routerAddress: return "That looks like your router's address, not your PC's" case .noNetwork: return "This iPad isn't connected to a network" + case .wifiOff: return "Wi-Fi is off" case .localNetworkBlocked: return "Local Network is turned off for EternalMonitor" case .couldNotConnect: return "Couldn't connect to your PC" case .usbFailed: return "The connection over the cable stopped" @@ -75,10 +130,19 @@ enum ConnectionProblem: Equatable { case .lost(.usbTurnedOff): return "USB connections were turned off" case .reconnecting: return "Reconnecting to your PC…" case .reconnectFailed: return "Couldn't reconnect to your PC" - case .pcEndedSession, .pcEndedSessionOnError: return "Your PC ended the session" + case .pcEndedSession: return "Your PC ended the session" + // Also what a PC that restarts by itself sends: true for both. + case .pcEndedSessionOnError: return "The connection to your PC ended" + case .pcAsleep: return "Your PC went to sleep" + case .pcPoweringOff: return "Your PC is shutting down or signing out" case .pcBusy: return "Your PC is already streaming to another device" case .pcUpdating(.reconnecting), .pcUpdating(.manual): return "Your PC is updating EternalMonitor" case .pcUpdating(.notBack): return "Your PC hasn't come back from its update yet" + case .pcNotStreaming(.stopped): return "Your PC isn't streaming" + case .pcNotStreaming(.restarting): return "Your PC is restarting the stream…" + case .pcNotStreaming(.failed): return "Your PC stopped streaming after repeated errors" + case .pcGraphicsCard(.screen): return "Your PC can't share its screen from its current graphics card" + case .pcGraphicsCard(.secondScreen): return "The second screen didn't start, and your PC can't share its main screen" } } @@ -88,6 +152,10 @@ enum ConnectionProblem: Equatable { return "Check that EternalMonitor is open on the PC and says Ready for your iPad, that the PC is awake, " + "and that both are on the same Wi-Fi, not a guest network. If all of that is right, Windows Firewall " + "or security software on the PC may be blocking EternalMonitor: run the PC installer again, or connect with a USB cable." + case .noReplyAtNewAddress(let address): + return "This iPad found your PC on the network at a new address, \(address), but EternalMonitor didn't answer there. " + + "Check that the PC says Ready for your iPad. If it does, Windows Firewall or security software on the PC " + + "may be blocking EternalMonitor: run the PC installer again, or connect with a USB cable." case .noReply(usb: true): return "Check that EternalMonitor and Apple Devices are both open on the PC, then unplug the cable and plug it in again." case .noPicture: @@ -103,7 +171,7 @@ enum ConnectionProblem: Equatable { case .routerAddress: return "Addresses ending in .1 usually belong to the Wi-Fi router. Type the Address shown in the " + "EternalMonitor window on the PC, for example 192.168.1.20." - case .noNetwork: + case .noNetwork, .wifiOff: return "Turn on Wi-Fi and join the same network as your PC, or connect with a USB cable." case .localNetworkBlocked: return "iPadOS is stopping this app from reaching your PC over Wi-Fi. " @@ -126,6 +194,10 @@ enum ConnectionProblem: Equatable { return "Someone clicked Stop on the PC, changed a setting that restarts the stream, or closed EternalMonitor. Try again when the PC says Ready for your iPad." case .pcEndedSessionOnError: return "The connection to your PC dropped, or the PC ran into a problem. Tap Try again to reconnect." + case .pcAsleep: + return "Wake the PC, then tap Try again." + case .pcPoweringOff: + return "Tap Try again once the PC is on, someone has signed in, and EternalMonitor is open." case .pcUpdating(.reconnecting): return "This iPad reconnects when it's back. If it doesn't, check that EternalMonitor opened again on the PC, then try again." case .pcUpdating(.manual): @@ -134,6 +206,19 @@ enum ConnectionProblem: Equatable { return "Check that EternalMonitor opened again on the PC, then try again. Windows may still be waiting for you to approve the update there." case .pcBusy: return "Disconnect the other iPad first, then try again." + case .pcNotStreaming(.stopped): + return "Click Start streaming in EternalMonitor on the PC. This iPad connects as soon as it starts." + case .pcNotStreaming(.restarting): + return "This iPad connects as soon as it's back." + case .pcNotStreaming(.failed(let token)): + let start = "Click Start streaming in EternalMonitor on the PC to try again. This iPad connects as soon as it starts." + return StreamWait.failedReason(token).map { "\($0). \(start)" } ?? start + case .pcGraphicsCard(.screen): + return "Windows won't let EternalMonitor share the screen from the graphics card it runs on. " + + "The EternalMonitor window on the PC says how to switch it. Then try again." + case .pcGraphicsCard(.secondScreen): + return "Windows won't let EternalMonitor share the PC's main screen from the graphics card it runs on, " + + "so it can't mirror it instead. The EternalMonitor window on the PC says how to switch the card. Then try again." } } @@ -152,7 +237,8 @@ enum ConnectionProblem: Equatable { var canReport: Bool { switch self { case .reconnecting, .pcUpdating(.reconnecting), .pcUpdating(.manual), - .lost(.cableUnplugged), .lost(.usbTurnedOff), .pcBusy, .pairingCodeAsAddress: + .lost(.cableUnplugged), .lost(.usbTurnedOff), .pcBusy, .pairingCodeAsAddress, + .pcNotStreaming(.stopped), .pcNotStreaming(.restarting), .pcAsleep, .pcPoweringOff: return false default: return true @@ -162,7 +248,8 @@ enum ConnectionProblem: Equatable { var actions: [Action] { switch self { case .localNetworkBlocked: return [.openSettings, .retry] - case .reconnecting, .lost(.cableUnplugged), .lost(.usbTurnedOff), .receiverNotReady, .pairingCodeAsAddress: return [] + case .reconnecting, .lost(.cableUnplugged), .lost(.usbTurnedOff), .receiverNotReady, .pairingCodeAsAddress, + .pcNotStreaming: return [] default: return [.retry] } } @@ -199,8 +286,8 @@ enum ConnectionProblem: Equatable { /// never answered. static func afterReconnecting(lastAttempt: ConnectionProblem?) -> ConnectionProblem { switch lastAttempt { - case .localNetworkBlocked, .noNetwork, .badAddress, .usbFailed, .noPicture, .pcBusy, .pcEndedSession, - .pcEndedSessionOnError, .pcUpdating: + case .localNetworkBlocked, .noNetwork, .wifiOff, .badAddress, .usbFailed, .noPicture, .pcBusy, .pcEndedSession, + .pcEndedSessionOnError, .pcUpdating, .pcGraphicsCard: return lastAttempt ?? .reconnectFailed default: return .reconnectFailed @@ -225,6 +312,8 @@ enum ConnectionProblem: Equatable { case .hostShuttingDown: return .pcUpdating(.reconnecting) case .error: return .pcEndedSessionOnError case .userDisconnect, .superseded, .appBackground: return .pcEndedSession + case .hostSleeping: return .pcAsleep + case .hostPoweringOff: return .pcPoweringOff } } @@ -247,6 +336,7 @@ enum ConnectionProblem: Equatable { // The cable is a local tunnel: Wi-Fi, DNS and Local Network don't apply. if isUSB { return .usbFailed } if error.kind == .localNetworkDenied { return .localNetworkBlocked } + if error.pathUnsatisfied { return .wifiOff } if let nw = error.underlying as? NWError { switch nw { case .dns(let code): @@ -346,9 +436,17 @@ struct NoPictureStreak: Equatable { /// What the USB cable card says for each listener status. enum USBStatusText { - static func describe(_ status: String) -> String { + /// `appleDevicesUnreachable`: when this iPad last connected, the PC + /// couldn't reach Apple's USB service. That was one sample, and a host + /// may leave the capability out for other reasons, so the card gives + /// it as what happened and what to try, not as the current state. + static func describe(_ status: String, appleDevicesUnreachable: Bool = false) -> String { switch status { case "USB: off": return L("Off. Turn on to connect with a cable.") + case "USB: waiting for a cable" where appleDevicesUnreachable: + // A problem: stays English, like the other problem text. + return "Last time this iPad connected, your PC couldn't reach Apple Devices. If the cable doesn't connect, " + + "open Apple Devices on the PC, or install it from the Microsoft Store. Wi-Fi still works." case "USB: waiting for a cable": return L("Plug this iPad into your PC. Keep Apple Devices open on the PC.") case "USB: disconnected": return L("Paused. Tap Connect USB to use the cable again.") case "USB: connecting": return L("Connecting over the cable…") @@ -363,3 +461,31 @@ enum USBStatusText { } } } + +/// Which screen Windows won't let the PC capture from its graphics card, as +/// its heartbeats say. +enum PCGraphicsCard: Equatable { + /// The screen it mirrors. + case screen + /// The main screen, in Second screen mode after the second screen + /// failed to start. + case secondScreen + + /// From a heartbeat's notice; nil for any other. + init?(notice: UInt8) { + switch notice { + case Heartbeat.noticeUnsupportedGPU: self = .screen + case Heartbeat.noticeSecondScreenUnsupportedGPU: self = .secondScreen + default: return nil + } + } + + /// What the iPad says while it waits, and over a held picture. + var short: String { + switch self { + case .screen: return L("Your PC can't share its screen from its current graphics card. Check the PC.") + case .secondScreen: + return L("The second screen didn't start, and your PC can't share its main screen from its current graphics card. Check the PC.") + } + } +} diff --git a/ios/EternalMonitor/App/DisplayView.swift b/ios/EternalMonitor/App/DisplayView.swift index ba21c5fc..3c1e5f39 100644 --- a/ios/EternalMonitor/App/DisplayView.swift +++ b/ios/EternalMonitor/App/DisplayView.swift @@ -51,6 +51,26 @@ struct DisplayView: View { reconnectingCard .transition(.opacity.combined(with: .scale(scale: 0.96))) .allowsHitTesting(false) + } else if connectionManager.pcLocked { + pcLockedPill + .transition(.opacity) + .allowsHitTesting(false) + } else if connectionManager.pcNoDisplay { + noDisplayPill + .transition(.opacity) + .allowsHitTesting(false) + } else if let card = connectionManager.pcGraphicsCard { + graphicsCardPill(card) + .transition(.opacity) + .allowsHitTesting(false) + } else if connectionManager.pcRestarting { + pcRestartingPill + .transition(.opacity) + .allowsHitTesting(false) + } else if connectionManager.secondScreenNeedsWindows11 { + needsWindows11Pill + .transition(.opacity) + .allowsHitTesting(false) } else if connectionManager.pictureRestoring { restoringPill .transition(.opacity) @@ -113,6 +133,11 @@ struct DisplayView: View { .persistentSystemOverlays(.hidden) .animation(.easeInOut(duration: 0.25), value: connectionManager.signalLost) .animation(.easeInOut(duration: 0.25), value: connectionManager.pictureRestoring) + .animation(.easeInOut(duration: 0.25), value: connectionManager.pcLocked) + .animation(.easeInOut(duration: 0.25), value: connectionManager.pcNoDisplay) + .animation(.easeInOut(duration: 0.25), value: connectionManager.pcGraphicsCard) + .animation(.easeInOut(duration: 0.25), value: connectionManager.pcRestarting) + .animation(.easeInOut(duration: 0.25), value: connectionManager.secondScreenNeedsWindows11) .animation(reduceMotion ? nil : .easeInOut(duration: 0.35), value: appeared) .onAppear { appeared = true @@ -410,6 +435,108 @@ struct DisplayView: View { .background(glass(RoundedRectangle(cornerRadius: 22, style: .continuous))) } + /// Over the held picture while the PC is locked or shows a permission + /// prompt, until it captures again. + private var pcLockedPill: some View { + HStack(spacing: 10) { + Image(systemName: "lock.fill") + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(Theme.text) + .accessibilityHidden(true) + Text(L("Your PC is locked. Unlock it to continue.")) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + } + .padding(.horizontal, 18) + .frame(minHeight: 44) + .background(glass(Capsule())) + .accessibilityElement(children: .ignore) + .accessibilityLabel(L("Your PC is locked. Unlock it to continue.")) + .accessibilityIdentifier("display.pcLocked") + } + + /// Over the held picture while the PC has no display turned on, until + /// one comes back. + private var noDisplayPill: some View { + HStack(spacing: 10) { + Image(systemName: "display") + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(Theme.text) + .accessibilityHidden(true) + Text(L("Your PC has no active display. Turn on a screen to continue.")) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + } + .padding(.horizontal, 18) + .frame(minHeight: 44) + .background(glass(Capsule())) + .accessibilityElement(children: .ignore) + .accessibilityLabel(L("Your PC has no active display. Turn on a screen to continue.")) + .accessibilityIdentifier("display.pcNoDisplay") + } + + /// Over the mirrored picture for a few seconds, when the PC chose Second + /// screen but its Windows can't run it. + private var needsWindows11Pill: some View { + HStack(spacing: 10) { + Image(systemName: "rectangle.on.rectangle") + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(Theme.text) + .accessibilityHidden(true) + Text(L("Second screen needs Windows 11. Showing your PC's screen instead.")) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + } + .padding(.horizontal, 18) + .frame(minHeight: 44) + .background(glass(Capsule())) + .accessibilityElement(children: .ignore) + .accessibilityLabel(L("Second screen needs Windows 11. Showing your PC's screen instead.")) + .accessibilityIdentifier("display.needsWindows11") + } + + /// Over the held picture while Windows won't let the PC capture from its + /// graphics card, until it can again. + private func graphicsCardPill(_ card: PCGraphicsCard) -> some View { + HStack(spacing: 10) { + Image(systemName: "display") + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(Theme.text) + .accessibilityHidden(true) + Text(card.short) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + .multilineTextAlignment(.center) + } + .padding(.horizontal, 18) + .padding(.vertical, 8) + .frame(minHeight: 44) + .background(glass(Capsule())) + .padding(.horizontal, 24) + .accessibilityElement(children: .ignore) + .accessibilityLabel(card.short) + .accessibilityIdentifier("display.pcGraphicsCard") + } + + /// Over the held picture while the PC restarts its stream, until the + /// first new frame. + private var pcRestartingPill: some View { + HStack(spacing: 10) { + ProgressView() + .tint(Theme.text) + .accessibilityHidden(true) + Text(L("Your PC is restarting the stream…")) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + } + .padding(.horizontal, 18) + .frame(minHeight: 44) + .background(glass(Capsule())) + .accessibilityElement(children: .ignore) + .accessibilityLabel(L("Your PC is restarting the stream…")) + .accessibilityIdentifier("display.pcRestarting") + } + /// The PC is still connected but a lost frame left the picture waiting /// for a fresh one; usually gone in well under a second. private var restoringPill: some View { diff --git a/ios/EternalMonitor/App/EternalMonitorApp.swift b/ios/EternalMonitor/App/EternalMonitorApp.swift index 2d4146f3..2f778928 100644 --- a/ios/EternalMonitor/App/EternalMonitorApp.swift +++ b/ios/EternalMonitor/App/EternalMonitorApp.swift @@ -33,7 +33,8 @@ struct EternalMonitorApp: App { // Automated end-to-end harness: EM_AUTOCONNECT=host:port // connects immediately, bypassing the connect screen. if let target = E2E.autoconnectTarget { - connectionManager.connect(host: target.host, port: target.port) + connectionManager.connect(host: target.host, port: target.port, token: target.token, + found: target.found) } #if DEBUG UIPreview.apply(to: connectionManager) diff --git a/ios/EternalMonitor/App/MovedPC.swift b/ios/EternalMonitor/App/MovedPC.swift new file mode 100644 index 00000000..5e0e0723 --- /dev/null +++ b/ios/EternalMonitor/App/MovedPC.swift @@ -0,0 +1,227 @@ +import CryptoKit +import Foundation +import Network +import os + +/// Finding a saved PC again after its address changed: the router gave it a +/// new one, or it moved between a cable and Wi-Fi. Only a paired PC that this +/// iPad found with Bonjour, on a home network address that the iPad reaches +/// directly over Wi-Fi or Ethernet, moves by itself; the iPad checks that it +/// is the same PC, at that address, before reconnecting. +enum MovedPC { + /// The Bonjour service a PC named `pcName` advertises. + static func serviceName(pcName: String) -> String { "EternalMonitor (\(pcName))" } + + /// Whether an attempt at `host` that got no answer may look for its PC + /// by name: the PC has a name and a pairing, and its address came from + /// Bonjour (`found`), not from typing, a QR code or an older version. + static func canLookUp(host: String, pcName: String?, found: Bool, paired: Bool, + homeAddress: (String) -> Bool = isHomeAddress) -> Bool { + guard found, paired, let pcName, !pcName.isEmpty else { return false } + return homeAddress(host) + } + + /// Where Bonjour found the PC named `pcName` instead of `host:port`, or + /// nil: not found, found at the same address, not on a home network + /// address, or not one PC. Another PC with the same name makes Bonjour + /// rename one of the two services, so any other service named for it, + /// resolved or not, makes it ambiguous. + static func newAddress(pcName: String, host: String, port: UInt16, found: [DiscoveredHost], + homeAddress: (String) -> Bool = isHomeAddress) -> (host: String, port: UInt16)? { + let name = serviceName(pcName: pcName) + let claims = found.filter { $0.name == name || $0.name.hasPrefix(name + " (") } + guard claims.count == 1, let pc = claims.first, pc.name == name, let address = pc.address, + homeAddress(address), address != host || pc.port != port else { return nil } + return (address, pc.port) + } + + /// A home network address, which the router hands out and may hand to + /// another device in time. VPN and Tailscale addresses never move; a VPN + /// in these ranges is caught by `isLocal`. + static func isHomeAddress(_ host: String) -> Bool { + guard host.split(separator: ".").count == 4, let bytes = IPv4Address(host)?.rawValue, bytes.count == 4 else { return false } + return bytes[0] == 10 || (bytes[0] == 172 && (16...31).contains(bytes[1])) || (bytes[0] == 192 && bytes[1] == 168) + || (bytes[0] == 169 && bytes[1] == 254) + } + + /// Looks for EternalMonitor PCs for up to `seconds`, stopping soon after + /// the service `name` has an address. The wait after it lets a second + /// service with the same name show up. Services still resolving count. + @MainActor + static func find(_ name: String, seconds: Double = 4) async -> [DiscoveredHost] { + let scanner = NetworkScanner() + scanner.retriesWhenEmpty = false + scanner.startScan() + let deadline = Date().addingTimeInterval(seconds) + var foundAt: Date? + while Date() < deadline { + try? await Task.sleep(nanoseconds: 250_000_000) + if foundAt == nil, scanner.hosts.contains(where: { $0.name == name && $0.address != nil }) { foundAt = Date() } + if let foundAt, Date().timeIntervalSince(foundAt) >= 0.75 { break } + } + let claims = scanner.claims + scanner.stopScan() + return claims + } + + /// How the iPad reaches an address: the interface it sends from, whether + /// the path is Wi-Fi or wired Ethernet and nothing else, and that + /// interface's IPv4 address and netmask. + struct Route: Equatable { + var interface: String + var direct: Bool + var address: String + var netmask: String + } + + /// Interfaces that are tunnels, whatever the path calls them. + private static let tunnels = ["utun", "ipsec", "ppp", "tun", "tap", "wg"] + + /// Whether `host` is on the local network `route` sends from: a direct + /// Wi-Fi or Ethernet path, not a tunnel, with `host` in its subnet. A + /// VPN that uses home network ranges fails here. + static func isLocal(_ host: String, over route: Route) -> Bool { + let ipv4 = { (text: String) -> UInt32? in + guard text.split(separator: ".").count == 4, let bytes = IPv4Address(text)?.rawValue, bytes.count == 4 else { return nil } + return bytes.reduce(0) { $0 << 8 | UInt32($1) } + } + guard route.direct, !tunnels.contains(where: route.interface.hasPrefix), + let address = ipv4(route.address), let mask = ipv4(route.netmask), let candidate = ipv4(host), + mask != 0, mask != .max else { return false } + return address & mask == candidate & mask + } + + /// The other home network addresses `identity` signed, in the PC's + /// order (those with a default gateway first), without `asked`. + static func otherAddresses(_ identity: WireIdentity, asked: String, + homeAddress: (String) -> Bool = isHomeAddress) -> [String] { + identity.addresses.map { $0.map(String.init).joined(separator: ".") } + .filter { $0 != asked && homeAddress($0) } + } + + /// How this iPad reaches `host:port` now, from a UDP path to it that + /// sends nothing; nil if no path is ready within 2 s. + static func route(to host: String, port: UInt16) async -> Route? { + guard let remotePort = NWEndpoint.Port(rawValue: port) else { return nil } + let connection = NWConnection(to: .hostPort(host: NWEndpoint.Host(host), port: remotePort), using: .udp) + let queue = DispatchQueue(label: "com.eternal.moved-pc.route") + let path: NWPath? = await withCheckedContinuation { continuation in + let done = OSAllocatedUnfairLock(initialState: false) + let finish: @Sendable (NWPath?) -> Void = { path in + guard done.withLock({ finished in defer { finished = true }; return !finished }) else { return } + connection.stateUpdateHandler = nil + connection.cancel() + continuation.resume(returning: path) + } + connection.stateUpdateHandler = { [weak connection] state in + switch state { + case .ready: finish(connection?.currentPath) + case .failed, .cancelled: finish(nil) + default: break + } + } + queue.asyncAfter(deadline: .now() + 2) { finish(nil) } + connection.start(queue: queue) + } + guard let path, case .hostPort(.ipv4(let local), _)? = path.localEndpoint, + var interface = path.availableInterfaces.first?.name else { return nil } + var direct = (path.usesInterfaceType(.wifi) || path.usesInterfaceType(.wiredEthernet)) + && !path.usesInterfaceType(.other) && !path.usesInterfaceType(.cellular) + if path.usesInterfaceType(.loopback), let stand = E2E.loopbackRouteInterface { + interface = stand + direct = true + } + let address = local.rawValue.map { String($0) }.joined(separator: ".") + guard let netmask = netmask(interface: interface, address: address) else { return nil } + return Route(interface: interface, direct: direct, address: address, netmask: netmask) + } + + /// The netmask of `address` on `interface`, from the interface list. + private static func netmask(interface: String, address: String) -> String? { + var list: UnsafeMutablePointer? + guard getifaddrs(&list) == 0, let first = list else { return nil } + defer { freeifaddrs(first) } + let text = { (pointer: UnsafeMutablePointer) -> String in + var host = [CChar](repeating: 0, count: Int(NI_MAXHOST)) + getnameinfo(pointer, socklen_t(pointer.pointee.sa_len), &host, socklen_t(host.count), nil, 0, NI_NUMERICHOST) + return String(cString: host) + } + for entry in sequence(first: first, next: { $0.pointee.ifa_next }) { + guard String(cString: entry.pointee.ifa_name) == interface, let addr = entry.pointee.ifa_addr, + addr.pointee.sa_family == sa_family_t(AF_INET), text(addr) == address, + let mask = entry.pointee.ifa_netmask else { continue } + return text(mask) + } + return nil + } + + /// What the PC holding `token` answers to `nonce`, receiving at `port` + /// on `addresses` (4 bytes each). + static func proof(token: Data, nonce: Data, port: UInt16, addresses: [Data]) -> Data { + Data(HMAC.authenticationCode(for: WireIdentity.message(nonce: nonce, port: port, addresses: addresses), + using: SymmetricKey(data: token))) + } + + /// Asks the PC at `host:port`, over `link` and nothing else, to show it + /// is the PC that `token` paired with, receiving at `host:port`. True + /// once its answer checks out and names that address; false after + /// `attempts` requests go without one, as from a PC before 0.3.5, or a + /// wrong proof. The token itself is never sent. + static func confirm(host: String, port: UInt16, token: Data, link: any MediaLink, attempts: Int = 8, + interval: Double = 0.25) async -> Bool { + await identify(host: host, port: port, token: token, link: link, attempts: attempts, interval: interval) != nil + } + + /// As `confirm`, with the PC's signed answer: the port it receives the + /// stream on and its home network addresses, one of them `host`. + static func identify(host: String, port: UInt16, token: Data, link: any MediaLink, attempts: Int = 8, + interval: Double = 0.25) async -> WireIdentity? { + var generator = SystemRandomNumberGenerator() + let nonce = Data((0..<16).map { _ in UInt8.random(in: .min ... .max, using: &generator) }) + let request = Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identify(WireIdentify(nonce: nonce))) + let key = SymmetricKey(data: token) + let asked = IPv4Address(host)?.rawValue + let box = LinkBox(link) + return await withCheckedContinuation { continuation in + let done = OSAllocatedUnfairLock(initialState: false) + let finish: @Sendable (WireIdentity?) -> Void = { result in + guard done.withLock({ finished in defer { finished = true }; return !finished }) else { return } + box.link.controlQueue.async { + box.link.onControlDatagram = nil + box.link.onListenerReady = nil + box.link.onError = nil + box.link.stop() + } + continuation.resume(returning: result) + } + link.onControlDatagram = { data in + guard case .identity(let identity)? = Wire.parseControl(data)?.message, + identity.version == WireIdentify.version, identity.nonce == nonce else { return } + let signed = WireIdentity.message(nonce: nonce, port: identity.port, addresses: identity.addresses) + guard HMAC.isValidAuthenticationCode(identity.proof, authenticating: signed, using: key) else { + return finish(nil) + } + // The PC signs where it receives the stream. An answer for + // another address isn't the saved PC at this one. + if identity.port == port, let asked, identity.addresses.contains(asked) { finish(identity) } + } + link.onError = { _ in finish(nil) } + link.onListenerReady = { _ in + for attempt in 0.. framesBeforeStep, sendingSince == nil { sendingSince = now } + } + + /// The session already runs at `size`: the steps go on with the sizes + /// below it. + mutating func skipSizes(downTo size: UInt16) { + sizesAsked = max(sizesAsked, Self.sizes.filter { $0 >= size }.count) + } + + /// The step to take at `now`, while there is no picture; nil while the + /// PC sends nothing, or within `wait` of its sending. + mutating func next(at now: TimeInterval) -> Step? { + guard !finished, let since = sendingSince, now - since >= wait else { return nil } + sendingSince = nil + framesBeforeStep = framesSent + guard sizesAsked < Self.sizes.count else { + finished = true + return .otherAddress + } + sizesAsked += 1 + return .smallerPackets(Self.sizes[sizesAsked - 1]) + } +} + +/// Where a PC's video didn't arrive, and the address of the same PC where it +/// did, so the next connect to the first goes to the second once the PC +/// there proves it holds the pairing. Each entry is bound to that pairing by +/// a digest of its token. Kept in UserDefaults. +enum NoVideoPaths { + private static let key = "noVideoPaths" + private static let kept = 16 + + struct Address: Codable, Equatable { + var host: String + var port: UInt16 + } + + private struct Entry: Codable { + var from: Address + var to: Address + /// SHA-256 of the pairing token, in hex. + var pairing: String + } + + /// The address where the PC paired with `pairing` at `host:port` showed + /// video instead. + static func better(than host: String, port: UInt16, pairing: Data) -> Address? { + let digest = digest(pairing) + return load().last { $0.from == Address(host: host, port: port) && $0.pairing == digest }?.to + } + + static func remember(from: Address, to: Address, pairing: Data) { + var entries = load().filter { $0.from != from && $0.from != to } + entries.append(Entry(from: from, to: to, pairing: digest(pairing))) + save(Array(entries.suffix(kept))) + } + + static func forget(from: Address) { + save(load().filter { $0.from != from }) + } + + /// SHA-256 of a pairing token, in hex. + static func digest(_ token: Data) -> String { + SHA256.hash(data: token).map { String(format: "%02x", $0) }.joined() + } + + static func forgetAll() { + UserDefaults.standard.removeObject(forKey: key) + } + + private static func load() -> [Entry] { + guard let data = UserDefaults.standard.data(forKey: key) else { return [] } + return (try? JSONDecoder().decode([Entry].self, from: data)) ?? [] + } + + private static func save(_ entries: [Entry]) { + guard let data = try? JSONEncoder().encode(entries) else { return } + UserDefaults.standard.set(data, forKey: key) + } +} + +/// The media datagram size a PC's video needed at an address, so the next +/// session there starts with it instead of waiting through the steps again. +/// Each entry is bound to the pairing by a digest of its token, and to the +/// network this iPad reached the PC over: one address may keep a size per +/// network. It lasts a week from when the size was last needed, and every +/// size kept for the address goes as soon as a session there shows a +/// picture at the PC's own size. Kept in UserDefaults. +enum NoVideoSizes { + static let lifetime: TimeInterval = 7 * 24 * 60 * 60 + private static let key = "noVideoSizes" + private static let kept = 16 + + /// A session's address, pairing and network. + struct Key: Equatable { + var address: NoVideoPaths.Address + var pairing: Data + /// `network(_:)` of this iPad's route to the address. + var network: String + } + + private struct Entry: Codable { + var address: NoVideoPaths.Address + /// SHA-256 of the pairing token, in hex. + var pairing: String + var network: String + var size: UInt16 + var at: Date + } + + /// This iPad's side of a route: its interface, address and netmask. + static func network(_ route: MovedPC.Route?) -> String { + route.map { "\($0.interface) \($0.address)/\($0.netmask)" } ?? "-" + } + + /// The size to start a session at `key` with: one kept for its address + /// and pairing, on the same network, less than `lifetime` old. + static func size(for key: Key, now: Date) -> UInt16? { + guard key.network != "-", let entry = load().last(where: { matches($0, key, network: true) }), + now >= entry.at, now.timeIntervalSince(entry.at) < lifetime else { return nil } + return entry.size + } + + static func remember(_ size: UInt16, for key: Key, now: Date) { + var entries = load().filter { !matches($0, key, network: true) } + entries.append(Entry(address: key.address, pairing: NoVideoPaths.digest(key.pairing), network: key.network, size: size, at: now)) + save(Array(entries.suffix(kept))) + } + + /// Forgets the sizes kept for `key`'s address and pairing, on every + /// network; true if there was one. + @discardableResult + static func forget(_ key: Key) -> Bool { + let entries = load() + let kept = entries.filter { !matches($0, key, network: false) } + guard kept.count != entries.count else { return false } + save(kept) + return true + } + + static func forgetAll() { + UserDefaults.standard.removeObject(forKey: key) + } + + /// Whether `entry` is kept for `key`'s address and pairing, and for its + /// network when `network` says so. + private static func matches(_ entry: Entry, _ key: Key, network: Bool) -> Bool { + entry.address == key.address && entry.pairing == NoVideoPaths.digest(key.pairing) + && (!network || entry.network == key.network) + } + + private static func load() -> [Entry] { + guard let data = UserDefaults.standard.data(forKey: key) else { return [] } + return (try? JSONDecoder().decode([Entry].self, from: data)) ?? [] + } + + private static func save(_ entries: [Entry]) { + guard let data = try? JSONEncoder().encode(entries) else { return } + UserDefaults.standard.set(data, forKey: key) + } +} diff --git a/ios/EternalMonitor/App/PCReturn.swift b/ios/EternalMonitor/App/PCReturn.swift new file mode 100644 index 00000000..8cd80776 --- /dev/null +++ b/ios/EternalMonitor/App/PCReturn.swift @@ -0,0 +1,117 @@ +import Combine +import Foundation + +/// Bonjour's list of PCs, kept up to date until stopped. Tests pass a fake. +@MainActor +protocol PCBrowser: AnyObject { + /// Called with the whole list each time it changes. + var onHosts: (([DiscoveredHost]) -> Void)? { get set } + func start() + func stop() +} + +/// A Bonjour search that runs until stopped. +@MainActor +final class BonjourPCBrowser: PCBrowser { + var onHosts: (([DiscoveredHost]) -> Void)? + private let scanner = NetworkScanner() + private var watch: AnyCancellable? + + init() { + scanner.retriesWhenEmpty = false + scanner.searchesUntilStopped = true + } + + func start() { + // One update replaces a PC's entry in steps; only the list it ends on counts. + watch = scanner.$hosts.debounce(for: .milliseconds(100), scheduler: RunLoop.main).sink { [weak self] _ in + guard let self else { return } + self.onHosts?(self.scanner.hosts) + } + scanner.startScan() + } + + func stop() { + watch = nil + scanner.stopScan() + } +} + +/// The PC that didn't answer, watched for on Bonjour from the start of the +/// attempt. Its HELLO2s go out for two seconds only, so a PC app that opens +/// after that never hears this iPad. While the card is up, each listing of +/// the PC is asked to prove it holds the saved pairing: at once when it +/// turns up or changes, then every few seconds, because a PC that closed +/// without saying goodbye stays listed, unchanged, after it opens again. +/// Every such listing is asked in turn, so one that never answers, such as +/// another PC at the saved address, can't keep the PC from being asked +/// where it is now. A listing that proves it gets one try; only a new or +/// changed one gets another. +struct PCReturn { + /// The PC's name, when this iPad knows it: its Bonjour service is the + /// PC, wherever it is. The service at the address tried is the PC too. + let name: String? + let host: String + let port: UInt16 + /// The PC's listings now, by name, address, port and version. + private(set) var listed: [String: DiscoveredHost] = [:] + /// Listings that had their try, while they stay listed. + private var tried = Set() + /// When the card went up and the watch's time limit started. + private(set) var armedAt: Date? + + init(name: String?, host: String, port: UInt16) { + self.name = name.flatMap { $0.isEmpty ? nil : $0 } + self.host = host + self.port = port + } + + var service: String? { name.map(MovedPC.serviceName) } + + func matches(_ pc: DiscoveredHost) -> Bool { + guard let address = pc.address else { return false } + return (service != nil && pc.name == service) || (address == host && pc.port == port) + } + + private static func key(_ pc: DiscoveredHost) -> String { + "\(pc.name)|\(pc.address ?? "")|\(pc.port)|\(pc.version ?? "")" + } + + /// Bonjour's list now. True when a listing of the PC turned up, or + /// changed address, port or version: it is asked at once. A listing + /// that leaves and comes back may have a try again. + mutating func heard(_ hosts: [DiscoveredHost]) -> Bool { + var current: [String: DiscoveredHost] = [:] + for pc in hosts where matches(pc) { current[Self.key(pc)] = pc } + let new = Set(current.keys).subtracting(listed.keys) + listed = current + tried.formIntersection(current.keys) + return !new.isEmpty + } + + /// The listings to ask, each in its turn: those that haven't had their + /// try. One that doesn't answer stays among them. + var candidates: [DiscoveredHost] { + listed.keys.sorted().filter { !tried.contains($0) }.compactMap { listed[$0] } + } + + /// Whether `pc` is still listed and hasn't had its try. + func awaits(_ pc: DiscoveredHost) -> Bool { + let key = Self.key(pc) + return listed[key] != nil && !tried.contains(key) + } + + /// `pc` had its try, or can't have one. + mutating func didTry(_ pc: DiscoveredHost) { + tried.insert(Self.key(pc)) + } + + /// The card is up: the time limit starts, once. + mutating func arm(at now: Date) { + if armedAt == nil { armedAt = now } + } + + func expired(at now: Date, limit: TimeInterval) -> Bool { + armedAt.map { now.timeIntervalSince($0) >= limit } ?? false + } +} diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index 2103755b..fc7d4407 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -13,6 +13,11 @@ enum AttemptEnd: Equatable { case pcGoodbye /// The PC turned the connection down. case pcRefused + /// The PC turned it down as Busy: it streams to another device. + case pcBusy + /// The PC turned it down as too old for this app. + case oldPC + /// A v0.1.x PC, too old to answer HELLO2. case legacyPC /// The connect watchdog ran out before a picture. case connectTimeout @@ -27,6 +32,19 @@ enum AttemptEnd: Equatable { case closedBeforePicture /// This iPad's network side failed, for example Local Network denied. case linkFailed + /// iPadOS found no usable network for the path: Wi-Fi is off or + /// joined to none (`ConnectionProblem.wifiOff`). + case noNetwork + /// No answer at the address tried, and the same PC is known at another: + /// Bonjour found it there, or its video arrived there before. A new + /// attempt goes there. + case pcMoved + /// The PC sent video that gave no picture at this address, even in + /// small datagrams, and showed it is the same PC at another: a new + /// attempt goes there. + case noVideoPath + /// The PC found at a saved PC's new address turned out to be another PC. + case otherPC case receiverFailed /// The PC asked for the pairing code, and a new HELLO2 sends it: the /// refused one is over, the attempt goes on. @@ -60,6 +78,10 @@ struct SessionSummary: Equatable { /// The PC asked for the pairing code and the attempt ended before a session. var awaitingPairing = false var datagramsReceived = 0 + /// The PC said it can't capture its screen from its graphics card. + var captureRefused = false + /// The PC's heartbeats counted video frames sent this attempt. + var pcSentVideo = false /// Seconds from the start to the first decoded frame. var firstFrame: TimeInterval? var duration: TimeInterval @@ -93,18 +115,26 @@ struct SessionSummary: Equatable { case .userDisconnect, .usbTurnedOff: return "user_ended" case .appBackground, .superseded, .pcGoodbye: return "app_ended" case .connectTimeout: return sessionId == nil ? "no_reply" : "no_picture" - case .closedBeforePicture: return "no_picture" + case .pcMoved: return "no_reply" + case .closedBeforePicture, .noVideoPath: return "no_picture" default: return "error" } } private var failure: String { switch end { - case .connectTimeout where sessionId == nil: return "no_reply" + case .connectTimeout where sessionId == nil, .pcMoved: return "no_reply" + // Only when no video went out, as the PC's summary has it: video + // that went out and gave no picture was lost or didn't decode. + case .connectTimeout where captureRefused && !pcSentVideo && datagramsReceived == 0: return "capture_failed" case .connectTimeout: return datagramsReceived == 0 ? "no_video" : "no_frame" + case .noVideoPath: return datagramsReceived == 0 ? "no_video" : "no_frame" case .pcStoppedResponding: return "signal_lost" case .cableUnplugged, .usbLinkFailed, .closedBeforePicture: return "usb_lost" - case .pcRefused, .legacyPC, .linkFailed, .receiverFailed, .pictureStalled: return "other" + case .noNetwork: return "no_network" + case .pcBusy: return "pc_busy" + case .oldPC, .legacyPC: return "old_pc" + case .pcRefused, .linkFailed, .receiverFailed, .pictureStalled, .otherPC: return "other" default: return "none" } } @@ -113,7 +143,7 @@ struct SessionSummary: Equatable { switch end { case .userDisconnect, .usbTurnedOff: return "user" case .appBackground: return "background" - case .superseded, .retriedWithCode: return "superseded" + case .superseded, .retriedWithCode, .pcMoved, .noVideoPath: return "superseded" case .pcGoodbye: return "goodbye" case .connectTimeout: return "timeout" case .pcStoppedResponding: return "liveness" diff --git a/ios/EternalMonitor/App/UIPreview.swift b/ios/EternalMonitor/App/UIPreview.swift index ca0e247a..a6af085f 100644 --- a/ios/EternalMonitor/App/UIPreview.swift +++ b/ios/EternalMonitor/App/UIPreview.swift @@ -4,7 +4,7 @@ import SwiftUI /// Debug builds only: put the UI in a state that normally needs a live PC, /// for screenshots and design review. Never opens a connection. /// -/// `EM_UI_PREVIEW=display|quality|restoring|connecting|error|error-no-picture|error-pairing-code|error-router|error-updating|error-local-network|mismatch|pairing|settings|settings-about|feedback|recent|searchFailed|searchBlocked|setup|setup-blocked`. +/// `EM_UI_PREVIEW=display|quality|restoring|needs-windows-11|connecting|error|error-no-picture|error-pairing-code|error-router|error-updating|error-local-network|error-wifi-off|mismatch|pairing|settings|settings-about|feedback|recent|old-pc|searchFailed|searchBlocked|setup|setup-blocked`. /// `setup` opens first-run setup whatever the saved state (launch with /// `-didSeeOnboarding NO`); `setup-blocked` also fails its search as if /// Local Network were off. @@ -12,7 +12,7 @@ import SwiftUI /// Launch with `-allowUSB NO` so the USB listener stays off as well. enum UIPreview { static let mode = ProcessInfo.processInfo.environment["EM_UI_PREVIEW"] ?? "" - static var showsBackdrop: Bool { mode == "display" || mode == "quality" || mode == "restoring" } + static var showsBackdrop: Bool { mode == "display" || mode == "quality" || mode == "restoring" || mode == "locked" || mode == "no-display" || mode == "needs-windows-11" } static var opensQuality: Bool { mode == "quality" } static var opensSettings: Bool { mode == "settings" || mode == "settings-about" } /// Scrolls Settings to Support and About. @@ -46,6 +46,18 @@ enum UIPreview { manager.transportMode = "WiFi" manager.state = .connected manager.pictureRestoring = true + case "locked": + manager.transportMode = "WiFi" + manager.state = .connected + manager.pcLocked = true + case "no-display": + manager.transportMode = "WiFi" + manager.state = .connected + manager.pcNoDisplay = true + case "needs-windows-11": + manager.transportMode = "WiFi" + manager.state = .connected + manager.secondScreenNeedsWindows11 = true case "connecting": manager.state = .connecting case "error": @@ -60,11 +72,15 @@ enum UIPreview { manager.problem = .pcUpdating(.reconnecting) case "error-local-network": manager.problem = .localNetworkBlocked + case "error-wifi-off": + manager.problem = .wifiOff case "mismatch": manager.versionMismatch = .iPadOlder manager.connectionError = VersionMismatch.iPadOlder.message case "pairing": manager.pairingPrompt = PairingSheetModel() + case "old-pc": + OldPCPrompt.shared.preview(OldPCPrompt.Card(key: "preview", pc: "Studio PC", version: "0.3.2")) case "recent": RecentConnectionStore.shared.add(host: "192.168.1.20", port: 9876, isUSB: false) RecentConnectionStore.shared.add(host: "Studio PC", port: 9877, isUSB: true) diff --git a/ios/EternalMonitor/App/UpdateBanner.swift b/ios/EternalMonitor/App/UpdateBanner.swift index 5b2149e5..a22a516c 100644 --- a/ios/EternalMonitor/App/UpdateBanner.swift +++ b/ios/EternalMonitor/App/UpdateBanner.swift @@ -65,6 +65,54 @@ struct UpdateBanner: View { } } +/// Connect screen: the PC this iPad found or streamed from runs an +/// EternalMonitor older than 0.3.4. Quiet, and only in the screen's flow. +struct OldPCCard: View { + let card: OldPCPrompt.Card + let close: () -> Void + + var body: some View { + let title = L("Your PC app is out of date") + let detail = L("Update EternalMonitor on \(card.pc) for faster, steadier streaming. Download it on the PC from eternalmonitor.dev.") + HStack(alignment: .top, spacing: 12) { + Image(systemName: "desktopcomputer") + .font(.system(size: 15, weight: .semibold)) + .foregroundStyle(Theme.accent) + .frame(width: 34, height: 34) + .background( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .fill(Theme.accent.opacity(0.1)) + ) + .accessibilityHidden(true) + VStack(alignment: .leading, spacing: 2) { + Text(title) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + Text(detail) + .font(.app(13, relativeTo: .footnote)) + .foregroundStyle(Theme.textMuted) + .fixedSize(horizontal: false, vertical: true) + } + .accessibilityElement(children: .combine) + .accessibilityIdentifier("connect.oldPC") + Spacer(minLength: 0) + Button(action: close) { + Image(systemName: "xmark") + .font(.system(size: 12, weight: .semibold)) + .foregroundStyle(Theme.textFaint) + .frame(width: 44, height: 44) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .padding(.top, -6) + .accessibilityLabel(Text(verbatim: L("Close"))) + .accessibilityIdentifier("connect.oldPCClose") + } + .card(padding: 12) + .transition(.opacity) + } +} + /// Connect screen: a small card over the screen when a newer build is ready. /// `allowed` is false while the user is busy (connecting, typing, or in a /// sheet). The Connect screen is never on screen during a stream. diff --git a/ios/EternalMonitor/Core/E2ESignpost.swift b/ios/EternalMonitor/Core/E2ESignpost.swift index 675d654b..de825419 100644 --- a/ios/EternalMonitor/Core/E2ESignpost.swift +++ b/ios/EternalMonitor/Core/E2ESignpost.swift @@ -33,15 +33,32 @@ enum E2E { #endif } + /// `EM_DRAWS_POINTER=0` leaves the pointer in the PC's video, as apps + /// before 0.3.5 did, for the end-to-end compatibility run. + static var drawsPointer: Bool { + ProcessInfo.processInfo.environment["EM_DRAWS_POINTER"] != "0" + } + + /// The pointer counters next to the decoded frame count: with the PC + /// sending the pointer apart, it moves while no frame decodes. + static func pointer(_ counters: PointerOverlay.Counters, decoded: Int) { + emit("E2E_POINTER states=\(counters.states) stale=\(counters.staleStates) shapes=\(counters.shapes) drawn=\(counters.drawn) moves=\(counters.drawnMoves) decoded=\(decoded)") + } + /// `EM_AUTOCONNECT=host:port` makes the app connect immediately on launch, - /// bypassing the connect screen — the harness's way in. - static var autoconnectTarget: (host: String, port: UInt16)? { - guard let spec = ProcessInfo.processInfo.environment["EM_AUTOCONNECT"], - let colon = spec.lastIndex(of: ":"), + /// bypassing the connect screen — the harness's way in. A pairing link + /// (`eternaldisplay://host:port?t=…`, the QR code's text) also brings its + /// token. `EM_AUTOCONNECT_FOUND=1` connects as a tap on a PC that Bonjour + /// listed would. + static var autoconnectTarget: (host: String, port: UInt16, token: Data?, found: Bool)? { + guard let spec = ProcessInfo.processInfo.environment["EM_AUTOCONNECT"] else { return nil } + let found = ProcessInfo.processInfo.environment["EM_AUTOCONNECT_FOUND"] == "1" + if case .success(let link) = PairingCode.parse(spec) { return (link.host, link.port, link.token, found) } + guard let colon = spec.lastIndex(of: ":"), let port = UInt16(spec[spec.index(after: colon)...]) else { return nil } let host = String(spec[.. String? { guard let data else { return nil } diff --git a/ios/EternalMonitor/Core/OldPC.swift b/ios/EternalMonitor/Core/OldPC.swift new file mode 100644 index 00000000..77ef894e --- /dev/null +++ b/ios/EternalMonitor/Core/OldPC.swift @@ -0,0 +1,319 @@ +import CryptoKit +import Foundation + +/// Tells a PC whose EternalMonitor is older than 0.3.4 from a current one, +/// using only what PCs already send. +/// +/// - Bonjour: every PC puts its version in the TXT record's `version`. +/// Only a plain release version counts; a beta's or a missing one doesn't. +/// - HELLO_ACK: every 0.3.4 and later PC sets `hostCapContiguousFrameSeq` in +/// every reply that accepts a session, whatever its hardware or settings, +/// and no earlier PC ever set that bit. This covers PCs reached over USB or +/// by address. Refusals and not-streaming replies may leave every cap out, +/// so they count for nothing either way. +/// +/// A PC is old only when nothing it sent says it is current. +enum OldPC { + static let current = AppVersion("0.3.4")! + + /// True for an old PC, false for a current one, nil when it can't tell. + /// `hostCaps` must come from a reply that accepted a session. + static func isOld(announced: String?, hostCaps: UInt16?) -> Bool? { + if let hostCaps, hostCaps & HelloAck.hostCapContiguousFrameSeq != 0 { return false } + if let version = announced.flatMap(AppVersion.init) { return version < current } + return hostCaps == nil ? nil : true + } + + /// The name a PC gives itself, from its Bonjour service name, with + /// Bonjour's collision number if it has one: "Lab (2)". + static func pcName(bonjour name: String) -> String { + let prefix = "EternalMonitor (" + guard name.hasPrefix(prefix), name.hasSuffix(")") else { return name } + var inner = name.dropFirst(prefix.count).dropLast() + var number = "" + if let open = inner.range(of: ") (", options: .backwards) { + let digits = inner[open.upperBound...] + if !digits.isEmpty, digits.allSatisfy({ $0.isASCII && $0.isNumber }) { + number = " (\(digits))" + inner = inner[..)", with or without a collision suffix such as +/// " (2)", for this launch only, and nothing while the search lists two or +/// more. Known limit: two PCs with one name on a network, one of them used +/// over USB, can get one extra or one missed card per PC every three days. +@MainActor +final class OldPCPrompt: ObservableObject { + static let shared = OldPCPrompt() + static let interval: TimeInterval = 3 * 86_400 + static let defaultsKey = "oldPCPromptMemory" + /// PCs remembered. + static let kept = 16 + + struct Card: Equatable { + /// Who the PC is to this memory. + let key: String + /// The name shown. + let pc: String + /// The release version it announced, if any. + let version: String? + } + + struct Memory: Codable, Equatable { + struct PC: Codable, Equatable { + var shownAt: Date? + /// What a session's accepted HELLO_ACK said: current or not. + var current: Bool? + var seenAt: Date + } + var pcs: [String: PC] = [:] + /// Bonjour service name to the PC a session through it answered as. + var services: [String: String] = [:] + } + + /// The card on screen. + @Published private(set) var card: Card? + /// An old PC waiting for the Connect screen. + private var waiting: Card? + /// The services the search lists now. + private var listed: [DiscoveredHost] = [] + /// The service the user tapped, and its address. + private var tapped: (service: String, address: String)? + /// This launch's sessions that no address linked, by PC name. + private var unlinked: [String: String] = [:] + /// Services linked by those sessions' names, for this launch only. + private var named: [String: String] = [:] + private let defaults: UserDefaults + private let now: () -> Date + private let log: (String) -> Void + + init(defaults: UserDefaults = .standard, now: @escaping () -> Date = { Date() }, + log: @escaping (String) -> Void = { SupportDiagnostics.shared.record("info", "update", $0) }) { + self.defaults = defaults + self.now = now + self.log = log + } + + var memory: Memory { + defaults.data(forKey: Self.defaultsKey).flatMap { try? JSONDecoder().decode(Memory.self, from: $0) } ?? Memory() + } + + private func save(_ memory: Memory) { + var memory = memory + let dropped = memory.pcs.sorted { $0.value.seenAt > $1.value.seenAt }.dropFirst(Self.kept).map(\.key) + for key in dropped { memory.pcs[key] = nil } + // Links to the PCs seen last, a few per PC at most. + let linked = memory.services.filter { memory.pcs[$0.value] != nil } + .sorted { (memory.pcs[$0.value]!.seenAt, $1.key) > (memory.pcs[$1.value]!.seenAt, $0.key) } + memory.services = Dictionary(uniqueKeysWithValues: linked.prefix(Self.kept * 2).map { ($0.key, $0.value) }) + defaults.set(try? JSONEncoder().encode(memory), forKey: Self.defaultsKey) + } + + static func serviceKey(_ name: String) -> String { "service:\(name)" } + + /// The PC behind a Bonjour service. + private func key(service: String, in memory: Memory) -> String { + memory.services[service] ?? named[service] ?? Self.serviceKey(service) + } + + /// Whether `service` is the name a PC called `hostName` registers, or + /// that name with a Bonjour collision suffix. + static func isOwnService(_ service: String, of hostName: String) -> Bool { + let own = "EternalMonitor (\(hostName))" + if service == own { return true } + guard service.hasPrefix(own + " ("), service.hasSuffix(")") else { return false } + let number = service.dropFirst(own.count + 2).dropLast() + return !number.isEmpty && number.allSatisfy { $0.isASCII && $0.isNumber } + } + + /// Links each unlinked session's PC to its own service while the search + /// lists exactly one, and to nothing otherwise. + private func linkNames() { + var memory = self.memory + var changed = false + named.removeAll() + for (hostName, key) in unlinked { + let matches = listed.map(\.name).filter { Self.isOwnService($0, of: hostName) } + guard matches.count == 1, let service = matches.first, memory.services[service] == nil else { continue } + let target = Self.serviceKey(service) + guard target == key || memory.pcs[target]?.current == nil else { continue } + named[service] = key + guard target != key else { continue } + // Both records keep the later showing, so a later launch, which + // has no link, holds the card back under either. Only the time + // is saved, never the link. + if let shown = [memory.pcs[target]?.shownAt, memory.pcs[key]?.shownAt].compactMap({ $0 }).max() { + if memory.pcs[key] != nil, memory.pcs[key]?.shownAt != shown { + memory.pcs[key]?.shownAt = shown + changed = true + } + if memory.pcs[target]?.shownAt != shown { + memory.pcs[target, default: Memory.PC(seenAt: now())].shownAt = shown + changed = true + } + } + if waiting?.key == target { waiting = nil } + if let shown = card, shown.key == target { card = Card(key: key, pc: shown.pc, version: shown.version) } + } + if changed { save(memory) } + } + + /// The services the search lists now, each with the version its own + /// TXT record announced. + func discovered(_ hosts: [DiscoveredHost]) { + listed = hosts + linkNames() + let memory = self.memory + for host in hosts { + let key = key(service: host.name, in: memory) + switch OldPC.isOld(announced: host.version, hostCaps: nil) { + case true? where memory.pcs[key]?.current != true: + queue(Card(key: key, pc: OldPC.pcName(bonjour: host.name), version: host.version), memory: memory) + case false?: + drop(key) + default: + break + } + } + } + + /// The user tapped a found PC at `address`. + func connecting(service: String, address: String) { + tapped = (service, address) + } + + /// A PC accepted a session. `hostCaps` is from that accepted HELLO_ACK; + /// `address` is nil over USB. + /// + /// The session's service is the one the user tapped at that address, + /// else the one the search lists there. A service already linked to a PC + /// with a proof of its own stays with it. Without either, the PC's name + /// may link its own service for this launch (see the type's note). + func answered(hostName: String, pairingToken: Data, hostCaps: UInt16, address: String?) { + var memory = self.memory + let found = address.flatMap { address in + tapped.flatMap { $0.address == address ? $0.service : nil } + ?? listed.first { $0.address == address }?.name + } + tapped = nil + var service = found + var linkedTo: String? + if let candidate = service { + let key = Self.key(hostName: hostName, pairingToken: pairingToken, service: candidate) + let target = memory.services[candidate] ?? Self.serviceKey(candidate) + if target == key || memory.pcs[target]?.current == nil { + linkedTo = target + } else { + service = nil + } + } + let key = Self.key(hostName: hostName, pairingToken: pairingToken, service: service) + var pc = memory.pcs[key] ?? Memory.PC(seenAt: now()) + if let service, let old = linkedTo { + let serviceKey = Self.serviceKey(service) + if old != key { + if let shown = memory.pcs[old]?.shownAt { pc.shownAt = max(pc.shownAt ?? shown, shown) } + if waiting?.key == old { waiting = nil } + if let shown = card, shown.key == old { card = Card(key: key, pc: shown.pc, version: shown.version) } + // Only a record without a proof of its own goes. + if old == serviceKey { memory.pcs[old] = nil } + } + memory.services[service] = key == serviceKey ? nil : key + } + let old = OldPC.isOld(announced: nil, hostCaps: hostCaps) == true + pc.current = !old + pc.seenAt = now() + memory.pcs[key] = pc + save(memory) + // A PC answers under one name at a time. + unlinked = unlinked.filter { $0.value != key } + if found == nil, !hostName.isEmpty { unlinked[hostName] = key } + linkNames() + if old { + queue(Card(key: key, pc: hostName, version: nil), memory: self.memory) + } else { + drop(key) + } + } + + /// The pairing token's hash when the PC has one, otherwise the service + /// it was found as, otherwise its name. + static func key(hostName: String, pairingToken: Data, service: String?) -> String { + if PairingStore.valid(pairingToken) { + return "pair:" + SHA256.hash(data: pairingToken).prefix(12).map { String(format: "%02x", $0) }.joined() + } + if let service { return serviceKey(service) } + return "name:\(hostName)" + } + + private func queue(_ next: Card, memory: Memory) { + guard card == nil, due(next.key, in: memory) else { return } + waiting = next + } + + private func drop(_ key: String) { + if waiting?.key == key { waiting = nil } + if card?.key == key { card = nil } + } + + /// The Connect screen is in view with nothing streaming: shows the + /// waiting card, if its PC is due. + func show() { + guard card == nil, let next = waiting else { return } + waiting = nil + var memory = self.memory + guard due(next.key, in: memory) else { return } + var pc = memory.pcs[next.key] ?? Memory.PC(seenAt: now()) + pc.shownAt = now() + pc.seenAt = now() + memory.pcs[next.key] = pc + // The services this launch links to it by name get the showing too. + for (service, key) in named where key == next.key { + memory.pcs[Self.serviceKey(service), default: Memory.PC(seenAt: now())].shownAt = now() + } + save(memory) + card = next + log("old_pc_prompt shown=true version=\(next.version ?? "unknown")") + } + + /// Closed, or the Connect screen gave way to a stream. + func close() { + card = nil + } + + /// Whether the PC `key` may get its card again. A showing dated after + /// now, from a clock that was ahead, doesn't hold it back. + func due(_ key: String) -> Bool { due(key, in: memory) } + + private func due(_ key: String, in memory: Memory) -> Bool { + guard let last = memory.pcs[key]?.shownAt else { return true } + let now = now() + return last > now || now.timeIntervalSince(last) >= Self.interval + } + + #if DEBUG + func preview(_ card: Card) { + self.card = card + } + #endif +} diff --git a/ios/EternalMonitor/Core/PairingStore.swift b/ios/EternalMonitor/Core/PairingStore.swift index 34bf6e72..5d43db38 100644 --- a/ios/EternalMonitor/Core/PairingStore.swift +++ b/ios/EternalMonitor/Core/PairingStore.swift @@ -75,6 +75,14 @@ final class PairingStore: ObservableObject { try storage.records().first { $0.account.hasSuffix("|\(address)") && Self.valid($0.token) }?.token } + /// The name of the PC paired at `address`, if it gave one. + func pcName(address: String) -> String? { + guard let account = (try? storage.records())?.first(where: { $0.account.hasSuffix("|\(address)") && Self.valid($0.token) })?.account + else { return nil } + let name = String(account.dropLast(address.count + 1)) + return name.isEmpty ? nil : name + } + func isPaired(host: String, port: UInt16) -> Bool { (try? token(address: Self.address(host: host, port: port))) != nil } diff --git a/ios/EternalMonitor/Core/UpdateChecker.swift b/ios/EternalMonitor/Core/UpdateChecker.swift index 88380b8c..91fb2078 100644 --- a/ios/EternalMonitor/Core/UpdateChecker.swift +++ b/ios/EternalMonitor/Core/UpdateChecker.swift @@ -53,6 +53,20 @@ struct UpdateManifest: Decodable, Equatable { else { return nil } return note } + + /// The Windows app's version, read on its own so that a malformed + /// Windows entry never hides the iPad's update, or the other way round. + static func windowsVersion(_ data: Data) -> String? { + struct Manifest: Decodable { + struct Windows: Decodable { let version: String } + let schema: Int + let windows: Windows? + } + guard let manifest = try? JSONDecoder().decode(Manifest.self, from: data), manifest.schema == 1, + let version = manifest.windows?.version, AppVersion(version) != nil + else { return nil } + return version + } } extension UpdateManifest.IPad { @@ -212,6 +226,7 @@ final class UpdateChecker: ObservableObject { static let rolloutDays = "updateAvailableRolloutDays" /// The staged build whose prompt this iPad has reached. static let rolloutReached = "updateRolloutReachedBuild" + static let windowsVersion = "updateWindowsVersion" static let saved = [version, build, testflight, recommended, note, rolloutStart, rolloutDays] } @@ -224,6 +239,8 @@ final class UpdateChecker: ObservableObject { @Published private(set) var available: UpdateManifest.IPad? @Published private(set) var dismissed: (build: Int, at: Date)? @Published private(set) var popupClosed: (build: Int, at: Date)? + /// The newest EternalMonitor for Windows, from the same manifest. + @Published private(set) var windowsVersion: String? @Published var automatic: Bool { didSet { defaults.set(automatic, forKey: Key.automatic) @@ -245,6 +262,7 @@ final class UpdateChecker: ObservableObject { self.installed = installed self.installationID = installationID automatic = defaults.object(forKey: Key.automatic) as? Bool ?? true + windowsVersion = defaults.string(forKey: Key.windowsVersion) if let version = defaults.string(forKey: Key.version) { let found = UpdateManifest.IPad( version: version, @@ -322,6 +340,13 @@ final class UpdateChecker: ObservableObject { defaults.set(now, forKey: Key.popupAt) } + /// A PC that announced `pcVersion` should update EternalMonitor. Only + /// plain release versions compare; a beta or an unknown version never does. + func pcNeedsUpdate(_ pcVersion: String?) -> Bool { + guard let pc = pcVersion.flatMap(AppVersion.init), let newest = windowsVersion.flatMap(AppVersion.init) else { return false } + return pc < newest + } + /// The TestFlight invite named by the manifest, or the public link. var joinURL: URL { if let link = available?.testflight, let url = URL(string: link), @@ -366,6 +391,8 @@ final class UpdateChecker: ObservableObject { defaults.set(now, forKey: Key.checkedAt) // The first-picture wait applies from the next launch. RemoteConfig.save(RemoteConfig.parse(data), to: defaults, now: now) + windowsVersion = UpdateManifest.windowsVersion(data) + defaults.set(windowsVersion, forKey: Key.windowsVersion) guard let found = UpdateManifest.parse(data), installed.isOlder(than: found) else { available = nil for key in Key.saved { defaults.removeObject(forKey: key) } diff --git a/ios/EternalMonitor/Decode/ReferenceContinuity.swift b/ios/EternalMonitor/Decode/ReferenceContinuity.swift index d37425ae..c64ceaf0 100644 --- a/ios/EternalMonitor/Decode/ReferenceContinuity.swift +++ b/ios/EternalMonitor/Decode/ReferenceContinuity.swift @@ -34,6 +34,23 @@ struct ReferenceContinuity { /// Set from the session's host caps: a forward gap in `frame_seq` is a loss. var frameSeqContiguous = false + /// Why a picture was turned down, for the log line of the hold it starts. + enum Loss: Equatable { + /// `frame_seq` skipped from `previous` to `seq` on a PC that numbers + /// contiguously. + case frameSeqGap(previous: UInt32, seq: UInt32) + /// Another epoch, or a lower `frame_seq` in this one, and this picture + /// is not its keyframe. + case streamRestarted(previousEpoch: UInt32, epoch: UInt32, previousSeq: UInt32, seq: UInt32) + /// H.264: `frame_num` is not the one after the latest reference picture. + case frameNumJump(expected: UInt32, got: UInt32) + /// HEVC: the POC moved by other than the learned step. + case pocStep(expected: UInt32, got: UInt32) + } + + /// Why the latest picture turned down was, until another is. + private(set) var lastLoss: Loss? + struct H264Parameters: Equatable { var log2MaxFrameNum: Int var separateColourPlane: Bool @@ -91,17 +108,21 @@ struct ReferenceContinuity { /// frame number, `epoch` its stream epoch; a lower `seq` or another epoch /// means the host restarted its stream. mutating func admitH264(firstSlice nal: Data, isKeyframe: Bool, seq: UInt32, epoch: UInt32) -> Bool { - let (restarted, gap) = noteStream(seq: seq, epoch: epoch) + let stream = noteStream(seq: seq, epoch: epoch) guard let params = h264, let frameNum = Self.h264FrameNum(nal, params) else { previousReferenceFrameNum = nil - return isKeyframe || !(restarted || gap) + return isKeyframe || admitStream(stream, seq: seq, epoch: epoch) } let isReference = (nal[nal.startIndex] >> 5) & 0x03 != 0 if !isKeyframe { - if restarted || gap { return false } + if !admitStream(stream, seq: seq, epoch: epoch) { return false } if let previous = previousReferenceFrameNum, !params.gapsAllowed { let maxFrameNum = UInt32(1) << params.log2MaxFrameNum - guard frameNum == (previous + 1) % maxFrameNum else { return false } + let expected = (previous + 1) % maxFrameNum + guard frameNum == expected else { + lastLoss = .frameNumJump(expected: expected, got: frameNum) + return false + } } } if isReference { previousReferenceFrameNum = frameNum } @@ -110,18 +131,19 @@ struct ReferenceContinuity { mutating func admitHEVC(firstSlice nal: Data, isKeyframe: Bool, seq: UInt32, epoch: UInt32) -> Bool { let contiguous = previousSeq.map { seq == $0 &+ 1 } ?? false - let (restarted, gap) = noteStream(seq: seq, epoch: epoch) + let stream = noteStream(seq: seq, epoch: epoch) guard let params = hevc, let poc = Self.hevcPocLsb(nal, params) else { previousPoc = nil - return isKeyframe || !(restarted || gap) + return isKeyframe || admitStream(stream, seq: seq, epoch: epoch) } if !isKeyframe { - if restarted || gap { return false } + if !admitStream(stream, seq: seq, epoch: epoch) { return false } if let previous = previousPoc, !pocUnreliable { let delta = (poc &- previous) & ((UInt32(1) << params.log2MaxPocLsb) - 1) if contiguous { if pocStep == nil { pocStep = delta } else if delta != pocStep { pocUnreliable = true } } else if let step = pocStep, delta != step { + lastLoss = .pocStep(expected: step, got: delta) return false } } @@ -130,21 +152,47 @@ struct ReferenceContinuity { return true } + /// What [`noteStream`] found, with the frame number and epoch before. + private struct StreamStep { + var restarted = false + var gap = false + var previousSeq: UInt32 = 0 + var previousEpoch: UInt32 = 0 + } + /// Records the wire frame number and epoch. `restarted`: another epoch, /// or a lower number in this one, so a new encoder whose counts, POC step /// and reliability start over. `gap`: a number skipped on a PC that /// numbers contiguously. - private mutating func noteStream(seq: UInt32, epoch: UInt32) -> (restarted: Bool, gap: Bool) { + private mutating func noteStream(seq: UInt32, epoch: UInt32) -> StreamStep { defer { previousSeq = seq; previousEpoch = epoch } - guard let previous = previousSeq else { return (false, false) } + guard let previous = previousSeq else { return StreamStep() } + var step = StreamStep(previousSeq: previous, previousEpoch: previousEpoch ?? epoch) guard previousEpoch == epoch, seq >= previous else { previousReferenceFrameNum = nil previousPoc = nil pocStep = nil pocUnreliable = false - return (true, false) + step.restarted = true + return step + } + step.gap = frameSeqContiguous && seq > previous &+ 1 + return step + } + + /// Whether a picture that isn't a keyframe may follow `step`; when not, + /// [`lastLoss`] says why. + private mutating func admitStream(_ step: StreamStep, seq: UInt32, epoch: UInt32) -> Bool { + if step.restarted { + lastLoss = .streamRestarted(previousEpoch: step.previousEpoch, epoch: epoch, + previousSeq: step.previousSeq, seq: seq) + return false + } + if step.gap { + lastLoss = .frameSeqGap(previous: step.previousSeq, seq: seq) + return false } - return (false, frameSeqContiguous && seq > previous &+ 1) + return true } // MARK: - Parsing diff --git a/ios/EternalMonitor/Decode/VideoDecoder.swift b/ios/EternalMonitor/Decode/VideoDecoder.swift index 9d81e4a8..48a202b9 100644 --- a/ios/EternalMonitor/Decode/VideoDecoder.swift +++ b/ios/EternalMonitor/Decode/VideoDecoder.swift @@ -84,6 +84,42 @@ final class VideoDecoder { var reportedStall = false } private let hold = OSAllocatedUnfairLock(initialState: nil) + /// When the decoder last put out a picture. + private let lastDecodedUs = OSAllocatedUnfairLock(initialState: nil) + + /// What started a hold, for the one line it logs. + private enum HoldCause { + /// The reference-continuity check turned a picture down. + case continuity(ReferenceContinuity.Loss?) + /// VideoToolbox refused a picture, at submission or in its output. + case decodeError(stage: String, status: OSStatus, keyframe: Bool) + /// A decoder session for new parameter sets didn't open. + case sessionFailed(OSStatus) + /// A new decoder session opened mid-stream, for new parameter sets, + /// and waits for its keyframe. + case sessionReplaced + + var fields: String { + switch self { + case .continuity(.frameSeqGap(let previous, _)?): + return "reason=frame_seq_gap previous_seq=\(previous)" + case .continuity(.streamRestarted(let previousEpoch, let epoch, let previousSeq, _)?): + return "reason=stream_restarted previous_epoch=\(previousEpoch) epoch=\(epoch) previous_seq=\(previousSeq)" + case .continuity(.frameNumJump(let expected, let got)?): + return "reason=frame_num_jump expected_frame_num=\(expected) frame_num=\(got)" + case .continuity(.pocStep(let expected, let got)?): + return "reason=poc_step expected_step=\(expected) step=\(got)" + case .continuity(nil): + return "reason=continuity" + case .decodeError(let stage, let status, let keyframe): + return "reason=decode_error stage=\(stage) status=\(status) keyframe=\(keyframe)" + case .sessionFailed(let status): + return "reason=decoder_session_failed status=\(status)" + case .sessionReplaced: + return "reason=decoder_session_replaced" + } + } + } private var recoveryTimer: DispatchSourceTimer? private let nowUs: () -> UInt64 /// Whether each picture follows a lost reference picture. @@ -174,14 +210,22 @@ final class VideoDecoder { decodeQueue.async { [weak self] in self?.continuity.frameSeqContiguous = contiguous } } - private func beginWaitingForKeyframe() { + /// `seq`: the wire frame number of the picture that started it, when one did. + private func beginWaitingForKeyframe(_ cause: HoldCause, seq: UInt32? = nil) { guard !waitingForSyncSample else { return } waitingForSyncSample = true let now = nowUs() // A keyframe that failed to decode continues the same hold. - hold.withLock { current in - if current == nil { current = Hold(start: now) } else { current?.resumeGeneration = nil } + let started = hold.withLock { current -> Bool in + if current == nil { current = Hold(start: now); return true } + current?.resumeGeneration = nil + return false } + guard started else { return } + // Once per hold: which check started it, whatever the hold then does. + let frame = seq.map { " frame_seq=\($0)" } ?? "" + let since = lastDecodedUs.withLock { $0 }.map { String((now &- $0) / 1000) } ?? "-" + onEvent?("Picture hold started \(cause.fields)\(frame) since_decoded_ms=\(since)") } /// One timer tick, queued behind the frames already given to `decode`. @@ -630,7 +674,7 @@ final class VideoDecoder { // only those) try again instead of matching it and being ignored. // Until then the last picture stays, with keyframe requests. self.formatDescription = nil - beginWaitingForKeyframe() + beginWaitingForKeyframe(.sessionFailed(status)) failedSessionAttempts += 1 if failedSessionAttempts == 1 { reportFailure("Failed to create VideoToolbox session status=\(status); retrying on the next keyframe") @@ -652,7 +696,7 @@ final class VideoDecoder { let hardwareKind = hardwareStatus == noErr ? (hardware ? "hardware" : "software") : "unknown" decompressionSession = session recoveryGeneration &+= 1 - beginWaitingForKeyframe() + beginWaitingForKeyframe(.sessionReplaced) if notReadyDrops > 0 { onEvent?("Dropped \(notReadyDrops) more access units because decoder was not ready") } @@ -697,7 +741,7 @@ final class VideoDecoder { if !admitted(nalUnits, isSyncSample: isSyncSample, seq: seq, epoch: epoch) { // A reference picture before this one never arrived (lost whole, // or dropped unfinished). Hold and ask for a keyframe now. - beginWaitingForKeyframe() + beginWaitingForKeyframe(.continuity(continuity.lastLoss), seq: seq) requestKeyframe(at: nowUs()) return } @@ -823,12 +867,13 @@ final class VideoDecoder { guard status == noErr else { self.decodeQueue.async { self.recoverFromDecodeFailure( - status, generation: generation, source: "Decoder callback", onSyncSample: isSyncSample + status, generation: generation, source: "Decoder callback", onSyncSample: isSyncSample, seq: seq ) } return } guard let pixelBuffer = imageBuffer else { return } + self.lastDecodedUs.withLock { $0 = self.nowUs() } let ended = self.hold.withLock { current -> UInt64? in guard let held = current, held.resumeGeneration == generation else { return nil } current = nil @@ -848,7 +893,7 @@ final class VideoDecoder { if status != noErr || infoFlags.contains(.frameDropped) { finish() } if status != noErr { - recoverFromDecodeFailure(status, generation: generation, source: "VTDecodeFrame", onSyncSample: isSyncSample) + recoverFromDecodeFailure(status, generation: generation, source: "VTDecodeFrame", onSyncSample: isSyncSample, seq: seq) } } @@ -872,9 +917,10 @@ final class VideoDecoder { /// Hold dependent frames until a replacement sync sample, and coalesce errors /// already in flight so one damaged frame cannot cause a request storm. private func recoverFromDecodeFailure(_ status: OSStatus, generation: UInt64, source: String, - onSyncSample: Bool) { + onSyncSample: Bool, seq: UInt32) { guard !isShutdown, generation == recoveryGeneration, !waitingForSyncSample else { return } - beginWaitingForKeyframe() + let stage = source == "VTDecodeFrame" ? "submit" : "output" + beginWaitingForKeyframe(.decodeError(stage: stage, status: status, keyframe: onSyncSample), seq: seq) reportFailure("\(source) status=\(status) codec=\(codec) depth=\(decodeDepth) generation=\(generation) sync=\(onSyncSample); waiting for a keyframe") if status == kVTInvalidSessionErr || onSyncSample { // The session died underneath us (typical after app backgrounding), diff --git a/ios/EternalMonitor/Network/ControlChannel.swift b/ios/EternalMonitor/Network/ControlChannel.swift index a65961e1..43b03e75 100644 --- a/ios/EternalMonitor/Network/ControlChannel.swift +++ b/ios/EternalMonitor/Network/ControlChannel.swift @@ -22,6 +22,9 @@ final class ControlChannel { var onSessionEstablished: ((SessionInfo) -> Void)? var onRejected: ((HelloStatus) -> Void)? + /// The PC answered that it isn't streaming, with its reason token. The + /// same HELLO2 keeps going out slowly and connects once the PC streams. + var onPCNotStreaming: ((HelloStatus, String) -> Void)? /// The last refusal's version mismatch, if any. Read inside `onRejected`. private(set) var lastMismatch: VersionMismatch? /// HELLO2 went unanswered for the whole retry budget. @@ -34,6 +37,8 @@ final class ControlChannel { var onDiagnostic: ((String) -> Void)? /// Snapshot provider for periodic receiver reports. var reportProvider: (() -> ReceiverReport)? + /// Takes the PC's pointer when the PC sends it apart from the video. + var pointer: PointerOverlay? private let queue: DispatchQueue private let send: (Data) -> Void @@ -47,6 +52,8 @@ final class ControlChannel { private var helloBytes = Data() private var helloTemplate: Hello2? private var helloAttempts = 0 + /// The PC said it isn't streaming: keep asking, without a retry budget. + private var waitingForStream = false private var helloTimer: DispatchSourceTimer? private var reportTimer: DispatchSourceTimer? private var pingTimer: DispatchSourceTimer? @@ -71,15 +78,22 @@ final class ControlChannel { private static let maxHelloAttempts = 8 private static let helloRetryMs = 250 + /// How often to ask a PC that answered it isn't streaming. A restart + /// takes a second or so; Start takes someone at the PC. + static let restartingRetryMs = 1000 + static let stoppedRetryMs = 2000 private static let pingIntervalMs = 2000 private static let pingBurstCount = 5 private static let pingBurstSpacingMs = 200 - /// The host grants one keyframe per 500 ms and ignores the rest. A WiFi + /// The host grants one keyframe per 500 ms, besides one for each loss + /// after a recovery (`sendKeyframeRequest`), and ignores the rest. A WiFi /// stall can damage many frames at once; ask once for all of them. static let keyframeRequestSpacingUs: UInt64 = 500_000 private let nowUs: () -> UInt64 private var lastKeyframeRequestUs: UInt64? + /// A keyframe was assembled since the last keyframe request went out. + private var keyframeSinceRequest = false init(queue: DispatchQueue, nowUs: @escaping () -> UInt64 = ControlChannel.clientNowUs, send: @escaping (Data) -> Void) { @@ -123,6 +137,7 @@ final class ControlChannel { hostCaps = 0 msgSeq = 0 helloAttempts = 0 + waitingForStream = false clientNonce = UInt32.random(in: 1...UInt32.max) lastHelloNonce = clientNonce let hello = Hello2( @@ -160,14 +175,15 @@ final class ControlChannel { hello.pairingCode = code helloTemplate = hello helloAttempts = 0 + waitingForStream = false helloBytes = Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .hello2(hello)) startHelloTimer() } } - private func startHelloTimer() { + private func startHelloTimer(everyMs: Int = ControlChannel.helloRetryMs, firstAfterMs: Int = 0) { let timer = DispatchSource.makeTimerSource(queue: queue) - timer.schedule(deadline: .now(), repeating: .milliseconds(Self.helloRetryMs)) + timer.schedule(deadline: .now() + .milliseconds(firstAfterMs), repeating: .milliseconds(everyMs)) timer.setEventHandler { [weak self] in self?.helloTick() } helloTimer?.cancel() helloTimer = timer @@ -180,6 +196,10 @@ final class ControlChannel { helloTimer = nil return } + if waitingForStream { + send(helloBytes) + return + } helloAttempts += 1 if helloAttempts > Self.maxHelloAttempts { helloTimer?.cancel() @@ -225,6 +245,14 @@ final class ControlChannel { // traffic. guard header.sessionId == sessionId, sessionId != 0 else { return } onStreamConfig?(config) + case .pointerShape(let chunk): + guard header.sessionId == sessionId, sessionId != 0, + hostCaps & HelloAck.hostCapPointer != 0 else { return } + pointer?.handle(chunk) + case .pointerState(let state): + guard header.sessionId == sessionId, sessionId != 0, + hostCaps & HelloAck.hostCapPointer != 0 else { return } + pointer?.handle(state, msgSeq: header.msgSeq) default: break } @@ -238,6 +266,23 @@ final class ControlChannel { helloTimer?.cancel() helloTimer = nil + if ack.status.pcNotStreaming { + // The nonce outlives the wait, so a late or reordered copy can + // come after the OK. The session is up: it means nothing now. + guard sessionId == 0 else { + onDiagnostic?("Ignored a late HELLO_ACK (\(ack.status)) for a session already established") + return + } + // Same nonce: the PC answers this HELLO2 with a session once it streams. + if !waitingForStream { onDiagnostic?("PC answered that it isn't streaming (\(ack.status))") } + waitingForStream = true + let everyMs = ack.status == .restarting ? Self.restartingRetryMs : Self.stoppedRetryMs + startHelloTimer(everyMs: everyMs, firstAfterMs: everyMs) + onPCNotStreaming?(ack.status, ack.reason) + return + } + waitingForStream = false + guard ack.status == .ok else { clientNonce = 0 lastMismatch = helloTemplate.flatMap { @@ -250,6 +295,7 @@ final class ControlChannel { sessionId = ack.sessionId hostCaps = ack.hostCaps + pointer?.reset() SupportDiagnostics.notePCName(ack.hostName) onDiagnostic?("Session \(ack.sessionId) established with \(ack.hostName)") lastHeartbeatAtUs.withLock { $0 = Self.clientNowUs() } @@ -292,17 +338,41 @@ final class ControlChannel { timer.resume() } - func sendKeyframeRequest(streamEpoch: UInt32, lastCompleteSeq: UInt32, reason: KeyframeReason) { + /// A keyframe arrived whole: the next frame lost is a new loss. + func noteKeyframeAssembled() { + queue.async { [self] in keyframeSinceRequest = true } + } + + /// `maxMediaDgram` asks the PC for media datagrams no larger, for the + /// rest of the session; only a PC with `HelloAck.hostCapMediaSize` gets it. + func sendKeyframeRequest(streamEpoch: UInt32, lastCompleteSeq: UInt32, reason: KeyframeReason, + maxMediaDgram: UInt16 = 0) { queue.async { [self] in guard sessionId != 0 else { return } let now = nowUs() - if let last = lastKeyframeRequestUs, now &- last < Self.keyframeRequestSpacingUs { return } - lastKeyframeRequestUs = now - onDiagnostic?("Requesting keyframe (\(reason))") + // Startup comes at most every 2 s, while no frame completes. The + // dropped frames' own requests then keep the spacing busy, and + // would always hide it from the PC's log. + let spaced = reason == .startup + || lastKeyframeRequestUs.map { now &- $0 >= Self.keyframeRequestSpacingUs } ?? true + // A frame lost after a keyframe arrived is a new loss, which a PC + // with `hostCapKeyframePerLoss` answers at once when the request + // names a frame from that keyframe on; only GapLoss names one. It + // goes once per keyframe and leaves the spacing as it was. An + // older PC moves its 500 ms window for any grant, so it isn't + // asked early. + let newLoss = reason == .gapLoss && keyframeSinceRequest + && hostCaps & HelloAck.hostCapKeyframePerLoss != 0 + guard spaced || newLoss else { return } + let maxMediaDgram = hostCaps & HelloAck.hostCapMediaSize != 0 ? maxMediaDgram : 0 + if spaced { lastKeyframeRequestUs = now } + keyframeSinceRequest = false + onDiagnostic?("Requesting keyframe (\(reason))" + (maxMediaDgram == 0 ? "" : " with media datagrams up to \(maxMediaDgram) bytes")) sendMessage(.keyframeRequest(KeyframeRequest( streamEpoch: streamEpoch, lastCompleteSeq: lastCompleteSeq, - reason: reason + reason: reason, + maxMediaDgram: maxMediaDgram ))) } } @@ -367,6 +437,7 @@ final class ControlChannel { helloTemplate = nil helloBytes = Data() clientNonce = 0 + waitingForStream = false sessionId = 0 hostCaps = 0 estimator.reset() diff --git a/ios/EternalMonitor/Network/FrameAssembler.swift b/ios/EternalMonitor/Network/FrameAssembler.swift index e75f7e74..a75117f7 100644 --- a/ios/EternalMonitor/Network/FrameAssembler.swift +++ b/ios/EternalMonitor/Network/FrameAssembler.swift @@ -17,6 +17,10 @@ final class FrameAssembler { var onInfo: ((String) -> Void)? var onNack: ((Nack) -> Void)? var onNeedsKeyframe: ((_ epoch: UInt32, _ lastCompleteSeq: UInt32) -> Void)? + /// Video has arrived for `noFrameYetUs` and no frame has completed, and + /// again after each further `noFrameYetUs` while video still arrives, + /// but never while a frame is still arriving in order. `missing` names the fragments the latest incomplete frames lack. + var onNoFrameYet: ((_ epoch: UInt32, _ missing: String) -> Void)? var repairEnabled = false var framePeriodUs: UInt64 = 16_667 var rttUs: () -> UInt64 = { 10_000 } @@ -62,6 +66,10 @@ final class FrameAssembler { /// Without repair, a partial frame is dropped when no fragment of it has /// arrived for this long. static let staleFrameUs: UInt64 = 100_000 + /// The first keyframe is the largest frame of a session. On lossy Wi-Fi, + /// or a path that drops full-size datagrams, it can lose fragments every + /// time while the frames after it wait for it. + static let noFrameYetUs: UInt64 = 2_000_000 /// Cumulative per-connection loss accounting, safe to read from any /// thread (feeds receiver reports and the HUD). @@ -118,6 +126,16 @@ final class FrameAssembler { private var staleEpochStreak: UInt32 = 0 private static let epochResyncThreshold: UInt32 = 512 + /// Until a frame completes: when the first fragment arrived, then when + /// `onNoFrameYet` last ran. The incomplete frames dropped so far, as + /// `onNoFrameYet` words them, and the largest fragment payload, which + /// is smaller than the PC sends when full-size datagrams are lost. + /// `reset()` keeps them: they span the attempt, not one stream epoch. + private var noFrameSince: UInt64? + private var deliveredAny = false + private var droppedBeforeFirstFrame: [String] = [] + private var largestPayload = 0 + struct PendingFrame { let fragmentCount: UInt16 let isKeyframe: Bool @@ -133,11 +151,14 @@ final class FrameAssembler { var deadline: UInt64? var retryAt: UInt64? var requestedAt: UInt64? + /// The latest request for this frame; see `keyframeRetryDue(_:at:)`. + var askedAt: UInt64? var completedAt: UInt64? var stallAllowanceUsed: UInt64 = 0 var stallCountedUntil: UInt64 = 0 /// Indices already requested; each is requested when first seen - /// missing and at most once more by the frame's single retry. + /// missing and at most once more by the frame's single retry. A + /// keyframe retries until it completes; see `keyframeRetryDue(_:at:)`. var requested = Set() var retried = false var keyframeRequested = false @@ -199,6 +220,7 @@ final class FrameAssembler { if repairEnabled { resumeAfterStall(at: now) } lastArrivalUs = now if isRetransmit { lastRepairUs = now } + if !deliveredAny && noFrameSince == nil { noFrameSince = now } let latestRetired = max(latestCompletedSeq, retiredSeq) if latestRetired > 0 { @@ -266,7 +288,7 @@ final class FrameAssembler { // A held frame now delays a newer one: one more repair window // once a complete frame waits. let latest = now + repairWindow() - for held in pending.keys where pending[held]?.holdRetryUs ?? 0 > 0 { + for held in pending.keys where pending[held]?.holdRetryUs ?? 0 > 0 && pending[held]?.isKeyframe == false { let dropBy = min(pending[held]?.dropBy ?? latest, latest) pending[held]?.dropBy = dropBy } @@ -275,6 +297,7 @@ final class FrameAssembler { frame.lastFragmentAt = now frame.byteCount += payload.count frame.highestIndex = max(frame.highestIndex, index) + largestPayload = max(largestPayload, payload.count) while frame.contiguous < count && frame.fragments[frame.contiguous] != nil { frame.contiguous += 1 } @@ -359,7 +382,13 @@ final class FrameAssembler { // Retire in sequence order. Expiring a later partial // first must not make an older repair look stale. if pending.keys.min() == seq { - let blocking = pending.values.contains { $0.isComplete } + // Every frame up to the next keyframe refers to this + // keyframe, so a complete one behind it is no reason + // to drop it: the decoder would hold the picture + // until another keyframe, and on lossy WiFi each + // replacement arrived incomplete too, for seconds. + // It is held, as a frame alone is. + let blocking = !frame.isKeyframe && pending.values.contains { $0.isComplete } if !blocking, let gap = frame.gapAt, now < gap + Self.maxHoldUs { hold(seq, now: now) } else if blocking { @@ -381,10 +410,12 @@ final class FrameAssembler { } } else if frame.gapAt == nil, now >= frame.createdAt + framePeriodUs { beginGap(seq, through: frame.fragmentCount - 1, now: now) - } else if let retry = frame.retryAt, now >= retry, !frame.retried { - pending[seq]?.retried = true - request(seq, through: frame.fragmentCount - 1, again: true, now: now) + } else if !frame.isKeyframe, let retryAt = frame.retryAt, now >= retryAt, !frame.retried { + retry(seq, now: now) } + // A keyframe's requests follow their own schedule, apart from + // its hold: one request per retry, never two in one interval. + if keyframeRetryDue(seq, at: now) { retry(seq, now: now) } } else if now >= frame.lastFragmentAt + Self.staleFrameUs { // Timed from the latest fragment: over a slow USB tunnel a // big keyframe is still arriving long after its first one. @@ -393,6 +424,52 @@ final class FrameAssembler { } if repairEnabled { drainReady(at: now) } updateDepth() + reportNoFrameYet(at: now) + } + + /// See `onNoFrameYet`. Repeats only while fragments still arrive: a PC + /// that went silent is the liveness and connect timeouts' business. + private func reportNoFrameYet(at now: UInt64) { + guard !deliveredAny, let since = noFrameSince, now >= since + Self.noFrameYetUs, + let arrived = lastArrivalUs, arrived >= since else { return } + // A frame that has been arriving in order for a while and still is, + // as a large keyframe does over a slow cable, isn't lost: another + // keyframe would only queue behind it. A frame with a hole, or one + // whose fragments all came at once, doesn't count. + let stillArriving = pending.values.contains { frame in + !frame.isComplete && Int(frame.contiguous) == frame.fragments.count + && now < frame.lastFragmentAt + Self.staleFrameUs && now >= frame.createdAt + Self.staleFrameUs + } + guard !stillArriving else { return } + noFrameSince = now + let waiting = pending.keys.sorted().compactMap { seq in + pending[seq].flatMap { $0.isComplete ? nil : Self.describe(seq, $0, dropped: false) } + } + let frames = (droppedBeforeFirstFrame + waiting).suffix(3) + onNoFrameYet?(currentEpoch ?? 0, (frames.isEmpty ? "no incomplete frame" : frames.joined(separator: "; ")) + + "; largest fragment \(largestPayload) bytes") + } + + /// One incomplete frame for the support log, such as + /// "seq=1 keyframe dropped with 95/97 fragments, missing 3,90". + static func describe(_ seq: UInt32, _ frame: PendingFrame, dropped: Bool) -> String { + let missing = (0.. String { + var runs: [ClosedRange] = [] + for index in indices { + if let last = runs.last, last.upperBound &+ 1 == index { + runs[runs.count - 1] = last.lowerBound...index + } else { + runs.append(index...index) + } + } + let shown = runs.prefix(8).map { $0.count == 1 ? "\($0.lowerBound)" : "\($0.lowerBound)-\($0.upperBound)" } + return shown.joined(separator: ",") + (runs.count > 8 ? " and \(runs.count - 8) more ranges" : "") } /// How long this waiting frame has been stalled, not yet counted, within @@ -437,6 +514,31 @@ final class FrameAssembler { request(seq, through: index, again: false, now: now) } + /// A keyframe has many fragments, and at 10% loss one round of requests + /// and repairs leaves some missing, so a keyframe asks again every RTT + /// plus 5 ms, not once: sooner, its repair can't have come back, and the + /// PC resends a fragment at most every 5 ms. This one schedule covers all + /// its repeated requests; its hold only moves its deadline. After its + /// first retry it asks only once something has arrived since it last + /// asked: while WiFi holds traffic, its request and the repair are still + /// queued, and with nothing arriving it does not ask again. A fragment + /// arriving now may be the missing one, so it asks only on a later tick. + private func keyframeRetryDue(_ seq: UInt32, at now: UInt64) -> Bool { + guard let frame = pending[seq], frame.isKeyframe, !frame.isComplete, let retryAt = frame.retryAt, + now >= retryAt else { return false } + guard frame.retried else { return true } + guard let asked = frame.askedAt, let arrival = lastArrivalUs else { return false } + return arrival > asked && arrival < now + } + + /// Asks again for every missing fragment and schedules the next retry. + private func retry(_ seq: UInt32, now: UInt64) { + guard let frame = pending[seq] else { return } + pending[seq]?.retried = true + pending[seq]?.retryAt = now + min(rttUs(), 1_000_000) + 5_000 + request(seq, through: frame.fragmentCount - 1, again: true, now: now) + } + private func repairWindow() -> UInt64 { max(8_000, min(25_000, 2 * min(rttUs(), 1_000_000) + framePeriodUs * 3 / 2)) } @@ -449,15 +551,18 @@ final class FrameAssembler { /// keyframes in an afternoon. A starved PC can also stop sending for /// several hundred milliseconds with two frames incomplete and then send /// both repairs together. Ask again, less often each time, until a newer - /// frame completes or the frame has waited `maxHoldUs`. + /// frame completes or the frame has waited `maxHoldUs`. A keyframe waits + /// for a newer keyframe instead of a newer frame, and its hold only moves + /// its deadline: its requests follow `keyframeRetryDue(_:at:)`. private func hold(_ seq: UInt32, now: UInt64) { guard let frame = pending[seq] else { return } let wait = frame.holdRetryUs == 0 ? repairWindow() : min(frame.holdRetryUs * 2, Self.maxHoldRetryUs) pending[seq]?.holdRetryUs = wait pending[seq]?.deadline = now + wait - if pending.count > 1 { pending[seq]?.dropBy = min(frame.dropBy ?? now, now) } + if pending.count > 1 && !frame.isKeyframe { pending[seq]?.dropBy = min(frame.dropBy ?? now, now) } pending[seq]?.stallAllowanceUsed = Self.maxStallAllowanceUs + guard !frame.isKeyframe else { return } pending[seq]?.retried = true request(seq, through: frame.fragmentCount - 1, again: true, now: now) } @@ -473,6 +578,7 @@ final class FrameAssembler { guard !missing.isEmpty else { return } pending[seq]?.requested.formUnion(missing) if pending[seq]?.requestedAt == nil { pending[seq]?.requestedAt = now } + pending[seq]?.askedAt = now for start in stride(from: 0, to: missing.count, by: 64) { counters.withLock { $0.nacksSent += 1 } onNack?(Nack(streamEpoch: currentEpoch ?? 0, frameSeq: seq, fragCount: frame.fragmentCount, @@ -516,6 +622,8 @@ final class FrameAssembler { } latestCompletedSeq = seq retiredSeq = max(retiredSeq, seq) + deliveredAny = true + droppedBeforeFirstFrame = [] counters.withLock { $0.framesComplete += 1 $0.fragsRepaired += frame.repaired @@ -526,6 +634,9 @@ final class FrameAssembler { private func dropPending(_ seq: UInt32) { requestKeyframe(seq) if let frame = pending.removeValue(forKey: seq) { + if !deliveredAny && !frame.isComplete { + droppedBeforeFirstFrame = Array(droppedBeforeFirstFrame.suffix(2)) + [Self.describe(seq, frame, dropped: true)] + } pendingBytes -= frame.byteCount counters.withLock { $0.framesDropped += 1 diff --git a/ios/EternalMonitor/Network/MediaLink.swift b/ios/EternalMonitor/Network/MediaLink.swift index 75c7a626..1dc99caf 100644 --- a/ios/EternalMonitor/Network/MediaLink.swift +++ b/ios/EternalMonitor/Network/MediaLink.swift @@ -21,6 +21,10 @@ struct LinkError: Error, CustomStringConvertible { let kind: Kind let message: String var underlying: Error? + /// The path was unsatisfied for want of any usable network when this + /// was reported. Only that is "Wi-Fi is off"; a path that is up but + /// can't reach the PC (no route to it) is not. + var pathUnsatisfied = false init(_ kind: Kind, _ message: String, underlying: Error? = nil) { self.kind = kind diff --git a/ios/EternalMonitor/Network/NetworkScanner.swift b/ios/EternalMonitor/Network/NetworkScanner.swift index c63e6848..99056438 100644 --- a/ios/EternalMonitor/Network/NetworkScanner.swift +++ b/ios/EternalMonitor/Network/NetworkScanner.swift @@ -11,20 +11,23 @@ struct DiscoveredHost: Identifiable, Equatable { let port: UInt16 /// Still resolving. False once resolution gave up without IPv4. var finding = false + /// The `version` this service's own TXT record announced. + var version: String? - init(name: String, address: String?, port: UInt16, finding: Bool = false) { + init(name: String, address: String?, port: UInt16, finding: Bool = false, version: String? = nil) { self.name = name self.address = address self.port = port self.finding = finding + self.version = version } /// The service as far as it has resolved: its first IPv4 address, or a /// PC still being found when only IPv6 (or nothing usable) came back. - static func resolved(name: String, port: Int, addresses: [Data]) -> DiscoveredHost { + static func resolved(name: String, port: Int, addresses: [Data], version: String? = nil) -> DiscoveredHost { let port = port > 0 ? UInt16(clamping: port) : 9876 let ipv4 = addresses.lazy.compactMap { NetworkScanner.ipAddress(from: $0, family: AF_INET) }.first - return DiscoveredHost(name: name, address: ipv4, port: port, finding: ipv4 == nil) + return DiscoveredHost(name: name, address: ipv4, port: port, finding: ipv4 == nil, version: version) } } @@ -75,6 +78,8 @@ final class NetworkScanner: NSObject, ObservableObject { private let browser = NetServiceBrowser() private var resolvingServices: [String: NetService] = [:] + /// Every service name this search heard of, until it leaves. + private var serviceNames = Set() /// Services whose IPv4 address is being looked up by host name, each /// with the lookup whose answer still counts. A removed service, a new /// scan or a newer address drops its entry, so a late answer can't list @@ -86,6 +91,11 @@ final class NetworkScanner: NSObject, ObservableObject { private var scanTask: Task? // track auto-retry state so we don't bounce forever when discovery is broken. private var didAutoRetry = false + /// Search once more when a search finds nothing. Off for a one-off + /// lookup that stops the search itself. + var retriesWhenEmpty = true + /// Keep searching until `stopScan`, for a watch rather than a lookup. + var searchesUntilStopped = false override init() { super.init() @@ -103,6 +113,7 @@ final class NetworkScanner: NSObject, ObservableObject { searchProblem = nil hosts = [] resolvingServices.removeAll() + serviceNames.removeAll() ipv4Lookups.removeAll() status = .looking @@ -113,6 +124,7 @@ final class NetworkScanner: NSObject, ObservableObject { } #endif browser.searchForServices(ofType: "_eternaldisplay._udp.", inDomain: "local.") + guard !searchesUntilStopped else { return } scanTask = Task { @MainActor in // 10s timeout — many routers are slow to forward the first mDNS @@ -124,6 +136,14 @@ final class NetworkScanner: NSObject, ObservableObject { } } + /// Every PC this search heard of: those listed, and those whose address + /// never came or is still coming, without one. A lookup that must not + /// mistake one PC for another counts them all. + var claims: [DiscoveredHost] { + hosts + serviceNames.filter { name in !hosts.contains { $0.name == name } }.sorted() + .map { DiscoveredHost(name: $0, address: nil, port: 0, finding: true) } + } + func stopScan() { browser.stop() resolvingServices.values.forEach { service in @@ -139,7 +159,7 @@ final class NetworkScanner: NSObject, ObservableObject { } if hosts.isEmpty { - if !didAutoRetry { + if !didAutoRetry && retriesWhenEmpty { // auto-retry once after 2s. Bonjour browse occasionally // misses the first multicast burst. didAutoRetry = true @@ -167,6 +187,7 @@ extension NetworkScanner: NetServiceBrowserDelegate, NetServiceDelegate { ) { Task { @MainActor in let key = Self.serviceKey(service) + serviceNames.insert(service.name) guard resolvingServices[key] == nil else { return } service.delegate = self @@ -192,6 +213,7 @@ extension NetworkScanner: NetServiceBrowserDelegate, NetServiceDelegate { resolving.stop() } ipv4Lookups.removeValue(forKey: service.name) + serviceNames.remove(service.name) hosts.removeAll { $0.name == service.name } if !moreComing { @@ -216,11 +238,13 @@ extension NetworkScanner: NetServiceBrowserDelegate, NetServiceDelegate { Task { @MainActor in let key = Self.serviceKey(sender) guard isCurrent(sender), sender.addresses?.isEmpty == false else { return } - let host = DiscoveredHost.resolved(name: sender.name, port: sender.port, addresses: sender.addresses ?? []) + let version = PCVersions.version(fromTXT: sender.txtRecordData()) + let host = DiscoveredHost.resolved(name: sender.name, port: sender.port, addresses: sender.addresses ?? [], + version: version) // discovery diagnostic — surface every resolved host so we // can tell whether mDNS is finding the right machine. print("[mDNS] Found: \(host.name) at \(host.address ?? "no IPv4 yet"):\(host.port)") - if let version = PCVersions.version(fromTXT: sender.txtRecordData()) { + if let version { PCVersions.record(version, name: host.name, address: host.address) } merge(host) @@ -261,7 +285,7 @@ extension NetworkScanner: NetServiceBrowserDelegate, NetServiceDelegate { guard self.ipv4Lookups[service.name] == lookup else { return } self.ipv4Lookups.removeValue(forKey: service.name) if let address { - self.merge(DiscoveredHost(name: service.name, address: address, port: service.port)) + self.merge(DiscoveredHost(name: service.name, address: address, port: service.port, version: service.version)) } else if !self.isScanning, let index = self.hosts.firstIndex(where: { $0.name == service.name && $0.address == nil }) { self.hosts[index].finding = false } diff --git a/ios/EternalMonitor/Network/UDPReceiver.swift b/ios/EternalMonitor/Network/UDPReceiver.swift index 8b8dac30..462de2a1 100644 --- a/ios/EternalMonitor/Network/UDPReceiver.swift +++ b/ios/EternalMonitor/Network/UDPReceiver.swift @@ -112,9 +112,8 @@ final class UDPReceiver: MediaLink { case .waiting(let error): guard !self.reportedWaiting else { return } self.reportedWaiting = true - let denied = connection.currentPath?.unsatisfiedReason == .localNetworkDenied - self.onError?(LinkError(denied ? .localNetworkDenied : .pathUnavailable, - "UDP path not ready: \(error.localizedDescription)", underlying: error)) + let path = connection.currentPath + self.onError?(Self.waitingError(error, status: path?.status, reason: path?.unsatisfiedReason)) case .failed(let error): SupportDiagnostics.shared.record("error", "udp", "Network.framework failed: \(error)") self.onError?(LinkError(.connectionFailed, "UDP connection failed: \(error.localizedDescription)", underlying: error)) @@ -130,6 +129,16 @@ final class UDPReceiver: MediaLink { connection.start(queue: queue) } + /// What a connection waiting for its path reports: Local Network + /// denied, or a path that isn't ready. Only a path iPadOS calls + /// unsatisfied with no usable network is `pathUnsatisfied`. + static func waitingError(_ error: NWError, status: NWPath.Status?, reason: NWPath.UnsatisfiedReason?) -> LinkError { + var link = LinkError(reason == .localNetworkDenied ? .localNetworkDenied : .pathUnavailable, + "UDP path not ready: \(error.localizedDescription)", underlying: error) + link.pathUnsatisfied = status == .unsatisfied && reason == .notAvailable + return link + } + private func startBSD(host: String) { var hints = addrinfo() hints.ai_family = AF_UNSPEC diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index a48396fa..6aa06987 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -29,7 +29,11 @@ enum Wire { case pong = 0x17 case streamConfig = 0x18 case nack = 0x19 + case pointerShape = 0x1C + case pointerState = 0x1D case inputEvent = 0x20 + case identify = 0x1A + case identity = 0x1B case error = 0x7F } @@ -253,8 +257,25 @@ struct Hello2: Equatable { static let featureWantsInput: UInt16 = 1 << 0 static let featureWantsAudio: UInt16 = 1 << 1 static let featureSupportsNack: UInt16 = 1 << 2 + /// This app understands the stopped, restarting and failed statuses. A PC + /// sends them only to an iPad that sets it. + static let featureStandbyStatus: UInt16 = 1 << 3 + /// A HELLO2 that sets `featureStandbyStatus` is padded with zeros to at + /// least this many bytes: the longest HELLO_ACK there is. A PC answers + /// while it isn't streaming only with no more bytes than it was sent. + static let standbyLength = 164 /// This app checks for its own updates, so the PC need not tell the user. 0.3.1 and older never set it. static let featureChecksUpdates: UInt16 = 1 << 4 + /// This app draws the pointer from POINTER_SHAPE and POINTER_STATE, so + /// a PC with `HelloAck.hostCapPointer` leaves it out of the video. + static let featureDrawsPointer: UInt16 = 1 << 5 + /// This iPad may ask for smaller media datagrams in a keyframe request, + /// and wants heartbeats to count the frames the PC sent. Bit 5 is + /// `featureDrawsPointer`. + static let featureMediaSize: UInt16 = 1 << 6 + /// This app understands the goodbyes a PC sends as it goes to sleep or + /// shuts down. Older apps drop a goodbye they can't read. + static let featurePowerBye: UInt16 = 1 << 7 static let maxNameLength = 64 var protoMin: UInt8 = 2 @@ -282,6 +303,15 @@ enum HelloStatus: UInt8 { case error = 3 case unauthorized = 4 case rateLimited = 5 + /// The PC answered but isn't streaming: someone clicked Stop on it, + case stopped = 6 + /// it is restarting its stream, + case restarting = 7 + /// or it stopped after repeated errors (`HelloAck.reason` says why). + case failed = 8 + + /// The PC is there and may stream again: keep asking. + var pcNotStreaming: Bool { self == .stopped || self == .restarting || self == .failed } } struct HelloAck: Equatable { @@ -292,7 +322,19 @@ struct HelloAck: Equatable { static let hostCapPen: UInt16 = 1 << 4 /// Media frame_seq counts the frames sent in a stream epoch without gaps, /// so a forward gap is a frame lost in transit. Older iPads ignore it. + /// Every 0.3.4 and later PC sets it in replies that accept a session; + /// see `OldPC`. static let hostCapContiguousFrameSeq: UInt16 = 1 << 5 + /// The PC sends the pointer apart from the video to an iPad that said + /// it draws it. Older iPads ignore it. + static let hostCapPointer: UInt16 = 1 << 6 + /// The PC takes `KeyframeRequest.maxMediaDgram` for the rest of the + /// session, and its heartbeats carry `framesSent`. Only an iPad that + /// sets `Hello2.featureMediaSize` hears it. + static let hostCapMediaSize: UInt16 = 1 << 7 + /// The PC answers a GapLoss request at once when it names a frame from + /// the PC's last recovery keyframe on (`ControlChannel.sendKeyframeRequest`). + static let hostCapKeyframePerLoss: UInt16 = 1 << 8 var status: HelloStatus var acceptedVersion: UInt8 var clientNonce: UInt32 @@ -304,11 +346,41 @@ struct HelloAck: Equatable { var hostName: String var authToken = Data(repeating: 0, count: 16) var hostCaps: UInt16 = 0 + /// Why the PC isn't streaming, for `.failed`: a token such as + /// "display_changing". Empty otherwise. + var reason = "" + static let maxReasonLength = 32 } struct Heartbeat: Equatable { var hostTimeUs: UInt64 var streamConfig: StreamConfig + /// What to say over the picture, for as long as it lasts. PCs older than + /// the field leave it off: `noticeNone`. Unknown values are ignored. + var notice: UInt8 = Heartbeat.noticeNone + + static let noticeNone: UInt8 = 0 + /// The PC is locked or shows a permission prompt, so it can't capture. + static let noticePCLocked: UInt8 = 1 + /// No display is on at the PC (a closed lid, screens off or unplugged). + static let noticeNoDisplay: UInt8 = 2 + /// Windows won't let EternalMonitor capture the PC's screen from the + /// graphics card it runs on. The PC's window says how to switch. + static let noticeUnsupportedGPU: UInt8 = 3 + /// The same in Second screen mode, after the second screen failed to + /// start: the main screen it fell back to can't be captured either. + static let noticeSecondScreenUnsupportedGPU: UInt8 = 4 + /// The PC is restarting its stream and keeps this session: the picture + /// comes back by itself. Sent only to an iPad that sets + /// `Hello2.featureStandbyStatus`. (3 and 4 are the graphics card notices.) + static let noticeRestarting: UInt8 = 5 + /// Second screen needs Windows 11, so the PC shows its main screen + /// instead. Sent for a few seconds, over a picture that works. + static let noticeSecondScreenNeedsWindows11: UInt8 = 6 + /// The video frames the PC sent this session, from a PC that offered + /// `HelloAck.hostCapMediaSize`; nil from others. Appended after + /// `notice`, which is then always written. + var framesSent: UInt32? } enum ByeReason: UInt8 { @@ -317,6 +389,11 @@ enum ByeReason: UInt8 { case hostShuttingDown = 2 case error = 3 case superseded = 4 + /// The PC is going to sleep. Only to an iPad that sets `featurePowerBye`. + case hostSleeping = 5 + /// The PC is shutting down, restarting or signing out. Only to an iPad + /// that sets `featurePowerBye`. + case hostPoweringOff = 6 } enum KeyframeReason: UInt8 { @@ -330,6 +407,11 @@ struct KeyframeRequest: Equatable { var streamEpoch: UInt32 var lastCompleteSeq: UInt32 var reason: KeyframeReason + /// The largest media datagram to send from now on in this session; 0 + /// asks nothing. Appended: only for a PC with `HelloAck.hostCapMediaSize`. + var maxMediaDgram: UInt16 = 0 + /// The smallest a request may ask for, as the PC's own setting. + static let minMediaDgram: UInt16 = 576 } struct Nack: Equatable { @@ -362,6 +444,47 @@ struct ReceiverReport: Equatable { var nacksSent: UInt32 = 0 var audioPacketsLost: UInt32 = 0 var audioBufferMs: UInt16 = 0 + /// The last pointer image held complete, 0 for none. + var pointerShapeId: UInt32 = 0 +} + +/// One chunk of a pointer image (see the Rust `PointerShapeChunk`). The +/// image is `height` rows of `width` premultiplied BGRA pixels, then, with +/// `invertFlag`, `height` rows of `(width + 7) / 8` mask bytes, most +/// significant bit first: a set bit inverts the picture under that pixel. +struct PointerShapeChunk: Equatable { + static let version: UInt8 = 1 + static let invertFlag: UInt8 = 1 << 0 + static let maxSide: UInt16 = 512 + + var shapeId: UInt32 + var sourceType: UInt8 + var flags: UInt8 + var width: UInt16 + var height: UInt16 + var hotX: UInt16 + var hotY: UInt16 + var totalLen: UInt32 + var offset: UInt32 + var data: Data + + static func imageLength(width: UInt16, height: UInt16, flags: UInt8) -> Int { + let (w, h) = (Int(width), Int(height)) + return w * h * 4 + (flags & invertFlag != 0 ? (w + 7) / 8 * h : 0) + } +} + +/// Where the PC's pointer is: the image's top-left corner in video pixels. +struct PointerState: Equatable { + static let visibleFlag: UInt8 = 1 << 0 + + var flags: UInt8 + var shapeId: UInt32 + var x: Int32 + var y: Int32 + var hostTimeUs: UInt64 + + var visible: Bool { flags & Self.visibleFlag != 0 } } struct WirePing: Equatable { @@ -374,6 +497,42 @@ struct WirePong: Equatable { var t3Us: UInt64 } +/// iPad → PC, session 0: before reconnecting to a PC it found at a new +/// address, the iPad checks that it is the same PC, at that address. The PC +/// answers with a `WireIdentity`. Reserved bytes keep the request as long as +/// the answer. +struct WireIdentify: Equatable { + static let version: UInt8 = 1 + /// What the proof signs after the nonce. + static let context = Data("EternalMonitor identify v1".utf8) + /// Addresses an answer carries at most. + static let maxAddresses = 4 + /// Both bodies: version, nonce, port, count, address slots and proof. + static let bodySize = 1 + 16 + 2 + 1 + 4 * maxAddresses + 32 + var version = WireIdentify.version + var nonce: Data +} + +/// PC → iPad, session 0: the port the PC receives the stream on, its home +/// network IPv4 addresses (4 bytes each), and HMAC-SHA256 keyed with the +/// pairing token over `message`. +struct WireIdentity: Equatable { + var version = WireIdentify.version + var nonce: Data + var port: UInt16 + var addresses: [Data] + var proof: Data + + /// What the proof signs: `nonce ‖ context ‖ port (LE) ‖ count ‖ addresses`. + static func message(nonce: Data, port: UInt16, addresses: [Data]) -> Data { + var message = nonce + WireIdentify.context + message.appendLE(port) + message.append(UInt8(addresses.count)) + for address in addresses { message.append(address) } + return message + } +} + struct WireInputEvent: Equatable { var inputVer: UInt8 = 1 var kind: UInt8 @@ -404,6 +563,10 @@ enum ControlMessage: Equatable { case pong(WirePong) case streamConfig(StreamConfig) case inputEvent(WireInputEvent) + case pointerShape(PointerShapeChunk) + case pointerState(PointerState) + case identify(WireIdentify) + case identity(WireIdentity) var packetType: Wire.PacketType { switch self { @@ -418,6 +581,10 @@ enum ControlMessage: Equatable { case .pong: return .pong case .streamConfig: return .streamConfig case .inputEvent: return .inputEvent + case .pointerShape: return .pointerShape + case .pointerState: return .pointerState + case .identify: return .identify + case .identity: return .identity } } } @@ -449,6 +616,10 @@ extension Wire { body.append(h.authToken) body.appendLE(h.pairingCode) } + let padded = Hello2.standbyLength - ControlHeader.size + if h.featureCaps & Hello2.featureStandbyStatus != 0, body.count < padded { + body.append(Data(count: padded - body.count)) + } case .helloAck(let a): let name = Data(a.hostName.utf8).prefixOnCharBoundary(a.hostName, max: Hello2.maxNameLength) body.append(a.status.rawValue) @@ -462,19 +633,27 @@ extension Wire { body.append(UInt8(name.count)) body.append(name) precondition(a.authToken.count == 16) - if a.hostCaps != 0 || a.authToken.contains(where: { $0 != 0 }) { + let reason = Data(a.reason.utf8).prefixOnCharBoundary(a.reason, max: HelloAck.maxReasonLength) + if a.hostCaps != 0 || a.authToken.contains(where: { $0 != 0 }) || !reason.isEmpty { body.append(a.authToken) body.appendLE(a.hostCaps) } + if !reason.isEmpty { + body.append(UInt8(reason.count)) + body.append(reason) + } case .heartbeat(let hb): body.appendLE(hb.hostTimeUs) hb.streamConfig.encode(into: &body) + if hb.notice != Heartbeat.noticeNone || hb.framesSent != nil { body.append(hb.notice) } + if let frames = hb.framesSent { body.appendLE(frames) } case .bye(let reason): body.append(reason.rawValue) case .keyframeRequest(let k): body.appendLE(k.streamEpoch) body.appendLE(k.lastCompleteSeq) body.append(k.reason.rawValue) + if k.maxMediaDgram != 0 { body.appendLE(k.maxMediaDgram) } case .nack(let n): precondition(n.isValid) body.appendLE(n.streamEpoch) @@ -496,12 +675,15 @@ extension Wire { body.appendLE(r.e2eLatencyMsX10) body.appendLE(r.rttMsX10) if r.fragsRepaired != 0 || r.nacksSent != 0 - || r.audioPacketsLost != 0 || r.audioBufferMs != 0 { + || r.audioPacketsLost != 0 || r.audioBufferMs != 0 || r.pointerShapeId != 0 { body.appendLE(r.fragsRepaired) body.appendLE(r.nacksSent) body.appendLE(r.audioPacketsLost) body.appendLE(r.audioBufferMs) } + if r.pointerShapeId != 0 { + body.appendLE(r.pointerShapeId) + } case .ping(let p): body.appendLE(p.t1Us) case .pong(let p): @@ -510,6 +692,22 @@ extension Wire { body.appendLE(p.t3Us) case .streamConfig(let cfg): cfg.encode(into: &body) + case .identify(let i): + precondition(i.nonce.count == 16) + body.append(i.version) + body.append(i.nonce) + body.append(Data(repeating: 0, count: WireIdentify.bodySize - 17)) + case .identity(let i): + precondition(i.nonce.count == 16 && i.proof.count == 32 && i.addresses.count <= WireIdentify.maxAddresses + && i.addresses.allSatisfy { $0.count == 4 }) + body.append(i.version) + body.append(i.nonce) + body.appendLE(i.port) + body.append(UInt8(i.addresses.count)) + for slot in 0..= 4 ? r.readU32() : nil)) case .bye: guard let raw = r.readU8(), let reason = ByeReason(rawValue: raw) else { return nil } message = .bye(reason) @@ -582,8 +804,8 @@ extension Wire { let last: UInt32 = r.readU32(), let raw = r.readU8(), let reason = KeyframeReason(rawValue: raw) else { return nil } - message = .keyframeRequest( - KeyframeRequest(streamEpoch: epoch, lastCompleteSeq: last, reason: reason)) + message = .keyframeRequest(KeyframeRequest(streamEpoch: epoch, lastCompleteSeq: last, reason: reason, + maxMediaDgram: r.remaining >= 2 ? r.readU16() ?? 0 : 0)) case .receiverReport: message = parseReceiverReport(&r) case .nack: @@ -609,6 +831,26 @@ extension Wire { message = .streamConfig(cfg) case .inputEvent: message = parseInputEvent(&r) + case .pointerShape: + message = parsePointerShape(&r) + case .pointerState: + guard r.readU8() == PointerShapeChunk.version, + let flags = r.readU8(), let _ = r.readU16(), + let shapeId = r.readU32(), let x = r.readU32(), let y = r.readU32(), + let time = r.readU64() else { return nil } + message = .pointerState(PointerState( + flags: flags, shapeId: shapeId, x: Int32(bitPattern: x), y: Int32(bitPattern: y), + hostTimeUs: time)) + case .identify: + guard let version = r.readU8(), let nonce = r.take(16), r.take(WireIdentify.bodySize - 17) != nil else { return nil } + message = .identify(WireIdentify(version: version, nonce: Data(nonce))) + case .identity: + guard let version = r.readU8(), let nonce = r.take(16), let port = r.readU16(), let count = r.readU8(), + Int(count) <= WireIdentify.maxAddresses, let slots = r.take(4 * WireIdentify.maxAddresses), + let proof = r.take(32) else { return nil } + let addresses = (0.. 0 { + guard let length = r.readU8(), Int(length) <= HelloAck.maxReasonLength, let bytes = r.take(Int(length)), + let reason = String(bytes: bytes, encoding: .utf8) else { return nil } + ack.reason = reason + } return .helloAck(ack) } @@ -699,9 +946,34 @@ extension Wire { report.audioPacketsLost = audioLost report.audioBufferMs = audioBuffer } + if r.remaining >= 4, let shape = r.readU32() { + report.pointerShapeId = shape + } return .receiverReport(report) } + /// Enforces what the Rust parser does: a nonzero id, a size within the + /// cap, the length that size implies, and a chunk inside it. + private static func parsePointerShape(_ r: inout WireReader) -> ControlMessage? { + guard r.readU8() == PointerShapeChunk.version, + let sourceType = r.readU8(), let flags = r.readU8(), let _ = r.readU8(), + let shapeId = r.readU32(), shapeId != 0, + let width = r.readU16(), let height = r.readU16(), + (1...PointerShapeChunk.maxSide).contains(width), + (1...PointerShapeChunk.maxSide).contains(height), + let hotX = r.readU16(), let hotY = r.readU16(), + let totalLen = r.readU32(), + Int(totalLen) == PointerShapeChunk.imageLength(width: width, height: height, flags: flags), + let offset = r.readU32(), + let dataLen = r.readU16(), dataLen > 0, + Int(offset) + Int(dataLen) <= Int(totalLen), + let data = r.take(Int(dataLen)) + else { return nil } + return .pointerShape(PointerShapeChunk( + shapeId: shapeId, sourceType: sourceType, flags: flags, width: width, height: height, + hotX: hotX, hotY: hotY, totalLen: totalLen, offset: offset, data: Data(data))) + } + private static func parseInputEvent(_ r: inout WireReader) -> ControlMessage? { guard let inputVer = r.readU8(), let kind = r.readU8(), diff --git a/ios/EternalMonitor/Render/MetalView.swift b/ios/EternalMonitor/Render/MetalView.swift index b20cce4a..674f8c53 100644 --- a/ios/EternalMonitor/Render/MetalView.swift +++ b/ios/EternalMonitor/Render/MetalView.swift @@ -31,6 +31,7 @@ struct MetalView: UIViewRepresentable { renderer.pictureFit = settings.pictureFit context.coordinator.renderer = renderer context.coordinator.frameSlot = connectionManager.frameSlot + context.coordinator.pointer = connectionManager.pointerOverlay context.coordinator.attach(view: view) view.delegate = context.coordinator @@ -59,6 +60,8 @@ struct MetalView: UIViewRepresentable { var renderer: MetalRenderer? /// Direct reference to the shared FrameSlot — avoids crossing @MainActor boundary on the render thread. var frameSlot: FrameSlot? + /// The PC's pointer when this app draws it; each move redraws. + var pointer: PointerOverlay? private weak var view: MTKView? private var memoryWarning: NSObjectProtocol? @@ -71,6 +74,9 @@ struct MetalView: UIViewRepresentable { frameSlot?.onFrameStored = { [weak self] in self?.scheduleRedraw() } + pointer?.onChange = { [weak self] in + self?.scheduleRedraw() + } // The next decoded frame redraws within a frame period, or the // PC's idle keepalive; the screen keeps showing the last one. memoryWarning = NotificationCenter.default.addObserver( @@ -83,6 +89,7 @@ struct MetalView: UIViewRepresentable { func detach() { frameSlot?.onFrameStored = nil + pointer?.onChange = nil view = nil if let memoryWarning { NotificationCenter.default.removeObserver(memoryWarning) } memoryWarning = nil @@ -111,7 +118,7 @@ struct MetalView: UIViewRepresentable { func draw(in view: MTKView) { guard let renderer else { return } let pixelBuffer = frameSlot?.take() - renderer.draw(in: view, pixelBuffer: pixelBuffer) + renderer.draw(in: view, pixelBuffer: pixelBuffer, pointer: pointer) } } } @@ -152,6 +159,17 @@ private final class InFlightFrame { deinit { frame = nil } } +private struct PointerUniforms { + var rect: SIMD4 +} + +/// The current pointer image's textures, made once per image. +private struct PointerTextures { + let id: UInt32 + let image: MTLTexture + let invertMask: MTLTexture? +} + private struct DisplayUniforms { var scale: SIMD2 var matrixIndex: UInt32 @@ -163,6 +181,9 @@ final class MetalRenderer { private let device: MTLDevice private let commandQueue: MTLCommandQueue private let pipelineState: MTLRenderPipelineState + private let pointerPipeline: MTLRenderPipelineState + private let pointerInvertPipeline: MTLRenderPipelineState + private var pointerTextures: PointerTextures? private(set) var textureCache: CVMetalTextureCache? private var currentFrame: FrameTextures? /// Read on the main thread, where the paused MTKView draws. @@ -196,6 +217,10 @@ final class MetalRenderer { do { pipelineState = try device.makeRenderPipelineState(descriptor: pipelineDescriptor) + pointerPipeline = try device.makeRenderPipelineState( + descriptor: Self.pointerDescriptor(library, fragment: "pointer_fragment", view: view, invert: false)) + pointerInvertPipeline = try device.makeRenderPipelineState( + descriptor: Self.pointerDescriptor(library, fragment: "pointer_invert_fragment", view: view, invert: true)) } catch { fatalError("Failed to create pipeline state: \(error)") } @@ -209,7 +234,27 @@ final class MetalRenderer { // The held frame's textures go before the cache that made them. deinit { currentFrame = nil } - func draw(in view: MTKView, pixelBuffer: CVPixelBuffer?) { + /// Over: source + destination x (1 - source alpha). Invert: source + /// x (1 - destination) + destination x (1 - source), so white inverts + /// and black leaves the picture alone. + private static func pointerDescriptor(_ library: MTLLibrary, fragment: String, view: MTKView, + invert: Bool) -> MTLRenderPipelineDescriptor { + let descriptor = MTLRenderPipelineDescriptor() + descriptor.vertexFunction = library.makeFunction(name: "pointer_vertex") + descriptor.fragmentFunction = library.makeFunction(name: fragment) + let color = descriptor.colorAttachments[0]! + color.pixelFormat = view.colorPixelFormat + color.isBlendingEnabled = true + color.rgbBlendOperation = .add + color.alphaBlendOperation = .add + color.sourceRGBBlendFactor = invert ? .oneMinusDestinationColor : .one + color.destinationRGBBlendFactor = invert ? .oneMinusSourceColor : .oneMinusSourceAlpha + color.sourceAlphaBlendFactor = .zero + color.destinationAlphaBlendFactor = .one + return descriptor + } + + func draw(in view: MTKView, pixelBuffer: CVPixelBuffer?, pointer: PointerOverlay? = nil) { if let pixelBuffer, let frame = makeFrameTextures(from: pixelBuffer) { currentFrame = frame // Let the cache recycle textures released by completed draws. @@ -243,6 +288,10 @@ final class MetalRenderer { encoder.setFragmentTexture(frame.luma, index: 0) encoder.setFragmentTexture(frame.chroma, index: 1) encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4) + if let pointer { + drawPointer(pointer, encoder: encoder, video: CGSize(width: frame.width, height: frame.height), + drawable: view.drawableSize) + } encoder.endEncoding() // Keep the pixel buffer + texture wrappers alive until the GPU is done @@ -262,6 +311,48 @@ final class MetalRenderer { commandBuffer.commit() } + private func drawPointer(_ pointer: PointerOverlay, encoder: MTLRenderCommandEncoder, video: CGSize, + drawable: CGSize) { + let snapshot = pointer.snapshot() + guard snapshot.visible, let image = snapshot.image, let textures = textures(for: image), + let clip = PointerGeometry.clip(video: video, view: drawable, fit: pictureFit) + else { return } + let rect = PointerGeometry.rect(x: snapshot.x, y: snapshot.y, width: image.width, height: image.height, + video: video, view: drawable, fit: pictureFit) + var uniforms = PointerUniforms(rect: PointerGeometry.ndc(rect, view: drawable)) + // Both passes stay on the picture, out of the letterbox bars. + encoder.setScissorRect(MTLScissorRect(x: clip.x, y: clip.y, width: clip.width, height: clip.height)) + encoder.setRenderPipelineState(pointerPipeline) + encoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 0) + encoder.setFragmentTexture(textures.image, index: 0) + encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4) + if let mask = textures.invertMask { + encoder.setRenderPipelineState(pointerInvertPipeline) + encoder.setFragmentTexture(mask, index: 0) + encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4) + } + pointer.noteDrawn(x: snapshot.x, y: snapshot.y) + } + + private func textures(for image: PointerOverlay.Image) -> PointerTextures? { + if let cached = pointerTextures, cached.id == image.id { return cached } + func texture(_ format: MTLPixelFormat, bytes: Data, bytesPerPixel: Int) -> MTLTexture? { + let descriptor = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: format, width: image.width, height: image.height, mipmapped: false) + descriptor.usage = .shaderRead + guard let texture = device.makeTexture(descriptor: descriptor) else { return nil } + bytes.withUnsafeBytes { raw in + texture.replace(region: MTLRegionMake2D(0, 0, image.width, image.height), mipmapLevel: 0, + withBytes: raw.baseAddress!, bytesPerRow: image.width * bytesPerPixel) + } + return texture + } + guard let color = texture(.bgra8Unorm, bytes: image.pixels, bytesPerPixel: 4) else { return nil } + let mask = image.invertMask.flatMap { texture(.r8Unorm, bytes: $0, bytesPerPixel: 1) } + pointerTextures = PointerTextures(id: image.id, image: color, invertMask: mask) + return pointerTextures + } + /// Whether a decoded frame is held for redraws (resize, rotation). var holdsFrame: Bool { currentFrame != nil } diff --git a/ios/EternalMonitor/Render/PointerOverlay.swift b/ios/EternalMonitor/Render/PointerOverlay.swift new file mode 100644 index 00000000..2ddc860b --- /dev/null +++ b/ios/EternalMonitor/Render/PointerOverlay.swift @@ -0,0 +1,201 @@ +import CoreGraphics +import Foundation +import os + +/// The PC's pointer, which this app draws over the video when the PC sends +/// it apart (this app sets `Hello2.featureDrawsPointer`, the PC answers +/// with `HelloAck.hostCapPointer`). Pointer motion then never waits for a +/// video frame. Messages arrive on the control queue; the renderer reads a +/// snapshot on the main thread. +final class PointerOverlay { + /// A complete pointer image: premultiplied BGRA rows, and one byte per + /// pixel (0 or 255) where the picture under it is inverted. + struct Image: Equatable { + let id: UInt32 + let width: Int + let height: Int + let pixels: Data + let invertMask: Data? + } + + struct Snapshot: Equatable { + var image: Image? + var x: Int32 = 0 + var y: Int32 = 0 + var visible = false + } + + struct Counters: Equatable { + var states = 0 + var staleStates = 0 + var shapes = 0 + /// Draws with the pointer, and how many of them moved it. + var drawn = 0 + var drawnMoves = 0 + var lastDrawn: (x: Int32, y: Int32)? + + static func == (a: Counters, b: Counters) -> Bool { + (a.states, a.staleStates, a.shapes, a.drawn, a.drawnMoves) + == (b.states, b.staleStates, b.shapes, b.drawn, b.drawnMoves) + } + } + + /// A position applied or an image completed: redraw. Called on the + /// control queue. + var onChange: (() -> Void)? + + private struct Assembly { + let chunk: PointerShapeChunk + var bytes: [UInt8] + var offsets: Set = [] + var received = 0 + } + + private struct State { + var assembly: Assembly? + var snapshot = Snapshot() + var lastSeq: UInt32? + var counters = Counters() + } + + private let state = OSAllocatedUnfairLock(initialState: State()) + + /// A new session: nothing from the last one applies. + func reset() { + state.withLock { $0 = State() } + onChange?() + } + + /// The image the receiver reports name, so the PC sends it again when + /// a chunk was lost. + var completeShapeId: UInt32 { state.withLock { $0.snapshot.image?.id ?? 0 } } + + var counters: Counters { state.withLock { $0.counters } } + + func snapshot() -> Snapshot { state.withLock { $0.snapshot } } + + /// The renderer drew the pointer here. + func noteDrawn(x: Int32, y: Int32) { + state.withLock { s in + s.counters.drawn += 1 + if let last = s.counters.lastDrawn, last != (x, y) { s.counters.drawnMoves += 1 } + s.counters.lastDrawn = (x, y) + } + } + + /// Latest wins by the control header's `msgSeq`, which counts every + /// control message the PC sends. + func handle(_ pointer: PointerState, msgSeq: UInt32) { + let applied = state.withLock { s -> Bool in + if let last = s.lastSeq, Int32(bitPattern: msgSeq &- last) <= 0 { + s.counters.staleStates += 1 + return false + } + s.lastSeq = msgSeq + s.counters.states += 1 + s.snapshot.x = pointer.x + s.snapshot.y = pointer.y + s.snapshot.visible = pointer.visible + return true + } + if applied { onChange?() } + } + + func handle(_ chunk: PointerShapeChunk) { + let completed = state.withLock { s -> Bool in + // A chunk of the held image or an older one changes nothing: a + // late, reordered chunk must not bring back an old image. + if let held = s.snapshot.image?.id, !Self.isNewer(chunk.shapeId, than: held) { return false } + if s.assembly?.chunk.shapeId != chunk.shapeId { + if let pending = s.assembly, !Self.isNewer(chunk.shapeId, than: pending.chunk.shapeId) { + return false + } + s.assembly = Assembly(chunk: chunk, bytes: [UInt8](repeating: 0, count: Int(chunk.totalLen))) + } + guard var assembly = s.assembly, + assembly.chunk.width == chunk.width, assembly.chunk.height == chunk.height, + assembly.chunk.flags == chunk.flags, + !assembly.offsets.contains(chunk.offset) + else { return false } + let start = Int(chunk.offset) + assembly.bytes.replaceSubrange(start..= assembly.bytes.count else { + s.assembly = assembly + return false + } + s.assembly = nil + s.snapshot.image = Self.image(assembly.chunk, bytes: assembly.bytes) + s.counters.shapes += 1 + return true + } + if completed { onChange?() } + } + + /// Image ids count up from 1 and wrap: `id` is newer when it is less + /// than half the counter ahead. + static func isNewer(_ id: UInt32, than other: UInt32) -> Bool { + Int32(bitPattern: id &- other) > 0 + } + + static func image(_ chunk: PointerShapeChunk, bytes: [UInt8]) -> Image { + let (w, h) = (Int(chunk.width), Int(chunk.height)) + let pixels = Data(bytes[0..> (x % 8)) != 0 { + expanded[y * w + x] = 255 + } + } + mask = expanded + } + return Image(id: chunk.shapeId, width: w, height: h, pixels: pixels, invertMask: mask) + } +} + +/// Where the pointer is drawn, from the same content rect the touch relay +/// normalizes against (`ContentRectMapper`), so a tap on the drawn hotspot +/// lands where the PC has the pointer. Pure and unit-tested. +enum PointerGeometry { + /// The pointer image's rect in view coordinates. `x`/`y` and the image + /// size are video pixels; the image scales with the picture. + static func rect(x: Int32, y: Int32, width: Int, height: Int, video: CGSize, view: CGSize, + fit: PictureFit) -> CGRect { + let content = ContentRectMapper(viewSize: view, videoSize: video, fit: fit).contentRect + guard content.width > 0, content.height > 0 else { return .zero } + let sx = content.width / video.width + let sy = content.height / video.height + return CGRect(x: content.minX + CGFloat(x) * sx, y: content.minY + CGFloat(y) * sy, + width: CGFloat(width) * sx, height: CGFloat(height) * sy) + } + + /// The pixels the pointer may cover: the picture, inside the drawable. + /// A pointer partly past the picture's edge must not draw into Fit's + /// letterbox bars. `nil` while there is no picture. + static func clip(video: CGSize, view: CGSize, fit: PictureFit) -> (x: Int, y: Int, width: Int, height: Int)? { + let content = ContentRectMapper(viewSize: view, videoSize: video, fit: fit).contentRect + let visible = content.intersection(CGRect(origin: .zero, size: view)) + guard !visible.isNull, visible.width > 0, visible.height > 0 else { return nil } + let x0 = max(0, Int(visible.minX.rounded(.down))) + let y0 = max(0, Int(visible.minY.rounded(.down))) + let x1 = min(Int(view.width), Int(visible.maxX.rounded(.up))) + let y1 = min(Int(view.height), Int(visible.maxY.rounded(.up))) + guard x1 > x0, y1 > y0 else { return nil } + return (x0, y0, x1 - x0, y1 - y0) + } + + /// `rect` in Metal's normalized device coordinates: (minX, minY, maxX, + /// maxY) with y up, as the renderer's quad takes it. + static func ndc(_ rect: CGRect, view: CGSize) -> SIMD4 { + guard view.width > 0, view.height > 0 else { return .zero } + let x0 = rect.minX / view.width * 2 - 1 + let x1 = rect.maxX / view.width * 2 - 1 + let y0 = 1 - rect.maxY / view.height * 2 + let y1 = 1 - rect.minY / view.height * 2 + return SIMD4(Float(x0), Float(y0), Float(x1), Float(y1)) + } +} diff --git a/ios/EternalMonitor/Render/Shaders.metal b/ios/EternalMonitor/Render/Shaders.metal index 35d6b7ee..b035a183 100644 --- a/ios/EternalMonitor/Render/Shaders.metal +++ b/ios/EternalMonitor/Render/Shaders.metal @@ -72,3 +72,43 @@ fragment half4 display_fragment(VertexOut in [[stage_in]], return half4(half3(saturate(rgb)), 1.0h); } + +// The PC's pointer, drawn over the video (PointerOverlay.swift). `rect` is +// the image's quad in normalized device coordinates: minX, minY, maxX, maxY. +struct PointerUniforms { + float4 rect; +}; + +vertex VertexOut pointer_vertex(uint vid [[vertex_id]], + constant PointerUniforms &uniforms [[buffer(0)]]) { + // Texture coordinates of the strip's corners: bottom-left, bottom-right, + // top-left, top-right (texture rows run downward). + const float2 corners[4] = { + float2(0.0, 1.0), + float2(1.0, 1.0), + float2(0.0, 0.0), + float2(1.0, 0.0), + }; + float2 t = corners[vid]; + VertexOut out; + out.position = float4(mix(uniforms.rect.x, uniforms.rect.z, t.x), + mix(uniforms.rect.w, uniforms.rect.y, t.y), 0.0, 1.0); + out.texCoord = t; + return out; +} + +// Premultiplied BGRA, blended "over" the picture. +fragment half4 pointer_fragment(VertexOut in [[stage_in]], + texture2d image [[texture(0)]]) { + constexpr sampler s(filter::linear, address::clamp_to_edge); + return half4(image.sample(s, in.texCoord)); +} + +// The invert mask as white: the pipeline's blend (1 - destination) * source +// + destination * (1 - source) turns it into "invert the picture here". +fragment half4 pointer_invert_fragment(VertexOut in [[stage_in]], + texture2d mask [[texture(0)]]) { + constexpr sampler s(filter::linear, address::clamp_to_edge); + float m = mask.sample(s, in.texCoord).r; + return half4(m, m, m, m); +} diff --git a/ios/EternalMonitor/Resources/Localizable.xcstrings b/ios/EternalMonitor/Resources/Localizable.xcstrings index a4d4fa60..a290b0ba 100644 --- a/ios/EternalMonitor/Resources/Localizable.xcstrings +++ b/ios/EternalMonitor/Resources/Localizable.xcstrings @@ -6835,6 +6835,18 @@ } } }, + "Second screen needs Windows 11. Showing your PC's screen instead." : { + "comment" : "Over the mirrored picture for a few seconds, when the PC chose Second screen but runs Windows 10, which Second screen doesn't support", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Second screen needs Windows 11. Showing your PC's screen instead." + } + } + } + }, "Second screen or mirror" : { "extractionState" : "manual", "localizations" : { @@ -8188,6 +8200,18 @@ } } }, + "The second screen didn't start, and your PC can't share its main screen from its current graphics card. Check the PC." : { + "comment" : "On the connecting card and over the held picture, in Second screen mode after the second screen failed to start: Windows won't let the PC app capture its main screen from its graphics card either, so it can't mirror it.", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "The second screen didn't start, and your PC can't share its main screen from its current graphics card. Check the PC." + } + } + } + }, "Try again in %lld s" : { "comment" : "Button countdown after wrong codes; %lld is seconds", "extractionState" : "manual", @@ -8544,6 +8568,78 @@ } } }, + "Update EternalMonitor on %@ for faster, steadier streaming. Download it on the PC from eternalmonitor.dev." : { + "comment" : "Connect screen card under \"Your PC app is out of date\". The value is the PC's name. Keep eternalmonitor.dev as is", + "extractionState" : "manual", + "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "حدّث EternalMonitor على %@ لبث أسرع وأكثر استقرارًا. نزّله على PC من eternalmonitor.dev." + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "Aktualisiere EternalMonitor auf %@ für schnelleres, stabileres Streaming. Lade es auf dem PC von eternalmonitor.dev herunter." + } + }, + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Update EternalMonitor on %@ for faster, steadier streaming. Download it on the PC from eternalmonitor.dev." + } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "Actualiza EternalMonitor en %@ para una transmisión más rápida y estable. Descárgalo en el PC desde eternalmonitor.dev." + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "Mettez à jour EternalMonitor sur %@ pour une diffusion plus rapide et stable. Téléchargez-le sur le PC depuis eternalmonitor.dev." + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "तेज़ और स्थिर स्ट्रीमिंग के लिए %@ पर EternalMonitor अपडेट करें। इसे PC पर eternalmonitor.dev से डाउनलोड करें।" + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "より速く安定したストリーミングのために、%@ の EternalMonitor をアップデートしてください。PC で eternalmonitor.dev からダウンロードできます。" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "더 빠르고 안정적인 스트리밍을 위해 %@에서 EternalMonitor를 업데이트하세요. PC에서 eternalmonitor.dev를 통해 다운로드하세요." + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "Atualize o EternalMonitor em %@ para uma transmissão mais rápida e estável. Baixe-o no PC em eternalmonitor.dev." + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "Обновите EternalMonitor на %@ для более быстрой и стабильной трансляции. Скачайте его на PC с eternalmonitor.dev." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "更新 %@ 上的 EternalMonitor，让串流更快更稳定。在 PC 上从 eternalmonitor.dev 下载。" + } + } + } + }, "Updates" : { "extractionState" : "manual", "localizations" : { @@ -9043,6 +9139,246 @@ } } }, + "Your PC app is out of date" : { + "comment" : "Title of the Connect screen card shown when the PC runs an EternalMonitor older than 0.3.4", + "extractionState" : "manual", + "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "تطبيق PC لديك قديم" + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "Deine PC-App ist veraltet" + } + }, + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Your PC app is out of date" + } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "La app de tu PC está desactualizada" + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "L’application sur votre PC n’est pas à jour" + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "आपके PC का ऐप पुराना है" + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC アプリが古くなっています" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC 앱이 오래되었습니다" + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "O app do seu PC está desatualizado" + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "Приложение на PC устарело" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC 应用已过期" + } + } + } + }, + "Your PC can't share its screen from its current graphics card. Check the PC." : { + "comment" : "On the connecting card and over the held picture: Windows won't let the PC app capture its screen from the graphics card it runs on (laptops with two graphics cards). The PC app's window says how to switch.", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Your PC can't share its screen from its current graphics card. Check the PC." + } + } + } + }, + "Your PC has no active display. Turn on a screen to continue." : { + "comment" : "Over the held picture while no display is on at the PC: a closed laptop lid, or screens that are off or unplugged", + "extractionState" : "manual", + "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "لا توجد شاشة نشطة على PC. شغّل شاشة للمتابعة." + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "Dein PC hat keinen aktiven Bildschirm. Schalte einen Bildschirm ein, um fortzufahren." + } + }, + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Your PC has no active display. Turn on a screen to continue." + } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "Tu PC no tiene una pantalla activa. Enciende una pantalla para continuar." + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "Votre PC n’a pas d’écran actif. Allumez un écran pour continuer." + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "आपके PC पर कोई स्क्रीन चालू नहीं है। जारी रखने के लिए एक स्क्रीन चालू करें।" + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC に有効な画面がありません。続行するには画面をオンにしてください。" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC에 활성 화면이 없습니다. 계속하려면 화면을 켜세요." + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "Seu PC não tem uma tela ativa. Ligue uma tela para continuar." + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "На PC нет активного экрана. Включите экран, чтобы продолжить." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC 没有活动的显示器。打开显示器后即可继续。" + } + } + } + }, + "Your PC is locked. Unlock it to continue." : { + "comment" : "Over the held picture while the PC shows its lock screen or a permission prompt", + "extractionState" : "manual", + "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC مقفل. افتح قفله للمتابعة." + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "Dein PC ist gesperrt. Entsperre ihn, um fortzufahren." + } + }, + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Your PC is locked. Unlock it to continue." + } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "Tu PC está bloqueado. Desbloquéalo para continuar." + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "Votre PC est verrouillé. Déverrouillez-le pour continuer." + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "आपका PC लॉक है। जारी रखने के लिए उसे अनलॉक करें।" + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC がロックされています。続行するにはロックを解除してください。" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC가 잠겨 있습니다. 계속하려면 잠금을 해제하세요." + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "Seu PC está bloqueado. Desbloqueie-o para continuar." + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC заблокирован. Разблокируйте его, чтобы продолжить." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC 已锁定。解锁后即可继续。" + } + } + } + }, + "Your PC is restarting the stream…" : { + "comment" : "Over the held picture while the PC restarts its stream and keeps the session", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Your PC is restarting the stream…" + } + } + } + }, "⌘ key acts as" : { "comment" : "Values are Ctrl or Win", "extractionState" : "manual", diff --git a/ios/EternalMonitorTests/ConnectionProblemTests.swift b/ios/EternalMonitorTests/ConnectionProblemTests.swift index 7450d989..865f84bd 100644 --- a/ios/EternalMonitorTests/ConnectionProblemTests.swift +++ b/ios/EternalMonitorTests/ConnectionProblemTests.swift @@ -14,6 +14,34 @@ final class ConnectionProblemTests: XCTestCase { XCTAssertEqual(ConnectionProblem.timeout(listenerReady: true, helloAttempts: 1, handshakeDone: true, datagramsReceived: 0, isUSB: true), .noPicture) } + /// The PC's not-streaming answers, in plain words. None offers Try + /// again: the iPad is already asking again by itself. + func testNotStreamingAnswersSayWhatToDo() { + typealias Wait = ConnectionProblem.StreamWait + XCTAssertEqual(Wait(.stopped, reason: ""), .stopped) + XCTAssertEqual(Wait(.restarting, reason: "ignored"), .restarting) + XCTAssertEqual(Wait(.failed, reason: "encoder"), .failed("encoder")) + for status: HelloStatus in [.ok, .busy, .versionUnsupported, .error, .unauthorized, .rateLimited] { + XCTAssertNil(Wait(status, reason: ""), "\(status)") + } + XCTAssertEqual(ConnectionProblem.pcNotStreaming(.restarting).detail, "This iPad connects as soon as it's back.") + XCTAssertEqual(ConnectionProblem.pcNotStreaming(.failed("encoder")).detail, + "Encoding the video kept failing. Click Start streaming in EternalMonitor on the PC to try again. " + + "This iPad connects as soon as it starts.") + // Every token the PC sends has words here. + for token in ["capture", "encoder", "transport", "secure_desktop", "display_changing", "no_display", + "unsupported_gpu", "too_many_capture_apps", "session_disconnected", "driver_not_responding"] { + XCTAssertNotNil(Wait.failedReason(token), token) + } + for wait: Wait in [.stopped, .restarting, .failed("encoder")] { + XCTAssertEqual(ConnectionProblem.pcNotStreaming(wait).actions, []) + } + // Only a PC that ran into errors offers Report this problem. + XCTAssertFalse(ConnectionProblem.pcNotStreaming(.stopped).canReport) + XCTAssertFalse(ConnectionProblem.pcNotStreaming(.restarting).canReport) + XCTAssertTrue(ConnectionProblem.pcNotStreaming(.failed("")).canReport) + } + /// PCs on 0.3.1 and 0.3.2 answer but send no picture in Second screen /// mode. The card says to mirror, naming the choice as both versions do. func testNoPictureSaysToMirrorTheScreen() { @@ -162,6 +190,9 @@ final class ConnectionProblemTests: XCTestCase { XCTAssertEqual(wifi(LinkError(.connectionFailed, "UDP connection failed: PolicyDenied", underlying: NWError.dns(-65570))), .localNetworkBlocked) // Offline. XCTAssertEqual(wifi(LinkError(.pathUnavailable, "UDP path not ready: Network is down", underlying: NWError.posix(.ENETDOWN))), .noNetwork) + XCTAssertEqual(ConnectionProblem.noNetwork.title, "This iPad isn't connected to a network") + XCTAssertEqual(ConnectionProblem.wifiOff.title, "Wi-Fi is off") + XCTAssertTrue(ConnectionProblem.wifiOff.detail.contains("Turn on Wi-Fi"), ConnectionProblem.wifiOff.detail) // Unknown failures get general advice, never the raw error. let unknown = LinkError(.connectionFailed, "UDP socket failed: Address already in use", underlying: POSIXError(.EADDRINUSE)) XCTAssertEqual(wifi(unknown), .couldNotConnect) @@ -191,10 +222,38 @@ final class ConnectionProblemTests: XCTestCase { } } + /// A connection waiting for its path. Only a path iPadOS calls + /// unsatisfied with no usable network is "Wi-Fi is off". No route to + /// the PC or its network, on a path that is up or not yet known, keeps + /// the advice it had before. + func testOnlyAnUnsatisfiedPathIsWiFiOff() { + let wifi = { (error: LinkError) in ConnectionProblem.linkFailure(error, isUSB: false, host: "studio-pc") } + let off = UDPReceiver.waitingError(.posix(.ENETDOWN), status: .unsatisfied, reason: .notAvailable) + XCTAssertEqual(off.kind, .pathUnavailable) + XCTAssertTrue(off.pathUnsatisfied) + XCTAssertEqual(wifi(off), .wifiOff) + for code: POSIXErrorCode in [.EHOSTUNREACH, .ENETUNREACH] { + for status: NWPath.Status? in [.satisfied, .requiresConnection, nil] { + let waiting = UDPReceiver.waitingError(.posix(code), status: status, reason: .notAvailable) + XCTAssertFalse(waiting.pathUnsatisfied, "\(code) \(String(describing: status))") + XCTAssertNotEqual(wifi(waiting), .wifiOff, "\(code) \(String(describing: status))") + XCTAssertEqual(wifi(waiting), .noNetwork, "unchanged: \(code) \(String(describing: status))") + } + } + // Unsatisfied for a reason other than no network: not Wi-Fi off. + XCTAssertFalse(UDPReceiver.waitingError(.posix(.ENETDOWN), status: .unsatisfied, reason: .wifiDenied).pathUnsatisfied) + let denied = UDPReceiver.waitingError(.posix(.EPERM), status: .unsatisfied, reason: .localNetworkDenied) + XCTAssertEqual(denied.kind, .localNetworkDenied) + XCTAssertFalse(denied.pathUnsatisfied) + XCTAssertEqual(wifi(denied), .localNetworkBlocked) + // Over the cable it is always the cable. + XCTAssertEqual(ConnectionProblem.linkFailure(off, isUSB: true, host: "USB"), .usbFailed) + } + func testGivingUpOnReconnectingKeepsASpecificLastFailure() { // A classified failure in the last attempt is more useful than // "Couldn't reconnect"; a PC that just never answered is not. - for specific: ConnectionProblem in [.localNetworkBlocked, .noNetwork, .badAddress("pc"), .usbFailed, .noPicture, + for specific: ConnectionProblem in [.localNetworkBlocked, .noNetwork, .wifiOff, .badAddress("pc"), .usbFailed, .noPicture, .pcBusy, .pcEndedSession, .pcEndedSessionOnError] { XCTAssertEqual(ConnectionProblem.afterReconnecting(lastAttempt: specific), specific) } @@ -205,10 +264,10 @@ final class ConnectionProblemTests: XCTestCase { func testEveryProblemSaysWhatToDoWithoutTransportJargon() { let all: [ConnectionProblem] = [ - .noReply(usb: false), .noReply(usb: true), .noPicture, .receiverNotReady, .badAddress("pc"), - .noNetwork, .localNetworkBlocked, .couldNotConnect, .usbFailed, .lost(.pcStoppedResponding), .lost(.cableUnplugged), + .noReply(usb: false), .noReply(usb: true), .noReplyAtNewAddress("192.0.2.31"), .noPicture, .receiverNotReady, + .badAddress("pc"), .noNetwork, .wifiOff, .localNetworkBlocked, .couldNotConnect, .usbFailed, .lost(.pcStoppedResponding), .lost(.cableUnplugged), .lost(.usbTurnedOff), .reconnecting(attempt: 2, of: 6), .reconnectFailed, .pcEndedSession, - .pcEndedSessionOnError, .pcBusy, .pairingCodeAsAddress, .routerAddress, + .pcEndedSessionOnError, .pcAsleep, .pcPoweringOff, .pcBusy, .pairingCodeAsAddress, .routerAddress, .pcUpdating(.reconnecting), .pcUpdating(.manual), .pcUpdating(.notBack), ] for problem in all { @@ -230,7 +289,7 @@ final class ConnectionProblemTests: XCTestCase { /// someone stopped the stream; the other reasons keep saying so. func testEachGoodbyeFromThePCMapsToItsProblemAndText() { let reasons = (0...UInt8.max).compactMap(ByeReason.init(rawValue:)) - XCTAssertEqual(reasons.count, 5, "a new goodbye reason needs its card here") + XCTAssertEqual(reasons.count, 7, "a new goodbye reason needs its card here") let userAction = "Someone clicked Stop on the PC, changed a setting that restarts the stream, or closed EternalMonitor. " + "Try again when the PC says Ready for your iPad." let expected: [ByeReason: (ConnectionProblem, String)] = [ @@ -241,6 +300,8 @@ final class ConnectionProblemTests: XCTestCase { "The connection to your PC dropped, or the PC ran into a problem. Tap Try again to reconnect."), .hostShuttingDown: (.pcUpdating(.reconnecting), "This iPad reconnects when it's back. If it doesn't, check that EternalMonitor opened again on the PC, then try again."), + .hostSleeping: (.pcAsleep, "Wake the PC, then tap Try again."), + .hostPoweringOff: (.pcPoweringOff, "Tap Try again once the PC is on, someone has signed in, and EternalMonitor is open."), ] for reason in reasons { let problem = ConnectionProblem.pcGoodbye(reason) @@ -248,12 +309,24 @@ final class ConnectionProblemTests: XCTestCase { XCTAssertEqual(problem.detail, expected[reason]?.1, "\(reason)") } let error = ConnectionProblem.pcEndedSessionOnError - XCTAssertEqual(error.title, ConnectionProblem.pcEndedSession.title, "one title, as SETUP.md lists it") + // A PC that restarts by itself also says error: the title must be + // true for a restart as well as a fault, and never blame the PC's user. + XCTAssertEqual(error.title, "The connection to your PC ended") + XCTAssertEqual(ConnectionProblem.pcEndedSession.title, "Your PC ended the session") XCTAssertFalse(error.detail.contains("Stop") || error.detail.contains("setting"), error.detail) XCTAssertEqual(error.actions, [.retry]) XCTAssertTrue(error.canReport) XCTAssertEqual(error.withoutReconnecting, error) XCTAssertEqual(error.afterTheCycle, error) + // The PC going down is something that happened, not a fault to report. + for down: ConnectionProblem in [.pcAsleep, .pcPoweringOff] { + XCTAssertEqual(down.actions, [.retry], "\(down)") + XCTAssertFalse(down.canReport, "\(down)") + XCTAssertEqual(down.withoutReconnecting, down) + XCTAssertEqual(down.afterTheCycle, down) + } + XCTAssertEqual(ConnectionProblem.pcAsleep.title, "Your PC went to sleep") + XCTAssertEqual(ConnectionProblem.pcPoweringOff.title, "Your PC is shutting down or signing out") } func testAnUpdateStaysTheReasonWhileThePCRestarts() { @@ -301,6 +374,19 @@ final class ConnectionProblemTests: XCTestCase { } } + /// The PC's last session said it can't reach Apple's USB service: the + /// waiting card says why the cable won't connect. Other states don't + /// change, and a cable session (which reaches it) clears the flag. + func testUSBCardSaysWhenThePCLacksAppleDevices() { + let waiting = USBStatusText.describe("USB: waiting for a cable", appleDevicesUnreachable: true) + XCTAssertEqual(waiting, "Last time this iPad connected, your PC couldn't reach Apple Devices. If the cable doesn't connect, " + + "open Apple Devices on the PC, or install it from the Microsoft Store. Wi-Fi still works.") + XCTAssertNotEqual(waiting, USBStatusText.describe("USB: waiting for a cable")) + for status in ["USB: off", "USB: disconnected", "USB: connecting", "USB: unavailable", "USB: connected to PC"] { + XCTAssertEqual(USBStatusText.describe(status, appleDevicesUnreachable: true), USBStatusText.describe(status), status) + } + } + /// A HELLO no one answered is a warning (a PC off, asleep or elsewhere) /// unless that PC answered at this address earlier in this launch. func testAnUnansweredHelloIsAnErrorOnlyWhereAPCAnsweredBefore() { diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index 273ac591..d8f98858 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -14,6 +14,19 @@ final class FakeUDPHost: @unchecked Sendable { private var answered = Set() /// The nonce of the HELLO2 answered last. private(set) var lastNonce: UInt32 = 0 + /// The HELLO2 answered last, and its size in bytes. + private(set) var lastHello: Hello2? + private(set) var lastHelloSize = 0 + /// The pairing token the HELLO2 answered last carried. + private(set) var lastAuthToken = Data() + /// When the HELLO2 answered last arrived, by the kernel's receive + /// time: how fast this fake read it doesn't move it. Nil if the kernel + /// gave none. + private(set) var lastHelloArrival: Date? + /// When the datagram read last arrived, by the kernel's receive time. + private var arrival: Date? + /// A HELLO2 came before the identify request was answered. + private(set) var helloBeforeIdentify = false init() { var address = sockaddr_in() @@ -31,33 +44,84 @@ final class FakeUDPHost: @unchecked Sendable { } var timeout = timeval(tv_sec: 0, tv_usec: 300_000) setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, socklen_t(MemoryLayout.size)) + var on: Int32 = 1 + setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &on, socklen_t(MemoryLayout.size)) } deinit { Darwin.close(fd) } private func receive() -> (Data, sockaddr_storage, socklen_t)? { var buffer = [UInt8](repeating: 0, count: 2048) + var control = [UInt8](repeating: 0, count: 64) var from = sockaddr_storage() var length = socklen_t(MemoryLayout.size) - let count = withUnsafeMutablePointer(to: &from) { pointer in - pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { - recvfrom(fd, &buffer, buffer.count, 0, $0, &length) + var controlLength = 0 + let count = buffer.withUnsafeMutableBytes { bytes in + control.withUnsafeMutableBytes { controlBytes in + withUnsafeMutablePointer(to: &from) { name in + var vector = iovec(iov_base: bytes.baseAddress, iov_len: bytes.count) + return withUnsafeMutablePointer(to: &vector) { vectors in + var message = msghdr( + msg_name: UnsafeMutableRawPointer(name), msg_namelen: length, + msg_iov: vectors, msg_iovlen: 1, + msg_control: controlBytes.baseAddress, msg_controllen: socklen_t(controlBytes.count), + msg_flags: 0) + let count = recvmsg(fd, &message, 0) + length = message.msg_namelen + controlLength = Int(message.msg_controllen) + return count + } + } } } - return count > 0 ? (Data(buffer.prefix(count)), from, length) : nil + guard count > 0 else { return nil } + arrival = Self.kernelTimestamp(control.prefix(controlLength)) + return (Data(buffer.prefix(count)), from, length) + } + + /// The SCM_TIMESTAMP in a datagram's control messages. Darwin aligns + /// each message and its data to 4 bytes after a 12-byte header. + private static func kernelTimestamp(_ control: ArraySlice) -> Date? { + let bytes = Array(control) + var offset = 0 + while offset + 12 <= bytes.count { + let (length, level, type) = bytes.withUnsafeBytes { + ($0.loadUnaligned(fromByteOffset: offset, as: UInt32.self), + $0.loadUnaligned(fromByteOffset: offset + 4, as: Int32.self), + $0.loadUnaligned(fromByteOffset: offset + 8, as: Int32.self)) + } + guard length >= 12 else { return nil } + if level == SOL_SOCKET, type == SCM_TIMESTAMP, + offset + 12 + MemoryLayout.size <= bytes.count { + let time = bytes.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset + 12, as: timeval.self) + } + return Date(timeIntervalSince1970: TimeInterval(time.tv_sec) + TimeInterval(time.tv_usec) / 1_000_000) + } + offset += (Int(length) + 3) & ~3 + } + return nil } /// Answers the next HELLO2 attempt, with session 41 unless `status` /// refuses it. False after thirty quiet seconds, a backstop that a - /// loaded machine's slow start does not reach. - func acceptSession(status: HelloStatus = .ok, acceptedVersion: UInt8 = 2, hostCaps: UInt16 = 0) -> Bool { + /// loaded machine's slow start does not reach. An attempt told the PC + /// isn't streaming keeps its nonce and can be answered again. + func acceptSession(status: HelloStatus = .ok, acceptedVersion: UInt8 = 2, hostCaps: UInt16 = 0, + hostName: String = "Test PC", authToken: Data = Data(17...32), reason: String = "") -> Bool { for _ in 0..<100 { guard let (data, from, length) = receive() else { continue } guard case .hello2(let hello) = Wire.parseControl(data)?.message, - answered.insert(hello.clientNonce).inserted else { continue } + !answered.contains(hello.clientNonce) else { continue } + if !status.pcNotStreaming { answered.insert(hello.clientNonce) } client = (from, length) lastNonce = hello.clientNonce - send(Self.ack(nonce: hello.clientNonce, status: status, acceptedVersion: acceptedVersion, hostCaps: hostCaps)) + lastHello = hello + lastHelloSize = data.count + lastAuthToken = hello.authToken + lastHelloArrival = arrival + send(Self.ack(nonce: hello.clientNonce, status: status, acceptedVersion: acceptedVersion, + hostName: hostName, authToken: authToken, hostCaps: hostCaps, reason: reason)) return true } return false @@ -66,11 +130,13 @@ final class FakeUDPHost: @unchecked Sendable { /// The HELLO_ACK for the attempt with `nonce` from the PC `hostName` /// with pairing token `authToken`: session 41, unless `status` refuses it. static func ack(nonce: UInt32, status: HelloStatus = .ok, acceptedVersion: UInt8 = 2, - hostName: String = "Test PC", authToken: Data = Data(17...32), hostCaps: UInt16 = 0) -> Data { + hostName: String = "Test PC", authToken: Data = Data(17...32), hostCaps: UInt16 = 0, + reason: String = "") -> Data { Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .helloAck(HelloAck( status: status, acceptedVersion: acceptedVersion, clientNonce: nonce, sessionId: status == .ok ? 41 : 0, heartbeatIntervalMs: 1000, reportIntervalMs: 500, livenessTimeoutMs: 3000, - streamConfig: StreamConfig(), hostName: hostName, authToken: authToken, hostCaps: hostCaps))) + streamConfig: StreamConfig(), hostName: hostName, authToken: authToken, hostCaps: hostCaps, + reason: reason))) } private func send(_ datagram: Data) { @@ -84,15 +150,79 @@ final class FakeUDPHost: @unchecked Sendable { } } + /// A HELLO_ACK for `nonce`, sent again late, as Wi-Fi can reorder it. + func sendAck(nonce: UInt32, status: HelloStatus) { + send(Self.ack(nonce: nonce, status: status)) + } + + /// Answers the next identify request as the PC holding `token` would, + /// receiving at this fake's port on `addresses` (loopback unless given), + /// or with `proof` instead. False after thirty quiet seconds. + func answerIdentify(token: Data, proof: Data? = nil, addresses: [Data] = [Data([127, 0, 0, 1])]) -> Bool { + for _ in 0..<100 { + guard let (data, from, length) = receive() else { continue } + let message = Wire.parseControl(data)?.message + if case .hello2? = message { helloBeforeIdentify = true } + guard case .identify(let request)? = message else { continue } + client = (from, length) + send(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identity(WireIdentity( + nonce: request.nonce, port: port, addresses: addresses, + proof: proof ?? MovedPC.proof(token: token, nonce: request.nonce, port: port, addresses: addresses))))) + return true + } + return false + } + + /// Hands the next identify request to `pc` and gives the iPad whatever + /// `pc` answers, from this fake's port. False after thirty quiet seconds. + func forwardIdentify(to pc: FakeUDPHost) -> Bool { + var target = sockaddr_in() + target.sin_len = UInt8(MemoryLayout.size) + target.sin_family = sa_family_t(AF_INET) + target.sin_addr.s_addr = inet_addr("127.0.0.1") + target.sin_port = pc.port.bigEndian + for _ in 0..<100 { + guard let (data, from, length) = receive() else { continue } + let message = Wire.parseControl(data)?.message + if case .hello2? = message { helloBeforeIdentify = true } + if case .identity? = message, client != nil { + send(data) + return true + } + guard case .identify? = message else { continue } + client = (from, length) + _ = data.withUnsafeBytes { bytes in + withUnsafePointer(to: &target) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { + sendto(fd, bytes.baseAddress, bytes.count, 0, $0, socklen_t(MemoryLayout.size)) + } + } + } + } + return false + } + + /// The control messages received within `seconds`, by type. + func received(within seconds: Double) -> [Wire.PacketType] { + var types: [Wire.PacketType] = [] + let deadline = Date().addingTimeInterval(seconds) + while Date() < deadline { + guard let (data, _, _) = receive(), let (header, _) = Wire.parseControl(data) else { continue } + types.append(header.packetType) + } + return types + } + /// Ends session 41 from the PC's side. func sendBye(_ reason: ByeReason) { send(Wire.encodeControl(sessionId: 41, msgSeq: 2, message: .bye(reason))) } - /// A session 41 heartbeat, as the PC sends one each second. - func sendHeartbeat() { + /// A session 41 heartbeat, as the PC sends one each second, counting + /// `framesSent` video frames when it is given. + func sendHeartbeat(notice: UInt8 = Heartbeat.noticeNone, framesSent: UInt32? = nil) { send(Wire.encodeControl(sessionId: 41, msgSeq: 3, message: .heartbeat( - Heartbeat(hostTimeUs: 0, streamConfig: StreamConfig())))) + Heartbeat(hostTimeUs: 0, streamConfig: StreamConfig(), notice: notice, framesSent: framesSent)))) } /// The 64x64 H.264 keyframe fixture. @@ -132,6 +262,18 @@ final class FakeUDPHost: @unchecked Sendable { timedKeyframeRequests(within: seconds, count: count).map(\.reason) } + /// The reasons of the keyframe requests received within `seconds`, up to + /// and including the first for which `last` is true. + func keyframeRequests(within seconds: Double, until last: (KeyframeReason) -> Bool) -> [KeyframeReason] { + var reasons: [KeyframeReason] = [] + let deadline = Date().addingTimeInterval(seconds) + while Date() < deadline, !(reasons.last.map(last) ?? false) { + guard let (data, _, _) = receive() else { continue } + if case .keyframeRequest(let request) = Wire.parseControl(data)?.message { reasons.append(request.reason) } + } + return reasons + } + /// The keyframe requests received within `seconds`, with when each came, /// or the first `count` of them as soon as they have come. func timedKeyframeRequests(within seconds: Double, count: Int = .max) -> [(reason: KeyframeReason, at: Date)] { @@ -144,6 +286,84 @@ final class FakeUDPHost: @unchecked Sendable { return requests } + /// What `servePath` saw the iPad do. + struct PathRun { + /// The media datagram sizes the iPad asked for, in turn. + var sizesAsked: [UInt16] = [] + /// Keyframe requests that asked for no size. + var plainKeyframeRequests = 0 + var identifyAnswers = 0 + } + + /// Plays a PC whose media reaches the iPad only in datagrams of at most + /// `limit` bytes (0: none at all), as over a path that drops larger ones. + /// Accepts the next HELLO2 with `hostCaps`, then until `stop` says so, + /// about every 100 ms, sends a heartbeat (counting the frames sent when + /// `countFrames`) and `frame` as the next keyframe, cut at the size the + /// iPad asked for. Answers identify requests as the PC holding `token` + /// receiving on `addresses`, to whichever socket asks. + /// Takes the next HELLO2 as a new client's, as a PC does for the iPad's + /// next session. + func forgetClient() { + client = nil + } + + func servePath(limit: Int, frame: Data, hostCaps: UInt16, countFrames: Bool = true, token: Data = Data(17...32), + addresses: [Data] = [Data([127, 0, 0, 1])], until stop: () -> Bool) -> PathRun { + var run = PathRun() + var size = Wire.maxDatagramSize + var framesSent: UInt32 = 0 + var lastSent = Date.distantPast + let deadline = Date().addingTimeInterval(60) + while !stop(), Date() < deadline { + if let (data, from, length) = receive() { + switch Wire.parseControl(data)?.message { + case .hello2(let hello)? where client == nil: + client = (from, length) + lastNonce = hello.clientNonce + send(Self.ack(nonce: hello.clientNonce, authToken: token, hostCaps: hostCaps)) + case .keyframeRequest(let request)?: + if request.maxMediaDgram == 0 { + run.plainKeyframeRequests += 1 + } else { + run.sizesAsked.append(request.maxMediaDgram) + size = min(size, Int(request.maxMediaDgram)) + } + case .identify(let request)?: + run.identifyAnswers += 1 + let answer = Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identity(WireIdentity( + nonce: request.nonce, port: port, addresses: addresses, + proof: MovedPC.proof(token: token, nonce: request.nonce, port: port, addresses: addresses)))) + var to = from + _ = answer.withUnsafeBytes { bytes in + withUnsafePointer(to: &to) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { + sendto(fd, bytes.baseAddress, bytes.count, 0, $0, length) + } + } + } + default: + break + } + } + guard client != nil, Date().timeIntervalSince(lastSent) >= 0.1 else { continue } + lastSent = Date() + framesSent += 1 + let payload = size - MediaHeader.size + let count = (frame.count + payload - 1) / payload + for index in 0.. [KeyframeReason] { + try await sent(requests.map { ($0.0, Event.request($0.1)) }).map(\.reason) + } + + /// The keyframe requests a session's channel sends for `events`, each at + /// its time on the channel's clock, with the time each was sent. + private func sent(_ events: [(UInt64, Event)], hostCaps: UInt16 = 0) async throws + -> [(at: UInt64, reason: KeyframeReason)] { let queue = DispatchQueue(label: "keyframe-request-test") let clock = OSAllocatedUnfairLock(initialState: 1_000_000) let (stream, continuation) = AsyncStream.makeStream() - let channel = ControlChannel(queue: queue, nowUs: { clock.withLock { $0 } }) { continuation.yield($0) } + let sentAt = OSAllocatedUnfairLock<[UInt64]>(initialState: []) + let channel = ControlChannel(queue: queue, nowUs: { clock.withLock { $0 } }) { data in + if case .keyframeRequest = Wire.parseControl(data)?.message { + sentAt.withLock { $0.append(clock.withLock { $0 }) } + } + continuation.yield(data) + } defer { channel.stop(); continuation.finish() } channel.startHandshake(listenPort: 50000, identity: ControlChannel.ClientIdentity( deviceName: "Test iPad", screenPxW: 640, screenPxH: 360, screenPtW: 640, screenPtH: 360, refreshHz: 60)) var messages = stream.makeAsyncIterator() let first = await messages.next() guard case .hello2(let hello) = try XCTUnwrap(Wire.parseControl(XCTUnwrap(first))).message else { - return XCTFail("missing handshake") + XCTFail("missing handshake") + return [] } queue.sync { channel.handleControl(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .helloAck(HelloAck( status: .ok, acceptedVersion: 2, clientNonce: hello.clientNonce, sessionId: 41, heartbeatIntervalMs: 1000, reportIntervalMs: 500, livenessTimeoutMs: 3000, - streamConfig: StreamConfig(), hostName: "Test PC", authToken: Data(17...32), hostCaps: 0)))) + streamConfig: StreamConfig(), hostName: "Test PC", authToken: Data(17...32), hostCaps: hostCaps)))) } - for (at, reason) in [(1_000_000, KeyframeReason.gapLoss), (1_010_000, .gapLoss), - (1_400_000, .decodeError), (1_500_000, .gapLoss)] as [(UInt64, KeyframeReason)] { + for (at, event) in events { clock.withLock { $0 = at } - channel.sendKeyframeRequest(streamEpoch: 1, lastCompleteSeq: 7, reason: reason) + switch event { + case .request(let reason): + channel.sendKeyframeRequest(streamEpoch: 1, lastCompleteSeq: 7, reason: reason) + case .keyframe: + channel.noteKeyframeAssembled() + } queue.sync {} } channel.stop() continuation.finish() - var requests: [KeyframeReason] = [] + var reasons: [KeyframeReason] = [] while let data = await messages.next() { - if case .keyframeRequest(let request) = Wire.parseControl(data)?.message { requests.append(request.reason) } + if case .keyframeRequest(let request) = Wire.parseControl(data)?.message { reasons.append(request.reason) } } - XCTAssertEqual(requests, [.gapLoss, .gapLoss], "one request per 500 ms window") + return Array(zip(sentAt.withLock { $0 }, reasons)) } } diff --git a/ios/EternalMonitorTests/LocalizationTests.swift b/ios/EternalMonitorTests/LocalizationTests.swift index 821580fb..721db406 100644 --- a/ios/EternalMonitorTests/LocalizationTests.swift +++ b/ios/EternalMonitorTests/LocalizationTests.swift @@ -20,14 +20,24 @@ final class LocalizationTests: XCTestCase { .map { String(text[Range($0.range(at: 1), in: text)!]) }.sorted() } + /// English strings whose translations come in a separate pass. Until + /// then every language shows them in English. + private let awaitingTranslation: Set = [ + "Your PC can't share its screen from its current graphics card. Check the PC.", + "The second screen didn't start, and your PC can't share its main screen from its current graphics card. Check the PC.", + "Your PC is restarting the stream…", + "Second screen needs Windows 11. Showing your PC's screen instead.", + ] + /// Every language has exactly English's strings, none empty, each with /// the same placeholders. func testEveryLanguageHasEveryString() throws { let english = try table("en") XCTAssertGreaterThan(english.count, 100) + XCTAssertTrue(awaitingTranslation.isSubset(of: english.keys)) for language in AppLanguage.allCases where language != .en { let strings = try table(language.rawValue) - XCTAssertEqual(Set(strings.keys), Set(english.keys), language.rawValue) + XCTAssertEqual(Set(strings.keys), Set(english.keys).subtracting(awaitingTranslation), language.rawValue) for (key, value) in strings { XCTAssertFalse(value.trimmingCharacters(in: .whitespaces).isEmpty, "\(language.rawValue): \(key)") XCTAssertEqual(specifiers(value), specifiers(key), "\(language.rawValue): \(key)") diff --git a/ios/EternalMonitorTests/MovedPCTests.swift b/ios/EternalMonitorTests/MovedPCTests.swift new file mode 100644 index 00000000..4a8d95c3 --- /dev/null +++ b/ios/EternalMonitorTests/MovedPCTests.swift @@ -0,0 +1,433 @@ +import Combine +import XCTest +@testable import EternalMonitor + +/// Pairings kept in memory. `failSaves` makes every save fail, as a full +/// or locked Keychain does. +private final class MemoryPairingStorage: PairingStorage { + var values: [String: Data] = [:] + var failSaves = false + func records() throws -> [PairingRecord] { values.map { PairingRecord(account: $0.key, token: $0.value) } } + func save(account: String, token: Data) throws { + if failSaves { throw KeychainPairingStorage.Failure(status: errSecNotAvailable) } + values[account] = token + } + func remove(account: String?) throws { + if let account { values.removeValue(forKey: account) } else { values.removeAll() } + } +} + +/// A saved PC whose address changed: a new lease from the router, or the +/// PC moved between its cable and Wi-Fi. +@MainActor +final class MovedPCTests: XCTestCase { + nonisolated private static let token = Data(17...32) + /// The choice tests' home network: documentation addresses stand in. + private static let home: (String) -> Bool = { $0.hasPrefix("192.0.2.") || $0.hasPrefix("198.51.100.") } + + override func setUp() async throws { + let store = RecentConnectionStore.shared + let saved = store.connections + let defaults = ["recentConnections", "lastHost", "lastPort"].map { ($0, UserDefaults.standard.object(forKey: $0)) } + addTeardownBlock { @MainActor in + store.connections = saved + for (key, value) in defaults { UserDefaults.standard.set(value, forKey: key) } + } + } + + private func found(_ name: String, _ address: String?, port: UInt16 = 9876) -> DiscoveredHost { + DiscoveredHost(name: name, address: address, port: port) + } + + // MARK: - The choice + + func testAStaleSavedAddressMovesToTheFreshBonjourAddress() { + let fresh = [found("EternalMonitor (Office PC)", "192.0.2.40"), found("EternalMonitor (Studio PC)", "192.0.2.31")] + let moved = MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.20", port: 9876, found: fresh, homeAddress: Self.home) + XCTAssertEqual(moved?.host, "192.0.2.31") + XCTAssertEqual(moved?.port, 9876) + // A new port is a new address too. + let port = MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.31", port: 9876, + found: [found("EternalMonitor (Studio PC)", "192.0.2.31", port: 9900)], homeAddress: Self.home) + XCTAssertEqual(port?.port, 9900) + // Still at the saved address: it is just not answering. + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.31", port: 9876, found: fresh, homeAddress: Self.home)) + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.20", port: 9876, found: [], homeAddress: Self.home)) + } + + /// Only a paired PC that Bonjour listed, on a home network address, + /// moves by itself. Typed, scanned and older entries, and VPN or + /// Tailscale addresses on either side, never do. + func testOnlyAPairedPCFoundWithBonjourOnAHomeAddressIsLookedUp() throws { + XCTAssertTrue(MovedPC.canLookUp(host: "192.0.2.20", pcName: "Studio PC", found: true, paired: true, homeAddress: Self.home)) + XCTAssertFalse(MovedPC.canLookUp(host: "192.0.2.20", pcName: "Studio PC", found: false, paired: true, homeAddress: Self.home)) + XCTAssertFalse(MovedPC.canLookUp(host: "192.0.2.20", pcName: "Studio PC", found: true, paired: false, homeAddress: Self.home)) + XCTAssertFalse(MovedPC.canLookUp(host: "192.0.2.20", pcName: nil, found: true, paired: true, homeAddress: Self.home)) + XCTAssertFalse(MovedPC.canLookUp(host: "203.0.113.20", pcName: "Studio PC", found: true, paired: true, homeAddress: Self.home)) + + for home in ["10.0.0.1", "10.255.255.254", "172.16.0.1", "172.31.255.254", "192.168.0.1", "192.168.255.254", "169.254.0.1"] { + XCTAssertTrue(MovedPC.isHomeAddress(home), home) + } + for other in ["100.64.0.1", "100.127.255.254", "172.15.255.254", "172.32.0.1", "127.0.0.1", "192.0.2.1", + "studio-pc.local", "fe80::1", "192.168.1"] { + XCTAssertFalse(MovedPC.isHomeAddress(other), other) + XCTAssertFalse(MovedPC.canLookUp(host: other, pcName: "Studio PC", found: true, paired: true), other) + } + // Bonjour giving a VPN address is no move either. + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "10.255.255.254", port: 9876, + found: [found("EternalMonitor (Studio PC)", "100.64.0.9")])) + + // An entry saved before this version doesn't say where it came from. + let legacy = #"[{"host":"192.0.2.20","port":9876,"lastUsed":0,"isUSB":false}]"# + let decoded = try JSONDecoder().decode([RecentConnection].self, from: Data(legacy.utf8)) + XCTAssertNil(decoded.first?.found) + + let store = RecentConnectionStore.shared + store.connections = [] + store.add(host: "192.0.2.20", port: 9876, isUSB: false, name: "Studio PC", found: true) + store.add(host: "192.0.2.20", port: 9876, isUSB: false) + XCTAssertEqual(store.entry(host: "192.0.2.20", port: 9876)?.found, true, "picked again from Recent, still found") + XCTAssertEqual(store.entry(host: "192.0.2.20", port: 9876)?.name, "Studio PC") + store.add(host: "192.0.2.21", port: 9876, isUSB: false, name: "Studio PC") + XCTAssertEqual(store.entry(host: "192.0.2.21", port: 9876)?.found, false) + + store.moved(from: ("192.0.2.20", 9876), to: ("192.0.2.31", 9876), name: "Studio PC", replacesOld: false) + XCTAssertNotNil(store.entry(host: "192.0.2.20", port: 9876)) + store.moved(from: ("192.0.2.20", 9876), to: ("192.0.2.32", 9876), name: "Studio PC", replacesOld: true) + XCTAssertNil(store.entry(host: "192.0.2.20", port: 9876)) + XCTAssertEqual(store.connections.first?.host, "192.0.2.32") + XCTAssertEqual(store.connections.first?.found, true) + } + + func testTwoPCsWithTheSameNameAreNeverConfused() { + let ours = found("EternalMonitor (Studio PC)", "192.0.2.31") + // Bonjour renamed the second service: either one may be the saved PC. + let renamed = found("EternalMonitor (Studio PC) (2)", "192.0.2.40") + let choose = { MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.20", port: 9876, found: $0, homeAddress: Self.home) } + XCTAssertNil(choose([ours, renamed])) + XCTAssertNil(choose([renamed])) + XCTAssertNil(choose([ours, found("EternalMonitor (Studio PC) (2)", nil)])) + // Other PCs, however alike their names, are no claim. + let others = [found("EternalMonitor (Studio PC 2)", "192.0.2.41"), found("EternalMonitor (Studio)", "192.0.2.42")] + XCTAssertNil(choose(others)) + XCTAssertEqual(choose(others + [ours])?.host, "192.0.2.31") + } + + /// The real scanner: a second service still resolving when the lookup + /// ends has no address yet, and still makes the choice ambiguous. + func testAServiceStillResolvingCountsAsAClaim() async throws { + let scanner = NetworkScanner() + defer { scanner.stopScan() } + let choose = { MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.20", port: 9876, found: scanner.claims, + homeAddress: Self.home) } + scanner.merge(found("EternalMonitor (Studio PC)", "192.0.2.31")) + XCTAssertEqual(choose()?.host, "192.0.2.31") + scanner.netServiceBrowser(NetServiceBrowser(), didFind: NetService(domain: "local.", type: "_eternaldisplay._udp.", + name: "EternalMonitor (Studio PC) (2)"), moreComing: false) + let deadline = Date().addingTimeInterval(5) + while scanner.claims.count < 2, Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertEqual(scanner.hosts.map(\.name), ["EternalMonitor (Studio PC)"], "not listed: no address yet") + XCTAssertEqual(scanner.claims.map(\.name), ["EternalMonitor (Studio PC)", "EternalMonitor (Studio PC) (2)"]) + XCTAssertNil(choose()) + } + + /// The same known answer as the host's pairing tests and the wire's + /// golden `identity` vector. The proof signs the PC's port and addresses. + func testTheIdentityProofIsTheSharedKnownAnswer() { + let addresses = [Data([192, 0, 2, 10]), Data([198, 51, 100, 10])] + let proof = { (token: Data, port: UInt16, addresses: [Data]) in + MovedPC.proof(token: token, nonce: Data(0xA0...0xAF), port: port, addresses: addresses) + } + let known = proof(Data(1...16), 9876, addresses) + XCTAssertEqual(known.map { String(format: "%02x", $0) }.joined(), + "18fb81d76bbb96e91101119775ad9766b0c332b2c49441e6f07ec33be1c6d94e") + XCTAssertNotEqual(proof(Data(2...17), 9876, addresses), known) + XCTAssertNotEqual(proof(Data(1...16), 9877, addresses), known) + XCTAssertNotEqual(proof(Data(1...16), 9876, [addresses[0]]), known) + } + + /// Only an address this iPad reaches directly, over Wi-Fi or Ethernet + /// and in that network's subnet, may be a moved PC. A VPN in a home + /// network range is not. + func testOnlyAnAddressOnTheLocalNetworkIsLocal() { + let wifi = MovedPC.Route(interface: "en0", direct: true, address: "192.168.1.23", netmask: "255.255.255.0") + XCTAssertTrue(MovedPC.isLocal("192.168.1.40", over: wifi)) + XCTAssertFalse(MovedPC.isLocal("192.168.2.40", over: wifi), "another subnet") + XCTAssertFalse(MovedPC.isLocal("10.0.0.40", over: wifi)) + XCTAssertFalse(MovedPC.isLocal("studio-pc.local", over: wifi)) + let wide = MovedPC.Route(interface: "en0", direct: true, address: "10.20.0.5", netmask: "255.255.0.0") + XCTAssertTrue(MovedPC.isLocal("10.20.200.9", over: wide)) + + // A VPN in a home network range: the path isn't Wi-Fi or Ethernet, + // or the interface is a tunnel, or it is a point-to-point address. + let vpn = MovedPC.Route(interface: "utun4", direct: false, address: "10.8.0.6", netmask: "255.255.255.0") + XCTAssertFalse(MovedPC.isLocal("10.8.0.1", over: vpn)) + XCTAssertFalse(MovedPC.isLocal("10.8.0.1", over: MovedPC.Route(interface: "utun4", direct: true, address: "10.8.0.6", + netmask: "255.255.255.0"))) + XCTAssertFalse(MovedPC.isLocal("192.168.1.40", over: MovedPC.Route(interface: "ipsec0", direct: true, + address: "192.168.1.23", netmask: "255.255.255.0"))) + XCTAssertFalse(MovedPC.isLocal("192.168.1.23", over: MovedPC.Route(interface: "en0", direct: true, + address: "192.168.1.23", netmask: "255.255.255.255"))) + XCTAssertFalse(MovedPC.isLocal("192.168.1.40", over: MovedPC.Route(interface: "en0", direct: false, + address: "192.168.1.23", netmask: "255.255.255.0"))) + } + + // MARK: - Connecting + + private struct Setup { + let manager: ConnectionManager + let storage: MemoryPairingStorage + let pairings: PairingStore + let asked: () -> [String] + } + + /// A saved PC named "Test PC" at `old`, Bonjour-found unless `found` says + /// otherwise, paired with `token`; Bonjour has it at `new`. Loopback + /// counts as a home address here. + private func setUp(old: FakeUDPHost, new: FakeUDPHost, token: Data?, found: Bool? = true, + route: MovedPC.Route? = MovedPC.Route(interface: "en0", direct: true, address: "127.0.0.1", + netmask: "255.0.0.0"), + connectTimeout: Double) throws -> Setup { + RecentConnectionStore.shared.connections = [ + RecentConnection(host: "127.0.0.1", port: old.port, lastUsed: .now, isUSB: false, name: "Test PC", found: found), + ] + let storage = MemoryPairingStorage() + let pairings = PairingStore(storage: storage) + if let token { try pairings.save(token: token, hostName: "Test PC", address: PairingStore.address(host: "127.0.0.1", port: old.port)) } + let manager = ConnectionManager(timing: .init(connectTimeoutSeconds: connectTimeout, reconnectGraceSeconds: 0.1, + reconnectBackoffSeconds: { _ in 0.05 }), pairingStore: pairings) + manager.homeAddress = { _ in true } + manager.localRoute = { _, _ in route } + var asked: [String] = [] + manager.findPCs = { name in + asked.append(name) + return [DiscoveredHost(name: "EternalMonitor (Test PC)", address: "127.0.0.1", port: new.port)] + } + return Setup(manager: manager, storage: storage, pairings: pairings, asked: { asked }) + } + + private func until(_ description: String, seconds: Double, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + private func address(_ host: FakeUDPHost) -> String { PairingStore.address(host: "127.0.0.1", port: host.port) } + + private func stream(_ setup: Setup, from new: FakeUDPHost) async throws { + let accepted = await Task.detached { new.answerIdentify(token: Self.token) && new.acceptSession(authToken: Self.token) }.value + XCTAssertTrue(accepted, "the iPad never checked and tried the new address") + try await until("handshake", seconds: 60) { setup.manager.hostInfo != nil } + new.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming", seconds: 60) { setup.manager.state == .connected } + } + + /// The saved address stays silent. Bonjour has the same PC at another + /// port; it shows it holds the pairing before the token goes anywhere, + /// and the iPad streams from there with no tap. + func testASilentSavedAddressReconnectsOnceThePCShowsItIsTheSame() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 60) + defer { setup.manager.cancel() } + // Every line logged: the new attempt clears Connection details. + var lines: [String] = [] + let log = setup.manager.$diagnostics.sink { if let line = $0.last?.message { lines.append(line) } } + defer { log.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + try await stream(setup, from: new) + XCTAssertEqual(setup.asked(), ["EternalMonitor (Test PC)"]) + XCTAssertFalse(new.helloBeforeIdentify, "nothing with the token before the PC showed it is the same") + XCTAssertEqual(new.lastAuthToken, Self.token, "then the saved pairing went along") + XCTAssertTrue(lines.contains("Disconnect state=connecting reason=pcMoved"), lines.joined(separator: "\n")) + let summary = lines.first { $0.hasPrefix("session_summary ") } ?? "" + XCTAssertTrue(summary.contains(" stage=hello_sent outcome=no_reply failure=no_reply fallback=none end=superseded "), summary) + + let saved = RecentConnectionStore.shared.connections.first + XCTAssertEqual(saved?.port, new.port) + XCTAssertEqual(saved?.name, "Test PC") + XCTAssertEqual(saved?.found, true) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: old.port)) + XCTAssertEqual(try setup.pairings.token(address: address(new)), Self.token) + XCTAssertNil(try setup.pairings.token(address: address(old))) + XCTAssertEqual(UserDefaults.standard.integer(forKey: "lastPort"), Int(new.port)) + } + + /// A PC that gives a wrong answer, or none (a PC before 0.3.5), gets + /// nothing more: no HELLO2, no token. The attempt ends as it always did. + func testAPCThatDoesNotShowItIsTheSameIsLeftAlone() async throws { + for answer in ["wrong proof", "no answer"] { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 7) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let types = await Task.detached { () -> [Wire.PacketType] in + if answer == "wrong proof", !new.answerIdentify(token: Data(2...17)) { return [] } + return new.received(within: 6) + }.value + XCTAssertFalse(types.contains(.hello2), answer) + XCTAssertFalse(new.helloBeforeIdentify, answer) + if answer == "no answer" { XCTAssertEqual(Set(types), [.identify], answer) } + try await until("timed out: \(answer)", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.asked().count, 1, answer) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false), answer) + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token, answer) + XCTAssertNotNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: old.port), answer) + } + } + + /// The answer at the address Bonjour gave comes from the saved PC at + /// another address, and says so: it isn't the saved PC at this address. + /// No HELLO2, no token. + func testAnAnswerFromAnotherAddressIsNotTheSamePC() async throws { + let old = FakeUDPHost(), forwarder = FakeUDPHost(), pc = FakeUDPHost() + let setup = try setUp(old: old, new: forwarder, token: Self.token, connectTimeout: 7) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + async let answered = Task.detached { pc.answerIdentify(token: Self.token) }.value + let types = await Task.detached { () -> [Wire.PacketType] in + guard forwarder.forwardIdentify(to: pc) else { return [] } + return forwarder.received(within: 4) + }.value + let pcAnswered = await answered + XCTAssertTrue(pcAnswered, "the request reached the saved PC") + XCTAssertFalse(types.isEmpty, "the iPad asked the forwarder") + XCTAssertFalse(types.contains(.hello2), "\(types)") + XCTAssertFalse(forwarder.helloBeforeIdentify) + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: forwarder.port)) + } + + /// The PC at the address and port asked proves it holds the pairing but + /// signs other addresses, or none: it isn't the saved PC at this + /// address. No HELLO2, no token, and the pairing stays. + func testAnAnswerSigningOtherAddressesIsNotTheSamePC() async throws { + for signed in [[Data([192, 0, 2, 7])], []] { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 7) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let types = await Task.detached { () -> [Wire.PacketType] in + guard new.answerIdentify(token: Self.token, addresses: signed) else { return [] } + return new.received(within: 4) + }.value + XCTAssertFalse(types.isEmpty, "\(signed): the iPad kept asking") + XCTAssertFalse(types.contains(.hello2), "\(signed): \(types)") + XCTAssertFalse(new.helloBeforeIdentify, "\(signed)") + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.manager.problem, .noReply(usb: false), "\(signed)") + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token, "\(signed)") + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: new.port), "\(signed)") + } + } + + /// Bonjour has the PC at a home network address that this iPad reaches + /// over a VPN, or by no ready path: it isn't asked anything. + func testAPCReachedOverAVPNIsNeverAsked() async throws { + let vpn = MovedPC.Route(interface: "utun4", direct: false, address: "127.0.0.2", netmask: "255.0.0.0") + for route: MovedPC.Route? in [vpn, nil] { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, route: route, connectTimeout: 6) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let types = await Task.detached { new.received(within: 5) }.value + XCTAssertEqual(types, [], "\(String(describing: route))") + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.asked().count, 1) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + XCTAssertTrue(setup.manager.diagnostics.contains { $0.message.contains("not on this iPad's local network") }) + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token) + } + } + + /// Unpaired, nothing proves which PC is which: with the saved PC gone, a + /// same-named PC with pairing off isn't even looked for. + func testAnUnpairedSavedPCIsNeverMoved() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: nil, connectTimeout: 5) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let types = await Task.detached { new.received(within: 4) }.value + XCTAssertEqual(types, []) + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.asked(), []) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: new.port)) + } + + /// Entries that didn't come from Bonjour's list: typed, scanned, or saved + /// by an older version, which doesn't say. + func testAnEntryNotFoundWithBonjourIsNeverLookedUp() async throws { + for found: Bool? in [nil, false] { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, found: found, connectTimeout: 4) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.asked(), [], "\(String(describing: found))") + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + } + } + + /// The PC at the new address showed it holds the pairing, then wasn't + /// the saved PC by name. Try again goes back to the saved address and + /// checks again; it never goes straight to the other PC. + func testTryAgainAfterARefusalGoesBackToTheSavedAddress() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 60) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let accepted = await Task.detached { + new.answerIdentify(token: Self.token) && new.acceptSession(hostName: "Other PC", authToken: Data(repeating: 0, count: 16)) + }.value + XCTAssertTrue(accepted) + try await until("left it", seconds: 30) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + + setup.manager.retry() + let (atOld, atNew) = await Task.detached { + (old.helloAttempts(within: 1.5), new.received(within: 1)) + }.value + XCTAssertGreaterThan(atOld, 0, "Try again goes to the saved address") + XCTAssertFalse(atNew.contains(.hello2), "and not straight to the other PC") + XCTAssertNil(setup.manager.pairingPrompt) + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token) + } + + /// The new address's pairing can't be saved: the stream goes on, and the + /// saved entry and pairing stay where they were. + func testTheSavedEntryStaysWhenTheNewPairingCantBeSaved() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 60) + defer { setup.manager.cancel() } + UserDefaults.standard.set("127.0.0.1", forKey: "lastHost") + UserDefaults.standard.set(Int(old.port), forKey: "lastPort") + setup.storage.failSaves = true + setup.manager.connect(host: "127.0.0.1", port: old.port) + try await stream(setup, from: new) + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token) + XCTAssertNil(try setup.pairings.token(address: address(new))) + XCTAssertNotNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: old.port)) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: new.port)) + XCTAssertEqual(UserDefaults.standard.integer(forKey: "lastPort"), Int(old.port)) + XCTAssertTrue(setup.manager.diagnostics.contains { $0.message.hasPrefix("The PC's new address wasn't saved") }) + } + + /// The PC showed it is the same at its new address, then didn't answer + /// HELLO2 there either: the card says where the iPad found it. + func testNoAnswerAtTheNewAddressSaysWhereItLooked() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 7) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let answered = await Task.detached { new.answerIdentify(token: Self.token) }.value + XCTAssertTrue(answered) + let tried = await Task.detached { new.helloAttempts(within: 3) }.value + XCTAssertGreaterThan(tried, 0) + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.asked().count, 1, "looked once, not again from the new address") + XCTAssertEqual(setup.manager.problem, .noReplyAtNewAddress("127.0.0.1")) + XCTAssertEqual(setup.manager.problem?.actions, [.retry]) + } +} diff --git a/ios/EternalMonitorTests/NoPictureTimeoutTests.swift b/ios/EternalMonitorTests/NoPictureTimeoutTests.swift index a205845a..90b773a1 100644 --- a/ios/EternalMonitorTests/NoPictureTimeoutTests.swift +++ b/ios/EternalMonitorTests/NoPictureTimeoutTests.swift @@ -73,6 +73,68 @@ final class NoPictureTimeoutTests: XCTestCase { XCTAssertNil(logged(manager, "Disconnect state=connecting reason=userDisconnect")) } + /// The hybrid laptops: the PC accepted, then its heartbeats said + /// Windows won't let it capture from its graphics card. The iPad showed + /// "Connecting…" for 10 s, then "no picture". Now it says why while it + /// waits, its card says what to do on the PC when the wait runs out, and + /// its summary names a capture failure, as the PC's does. + func testAPCThatCantCaptureFromItsGraphicsCardSaysWhyInsteadOfNoPicture() async throws { + let host = FakeUDPHost() + var timing = fast + timing.connectTimeoutSeconds = 3 + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + host.sendHeartbeat(notice: Heartbeat.noticeUnsupportedGPU) + try await until("the PC says why") { manager.pcGraphicsCard == .screen } + XCTAssertEqual(manager.state, .connecting, "still waiting for a picture") + try await until("timed out", seconds: 15) { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .pcGraphicsCard(.screen)) + XCTAssertEqual(manager.problem?.actions, [.retry]) + let summary = logged(manager, "session_summary v=1 role=ipad") ?? "" + XCTAssertTrue(summary.contains(" stage=accepted outcome=no_picture failure=capture_failed "), summary) + XCTAssertNil(manager.pcGraphicsCard, "the next attempt starts without it") + } + + /// The PC counted video it sent before Windows refused its card, and no + /// picture came: the card still says why, and the summary keeps the + /// lost video, as the PC's does. + func testVideoSentBeforeTheRefusalIsStillCountedAsLost() async throws { + let host = FakeUDPHost() + var timing = fast + timing.connectTimeoutSeconds = 3 + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + host.sendHeartbeat(framesSent: 5) + host.sendHeartbeat(notice: Heartbeat.noticeUnsupportedGPU, framesSent: 5) + try await until("the PC says why") { manager.pcGraphicsCard == .screen } + try await until("timed out", seconds: 15) { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .pcGraphicsCard(.screen)) + let summary = logged(manager, "session_summary v=1 role=ipad") ?? "" + XCTAssertTrue(summary.contains(" outcome=no_picture failure=no_video "), summary) + } + + /// The same in Second screen mode, after the second screen failed. + func testThePCsSecondScreenModeKeepsItsOwnCard() async throws { + let host = FakeUDPHost() + var timing = fast + timing.connectTimeoutSeconds = 3 + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + host.sendHeartbeat(notice: Heartbeat.noticeSecondScreenUnsupportedGPU) + try await until("the PC says why") { manager.pcGraphicsCard == .secondScreen } + try await until("timed out", seconds: 15) { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .pcGraphicsCard(.secondScreen)) + } + func testTheFirstFrameLineNamesTheSessionAndTheWait() async throws { let host = FakeUDPHost() let manager = try await streaming(host) diff --git a/ios/EternalMonitorTests/NoVideoFallbackTests.swift b/ios/EternalMonitorTests/NoVideoFallbackTests.swift new file mode 100644 index 00000000..78f362c5 --- /dev/null +++ b/ios/EternalMonitorTests/NoVideoFallbackTests.swift @@ -0,0 +1,573 @@ +import Combine +import XCTest +import os +@testable import EternalMonitor + +/// Pairings kept in memory. +private final class MemoryPairings: PairingStorage { + var values: [String: Data] = [:] + func records() throws -> [PairingRecord] { values.map { PairingRecord(account: $0.key, token: $0.value) } } + func save(account: String, token: Data) throws { values[account] = token } + func remove(account: String?) throws { + if let account { values.removeValue(forKey: account) } else { values.removeAll() } + } +} + +/// A PC that accepts the session and sends video, over a path that drops +/// large datagrams or all of them: the iPad asks for smaller ones, then +/// tries the PC's other address. +@MainActor +final class NoVideoFallbackTests: XCTestCase { + nonisolated private static let token = Data(17...32) + /// The other address the first fake signs; the manager dials it at the + /// second fake. + nonisolated private static let otherAddress = Data([192, 0, 2, 7]) + nonisolated private static let bothAddresses = [Data([127, 0, 0, 1]), otherAddress] + + override func setUp() async throws { + let store = RecentConnectionStore.shared + let saved = store.connections + let defaults = ["recentConnections", "lastHost", "lastPort", "noVideoPaths", "noVideoSizes"] + .map { ($0, UserDefaults.standard.object(forKey: $0)) } + addTeardownBlock { @MainActor in + store.connections = saved + for (key, value) in defaults { UserDefaults.standard.set(value, forKey: key) } + } + UserDefaults.standard.removeObject(forKey: "noVideoPaths") + UserDefaults.standard.removeObject(forKey: "noVideoSizes") + store.connections = [] + } + + private func until(_ description: String, seconds: Double = 60, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + /// A manager whose steps come 0.3 s apart, paired with "Test PC" at + /// `pc`, reaching every address directly, and dialling the other + /// address the PC signs at `other`. + private func manager(pairedAt pc: FakeUDPHost, other: FakeUDPHost? = nil, + connectTimeout: Double = 60) throws -> (ConnectionManager, PairingStore) { + let pairings = PairingStore(storage: MemoryPairings()) + try pairings.save(token: Self.token, hostName: "Test PC", address: PairingStore.address(host: "127.0.0.1", port: pc.port)) + var timing = ConnectionManager.Timing(connectTimeoutSeconds: connectTimeout, reconnectGraceSeconds: 0.1, + reconnectBackoffSeconds: { _ in 0.05 }) + timing.noVideoStepSeconds = 0.3 + let manager = ConnectionManager(timing: timing, pairingStore: pairings) + manager.homeAddress = { _ in true } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "127.0.0.1", netmask: "255.0.0.0") } + if let other { + manager.dialAddress = { host, port in host == "192.0.2.7" ? ("127.0.0.1", other.port) : (host, port) } + } + return (manager, pairings) + } + + /// Every line the manager logs, across attempts (each clears the list). + private func lines(_ manager: ConnectionManager) -> (() -> [String], AnyCancellable) { + var lines: [String] = [] + let sink = manager.$diagnostics.sink { if let line = $0.last?.message { lines.append(line) } } + return ({ lines }, sink) + } + + // MARK: - The steps + + /// Each step waits for frames the PC says it sent after the last one, + /// and none comes while the PC sends nothing. + func testStepsWaitForVideoThePCSaysItSent() { + var steps = NoVideoFallback(wait: 1.5) + XCTAssertNil(steps.next(at: 100)) + steps.heard(framesSent: 0, at: 0) + XCTAssertNil(steps.next(at: 100), "a PC that sent nothing yet") + steps.heard(framesSent: 3, at: 100) + XCTAssertNil(steps.next(at: 101)) + XCTAssertEqual(steps.next(at: 101.5), .smallerPackets(1200)) + XCTAssertNil(steps.next(at: 110)) + steps.heard(framesSent: 3, at: 110) + XCTAssertNil(steps.next(at: 120), "nothing sent since the step") + steps.heard(framesSent: 4, at: 120) + steps.heard(framesSent: 9, at: 121) + XCTAssertNil(steps.next(at: 121.4)) + XCTAssertEqual(steps.next(at: 121.5), .smallerPackets(576)) + steps.heard(framesSent: 12, at: 122) + XCTAssertEqual(steps.next(at: 123.5), .otherAddress) + steps.heard(framesSent: 20, at: 124) + XCTAssertNil(steps.next(at: 200), "then no more") + } + + // MARK: - The size kept for a path + + /// The route `manager(pairedAt:)` gives, and the one of another network. + nonisolated private static let homeRoute = MovedPC.Route(interface: "en0", direct: true, address: "127.0.0.1", + netmask: "255.0.0.0") + nonisolated private static let otherRoute = MovedPC.Route(interface: "en0", direct: true, address: "10.0.0.5", + netmask: "255.255.255.0") + + private func sizeKey(_ pc: FakeUDPHost, pairing: Data = NoVideoFallbackTests.token, + route: MovedPC.Route = NoVideoFallbackTests.homeRoute) -> NoVideoSizes.Key { + NoVideoSizes.Key(address: .init(host: "127.0.0.1", port: pc.port), pairing: pairing, + network: NoVideoSizes.network(route)) + } + + /// A kept size answers only for its address, pairing and network, for a + /// week from when it was needed, and goes with `forget` on any network. + func testAKeptSizeIsBoundToItsAddressPairingNetworkAndWeek() { + let pc = FakeUDPHost(), elsewhere = FakeUDPHost() + let at = Date(timeIntervalSince1970: 1_800_000_000) + let key = sizeKey(pc) + NoVideoSizes.remember(576, for: key, now: at) + XCTAssertEqual(NoVideoSizes.size(for: key, now: at), 576) + XCTAssertEqual(NoVideoSizes.size(for: key, now: at.addingTimeInterval(NoVideoSizes.lifetime - 1)), 576) + XCTAssertNil(NoVideoSizes.size(for: key, now: at.addingTimeInterval(NoVideoSizes.lifetime)), "a week on") + XCTAssertNil(NoVideoSizes.size(for: key, now: at.addingTimeInterval(-1)), "a clock set back") + XCTAssertNil(NoVideoSizes.size(for: sizeKey(pc, pairing: Data(2...17)), now: at), "another pairing") + XCTAssertNil(NoVideoSizes.size(for: sizeKey(elsewhere), now: at), "another address") + XCTAssertNil(NoVideoSizes.size(for: sizeKey(pc, route: Self.otherRoute), now: at), "another network") + XCTAssertNil(NoVideoSizes.size(for: NoVideoSizes.Key(address: key.address, pairing: key.pairing, + network: NoVideoSizes.network(nil)), now: at), "no route") + // Needed again later: the week starts over, at the newer size. + NoVideoSizes.remember(1200, for: key, now: at.addingTimeInterval(86_400)) + XCTAssertEqual(NoVideoSizes.size(for: key, now: at.addingTimeInterval(NoVideoSizes.lifetime)), 1200) + // Each network keeps its own size; `forget` drops them all. + let other = sizeKey(pc, route: Self.otherRoute) + NoVideoSizes.remember(576, for: other, now: at.addingTimeInterval(86_400)) + XCTAssertEqual(NoVideoSizes.size(for: key, now: at.addingTimeInterval(86_400)), 1200) + XCTAssertEqual(NoVideoSizes.size(for: other, now: at.addingTimeInterval(86_400)), 576) + XCTAssertFalse(NoVideoSizes.forget(sizeKey(pc, pairing: Data(2...17)))) + XCTAssertTrue(NoVideoSizes.forget(other)) + XCTAssertNil(NoVideoSizes.size(for: key, now: at.addingTimeInterval(86_400))) + XCTAssertNil(NoVideoSizes.size(for: other, now: at.addingTimeInterval(86_400))) + } + + /// The size a path needed is kept: the next session at that address + /// asks for it as soon as the PC accepts, with no steps, and streams. + func testTheNextSessionStartsWithTheSizeThePathNeeded() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let at = Date(timeIntervalSince1970: 1_800_000_000) + manager.now = { at } + // The app's own wait: a slow runner's picture at the kept size comes + // well before any further step. + manager.timing.noVideoStepSeconds = 1.5 + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let frame = try FakeUDPHost.keyframe() + var runs: [FakeUDPHost.PathRun] = [] + for session in 1...2 { + let streaming = OSAllocatedUnfairLock(initialState: false) + pc.forgetClient() + manager.connect(host: "127.0.0.1", port: pc.port) + let served = Task.detached { + pc.servePath(limit: 1000, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("session \(session) streaming") { manager.state == .connected } + streaming.withLock { $0 = true } + runs.append(await served.value) + manager.cancel() + } + XCTAssertEqual(runs[0].sizesAsked, [1200, 576]) + XCTAssertEqual(NoVideoSizes.size(for: sizeKey(pc), now: at), 576) + XCTAssertEqual(runs[1].sizesAsked, [576], "asked at once, once") + let steps = logged().filter { $0.hasPrefix("no_video_fallback ") } + XCTAssertEqual(steps, ["no_video_fallback step=smaller_packets size=1200 result=no_video", + "no_video_fallback step=smaller_packets size=576 result=video", + "no_video_fallback step=remembered_size size=576 result=asked"]) + } + + /// Each network the iPad reaches the PC over keeps its own size: one + /// path needed 576 bytes, another 1,200. Back on the first, the session + /// asks for 576 at once. + func testEachNetworkKeepsItsOwnSize() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let at = Date(timeIntervalSince1970: 1_800_000_000) + manager.now = { at } + manager.timing.noVideoStepSeconds = 1.5 + let frame = try FakeUDPHost.keyframe() + var runs: [FakeUDPHost.PathRun] = [] + // Network A drops datagrams over 1,000 bytes, network B over 1,250. + for (route, limit) in [(Self.homeRoute, 1000), (Self.otherRoute, 1250), (Self.homeRoute, 1000)] { + manager.localRoute = { _, _ in route } + let streaming = OSAllocatedUnfairLock(initialState: false) + pc.forgetClient() + manager.connect(host: "127.0.0.1", port: pc.port) + let served = Task.detached { + pc.servePath(limit: limit, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("streaming over \(route.address)") { manager.state == .connected } + streaming.withLock { $0 = true } + runs.append(await served.value) + manager.cancel() + } + XCTAssertEqual(runs[0].sizesAsked, [1200, 576]) + XCTAssertEqual(runs[1].sizesAsked, [1200]) + XCTAssertEqual(runs[2].sizesAsked, [576], "network A's size, at once") + XCTAssertEqual(NoVideoSizes.size(for: sizeKey(pc), now: at), 576) + XCTAssertEqual(NoVideoSizes.size(for: sizeKey(pc, route: Self.otherRoute), now: at), 1200) + } + + /// A size kept for this address over another network isn't used, and + /// once the PC's own size gives a picture, it is forgotten. + func testAKeptSizeIsForgottenOnceThePCsOwnSizeWorks() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoSizes.remember(576, for: sizeKey(pc, route: Self.otherRoute), now: Date()) + let streaming = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: .max, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("streaming") { manager.state == .connected } + try await until("forgotten", seconds: 10) { + logged().contains("no_video_fallback step=remembered_size result=forgotten") + } + streaming.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, []) + XCTAssertNil(NoVideoSizes.size(for: sizeKey(pc, route: Self.otherRoute), now: Date())) + XCTAssertFalse(NoVideoSizes.forget(sizeKey(pc))) + } + + /// A week after the size was needed, the next session starts at the + /// PC's own size again; where the path still needs less, the steps find + /// it and the week starts over. + func testAnExpiredSizeIsNotUsed() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let needed = Date(timeIntervalSince1970: 1_800_000_000) + let later = needed.addingTimeInterval(NoVideoSizes.lifetime + 60) + manager.now = { later } + NoVideoSizes.remember(576, for: sizeKey(pc), now: needed) + let streaming = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: 1000, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("streaming") { manager.state == .connected } + streaming.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, [1200, 576]) + XCTAssertEqual(NoVideoSizes.size(for: sizeKey(pc), now: later.addingTimeInterval(NoVideoSizes.lifetime - 120)), 576) + } + + /// A PC that doesn't take smaller sizes is asked for none, a kept size + /// or not, and the kept size stays as it was. + func testAnOlderPCIsNeverAskedForAKeptSize() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let at = Date() + NoVideoSizes.remember(576, for: sizeKey(pc), now: at) + let streaming = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: .max, frame: frame, hostCaps: 0, countFrames: false) { streaming.withLock { $0 } } + } + try await until("streaming") { manager.state == .connected } + try await Task.sleep(nanoseconds: 500_000_000) + streaming.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, []) + XCTAssertFalse(logged().contains { $0.hasPrefix("no_video_fallback ") }, logged().joined(separator: "\n")) + XCTAssertEqual(NoVideoSizes.size(for: sizeKey(pc), now: at), 576) + } + + // MARK: - Connecting + + /// Datagrams over 1,000 bytes never arrive; only each frame's short last + /// fragment does. The iPad asks for 1,200 bytes, then 576, and streams. + func testAPathThatDropsLargeDatagramsStreamsWithSmallerOnes() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let streaming = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: 1000, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("streaming") { manager.state == .connected } + streaming.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, [1200, 576]) + XCTAssertEqual(run.identifyAnswers, 0, "smaller datagrams were enough") + let steps = logged().filter { $0.hasPrefix("no_video_fallback ") } + XCTAssertEqual(steps, ["no_video_fallback step=smaller_packets size=1200 result=no_video", + "no_video_fallback step=smaller_packets size=576 result=video"]) + } + + /// The first address carries no video at all. Asked there, the PC signs + /// another of its addresses; at that one it shows again that it holds + /// the pairing before the token goes there, and the iPad streams from it + /// with no tap. Connecting to the first address later goes there first. + func testAnAddressThatDropsAllVideoMovesToTheVerifiedOtherAddress() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, pairings) = try manager(pairedAt: first, other: other) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let moved = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: first.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + first.servePath(limit: 0, frame: frame, hostCaps: HelloAck.hostCapMediaSize, + addresses: Self.bothAddresses) { moved.withLock { $0 } } + } + let accepted = await Task.detached { + other.answerIdentify(token: Self.token) && other.acceptSession(authToken: Self.token) + }.value + XCTAssertTrue(accepted, "the iPad never checked and tried the other address") + moved.withLock { $0 = true } + let run = await served.value + try await until("handshake there") { manager.hostInfo != nil && manager.debugState.port == other.port } + other.sendKeyframe(frame) + try await until("streaming") { manager.state == .connected } + + XCTAssertEqual(run.sizesAsked, [1200, 576]) + XCTAssertGreaterThan(run.identifyAnswers, 0) + XCTAssertFalse(other.helloBeforeIdentify, "nothing with the token before the PC showed it is the same there") + XCTAssertEqual(other.lastAuthToken, Self.token) + let all = logged() + let steps = all.filter { $0.hasPrefix("no_video_fallback ") } + XCTAssertEqual(steps, ["no_video_fallback step=smaller_packets size=1200 result=no_video", + "no_video_fallback step=smaller_packets size=576 result=no_video", + "no_video_fallback step=other_address result=moved"]) + XCTAssertTrue(all.contains("Disconnect state=connecting reason=noVideoPath"), all.joined(separator: "\n")) + let summary = all.first { $0.hasPrefix("session_summary ") } ?? "" + XCTAssertTrue(summary.contains(" stage=accepted outcome=no_picture failure=no_video fallback=none end=superseded "), summary) + + // The working address is the saved one now; the first keeps its + // pairing, as the PC still answers there. + XCTAssertEqual(RecentConnectionStore.shared.connections.first?.port, other.port) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: first.port)) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: other.port)), Self.token) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) + XCTAssertEqual(UserDefaults.standard.integer(forKey: "lastPort"), Int(other.port)) + + // Connecting to the first address later goes to the other first, + // once the PC there shows again that it holds the pairing. + manager.cancel() + manager.connect(host: "127.0.0.1", port: first.port) + async let atFirst = Task.detached { first.received(within: 1) }.value + let answered = await Task.detached { + other.answerIdentify(token: Self.token) && other.acceptSession(authToken: Self.token) + }.value + XCTAssertTrue(answered, "the address that showed video goes first") + XCTAssertFalse(other.helloBeforeIdentify, "nothing with the token before the proof") + let firstTypes = await atFirst + XCTAssertFalse(firstTypes.contains(.hello2), "\(firstTypes)") + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=tried"), logged().joined(separator: "\n")) + + // Where nothing at the other address shows the proof, the attempt + // goes to the first, and the other is forgotten. + manager.cancel() + manager.connect(host: "127.0.0.1", port: first.port) + let backAtFirst = await Task.detached { first.helloAttempts(within: 8) }.value + XCTAssertGreaterThan(backAtFirst, 0) + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=not_confirmed"), + logged().joined(separator: "\n")) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token)) + } + + /// The remembered address belongs to another PC now: one with pairing + /// on, which can't make this pairing's proof, or one with pairing off, + /// which would take any HELLO2 and doesn't answer the identity check. + /// The token never goes there, the pairing stays, and the attempt goes + /// to the address tapped. + func testARememberedAddressThatIsNowAnotherPCNeverGetsTheToken() async throws { + for pairingOn in [true, false] { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, pairings) = try manager(pairedAt: first) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: first.port), + to: .init(host: "127.0.0.1", port: other.port), pairing: Self.token) + manager.connect(host: "127.0.0.1", port: first.port) + async let atOther = Task.detached { () -> [Wire.PacketType] in + if pairingOn, !other.answerIdentify(token: Data(2...17)) { return [] } + return other.received(within: 4) + }.value + let accepted = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(accepted, "\(pairingOn): the tapped address got no HELLO2") + XCTAssertEqual(first.lastAuthToken, Self.token, "\(pairingOn)") + let types = await atOther + XCTAssertFalse(types.contains(.hello2), "\(pairingOn): \(types)") + XCTAssertFalse(other.helloBeforeIdentify, "\(pairingOn)") + if !pairingOn { XCTAssertEqual(Set(types), [.identify]) } + try await until("handshake at the tapped address") { manager.hostInfo != nil } + XCTAssertEqual(manager.debugState.port, first.port) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token)) + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=not_confirmed"), "\(pairingOn)") + } + // An entry kept for another pairing is never used for this one. + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: 1), to: .init(host: "127.0.0.1", port: 2), + pairing: Data(2...17)) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: 1, pairing: Self.token)) + XCTAssertEqual(NoVideoPaths.better(than: "127.0.0.1", port: 1, pairing: Data(2...17))?.port, 2) + } + + /// A remembered address that is outside this iPad's own network now + /// gets nothing: no identity check, no HELLO2. The attempt goes to the + /// address tapped, and Try again goes there too. + func testARememberedAddressOffThisNetworkIsNeverAsked() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, pairings) = try manager(pairedAt: first) + defer { manager.cancel() } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "10.0.0.5", netmask: "255.255.255.0") } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: first.port), + to: .init(host: "127.0.0.1", port: other.port), pairing: Self.token) + manager.connect(host: "127.0.0.1", port: first.port) + let accepted = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(accepted) + XCTAssertEqual(first.lastAuthToken, Self.token) + let types = await Task.detached { other.received(within: 1) }.value + XCTAssertEqual(types, []) + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=not_local"), logged().joined(separator: "\n")) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token)) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) + } + + /// The PC at the remembered address passed its proof, then the attempt + /// there failed: HELLO2 went unanswered, the session showed no frame, + /// or the PC that answered had another name. The entry is forgotten, + /// the attempt goes to the address tapped at once, and so does the + /// next tap, with no identity check at the remembered address. + func testARememberedAddressThatFailsAfterItsProofIsRetired() async throws { + for failure in ["no_reply", "no_picture", "other_pc"] { + let first = FakeUDPHost(), other = FakeUDPHost() + // Well past HELLO2's 2 s of retries, so an unanswered HELLO2 always + // gives up before the connect timeout, even on a slow runner. + let (manager, _) = try manager(pairedAt: first, connectTimeout: 6) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: first.port), + to: .init(host: "127.0.0.1", port: other.port), pairing: Self.token) + manager.connect(host: "127.0.0.1", port: first.port) + let atOther = await Task.detached { () -> Bool in + guard other.answerIdentify(token: Self.token) else { return false } + switch failure { + case "no_picture": return other.acceptSession(authToken: Self.token) + case "other_pc": return other.acceptSession(hostName: "Other PC", authToken: Data(repeating: 0, count: 16)) + default: return other.helloAttempts(within: 1) > 0 + } + }.value + XCTAssertTrue(atOther, "\(failure): the proof passed and HELLO2 went there") + let atFirst = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(atFirst, "\(failure): then to the address tapped") + try await until("\(failure): handshake at the address tapped", seconds: 20) { + manager.hostInfo != nil && manager.debugState.port == first.port + } + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=\(failure)"), logged().joined(separator: "\n")) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token), failure) + _ = await Task.detached { other.received(within: 0.5) }.value + + manager.cancel() + manager.connect(host: "127.0.0.1", port: first.port) + let again = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(again, "\(failure): the next tap goes to the address tapped") + let types = await Task.detached { other.received(within: 0.5) }.value + XCTAssertFalse(types.contains(.identify) || types.contains(.hello2), "\(failure): \(types)") + } + } + + /// The other address the PC signs is outside this iPad's own network: + /// it is never asked, as for a PC Bonjour moved there. + func testAnOtherAddressOffThisNetworkIsNeverAsked() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, _) = try manager(pairedAt: first, other: other, connectTimeout: 8) + defer { manager.cancel() } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "10.0.0.5", netmask: "255.255.255.0") } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let ended = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: first.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + first.servePath(limit: 0, frame: frame, hostCaps: HelloAck.hostCapMediaSize, + addresses: Self.bothAddresses) { ended.withLock { $0 } } + } + try await until("asked the PC for its addresses", seconds: 20) { + logged().contains("no_video_fallback step=other_address result=none") + } + let types = await Task.detached { other.received(within: 1) }.value + XCTAssertEqual(types, []) + try await until("timed out", seconds: 20) { manager.state == .disconnected } + ended.withLock { $0 = true } + let run = await served.value + XCTAssertGreaterThan(run.identifyAnswers, 0) + } + + /// The other address the PC signs answers without the pairing's proof, + /// as a forwarder or another device would: no HELLO2 and no token go + /// there, and the attempt ends as before. + func testAnOtherAddressThatDoesNotShowItIsTheSamePCIsLeftAlone() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, _) = try manager(pairedAt: first, other: other, connectTimeout: 8) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let ended = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: first.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + first.servePath(limit: 0, frame: frame, hostCaps: HelloAck.hostCapMediaSize, + addresses: Self.bothAddresses) { ended.withLock { $0 } } + } + let types = await Task.detached { () -> [Wire.PacketType] in + guard other.answerIdentify(token: Data(2...17)) else { return [] } + return other.received(within: 3) + }.value + XCTAssertFalse(types.contains(.hello2), "\(types)") + XCTAssertFalse(other.helloBeforeIdentify) + try await until("timed out", seconds: 20) { manager.state == .disconnected } + ended.withLock { $0 = true } + _ = await served.value + XCTAssertEqual(manager.problem, .noPicture) + XCTAssertTrue(logged().contains("no_video_fallback step=other_address result=none"), logged().joined(separator: "\n")) + XCTAssertNil(UserDefaults.standard.data(forKey: "noVideoPaths")) + } + + /// A PC that doesn't offer smaller datagrams, as every PC before this + /// version: the iPad asks for nothing new and asks no identity, and the + /// attempt times out as it always did. + func testAnOlderPCIsLeftAsBefore() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc, connectTimeout: 3) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let ended = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: 0, frame: frame, hostCaps: 0, countFrames: false) { ended.withLock { $0 } } + } + try await until("timed out", seconds: 20) { manager.state == .disconnected } + ended.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, []) + XCTAssertEqual(run.identifyAnswers, 0) + XCTAssertEqual(manager.problem, .noPicture) + let all = logged() + XCTAssertFalse(all.contains { $0.hasPrefix("no_video_fallback ") }, all.joined(separator: "\n")) + XCTAssertTrue(all.contains { $0.hasPrefix("Session accepted by 127.0.0.1:") && $0.contains("no video datagrams arrived") }) + } +} diff --git a/ios/EternalMonitorTests/OldPCTests.swift b/ios/EternalMonitorTests/OldPCTests.swift new file mode 100644 index 00000000..8d737852 --- /dev/null +++ b/ios/EternalMonitorTests/OldPCTests.swift @@ -0,0 +1,572 @@ +import Darwin +import XCTest +@testable import EternalMonitor + +/// Telling a PC older than 0.3.4 from a current one, and the Connect +/// screen's card that asks to update it. +@MainActor +final class OldPCTests: XCTestCase { + /// HELLO_ACKs recorded from hosts built at the tags v0.3.2, v0.3.3 and + /// v0.3.4 (macOS, synthetic capture, pairing off, PC name OLD-PC), + /// answering this app's HELLO2. 0.3.2 and 0.3.3 answer with host caps + /// 0x000F, 0.3.4 with 0x002F. + static let recordedAcks = [ + "0.3.2": "454d0211000039002fe161dd010000000002443322112fe161dde803f401b80b010000000005d0023c000000c0e1e400064f4c442d5043000000000000000000000000000000000f00", + "0.3.3": "454d0211000039001d64ed60010000000002443322111d64ed60e803f401b80b010000000005d0023c000000c0e1e400064f4c442d5043000000000000000000000000000000000f00", + "0.3.4": "454d0211000039000c8c4e43010000000002443322110c8c4e43e803f401b80b010000000005d0023c000000c0e1e400064f4c442d5043000000000000000000000000000000002f00", + ] + + /// The same hosts' Bonjour TXT records, as `dns-sd -L` showed them. + static func recordedTXT(_ version: String) -> Data { + NetService.data(fromTXTRecord: ["version": Data(version.utf8), "proto": Data("2".utf8), + "platform": Data("other".utf8)]) + } + + private struct NotAnAck: Error {} + + private func recordedAck(_ version: String) throws -> HelloAck { + let hex = Array(try XCTUnwrap(Self.recordedAcks[version])) + let data = Data(stride(from: 0, to: hex.count, by: 2).map { UInt8(String(hex[$0...$0 + 1]), radix: 16)! }) + guard case .helloAck(let ack) = Wire.parseControl(data)?.message else { throw NotAnAck() } + return ack + } + + func testRecordedRepliesFromTaggedPCs() throws { + for (version, old) in [("0.3.2", true), ("0.3.3", true), ("0.3.4", false)] { + let ack = try recordedAck(version) + XCTAssertEqual(ack.status, .ok, version) + XCTAssertEqual(ack.hostName, "OLD-PC", version) + // Over USB or a typed address: the HELLO_ACK alone. + XCTAssertEqual(OldPC.isOld(announced: nil, hostCaps: ack.hostCaps), old, version) + // Found on the network: the TXT record alone, and both. + let announced = PCVersions.version(fromTXT: Self.recordedTXT(version)) + XCTAssertEqual(announced, version) + XCTAssertEqual(OldPC.isOld(announced: announced, hostCaps: nil), old, version) + XCTAssertEqual(OldPC.isOld(announced: announced, hostCaps: ack.hostCaps), old, version) + } + } + + /// Before 0.3.4 a PC's optional capabilities (audio, USB, keyboard, pen) + /// vary with its hardware and settings. None makes it current; the + /// numbering bit always does. + func testOnlyTheNumberingBitMarksACurrentPC() { + for caps in UInt16(0)..<32 { + XCTAssertEqual(OldPC.isOld(announced: nil, hostCaps: caps), true, "caps \(caps)") + XCTAssertEqual(OldPC.isOld(announced: nil, hostCaps: caps | HelloAck.hostCapContiguousFrameSeq), false, "caps \(caps)") + } + XCTAssertEqual(OldPC.isOld(announced: nil, hostCaps: 0xFFFF), false, "bits a later PC adds") + } + + /// Never call a current PC old: whatever says current wins, and a beta's + /// or a missing version decides nothing by itself. + func testACurrentPCIsNeverOld() { + let current = HelloAck.hostCapNack | HelloAck.hostCapContiguousFrameSeq + XCTAssertEqual(OldPC.isOld(announced: "0.3.2", hostCaps: current), false, "another PC's version at a reused address") + XCTAssertEqual(OldPC.isOld(announced: "0.3.4", hostCaps: HelloAck.hostCapNack), false) + XCTAssertEqual(OldPC.isOld(announced: "0.3.5", hostCaps: nil), false) + XCTAssertEqual(OldPC.isOld(announced: "1.0.0", hostCaps: nil), false) + XCTAssertNil(OldPC.isOld(announced: "0.3.4-beta.20261029.1", hostCaps: nil)) + XCTAssertNil(OldPC.isOld(announced: "0.3.3-beta.20261020.2", hostCaps: nil), "a beta from before the bump") + XCTAssertNil(OldPC.isOld(announced: "", hostCaps: nil)) + XCTAssertNil(OldPC.isOld(announced: nil, hostCaps: nil)) + XCTAssertEqual(OldPC.isOld(announced: "0.3.3-beta.20261020.2", hostCaps: HelloAck.hostCapNack), true) + } + + func testThePCsNameFromItsBonjourName() { + XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor (OLD-PC)"), "OLD-PC") + XCTAssertEqual(OldPC.pcName(bonjour: "Studio"), "Studio") + XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor ()"), "EternalMonitor ()") + XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor (Lab) (2)"), "Lab (2)") + XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor (Lab (Home)) (12)"), "Lab (Home) (12)") + XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor (Lab (Home))"), "Lab (Home)") + } + + private func defaults() -> UserDefaults { + let name = "OldPCTests-\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: name)! + addTeardownBlock { UserDefaults().removePersistentDomain(forName: name) } + return defaults + } + + private let tokenA = Data(1...16), tokenB = Data(17...32), tokenC = Data(33...48) + private let oldCaps: UInt16 = 0x000F, currentCaps: UInt16 = 0x002F + + private func service(_ pc: String, at address: String?, version: String?) -> DiscoveredHost { + DiscoveredHost(name: "EternalMonitor (\(pc))", address: address, port: 9876, finding: address == nil, version: version) + } + + /// At most once per PC every three days, on an injected clock, across + /// launches; a clock that was ahead doesn't hold it back. + func testTheCardShowsOncePerPCEveryThreeDays() { + let store = defaults() + let start = Date(timeIntervalSince1970: 1_000_000) + var now = start + var lines: [String] = [] + let make = { OldPCPrompt(defaults: store, now: { now }, log: { lines.append($0) }) } + var prompt = make() + let studio = OldPCPrompt.key(hostName: "Studio", pairingToken: tokenA, service: nil) + + prompt.show() + XCTAssertNil(prompt.card, "nothing seen yet") + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + XCTAssertNil(prompt.card, "waits for the Connect screen") + prompt.show() + XCTAssertEqual(prompt.card, OldPCPrompt.Card(key: studio, pc: "Studio", version: nil)) + XCTAssertEqual(lines, ["old_pc_prompt shown=true version=unknown"]) + prompt.show() + XCTAssertEqual(lines.count, 1, "one line per showing") + + // Another old PC waits while a card is up, and isn't kept. + prompt.discovered([service("Office", at: "192.0.2.20", version: "0.3.2")]) + prompt.close() + XCTAssertNil(prompt.card) + prompt.show() + XCTAssertNil(prompt.card) + + // The same PC again, later that day and in a new launch: no card. + now = start + 3_600 + prompt = make() + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertNil(prompt.card) + + // Another PC has its own three days. Known only by its TXT record. + prompt.discovered([service("Office", at: "192.0.2.20", version: "0.3.2")]) + prompt.show() + XCTAssertEqual(prompt.card, OldPCPrompt.Card(key: OldPCPrompt.serviceKey("EternalMonitor (Office)"), + pc: "Office", version: "0.3.2")) + XCTAssertEqual(lines.last, "old_pc_prompt shown=true version=0.3.2") + prompt.close() + + now = start + OldPCPrompt.interval - 1 + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertNil(prompt.card, "a second short of three days") + now = start + OldPCPrompt.interval + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertEqual(prompt.card?.key, studio) + prompt.close() + + // Shown "tomorrow" by a clock that was ahead, then the clock is fixed. + now = start + 30 * 86_400 + prompt.discovered([service("Lab", at: "192.0.2.30", version: "0.3.3")]) + prompt.show() + prompt.close() + now = start + 29 * 86_400 + prompt.discovered([service("Lab", at: "192.0.2.30", version: "0.3.3")]) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Lab") + XCTAssertEqual(lines.count, 5) + } + + /// r1 P1: a PC whose accepted HELLO_ACK proved it current gets no card + /// from a Bonjour record that says otherwise afterwards, in this launch + /// or a later one. + func testProofThatAPCIsCurrentOutlastsItsSession() { + let store = defaults() + var lines: [String] = [] + var prompt = OldPCPrompt(defaults: store, now: { Date() }, log: { lines.append($0) }) + // Tapped in the list: the HELLO_ACK says current, then the TXT 0.3.3. + prompt.connecting(service: "EternalMonitor (Studio)", address: "192.0.2.10") + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: currentCaps, address: "192.0.2.10") + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.3")]) + prompt.show() + XCTAssertNil(prompt.card) + + prompt = OldPCPrompt(defaults: store, now: { Date() }, log: { lines.append($0) }) + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.3")]) + prompt.show() + XCTAssertNil(prompt.card, "a new launch") + + // Typed address, while the search lists that PC there; no pairing. + prompt.discovered([service("Lab", at: "192.0.2.11", version: nil)]) + prompt.answered(hostName: "Lab", pairingToken: Data(count: 16), hostCaps: currentCaps, address: "192.0.2.11") + prompt.discovered([service("Lab", at: "192.0.2.11", version: "0.3.2")]) + prompt.show() + XCTAssertNil(prompt.card) + XCTAssertEqual(lines, []) + + // Only those PCs: another old one still gets its card. + prompt.discovered([service("Lab", at: "192.0.2.11", version: "0.3.2"), + service("Office", at: "192.0.2.12", version: "0.3.3")]) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Office") + XCTAssertEqual(lines, ["old_pc_prompt shown=true version=0.3.3"]) + } + + /// r1 P2: a version stays with the service whose TXT record announced it. + /// A service first seen over IPv6 keeps its own version when its IPv4 + /// address resolves, and a service at an address another PC used before + /// never takes that PC's version. + func testAVersionStaysWithItsOwnService() async throws { + var addr = sockaddr_in6() + addr.sin6_len = UInt8(MemoryLayout.size) + addr.sin6_family = sa_family_t(AF_INET6) + inet_pton(AF_INET6, "fe80::1", &addr.sin6_addr) + let ipv6 = withUnsafeBytes(of: &addr) { Data($0) } + + // An old PC was found at 192.0.2.40 earlier and left. + PCVersions.record("0.3.2", name: "EternalMonitor (Gone)", address: "192.0.2.40") + addTeardownBlock { await MainActor.run { PCVersions.forget(name: "EternalMonitor (Gone)", address: "192.0.2.40") } } + let scanner = NetworkScanner() + scanner.resolveIPv4 = { _ in "192.0.2.40" } + let beta = DiscoveredHost.resolved(name: "EternalMonitor (New)", port: 9876, addresses: [ipv6], + version: "0.3.4-beta.20261107.1") + XCTAssertNil(beta.address) + scanner.merge(beta) + scanner.lookUpIPv4(hostName: "new.local.", service: beta) + let deadline = Date().addingTimeInterval(10) + while scanner.hosts.first?.address == nil, Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertNotNil(scanner.hosts.first?.address, "the IPv4 lookup didn't answer") + XCTAssertEqual(scanner.hosts.map(\.address), ["192.0.2.40"]) + XCTAssertEqual(scanner.hosts.first?.version, "0.3.4-beta.20261107.1") + + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + prompt.discovered(scanner.hosts) + prompt.show() + XCTAssertNil(prompt.card, "a beta at a reused address") + prompt.discovered([service("Quiet", at: "192.0.2.40", version: nil)]) + prompt.show() + XCTAssertNil(prompt.card, "no TXT version at a reused address") + + // The same path keeps an old version too. + let old = DiscoveredHost.resolved(name: "EternalMonitor (Old)", port: 9876, addresses: [ipv6], version: "0.3.2") + scanner.resolveIPv4 = { _ in "192.0.2.41" } + scanner.merge(old) + scanner.lookUpIPv4(hostName: "old.local.", service: old) + let oldDeadline = Date().addingTimeInterval(10) + while scanner.hosts.first(where: { $0.name == old.name })?.address == nil, Date() < oldDeadline { + try await Task.sleep(nanoseconds: 10_000_000) + } + prompt.discovered(scanner.hosts) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Old") + XCTAssertEqual(prompt.card?.version, "0.3.2") + } + + /// r1 P3: a PC is its pairing, not its name. Two PCs with one name each + /// get a card; a renamed PC doesn't get another; Bonjour's collision + /// suffix names a second PC, and a session links it to its pairing. + func testAPCIsItsPairingNotItsName() { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Studio") + prompt.close() + prompt.answered(hostName: "Studio", pairingToken: tokenB, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertNotNil(prompt.card, "another PC with the same name") + prompt.close() + prompt.answered(hostName: "Studio Renamed", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertNil(prompt.card, "the first PC, renamed") + + // Two PCs called Lab on one network: Bonjour lists the second as "Lab (2)". + let lab = service("Lab", at: "192.0.2.50", version: "0.3.2") + let lab2 = DiscoveredHost(name: "EternalMonitor (Lab) (2)", address: "192.0.2.51", port: 9876, version: "0.3.2") + prompt.discovered([lab]) + prompt.show() + XCTAssertEqual(prompt.card?.key, OldPCPrompt.serviceKey(lab.name)) + prompt.close() + prompt.discovered([lab, lab2]) + prompt.show() + XCTAssertEqual(prompt.card?.key, OldPCPrompt.serviceKey(lab2.name), "the second Lab is another PC") + prompt.close() + // Connecting to the second links its service to its pairing: no new card. + prompt.connecting(service: lab2.name, address: "192.0.2.51") + prompt.answered(hostName: "Lab", pairingToken: tokenC, hostCaps: oldCaps, address: "192.0.2.51") + prompt.show() + XCTAssertNil(prompt.card, "shown already, as its service") + let memory = prompt.memory + let keyC = OldPCPrompt.key(hostName: "Lab", pairingToken: tokenC, service: nil) + XCTAssertEqual(memory.services[lab2.name], keyC) + XCTAssertNotNil(memory.pcs[keyC]?.shownAt) + XCTAssertNil(memory.pcs[OldPCPrompt.serviceKey(lab2.name)]) + XCTAssertFalse(memory.pcs.keys.contains { $0.contains(tokenC.map { String(format: "%02x", $0) }.joined()) }, + "the token itself isn't kept") + } + + /// A current PC never gets a card, and its answer takes a waiting or + /// showing card for that PC away. + func testACurrentPCTakesItsCardAway() { + var lines: [String] = [] + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { lines.append($0) }) + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.4")]) + prompt.answered(hostName: "Den", pairingToken: tokenC, hostCaps: currentCaps, address: nil) + prompt.show() + XCTAssertNil(prompt.card) + + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.3")]) + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.4")]) + prompt.show() + XCTAssertNil(prompt.card, "updated before the card showed") + + prompt.discovered([service("Office", at: "192.0.2.20", version: "0.3.2")]) + prompt.show() + XCTAssertNotNil(prompt.card) + prompt.connecting(service: "EternalMonitor (Office)", address: "192.0.2.20") + prompt.answered(hostName: "Office", pairingToken: tokenB, hostCaps: currentCaps, address: "192.0.2.20") + XCTAssertNil(prompt.card) + XCTAssertEqual(lines, ["old_pc_prompt shown=true version=0.3.2"]) + } + + /// r2: a PC streaming over USB has no address to link, so its session + /// links the service every PC registers, "EternalMonitor ()". + /// The same PC over USB and in the search gets one card, in either + /// order, and a proof over USB holds against its stale TXT record. + func testAUSBSessionAndItsOwnServiceAreOnePC() { + for usbFirst in [true, false] { + var lines: [String] = [] + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { lines.append($0) }) + let steps: [() -> Void] = [ + { prompt.answered(hostName: "Studio", pairingToken: self.tokenA, hostCaps: self.oldCaps, address: nil) }, + { prompt.discovered([self.service("Studio", at: "192.0.2.10", version: "0.3.3")]) }, + ] + for step in usbFirst ? steps : steps.reversed() { + step() + prompt.show() + prompt.close() + } + XCTAssertEqual(lines.count, 1, "usbFirst \(usbFirst): \(lines)") + } + + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: currentCaps, address: nil) + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.3")]) + prompt.show() + XCTAssertNil(prompt.card, "proved current over USB") + } + + /// r3: a name links a service only when the search lists exactly one + /// for it. B and A both answer as "Lab"; Bonjour lists B as "Lab (2)". + /// B proving current over USB never hides A's card, in either order. + func testANameLinksOnlyAnUnambiguousService() { + let a = DiscoveredHost(name: "EternalMonitor (Lab)", address: "192.0.2.70", port: 9876, version: "0.3.2") + let b = DiscoveredHost(name: "EternalMonitor (Lab) (2)", address: "192.0.2.71", port: 9876, version: "0.3.4") + for usbFirst in [true, false] { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + let steps: [() -> Void] = [ + { prompt.answered(hostName: "Lab", pairingToken: self.tokenB, hostCaps: self.currentCaps, address: nil) }, + { prompt.discovered([a, b]) }, + ] + for step in usbFirst ? steps : steps.reversed() { step() } + prompt.show() + XCTAssertEqual(prompt.card?.key, OldPCPrompt.serviceKey(a.name), "usbFirst \(usbFirst): A's card") + } + + // With A gone, B's own suffixed service is the only one: it is B's, + // and B's proof holds against its stale TXT record. + let staleB = DiscoveredHost(name: b.name, address: b.address, port: 9876, version: "0.3.3") + for usbFirst in [true, false] { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + let steps: [() -> Void] = [ + { prompt.answered(hostName: "Lab", pairingToken: self.tokenB, hostCaps: self.currentCaps, address: nil) }, + { prompt.discovered([staleB]) }, + ] + for step in usbFirst ? steps : steps.reversed() { step() } + prompt.show() + XCTAssertNil(prompt.card, "usbFirst \(usbFirst): B proved current") + XCTAssertEqual(prompt.memory.services, [:], "kept for this launch only") + } + + XCTAssertTrue(OldPCPrompt.isOwnService("EternalMonitor (Lab)", of: "Lab")) + XCTAssertTrue(OldPCPrompt.isOwnService("EternalMonitor (Lab) (2)", of: "Lab")) + XCTAssertTrue(OldPCPrompt.isOwnService("EternalMonitor (Lab) (12)", of: "Lab")) + for other in ["EternalMonitor (Lab2)", "EternalMonitor (Lab) ()", "EternalMonitor (Lab) (x)", + "EternalMonitor (Lab) (2) (3)", "EternalMonitor (Lab)(2)", "Lab"] { + XCTAssertFalse(OldPCPrompt.isOwnService(other, of: "Lab"), other) + } + } + + /// r4: one old PC used over USB and seen by the search gets one card + /// every three days, also in later launches, which have no name link. + /// The search's record may come before the card or after it. + func testAPCLinkedByNameGetsOneCardAcrossLaunches() { + let lab = service("Lab", at: "192.0.2.90", version: "0.3.3") + for listedFirst in [true, false] { + let store = defaults() + let start = Date(timeIntervalSince1970: 1_000_000) + var now = start + var lines: [String] = [] + let make = { OldPCPrompt(defaults: store, now: { now }, log: { lines.append($0) }) } + var prompt = make() + if listedFirst { prompt.discovered([lab]) } + prompt.answered(hostName: "Lab", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertEqual(prompt.card?.key, OldPCPrompt.key(hostName: "Lab", pairingToken: tokenA, service: nil)) + prompt.close() + if !listedFirst { prompt.discovered([lab]) } + prompt.show() + XCTAssertNil(prompt.card) + + now = start + 86_400 + prompt = make() + prompt.discovered([lab]) + prompt.show() + XCTAssertNil(prompt.card, "listedFirst \(listedFirst): day 1, the search only") + + now = start + OldPCPrompt.interval + prompt = make() + prompt.discovered([lab]) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Lab", "listedFirst \(listedFirst): day 3") + XCTAssertEqual(lines.count, 2, "listedFirst \(listedFirst): \(lines)") + XCTAssertEqual(prompt.memory.services, [:]) + } + } + + /// Bonjour updates that change nothing write nothing. + func testUnchangedUpdatesWriteNothing() { + final class Counting: UserDefaults { + var writes = 0 + override func set(_ value: Any?, forKey defaultName: String) { + writes += 1 + super.set(value, forKey: defaultName) + } + } + let name = "OldPCTests-\(UUID().uuidString)" + let store = Counting(suiteName: name)! + addTeardownBlock { UserDefaults().removePersistentDomain(forName: name) } + let prompt = OldPCPrompt(defaults: store, now: { Date() }, log: { _ in }) + let lab = service("Lab", at: "192.0.2.90", version: "0.3.3") + prompt.discovered([lab]) + prompt.show() + prompt.close() + prompt.answered(hostName: "Lab", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + let writes = store.writes + for _ in 0..<5 { prompt.discovered([lab]) } + XCTAssertEqual(store.writes, writes) + } + + /// r3: a name a PC gave up is free for another PC: after a rename in the + /// same launch, and in any later launch. + func testANameIsNotKeptForAnotherPC() { + let store = defaults() + var prompt = OldPCPrompt(defaults: store, now: { Date() }, log: { _ in }) + prompt.discovered([service("Lab", at: "192.0.2.80", version: nil)]) + prompt.answered(hostName: "Lab", pairingToken: tokenA, hostCaps: currentCaps, address: nil) + prompt.discovered([service("Lab", at: "192.0.2.80", version: "0.3.3")]) + prompt.show() + XCTAssertNil(prompt.card, "linked while A is the only Lab") + XCTAssertEqual(prompt.memory.services, [:]) + + // A is renamed; another PC takes the name "Lab". + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: currentCaps, address: nil) + prompt.discovered([service("Lab", at: "192.0.2.81", version: "0.3.2")]) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Lab", "same launch, after the rename") + prompt.close() + + let other = defaults() + prompt = OldPCPrompt(defaults: other, now: { Date() }, log: { _ in }) + prompt.discovered([service("Lab", at: "192.0.2.80", version: nil)]) + prompt.answered(hostName: "Lab", pairingToken: tokenA, hostCaps: currentCaps, address: nil) + prompt = OldPCPrompt(defaults: other, now: { Date() }, log: { _ in }) + prompt.discovered([service("Lab", at: "192.0.2.81", version: "0.3.2")]) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Lab", "a later launch") + } + + /// r2: a link never takes a service, or a proof, from another PC. Here + /// the search still lists A at an address B answers at now. + func testALinkNeverTakesAnotherPCsProof() { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + let a = service("A", at: "192.0.2.60", version: nil) + prompt.discovered([a]) + prompt.answered(hostName: "A", pairingToken: Data(count: 16), hostCaps: currentCaps, address: "192.0.2.60") + let proof = OldPCPrompt.serviceKey(a.name) + XCTAssertEqual(prompt.memory.pcs[proof]?.current, true) + + prompt.answered(hostName: "B", pairingToken: tokenB, hostCaps: oldCaps, address: "192.0.2.60") + let memory = prompt.memory + XCTAssertEqual(memory.pcs[proof]?.current, true, "A's proof stays") + XCTAssertNil(memory.services[a.name], "A's service isn't B's") + prompt.show() + XCTAssertEqual(prompt.card?.pc, "B", "B is old") + prompt.close() + prompt.discovered([service("A", at: "192.0.2.61", version: "0.3.3")]) + prompt.show() + XCTAssertNil(prompt.card, "A proved current") + } + + /// Only the last few PCs are kept, with their links. + func testTheMemoryIsBounded() { + var now = Date(timeIntervalSince1970: 1_000_000) + let prompt = OldPCPrompt(defaults: defaults(), now: { now }, log: { _ in }) + for index in 0..<(OldPCPrompt.kept + 4) { + now += 60 + prompt.connecting(service: "EternalMonitor (PC \(index))", address: "192.0.2.\(index)") + prompt.answered(hostName: "PC \(index)", pairingToken: Data(repeating: UInt8(index + 1), count: 16), + hostCaps: oldCaps, address: "192.0.2.\(index)") + prompt.show() + prompt.close() + } + let memory = prompt.memory + XCTAssertEqual(memory.pcs.count, OldPCPrompt.kept) + XCTAssertEqual(memory.services.count, OldPCPrompt.kept) + XCTAssertNil(memory.services["EternalMonitor (PC 0)"]) + XCTAssertNotNil(memory.services["EternalMonitor (PC \(OldPCPrompt.kept + 3))"]) + + // One PC found under many names keeps a few links. + for index in 0..<(OldPCPrompt.kept * 3) { + prompt.connecting(service: "EternalMonitor (Alias \(index))", address: "198.51.100.\(index)") + prompt.answered(hostName: "Alias", pairingToken: tokenA, hostCaps: oldCaps, address: "198.51.100.\(index)") + } + XCTAssertEqual(prompt.memory.services.count, OldPCPrompt.kept * 2) + } + + /// One session with `host`, answered with `status` and `caps`. + private func session(_ manager: ConnectionManager, _ host: FakeUDPHost, status: HelloStatus = .ok, + caps: UInt16) async throws { + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession(status: status, hostCaps: caps) }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + let deadline = Date().addingTimeInterval(10) + if status == .ok { + while manager.hostInfo == nil, Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertEqual(manager.hostInfo?.hostCaps, caps) + } else { + try await Task.sleep(nanoseconds: 300_000_000) + XCTAssertNil(manager.hostInfo) + } + manager.cancel() + } + + /// A session's accepted HELLO_ACK reaches the prompt: an old PC's card + /// waits for the Connect screen, a current PC's doesn't. + func testASessionsAnswerReachesThePrompt() async throws { + for (caps, old) in [(oldCaps, true), (currentCaps, false)] { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + let manager = ConnectionManager() + manager.oldPCPrompt = prompt + addTeardownBlock { await MainActor.run { manager.cancel() } } + try await session(manager, FakeUDPHost(), caps: caps) + prompt.show() + XCTAssertEqual(prompt.card?.pc, old ? "Test PC" : nil, "caps \(caps)") + XCTAssertEqual(prompt.card?.key.hasPrefix("pair:"), old ? true : nil) + XCTAssertEqual(prompt.memory.pcs.values.map(\.current), [!old]) + } + } + + /// Refusals and not-streaming replies may carry no caps at all (the PC + /// keeps them within the HELLO2's size), so they say nothing about the + /// PC's age either way. + func testRefusalsAndNotStreamingRepliesCountForNothing() async throws { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + let manager = ConnectionManager() + manager.oldPCPrompt = prompt + addTeardownBlock { await MainActor.run { manager.cancel() } } + for status in [HelloStatus.stopped, .busy, .unauthorized] { + try await session(manager, FakeUDPHost(), status: status, caps: 0) + } + prompt.show() + XCTAssertNil(prompt.card) + XCTAssertEqual(prompt.memory, OldPCPrompt.Memory()) + + // The same PC accepting a session is what counts. + try await session(manager, FakeUDPHost(), caps: oldCaps) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Test PC") + } +} diff --git a/ios/EternalMonitorTests/PCNotStreamingTests.swift b/ios/EternalMonitorTests/PCNotStreamingTests.swift new file mode 100644 index 00000000..3d5a6013 --- /dev/null +++ b/ios/EternalMonitorTests/PCNotStreamingTests.swift @@ -0,0 +1,176 @@ +import XCTest +@testable import EternalMonitor + +/// A PC that answers but isn't streaming (Stop clicked, restarting, or +/// stopped after repeated errors) says so. The iPad shows why, keeps asking +/// slowly without spending reconnect attempts, and connects as soon as the +/// PC streams again. +@MainActor +final class PCNotStreamingTests: XCTestCase { + private func until(_ description: String, seconds: Double = 30, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + /// Longer than the slow HELLO2 retry, so a waiting attempt never times + /// out between two answers, even on a loaded machine. + private let timing = ConnectionManager.Timing( + connectTimeoutSeconds: 5, reconnectGraceSeconds: 0.1, reconnectBackoffSeconds: { _ in 0.05 }) + + func testAStoppedPCIsWaitedForAndConnectsOnceItStarts() async throws { + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let answered = await Task.detached { host.acceptSession(status: .stopped) }.value + XCTAssertTrue(answered, "the iPad never sent HELLO2") + let advertised = (host.lastHello?.featureCaps ?? 0) & Hello2.featureStandbyStatus + XCTAssertNotEqual(advertised, 0, "the PC must know this iPad understands the answer") + XCTAssertGreaterThanOrEqual(host.lastHelloSize, Hello2.standbyLength, "padded so any answer fits") + let firstNonce = host.lastNonce + try await until("told the PC isn't streaming") { manager.problem == .pcNotStreaming(.stopped) } + XCTAssertEqual(manager.problem?.title, "Your PC isn't streaming") + XCTAssertEqual(manager.problem?.detail, + "Click Start streaming in EternalMonitor on the PC. This iPad connects as soon as it starts.") + XCTAssertEqual(manager.state, .connecting) + + // It keeps asking, every two seconds rather than four times a + // second, with the same HELLO2, past the connect timeout. The gaps + // are between the HELLO2s the PC received, by their arrival times, + // from those sent after the iPad showed it heard the answer: a quick + // retry sent before that may still have been on its way. + let told = Date() + var askedAt: [Date] = [] + for _ in 0..<6 where askedAt.count < 3 { + let again = await Task.detached { host.acceptSession(status: .stopped) }.value + XCTAssertTrue(again, "the iPad stopped asking") + XCTAssertEqual(host.lastNonce, firstNonce) + let arrival = try XCTUnwrap(host.lastHelloArrival, "no kernel receive time for the HELLO2") + if arrival > told { askedAt.append(arrival) } + } + XCTAssertEqual(askedAt.count, 3, "the iPad stopped asking") + for (earlier, later) in zip(askedAt.dropLast(), askedAt.dropFirst()) { + XCTAssertGreaterThan(later.timeIntervalSince(earlier), 1.5, "asking too often") + } + XCTAssertEqual(manager.state, .connecting) + XCTAssertEqual(manager.problem, .pcNotStreaming(.stopped)) + + // Start on the PC: the same attempt connects. + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted) + XCTAssertEqual(host.lastNonce, firstNonce) + try await until("handshake") { manager.hostInfo != nil } + XCTAssertNil(manager.problem) + } + + func testAFailedPCSaysWhyInWords() async throws { + for (token, detail) in [ + ("display_changing", "Windows is changing the display. Click Start streaming in EternalMonitor on the PC " + + "to try again. This iPad connects as soon as it starts."), + ("a_reason_from_a_newer_pc", "Click Start streaming in EternalMonitor on the PC to try again. " + + "This iPad connects as soon as it starts.") + ] { + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + manager.connect(host: "127.0.0.1", port: host.port) + let answered = await Task.detached { host.acceptSession(status: .failed, reason: token) }.value + XCTAssertTrue(answered) + try await until("told why") { manager.problem == .pcNotStreaming(.failed(token)) } + XCTAssertEqual(manager.problem?.title, "Your PC stopped streaming after repeated errors") + XCTAssertEqual(manager.problem?.detail, detail) + XCTAssertEqual(manager.problem?.actions, []) + XCTAssertEqual(manager.state, .connecting) + manager.cancel() + } + } + + /// The PC was stopped, then closed: once it stops answering, the iPad + /// says it didn't answer, not that no picture arrived. + func testAPCThatStopsAnsweringTimesOutAsNoReply() async throws { + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let answered = await Task.detached { host.acceptSession(status: .stopped) }.value + XCTAssertTrue(answered) + try await until("timed out") { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .noReply(usb: false)) + } + + /// A restarting PC doesn't run the reconnect cycle out: the cycle's + /// attempt waits, and connects when the stream is back. + func testARestartingPCDoesNotSpendReconnectAttempts() async throws { + let previous = UserDefaults.standard.object(forKey: "autoReconnect") + UserDefaults.standard.set(true, forKey: "autoReconnect") + defer { UserDefaults.standard.set(previous, forKey: "autoReconnect") } + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let first = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(first) + host.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming") { manager.state == .connected } + + // The PC's stream fails and it restarts. + host.sendBye(.error) + try await until("reconnecting") { manager.reconnectAttempt == 1 } + var answeredAt: [Date] = [] + for _ in 0..<4 { + let answered = await Task.detached { host.acceptSession(status: .restarting) }.value + XCTAssertTrue(answered, "the iPad stopped asking") + answeredAt.append(Date()) + } + for (earlier, later) in zip(answeredAt.dropLast(), answeredAt.dropFirst()) { + XCTAssertGreaterThan(later.timeIntervalSince(earlier), 0.7, "asking too often") + } + XCTAssertEqual(manager.problem, .pcNotStreaming(.restarting)) + XCTAssertEqual(manager.problem?.title, "Your PC is restarting the stream…") + XCTAssertEqual(manager.reconnectAttempt, 1, "waiting must not spend attempts") + + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted) + try await until("handshake") { manager.hostInfo != nil } + host.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming again") { manager.state == .connected } + XCTAssertNil(manager.problem) + XCTAssertEqual(manager.reconnectAttempt, 0) + } + + /// The attempt keeps its nonce while it waits, so a Restarting answer + /// can arrive after the OK (Wi-Fi reordered it, or a late copy). Once the + /// session is up, it changes nothing: no card, no new timeout. + func testARestartingAnswerAfterTheSessionStartedIsIgnored() async throws { + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let waiting = await Task.detached { host.acceptSession(status: .restarting) }.value + XCTAssertTrue(waiting) + try await until("waiting") { manager.problem == .pcNotStreaming(.restarting) } + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted) + try await until("handshake") { manager.hostInfo != nil } + XCTAssertNil(manager.problem) + + host.sendAck(nonce: host.lastNonce, status: .restarting) + try await Task.sleep(nanoseconds: 500_000_000) + XCTAssertNil(manager.problem, "a late Restarting answer brought the card back") + host.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming") { manager.state == .connected } + XCTAssertNil(manager.problem) + } + + func testCancelWhileWaitingClearsTheMessage() async throws { + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + manager.connect(host: "127.0.0.1", port: host.port) + let answered = await Task.detached { host.acceptSession(status: .stopped) }.value + XCTAssertTrue(answered) + try await until("waiting") { manager.problem == .pcNotStreaming(.stopped) } + manager.cancel() + XCTAssertEqual(manager.state, .disconnected) + XCTAssertNil(manager.problem) + } +} diff --git a/ios/EternalMonitorTests/PCNoticeTests.swift b/ios/EternalMonitorTests/PCNoticeTests.swift new file mode 100644 index 00000000..ef994bd3 --- /dev/null +++ b/ios/EternalMonitorTests/PCNoticeTests.swift @@ -0,0 +1,119 @@ +import Combine +import XCTest +@testable import EternalMonitor + +/// What a session's PC tells this iPad about itself beyond the stream. +@MainActor +final class PCNoticeTests: XCTestCase { + private func until(_ description: String, seconds: Double = 10, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + /// One session with `host`, whose HELLO_ACK carries `caps`. + private func session(_ manager: ConnectionManager, _ host: FakeUDPHost, caps: UInt16) async throws { + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession(hostCaps: caps) }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + try await until("accepted") { manager.hostInfo?.hostCaps == caps } + manager.cancel() + } + + /// A PC that couldn't reach Apple's USB service (Apple Devices not + /// running or not installed) leaves the bit out of its HELLO_ACK. It is + /// kept for that PC after the session, and cleared by the same PC's + /// next session once the service is back. + func testTheSightingClearsWhenThePCsServiceRecovers() async throws { + let manager = ConnectionManager() + addTeardownBlock { await MainActor.run { manager.cancel() } } + XCTAssertNil(manager.appleDevicesUnreachable, "nothing known before a session") + let host = FakeUDPHost() + let before = Date() + try await session(manager, host, caps: 0) + let sighting = try XCTUnwrap(manager.appleDevicesUnreachable) + XCTAssertEqual(sighting.target, "127.0.0.1:\(host.port)") + XCTAssertTrue(sighting.applies(now: Date())) + XCTAssertGreaterThanOrEqual(sighting.at, before) + + try await session(manager, host, caps: HelloAck.hostCapUSB) + XCTAssertNil(manager.appleDevicesUnreachable) + } + + /// Connecting to another PC drops what the last one said, at once and + /// whether or not the new one answers. + func testSwitchingPCsDropsTheOtherPCsSighting() async throws { + let manager = ConnectionManager() + addTeardownBlock { await MainActor.run { manager.cancel() } } + let first = FakeUDPHost() + try await session(manager, first, caps: 0) + XCTAssertNotNil(manager.appleDevicesUnreachable) + + let second = FakeUDPHost() + manager.connect(host: "127.0.0.1", port: second.port) + XCTAssertNil(manager.appleDevicesUnreachable, "another PC's sighting") + manager.cancel() + XCTAssertNil(manager.appleDevicesUnreachable) + + // The same PC again keeps it until that PC answers. + try await session(manager, first, caps: 0) + manager.connect(host: "127.0.0.1", port: first.port) + XCTAssertNotNil(manager.appleDevicesUnreachable) + manager.cancel() + } + + /// A sighting is one sample: it applies from when it was seen until it + /// expires, and not under a clock set back. + func testASightingGoesStale() { + let at = Date(timeIntervalSince1970: 1_000_000) + let until = at + ConnectionManager.AppleDevicesSighting.lifetime + let sighting = ConnectionManager.AppleDevicesSighting(target: "192.168.1.20:9876", at: at, until: until) + XCTAssertTrue(sighting.applies(now: at)) + XCTAssertTrue(sighting.applies(now: until - 1)) + XCTAssertFalse(sighting.applies(now: until)) + XCTAssertFalse(sighting.applies(now: at - 60), "a clock set back") + } + + /// The Connect screen may sit idle after the session: the published + /// hint goes by itself when it expires, with nothing else happening. + func testThePublishedHintExpiresByItself() async throws { + let seen = Date(timeIntervalSince1970: 2_000_000) + var timing = ConnectionManager.Timing() + timing.appleDevicesNoteSeconds = 0.5 + let manager = ConnectionManager(timing: timing) + manager.now = { seen } + addTeardownBlock { await MainActor.run { manager.cancel() } } + var published: [ConnectionManager.AppleDevicesSighting?] = [] + let watch = manager.$appleDevicesUnreachable.dropFirst().sink { published.append($0) } + defer { watch.cancel() } + + try await session(manager, FakeUDPHost(), caps: 0) + let sighting = try XCTUnwrap(manager.appleDevicesUnreachable) + XCTAssertEqual(sighting.at, seen) + XCTAssertEqual(sighting.until, seen + 0.5) + try await until("expired", seconds: 10) { manager.appleDevicesUnreachable == nil } + XCTAssertEqual(published.last, .some(nil), "the change was published: \(published)") + XCTAssertEqual(USBStatusText.describe("USB: waiting for a cable", + appleDevicesUnreachable: manager.appleDevicesUnreachable != nil), + USBStatusText.describe("USB: waiting for a cable")) + } + + /// Back in the foreground the sighting is checked against the clock, + /// since a suspended app's timer may have missed its time. + func testComingBackChecksTheClock() async throws { + var clock = Date(timeIntervalSince1970: 3_000_000) + let manager = ConnectionManager() + manager.now = { clock } + addTeardownBlock { await MainActor.run { manager.cancel() } } + try await session(manager, FakeUDPHost(), caps: 0) + let sighting = try XCTUnwrap(manager.appleDevicesUnreachable) + XCTAssertEqual(sighting.until, clock + ConnectionManager.AppleDevicesSighting.lifetime) + + clock = sighting.until - 60 + manager.recheckAppleDevicesSighting() + XCTAssertEqual(manager.appleDevicesUnreachable, sighting, "a minute left") + clock = sighting.until + manager.recheckAppleDevicesSighting() + XCTAssertNil(manager.appleDevicesUnreachable) + } +} diff --git a/ios/EternalMonitorTests/PCReturnTests.swift b/ios/EternalMonitorTests/PCReturnTests.swift new file mode 100644 index 00000000..a7ac524c --- /dev/null +++ b/ios/EternalMonitorTests/PCReturnTests.swift @@ -0,0 +1,444 @@ +import Combine +import XCTest +@testable import EternalMonitor + +/// Bonjour's list as a test hands it over. A search that starts lists +/// what Bonjour knows already, as a real one does in its first moments. +@MainActor +private final class FakePCBrowser: PCBrowser { + var onHosts: (([DiscoveredHost]) -> Void)? + /// What Bonjour has now. + var current: [DiscoveredHost] = [] + private(set) var starts = 0 + private(set) var stops = 0 + var running: Bool { starts > stops } + func start() { + starts += 1 + onHosts?(current) + } + func stop() { stops += 1 } + func list(_ hosts: [DiscoveredHost]) { + current = hosts + onHosts?(hosts) + } +} + +/// The iPad shows "didn't answer" because the PC app wasn't running yet. +/// When the PC's service then shows up on the network, it connects once by +/// itself, while the card is up, in the foreground, for a few minutes, and +/// only to the PC that proves it holds the saved pairing. +@MainActor +final class PCReturnTests: XCTestCase { + nonisolated private static let token = Data(17...32) + private let start = Date(timeIntervalSince1970: 1_000_000) + + private func pc(_ name: String = "EternalMonitor (Test PC)", _ address: String? = "127.0.0.1", port: UInt16, + version: String? = "0.3.5") -> DiscoveredHost { + DiscoveredHost(name: name, address: address, port: port, version: version) + } + + // MARK: - Which listings are asked + + func testEachListingOfThePCGetsOneTry() { + var watch = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876) + let listed = pc(port: 9876) + XCTAssertTrue(watch.heard([listed]), "asked at once") + // Not proved yet: asked again, every few seconds. + XCTAssertFalse(watch.heard([listed])) + XCTAssertEqual(watch.candidates.first, listed) + watch.didTry(listed) + XCTAssertNil(watch.candidates.first, "one try per listing") + XCTAssertFalse(watch.heard([listed])) + XCTAssertNil(watch.candidates.first) + // It leaves (the app closed) and comes back. + XCTAssertFalse(watch.heard([])) + XCTAssertTrue(watch.heard([listed])) + XCTAssertEqual(watch.candidates.first, listed) + watch.didTry(listed) + // A change is a new listing, and the try goes where the PC is now. + XCTAssertTrue(watch.heard([pc(port: 9877)])) + XCTAssertEqual(watch.candidates.first?.port, 9877, "a new port") + watch.didTry(pc(port: 9877)) + XCTAssertTrue(watch.heard([pc(port: 9877, version: "0.3.6")])) + XCTAssertEqual(watch.candidates.first?.version, "0.3.6", "a new version") + } + + /// A listing that never answers stays to be asked, beside the others. + func testEveryUntriedListingIsAsked() { + var watch = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876) + let stale = pc("EternalMonitor (Another PC)", "192.0.2.20", port: 9876) + let moved = pc("EternalMonitor (Test PC)", "192.0.2.40", port: 9876) + XCTAssertTrue(watch.heard([stale, moved])) + XCTAssertEqual(watch.candidates, [stale, moved]) + XCTAssertTrue(watch.awaits(moved)) + watch.didTry(moved) + XCTAssertEqual(watch.candidates, [stale]) + XCTAssertFalse(watch.awaits(moved)) + XCTAssertFalse(watch.heard([stale])) + XCTAssertFalse(watch.awaits(moved), "no longer listed") + } + + func testOnlyThisPCCounts() { + var named = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876) + var unnamed = PCReturn(name: nil, host: "192.0.2.20", port: 9876) + let others = [pc("EternalMonitor (Other PC)", "192.0.2.30", port: 9876), + pc("EternalMonitor (Test PC 2)", "192.0.2.31", port: 9876), + pc("EternalMonitor (Other PC)", "192.0.2.20", port: 9900), + pc("EternalMonitor (Test PC)", nil, port: 9876)] + XCTAssertFalse(named.heard(others)) + XCTAssertFalse(unnamed.heard(others)) + XCTAssertNil(named.candidates.first) + XCTAssertNil(unnamed.candidates.first) + // By name at another address, or by the address tried. + XCTAssertTrue(named.heard(others + [pc("EternalMonitor (Test PC)", "192.0.2.40", port: 9876)])) + XCTAssertEqual(named.candidates.first?.address, "192.0.2.40") + XCTAssertTrue(unnamed.heard(others + [pc("EternalMonitor (Any)", "192.0.2.20", port: 9876)])) + XCTAssertEqual(unnamed.candidates.first?.name, "EternalMonitor (Any)") + } + + func testTheTimeLimitStartsWhenTheCardGoesUp() { + var watch = PCReturn(name: nil, host: "192.0.2.20", port: 9876) + XCTAssertFalse(watch.expired(at: start.addingTimeInterval(1_000), limit: 180), "not armed yet") + watch.arm(at: start.addingTimeInterval(10)) + watch.arm(at: start.addingTimeInterval(100)) + XCTAssertFalse(watch.expired(at: start.addingTimeInterval(189), limit: 180)) + XCTAssertTrue(watch.expired(at: start.addingTimeInterval(190), limit: 180), "a second card keeps the first limit") + } + + // MARK: - Connecting + + private final class Setup { + let manager: ConnectionManager + let browser: FakePCBrowser + let host: FakeUDPHost + let pairings: PairingStore + var lines: [String] = [] + var log: AnyCancellable? + var clock: Date + + init(manager: ConnectionManager, browser: FakePCBrowser, host: FakeUDPHost, pairings: PairingStore, clock: Date) { + self.manager = manager + self.browser = browser + self.host = host + self.pairings = pairings + self.clock = clock + } + + var tries: Int { lines.filter { $0 == "pc_return result=tried" }.count } + var attempts: Int { lines.filter { $0.hasPrefix("Starting connection attempt=") }.count } + var listing: DiscoveredHost { DiscoveredHost(name: "EternalMonitor (Test PC)", address: "127.0.0.1", port: host.port) } + } + + override func setUp() async throws { + let store = RecentConnectionStore.shared + let saved = store.connections + let defaults = ["recentConnections", "lastHost", "lastPort", "allowUSB", "autoReconnect"] + .map { ($0, UserDefaults.standard.object(forKey: $0)) } + // No cable listener in these tests. + UserDefaults.standard.set(false, forKey: "allowUSB") + addTeardownBlock { @MainActor in + store.connections = saved + for (key, value) in defaults { UserDefaults.standard.set(value, forKey: key) } + } + } + + /// A paired PC at loopback, saved under its name unless `named` is + /// false, that doesn't answer. Loopback counts as a home address here. + private func paired(named: Bool = true) throws -> Setup { + let host = FakeUDPHost() + let name = named ? "Test PC" : "" + RecentConnectionStore.shared.connections = [ + RecentConnection(host: "127.0.0.1", port: host.port, lastUsed: .now, isUSB: false, + name: named ? name : nil, found: false), + ] + let pairings = PairingStore(storage: MemoryPairingStorage()) + try pairings.save(token: Self.token, hostName: name, address: PairingStore.address(host: "127.0.0.1", port: host.port)) + let manager = ConnectionManager(timing: .init(connectTimeoutSeconds: 4, reconnectGraceSeconds: 0.1, + reconnectBackoffSeconds: { _ in 0.05 }, pcReturnDebounceSeconds: 0.1, + pcReturnProbeSeconds: 0.5), + pairingStore: pairings) + manager.homeAddress = { _ in true } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "127.0.0.1", netmask: "255.0.0.0") } + manager.findPCs = { _ in [] } + let browser = FakePCBrowser() + manager.makePCBrowser = { browser } + let setup = Setup(manager: manager, browser: browser, host: host, pairings: pairings, clock: start) + manager.now = { [unowned setup] in setup.clock } + setup.log = manager.$diagnostics.sink { [unowned setup] in + if let line = $0.last?.message { setup.lines.append(line) } + } + return setup + } + + /// The "didn't answer" card is up for `paired()`'s PC, and its watch armed. + private func noReply(named: Bool = true) async throws -> Setup { + let setup = try paired(named: named) + setup.manager.connect(host: "127.0.0.1", port: setup.host.port) + XCTAssertTrue(setup.browser.running, "the search runs from the attempt's start") + // Bonjour has nothing for the PC: its app isn't open. + try await until("didn't answer", seconds: 15) { setup.manager.problem == .noReply(usb: false) } + XCTAssertTrue(setup.manager.watchingForPC) + XCTAssertTrue(setup.browser.running, "the search runs while the card is up") + // Drop the unanswered HELLO2s, so only a new attempt's count. + _ = await Task.detached { [host = setup.host] in host.helloAttempts(within: 0.4) }.value + return setup + } + + private func until(_ description: String, seconds: Double, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + private func settle(_ seconds: Double = 1) async throws { + try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) + } + + /// The PC proves the pairing, then accepts and streams. As a real PC + /// does, it gives the pairing back only when it is new: not here. + private func stream(_ setup: Setup, from pc: FakeUDPHost) async throws { + let accepted = await Task.detached { + pc.answerIdentify(token: Self.token) && pc.acceptSession(authToken: Data(repeating: 0, count: 16)) + }.value + XCTAssertTrue(accepted, "no automatic attempt reached the PC") + XCTAssertFalse(pc.helloBeforeIdentify, "nothing with the pairing before the PC proved it holds it") + try await until("handshake", seconds: 15) { setup.manager.hostInfo != nil } + pc.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming", seconds: 30) { setup.manager.state == .connected } + } + + /// No answer; the PC app opens and its service shows up; the iPad + /// connects once, by itself, and streams. + func testThePCShowingUpConnectsOnceByItself() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + XCTAssertEqual(setup.attempts, 1) + setup.browser.list([setup.listing]) + // Updates while it waits count as the same appearance. + setup.browser.list([setup.listing]) + try await stream(setup, from: setup.host) + XCTAssertEqual(setup.tries, 1) + XCTAssertEqual(setup.attempts, 2) + XCTAssertFalse(setup.manager.watchingForPC) + XCTAssertFalse(setup.browser.running, "streaming ends the watch") + } + + /// The PC opened just after the attempt's HELLO2s stopped, before the + /// card: its listing is asked once the card is up. + func testAPCOpenedJustAfterTheHellosIsNotMissed() async throws { + let setup = try paired() + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: setup.host.port) + setup.browser.list([setup.listing]) + try await until("didn't answer", seconds: 15) { setup.manager.problem == .noReply(usb: false) } + _ = await Task.detached { [host = setup.host] in host.helloAttempts(within: 0.2) }.value + try await stream(setup, from: setup.host) + XCTAssertEqual(setup.tries, 1) + } + + /// The PC closed without a goodbye, so Bonjour still lists it, the + /// same as before, when its app opens again. The listing is asked + /// until the PC answers, with nothing sent that needs the pairing. + func testAPCStillListedFromBeforeIsAskedUntilItAnswers() async throws { + let setup = try paired() + defer { setup.manager.cancel() } + setup.browser.current = [setup.listing] + setup.manager.connect(host: "127.0.0.1", port: setup.host.port) + try await until("didn't answer", seconds: 15) { setup.manager.problem == .noReply(usb: false) } + _ = await Task.detached { [host = setup.host] in host.helloAttempts(within: 0.05) }.value + let asked = await Task.detached { [host = setup.host] in host.received(within: 6) }.value + XCTAssertGreaterThan(Set(asked).count, 0) + XCTAssertFalse(asked.contains(.hello2), "\(asked)") + XCTAssertGreaterThan(asked.filter { $0 == .identify }.count, 8, "asked more than once") + XCTAssertEqual(setup.tries, 0) + try await stream(setup, from: setup.host) + XCTAssertEqual(setup.tries, 1) + } + + /// Bonjour has the saved PC at a new port: the try goes there, after the + /// PC there proves the pairing, and the saved entry moves with it. + func testAPCBackAtANewPortIsTriedThere() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + let moved = FakeUDPHost() + setup.browser.list([DiscoveredHost(name: "EternalMonitor (Test PC)", address: "127.0.0.1", port: moved.port)]) + try await stream(setup, from: moved) + let atOld = await Task.detached { [host = setup.host] in host.helloAttempts(within: 0.3) }.value + XCTAssertEqual(atOld, 0, "nothing went to the old port") + XCTAssertEqual(RecentConnectionStore.shared.connections.first?.port, moved.port) + XCTAssertEqual(try setup.pairings.token(address: PairingStore.address(host: "127.0.0.1", port: moved.port)), Self.token) + } + + /// Another PC sits at the saved address and never answers; Bonjour has + /// the saved PC at a new port too, and that listing sorts after it. Both + /// are asked in turn, so the PC streams from its new port. + func testASilentListingDoesNotKeepThePCFromBeingAsked() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + let moved = FakeUDPHost() + let stale = DiscoveredHost(name: "EternalMonitor (Another PC)", address: "127.0.0.1", port: setup.host.port) + let pc = DiscoveredHost(name: "EternalMonitor (Test PC)", address: "127.0.0.1", port: moved.port) + setup.browser.list([stale, pc]) + try await stream(setup, from: moved) + let atStale = await Task.detached { [host = setup.host] in host.received(within: 0.3) }.value + XCTAssertFalse(atStale.contains(.hello2), "\(atStale)") + XCTAssertEqual(setup.tries, 1) + } + + /// Another PC took the saved address, with pairing off: it can't prove + /// the pairing, so nothing is sent to it. A session that gets past that + /// from a PC with another name, or that gives another pairing back, is + /// dropped before a picture, and the saved pairing stays. + func testADifferentPCAtTheSavedAddressGetsNoAutomaticSession() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + let other = DiscoveredHost(name: "EternalMonitor (Other PC)", address: "127.0.0.1", port: setup.host.port) + setup.browser.list([other]) + let types = await Task.detached { [host = setup.host] in host.received(within: 4) }.value + XCTAssertTrue(types.contains(.identify), "\(types)") + XCTAssertFalse(types.contains(.hello2), "\(types)") + XCTAssertEqual(setup.tries, 0) + XCTAssertEqual(setup.attempts, 1) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + + for (round, (name, pairing)) in [("Other PC", Data(repeating: 0, count: 16)), ("Test PC", Data(1...16))].enumerated() { + if round > 0 { + setup.browser.list([]) + setup.browser.list([other]) + } + let accepted = await Task.detached { [host = setup.host] in + host.answerIdentify(token: Self.token) && host.acceptSession(hostName: name, authToken: pairing) + }.value + XCTAssertTrue(accepted, name) + try await until("dropped: \(name)", seconds: 15) { + setup.lines.filter { $0 == "pc_return result=other_pc" }.count == round + 1 + } + XCTAssertEqual(setup.manager.state, .disconnected, name) + XCTAssertNil(setup.manager.hostInfo, name) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false), name) + } + XCTAssertEqual(try setup.pairings.token(address: PairingStore.address(host: "127.0.0.1", port: setup.host.port)), Self.token) + XCTAssertFalse(setup.lines.contains { $0.hasPrefix("Connected to") }) + } + + /// The automatic try gets no answer either: the card comes back and + /// waits for the PC to show up again. The same listing never brings + /// another try. + func testOneTryPerAppearance() async throws { + let setup = try await noReply(named: false) + defer { setup.manager.cancel() } + let listing = DiscoveredHost(name: "EternalMonitor (Unknown)", address: "127.0.0.1", port: setup.host.port) + setup.browser.list([listing]) + let proved = await Task.detached { [host = setup.host] in host.answerIdentify(token: Self.token) }.value + XCTAssertTrue(proved) + try await until("tried", seconds: 5) { setup.attempts == 2 } + try await until("didn't answer again", seconds: 15) { + setup.manager.state == .disconnected && setup.manager.problem == .noReply(usb: false) + } + XCTAssertTrue(setup.manager.watchingForPC, "still waiting for the PC") + // The same listing isn't asked again. + setup.browser.list([listing]) + let after = await Task.detached { [host = setup.host] in host.received(within: 2) }.value + XCTAssertFalse(after.contains(.identify), "\(after)") + XCTAssertEqual(setup.tries, 1) + XCTAssertEqual(setup.attempts, 2) + // It leaves and comes back: one more try. + setup.browser.list([]) + setup.browser.list([listing]) + let again = await Task.detached { [host = setup.host] in host.answerIdentify(token: Self.token) }.value + XCTAssertTrue(again) + try await until("tried again", seconds: 5) { setup.tries == 2 } + } + + func testNoTryAfterTheCardIsDismissed() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + setup.manager.problem = nil + XCTAssertFalse(setup.manager.watchingForPC) + XCTAssertFalse(setup.browser.running) + setup.browser.list([setup.listing]) + let asked = await Task.detached { [host = setup.host] in host.received(within: 1.5) }.value + XCTAssertEqual(asked, []) + XCTAssertEqual(setup.tries, 0) + } + + func testNoTryAfterTheTimeLimit() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + setup.clock = setup.clock.addingTimeInterval(ConnectionManager.Timing().pcReturnLimitSeconds) + setup.browser.list([setup.listing]) + try await settle() + XCTAssertEqual(setup.tries, 0) + XCTAssertFalse(setup.manager.watchingForPC) + XCTAssertFalse(setup.browser.running) + XCTAssertTrue(setup.lines.contains("pc_return result=expired")) + } + + /// Nothing starts in the background. Back in the foreground, a PC that + /// showed up meanwhile gets its try. + func testNoTryInTheBackground() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + setup.manager.handleAppBackgrounded() + XCTAssertFalse(setup.browser.running, "no search in the background") + setup.browser.current = [setup.listing] + let asked = await Task.detached { [host = setup.host] in host.received(within: 1.5) }.value + XCTAssertEqual(asked, [], "nothing asked in the background") + setup.manager.handleAppForegrounded() + XCTAssertTrue(setup.browser.running) + let proved = await Task.detached { [host = setup.host] in host.answerIdentify(token: Self.token) }.value + XCTAssertTrue(proved) + try await until("tried on return", seconds: 5) { setup.tries == 1 } + } + + /// An unpaired PC can't prove it is the same one: it isn't watched for. + func testAnUnpairedPCIsNotWatchedFor() async throws { + let setup = try paired() + defer { setup.manager.cancel() } + try setup.pairings.forgetAll() + setup.manager.connect(host: "127.0.0.1", port: setup.host.port) + XCTAssertEqual(setup.browser.starts, 0) + try await until("didn't answer", seconds: 15) { setup.manager.problem == .noReply(usb: false) } + XCTAssertFalse(setup.manager.watchingForPC) + XCTAssertEqual(setup.browser.starts, 0) + } + + func testPickingAnotherPCEndsTheWatch() async throws { + let setup = try await noReply() + let other = FakeUDPHost() + setup.manager.connect(host: "127.0.0.1", port: other.port) + defer { setup.manager.cancel() } + XCTAssertFalse(setup.browser.running) + setup.browser.list([setup.listing]) + try await settle() + XCTAssertEqual(setup.tries, 0) + } + + /// A PC that turns the automatic try down (busy here) ends the watch: + /// it never loops. + func testARefusalEndsTheWatch() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + setup.browser.list([setup.listing]) + let refused = await Task.detached { [host = setup.host] in + host.answerIdentify(token: Self.token) && host.acceptSession(status: .busy) + }.value + XCTAssertTrue(refused) + try await until("busy", seconds: 15) { setup.manager.problem == .pcBusy } + XCTAssertFalse(setup.browser.running) + setup.browser.list([]) + setup.browser.list([setup.listing]) + try await settle() + XCTAssertEqual(setup.tries, 1) + } +} + +/// Pairings kept in memory. +private final class MemoryPairingStorage: PairingStorage { + var values: [String: Data] = [:] + func records() throws -> [PairingRecord] { values.map { PairingRecord(account: $0.key, token: $0.value) } } + func save(account: String, token: Data) throws { values[account] = token } + func remove(account: String?) throws { + if let account { values.removeValue(forKey: account) } else { values.removeAll() } + } +} diff --git a/ios/EternalMonitorTests/PointerOverlayTests.swift b/ios/EternalMonitorTests/PointerOverlayTests.swift new file mode 100644 index 00000000..a8320e6a --- /dev/null +++ b/ios/EternalMonitorTests/PointerOverlayTests.swift @@ -0,0 +1,243 @@ +import XCTest +import os +@testable import EternalMonitor + +final class PointerOverlayTests: XCTestCase { + /// A 40x40 color image split as the PC splits it (1358 bytes a chunk). + private func chunks(id: UInt32, side: UInt16 = 40, fill: UInt8 = 7) -> [PointerShapeChunk] { + let total = PointerShapeChunk.imageLength(width: side, height: side, flags: 0) + let bytes = Data(repeating: fill, count: total) + return stride(from: 0, to: total, by: 1358).map { offset in + PointerShapeChunk(shapeId: id, sourceType: 2, flags: 0, width: side, height: side, hotX: 0, hotY: 0, + totalLen: UInt32(total), offset: UInt32(offset), + data: bytes.subdata(in: offset.. PointerState { + PointerState(flags: visible ? PointerState.visibleFlag : 0, shapeId: shape, x: x, y: 2, hostTimeUs: 0) + } + + func testAnImageCompletesFromChunksInAnyOrderOnce() { + let overlay = PointerOverlay() + var changes = 0 + overlay.onChange = { changes += 1 } + let parts = chunks(id: 1) + XCTAssertEqual(parts.count, 5) + for chunk in parts.dropFirst().reversed() { overlay.handle(chunk) } + overlay.handle(parts[4]) // a repeat adds nothing + XCTAssertNil(overlay.snapshot().image, "one chunk still missing") + XCTAssertEqual(overlay.completeShapeId, 0) + overlay.handle(parts[0]) + let image = try? XCTUnwrap(overlay.snapshot().image) + XCTAssertEqual(image?.id, 1) + XCTAssertEqual(image?.pixels.count, 40 * 40 * 4) + XCTAssertNil(image?.invertMask) + XCTAssertEqual(overlay.completeShapeId, 1) + XCTAssertEqual(changes, 1) + // The PC's resend of a held image changes nothing. + for chunk in parts { overlay.handle(chunk) } + XCTAssertEqual(overlay.counters.shapes, 1) + } + + func testANewImageReplacesTheOldButALateOlderChunkDoesNot() { + let overlay = PointerOverlay() + for chunk in chunks(id: 1) { overlay.handle(chunk) } + let newer = chunks(id: 2, fill: 9) + overlay.handle(newer[0]) + overlay.handle(chunks(id: 1)[1]) // a held image's late chunk + XCTAssertEqual(overlay.snapshot().image?.id, 1, "the old image stays until the new one is whole") + for chunk in newer.dropFirst() { overlay.handle(chunk) } + XCTAssertEqual(overlay.snapshot().image?.id, 2) + XCTAssertEqual(overlay.snapshot().image?.pixels.first, 9) + let third = chunks(id: 4) + overlay.handle(third[0]) + overlay.handle(chunks(id: 3)[0]) // older than the one assembling + for chunk in third.dropFirst() { overlay.handle(chunk) } + XCTAssertEqual(overlay.snapshot().image?.id, 4) + } + + /// Review r1: image 1's chunks arriving after image 2 completed brought + /// image 1 back. Older ids lose against the held image as well as the + /// one assembling, across the counter's wrap too. + func testAReorderedOlderImageNeverReplacesTheHeldOne() { + let overlay = PointerOverlay() + for chunk in chunks(id: 2, fill: 9) { overlay.handle(chunk) } + for chunk in chunks(id: 1, fill: 7) { overlay.handle(chunk) } + XCTAssertEqual(overlay.snapshot().image?.id, 2) + XCTAssertEqual(overlay.snapshot().image?.pixels.first, 9) + XCTAssertEqual(overlay.completeShapeId, 2) + XCTAssertEqual(overlay.counters.shapes, 1) + + let wrapped = PointerOverlay() + for chunk in chunks(id: 3) { wrapped.handle(chunk) } + for chunk in chunks(id: UInt32.max) { wrapped.handle(chunk) } + XCTAssertEqual(wrapped.snapshot().image?.id, 3, "max is older than 3 after the wrap") + for chunk in chunks(id: 4) { wrapped.handle(chunk) } + XCTAssertEqual(wrapped.snapshot().image?.id, 4, "a newer image still replaces it") + XCTAssertTrue(PointerOverlay.isNewer(1, than: UInt32.max)) + XCTAssertFalse(PointerOverlay.isNewer(UInt32.max, than: 1)) + } + + func testThePositionIsLatestWinsByMessageNumber() { + let overlay = PointerOverlay() + overlay.handle(state(10), msgSeq: 5) + overlay.handle(state(9), msgSeq: 4) // reordered on the way + XCTAssertEqual(overlay.snapshot().x, 10) + overlay.handle(state(11, visible: false), msgSeq: 6) + XCTAssertFalse(overlay.snapshot().visible) + overlay.handle(state(12), msgSeq: 6 &+ 0x7FFF_FFFF) + XCTAssertEqual(overlay.snapshot().x, 12, "numbers compare across a wrap") + XCTAssertEqual(overlay.counters.states, 3) + XCTAssertEqual(overlay.counters.staleStates, 1) + overlay.reset() + overlay.handle(state(1), msgSeq: 1) + XCTAssertEqual(overlay.snapshot().x, 1, "a new session numbers from 1") + XCTAssertNil(overlay.snapshot().image) + } + + func testTheInvertMaskExpandsToABytePerPixel() { + // 9x1: two mask bytes per row, bits 0 and 8 set. + let flags = PointerShapeChunk.invertFlag + let total = PointerShapeChunk.imageLength(width: 9, height: 1, flags: flags) + XCTAssertEqual(total, 9 * 4 + 2) + var bytes = [UInt8](repeating: 0, count: total) + bytes[36] = 0x80 + bytes[37] = 0x80 + let chunk = PointerShapeChunk(shapeId: 1, sourceType: 1, flags: flags, width: 9, height: 1, hotX: 0, + hotY: 0, totalLen: UInt32(total), offset: 0, data: Data(bytes)) + let mask = [UInt8](PointerOverlay.image(chunk, bytes: bytes).invertMask ?? Data()) + XCTAssertEqual(mask, [255, 0, 0, 0, 0, 0, 0, 0, 255]) + } + + // MARK: Geometry + + /// Taps on the drawn hotspot normalize to the hotspot's video pixel, in + /// Fit and Fill, for a wider and a taller picture. + func testThePointerIsDrawnWhereTouchesMapTo() { + let view = CGSize(width: 1194, height: 834) + for video in [CGSize(width: 3440, height: 1440), CGSize(width: 1080, height: 1920)] { + for fit in [PictureFit.fit, .fill] { + let (x, y, hotX, hotY): (Int32, Int32, CGFloat, CGFloat) = (500, 300, 3, 5) + let rect = PointerGeometry.rect(x: x, y: y, width: 32, height: 32, video: video, view: view, fit: fit) + let mapper = ContentRectMapper(viewSize: view, videoSize: video, fit: fit) + let content = mapper.contentRect + XCTAssertEqual(rect.width, 32 * content.width / video.width, accuracy: 1e-9) + let hot = CGPoint(x: rect.minX + hotX * content.width / video.width, + y: rect.minY + hotY * content.height / video.height) + guard let normalized = mapper.normalize(hot) else { + // Fill trims edges: this pointer may be off screen. + XCTAssertFalse(mapper.visibleRect.contains(hot), "\(video) \(fit)") + continue + } + let expectX = (CGFloat(x) + hotX) / video.width * 65535 + let expectY = (CGFloat(y) + hotY) / video.height * 65535 + XCTAssertEqual(Double(normalized.x), Double(expectX), accuracy: 2, "\(video) \(fit)") + XCTAssertEqual(Double(normalized.y), Double(expectY), accuracy: 2, "\(video) \(fit)") + } + } + } + + /// A pointer image the size of the video covers exactly the video's + /// quad, which the renderer scales by `MetalRenderer.scale`. + func testTheQuadMatchesTheVideoQuad() { + let view = CGSize(width: 2388, height: 1668) + for video in [CGSize(width: 3440, height: 1440), CGSize(width: 1080, height: 1920)] { + for fit in [PictureFit.fit, .fill] { + let rect = PointerGeometry.rect(x: 0, y: 0, width: Int(video.width), height: Int(video.height), + video: video, view: view, fit: fit) + let ndc = PointerGeometry.ndc(rect, view: view) + let scale = MetalRenderer.scale(fit, video: video, drawable: view) + XCTAssertEqual(ndc.x, -scale.x, accuracy: 1e-4) + XCTAssertEqual(ndc.z, scale.x, accuracy: 1e-4) + XCTAssertEqual(ndc.y, -scale.y, accuracy: 1e-4) + XCTAssertEqual(ndc.w, scale.y, accuracy: 1e-4) + } + } + // Partly off the left edge: negative positions keep their offset. + let rect = PointerGeometry.rect(x: -4, y: 0, width: 8, height: 8, video: CGSize(width: 1000, height: 500), + view: CGSize(width: 1000, height: 500), fit: .fit) + XCTAssertEqual(rect.minX, -4) + XCTAssertEqual(PointerGeometry.rect(x: 0, y: 0, width: 8, height: 8, video: .zero, view: view, fit: .fit), .zero) + } + + /// Review r1: a pointer past the picture's edge in Fit drew into the + /// letterbox bar. Both passes are clipped to the picture. + func testAPointerCrossingThePicturesEdgeStaysOutOfTheBars() throws { + let view = CGSize(width: 2388, height: 1668) + let video = CGSize(width: 3440, height: 1440) + let content = ContentRectMapper(viewSize: view, videoSize: video, fit: .fit).contentRect + let clip = try XCTUnwrap(PointerGeometry.clip(video: video, view: view, fit: .fit)) + // Letterboxed top and bottom: the clip is the picture, not the view. + XCTAssertEqual(clip.x, 0) + XCTAssertEqual(clip.width, 2388) + XCTAssertEqual(clip.y, Int(content.minY.rounded(.down))) + XCTAssertEqual(clip.y + clip.height, Int(content.maxY.rounded(.up))) + XCTAssertGreaterThan(clip.y, 0) + // A pointer hanging over the bottom edge: part of its quad is in the + // bar, and the clip cuts it at the picture's edge. + let rect = PointerGeometry.rect(x: 1000, y: 1430, width: 32, height: 32, video: video, view: view, fit: .fit) + XCTAssertGreaterThan(rect.maxY, content.maxY) + let shown = rect.intersection(CGRect(x: clip.x, y: clip.y, width: clip.width, height: clip.height)) + XCTAssertLessThanOrEqual(shown.maxY, CGFloat(clip.y + clip.height)) + XCTAssertGreaterThan(shown.height, 0, "the part on the picture still shows") + // Pillarboxed, and Fill (the picture covers the view). + let tall = try XCTUnwrap(PointerGeometry.clip(video: CGSize(width: 1080, height: 1920), view: view, fit: .fit)) + XCTAssertGreaterThan(tall.x, 0) + XCTAssertEqual(tall.height, 1668) + let fill = try XCTUnwrap(PointerGeometry.clip(video: video, view: view, fit: .fill)) + XCTAssertEqual([fill.x, fill.y, fill.width, fill.height], [0, 0, 2388, 1668]) + XCTAssertNil(PointerGeometry.clip(video: .zero, view: view, fit: .fit)) + } + + // MARK: Channel + + /// The channel hands pointer messages on only for its session, and only + /// from a PC that said it sends the pointer. + func testPointerMessagesNeedTheSessionAndTheCapability() async throws { + for hostCaps in [UInt16(0), HelloAck.hostCapPointer] { + let queue = DispatchQueue(label: "pointer-channel-test") + let (stream, continuation) = AsyncStream.makeStream() + let channel = ControlChannel(queue: queue) { continuation.yield($0) } + let overlay = PointerOverlay() + channel.pointer = overlay + channel.startHandshake(listenPort: 50000, identity: ControlChannel.ClientIdentity( + deviceName: "Test iPad", screenPxW: 640, screenPxH: 360, screenPtW: 640, screenPtH: 360, + refreshHz: 60)) + var messages = stream.makeAsyncIterator() + let first = await messages.next() + guard case .hello2(let hello) = try XCTUnwrap(Wire.parseControl(XCTUnwrap(first))).message else { + return XCTFail("missing handshake") + } + queue.sync { + channel.handleControl(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .helloAck(HelloAck( + status: .ok, acceptedVersion: 2, clientNonce: hello.clientNonce, sessionId: 41, + heartbeatIntervalMs: 1000, reportIntervalMs: 500, livenessTimeoutMs: 3000, + streamConfig: StreamConfig(), hostName: "Test PC", authToken: Data(17...32), + hostCaps: hostCaps)))) + channel.handleControl(Wire.encodeControl(sessionId: 40, msgSeq: 2, message: .pointerState(state(1)))) + for chunk in chunks(id: 1) { + channel.handleControl(Wire.encodeControl(sessionId: 41, msgSeq: 3, message: .pointerShape(chunk))) + } + channel.handleControl(Wire.encodeControl(sessionId: 41, msgSeq: 4, message: .pointerState(state(7)))) + } + channel.stop() + continuation.finish() + let snapshot = overlay.snapshot() + if hostCaps == 0 { + XCTAssertEqual(snapshot, PointerOverlay.Snapshot(), "a PC without the bit gets no pointer drawn") + } else { + XCTAssertEqual(snapshot.x, 7, "another session's state is ignored") + XCTAssertEqual(snapshot.image?.id, 1) + XCTAssertEqual(overlay.counters.states, 1) + } + } + } + + func testThisAppAsksToDrawThePointer() { + let caps = ConnectionManager.featureCaps(wantsInput: true, wantsAudio: false, isUSB: false) + XCTAssertNotEqual(caps & Hello2.featureDrawsPointer, 0) + XCTAssertEqual(Hello2.featureDrawsPointer & (Hello2.featureWantsInput | Hello2.featureWantsAudio + | Hello2.featureSupportsNack | Hello2.featureChecksUpdates), 0) + } +} diff --git a/ios/EternalMonitorTests/SessionSummaryTests.swift b/ios/EternalMonitorTests/SessionSummaryTests.swift index 9e911677..4ae04028 100644 --- a/ios/EternalMonitorTests/SessionSummaryTests.swift +++ b/ios/EternalMonitorTests/SessionSummaryTests.swift @@ -1,3 +1,4 @@ +import Network import XCTest @testable import EternalMonitor @@ -13,7 +14,7 @@ final class SessionSummaryTests: XCTestCase { + #" transport=(usb|wifi) mode=(mirror|second_screen|unknown)"# + #" stage=(started|hello_sent|accepted|video_sent|first_frame|steady)"# + #" outcome=(streamed|no_reply|no_picture|pairing|user_ended|app_ended|error)"# - + #" failure=(none|no_reply|no_video|no_frame|vdd_enable_failed|vdd_attach_timeout|vdd_not_attached|capture_failed|encoder_failed|usb_lost|signal_lost|other)"# + + #" failure=(none|no_reply|no_video|no_frame|vdd_enable_failed|vdd_attach_timeout|vdd_not_attached|capture_failed|encoder_failed|usb_lost|signal_lost|no_network|pc_busy|old_pc|other)"# + #" fallback=(none|attach_timeout|enable_failed|no_driver|no_frames|duplication_failed|remembered|other)"# + #" end=(goodbye|liveness|superseded|shutdown|timeout|user|background|error)"# + #" ttff_ms=(0|[1-9][0-9]{0,5}|-) duration_ms=(0|[1-9][0-9]{0,9})"# @@ -62,6 +63,8 @@ final class SessionSummaryTests: XCTestCase { func testEachWayAnAttemptEndsHasItsSummary() { var noFrame = summary(.connectTimeout, duration: 10) noFrame.datagramsReceived = 12 + var movedNoFrame = summary(.noVideoPath, duration: 7) + movedNoFrame.datagramsReceived = 3 var pairing = summary(.userDisconnect, session: nil, duration: 30) pairing.awaitingPairing = true var retried = summary(.retriedWithCode, session: nil, duration: 4) @@ -75,16 +78,25 @@ final class SessionSummaryTests: XCTestCase { (summary(.pcGoodbye), ["accepted", "app_ended", "none", "goodbye"]), (summary(.pcRefused, session: nil), ["hello_sent", "error", "other", "error"]), (pairing, ["hello_sent", "pairing", "none", "user"]), - (summary(.legacyPC, session: nil), ["hello_sent", "error", "other", "error"]), + (summary(.pcBusy, session: nil), ["hello_sent", "error", "pc_busy", "error"]), + (summary(.oldPC, session: nil), ["hello_sent", "error", "old_pc", "error"]), + (summary(.legacyPC, session: nil), ["hello_sent", "error", "old_pc", "error"]), (summary(.connectTimeout, session: nil, duration: 10), ["hello_sent", "no_reply", "no_reply", "timeout"]), + (summary(.pcMoved, session: nil, duration: 3), ["hello_sent", "no_reply", "no_reply", "superseded"]), + (summary(.otherPC), ["accepted", "error", "other", "error"]), (summary(.connectTimeout, duration: 10), ["accepted", "no_picture", "no_video", "timeout"]), (noFrame, ["accepted", "no_picture", "no_frame", "timeout"]), + // Moved to the PC's other address: as the PC says, nothing came, + // or no frame of what came completed. + (summary(.noVideoPath, duration: 7), ["accepted", "no_picture", "no_video", "superseded"]), + (movedNoFrame, ["accepted", "no_picture", "no_frame", "superseded"]), (summary(.pcStoppedResponding, firstFrame: 0.3, duration: 90), ["steady", "streamed", "signal_lost", "liveness"]), (summary(.pictureStalled, firstFrame: 0.3, duration: 90), ["steady", "streamed", "other", "error"]), (summary(.cableUnplugged, usb: true, firstFrame: 0.3), ["first_frame", "streamed", "usb_lost", "error"]), (summary(.usbLinkFailed, usb: true), ["accepted", "error", "usb_lost", "error"]), (summary(.closedBeforePicture, usb: true), ["accepted", "no_picture", "usb_lost", "error"]), (summary(.linkFailed, session: nil, nonce: 0, duration: 0.2), ["started", "error", "other", "error"]), + (summary(.noNetwork, session: nil, nonce: 0, duration: 0.2), ["started", "error", "no_network", "error"]), (summary(.receiverFailed, session: nil, nonce: 0, duration: 0), ["started", "error", "other", "error"]), (retried, ["hello_sent", "pairing", "none", "superseded"]), ] @@ -97,14 +109,42 @@ final class SessionSummaryTests: XCTestCase { XCTAssertTrue(summary(.cableUnplugged, usb: true).line.contains(" transport=usb ")) } + /// The PC said Windows won't let it capture from its graphics card, and + /// the wait for a picture ran out: a capture failure, as the PC's own + /// summary says, not lost video. A picture that came later keeps its tokens. + func testATimeoutAfterThePCSaidItCantCaptureIsACaptureFailure() { + var refused = summary(.connectTimeout, duration: 10) + refused.captureRefused = true + XCTAssertEqual(tokens(refused.line), ["accepted", "no_picture", "capture_failed", "timeout"]) + Self.assertContract(refused.line) + var streamed = summary(.pcStoppedResponding, firstFrame: 0.3, duration: 90) + streamed.captureRefused = true + XCTAssertEqual(tokens(streamed.line), ["steady", "streamed", "signal_lost", "liveness"]) + } + + /// Review r1: the PC sent video before Windows refused its card, and + /// none of it gave a picture. That is lost or undecodable video, as the + /// PC's summary says (it names a capture failure only when no video went + /// out), and the refusal mustn't hide it. + func testVideoThePCSentBeforeTheRefusalKeepsItsDeliveryFailure() { + var counted = summary(.connectTimeout, duration: 10) + (counted.captureRefused, counted.pcSentVideo) = (true, true) + XCTAssertEqual(tokens(counted.line), ["accepted", "no_picture", "no_video", "timeout"]) + var arrived = summary(.connectTimeout, duration: 10) + (arrived.captureRefused, arrived.datagramsReceived) = (true, 12) + XCTAssertEqual(tokens(arrived.line), ["accepted", "no_picture", "no_frame", "timeout"]) + Self.assertContract(counted.line) + Self.assertContract(arrived.line) + } + func testOnlyTheUsersOwnActionSaysUserDisconnectToThePC() { XCTAssertEqual(AttemptEnd.userDisconnect.bye, .userDisconnect) XCTAssertEqual(AttemptEnd.usbTurnedOff.bye, .userDisconnect) XCTAssertEqual(AttemptEnd.appBackground.bye, .appBackground) XCTAssertEqual(AttemptEnd.superseded.bye, .superseded) - for end in [AttemptEnd.pcGoodbye, .pcRefused, .legacyPC, .connectTimeout, .pcStoppedResponding, + for end in [AttemptEnd.pcGoodbye, .pcRefused, .pcBusy, .oldPC, .legacyPC, .connectTimeout, .pcStoppedResponding, .pictureStalled, .cableUnplugged, .usbLinkFailed, .closedBeforePicture, .linkFailed, - .receiverFailed, .retriedWithCode] { + .noNetwork, .receiverFailed, .retriedWithCode] { XCTAssertEqual(end.bye, .error, "\(end)") } XCTAssertEqual(AttemptEnd.linkFailed.name, "linkFailed") @@ -165,6 +205,44 @@ final class SessionSummaryTests: XCTestCase { Self.assertContract(line) } + /// Wi-Fi is off: Network.framework waits with "Network is down". The + /// card says so, and the summary gives it its own failure instead of + /// `other`. + func testWiFiOffHasItsOwnCardAndFailure() async throws { + let link = DeniedLink() + let manager = ConnectionManager(udpReceiver: { _ in link }) + manager.connect(host: "192.168.1.20", port: 9876) + link.onError?(UDPReceiver.waitingError(.posix(.ENETDOWN), status: .unsatisfied, reason: .notAvailable)) + try await until("ended") { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .wifiOff) + XCTAssertEqual(manager.problem?.title, "Wi-Fi is off") + XCTAssertEqual(logged(manager, "connection").filter { $0.hasPrefix("Disconnect ") }, + ["Disconnect state=connecting reason=noNetwork"]) + let line = logged(manager, "summary").first ?? "" + XCTAssertTrue(line.contains(" stage=started outcome=error failure=no_network fallback=none end=error "), line) + Self.assertContract(line) + } + + /// No route to the PC or its network while the path is up: the PC is + /// asleep or elsewhere, not this iPad's Wi-Fi off. It keeps the card + /// and the `other` failure it had before Wi-Fi off got its own. + func testUnreachableWhileWaitingIsNotWiFiOff() async throws { + for code: POSIXErrorCode in [.EHOSTUNREACH, .ENETUNREACH] { + let link = DeniedLink() + let manager = ConnectionManager(udpReceiver: { _ in link }) + manager.connect(host: "192.168.1.20", port: 9876) + link.onError?(UDPReceiver.waitingError(.posix(code), status: .satisfied, reason: nil)) + try await until("ended") { manager.state == .disconnected } + XCTAssertNotEqual(manager.problem, .wifiOff, "\(code)") + XCTAssertNotEqual(manager.problem?.title, "Wi-Fi is off", "\(code)") + XCTAssertEqual(logged(manager, "connection").filter { $0.hasPrefix("Disconnect ") }, + ["Disconnect state=connecting reason=linkFailed"], "\(code)") + let line = logged(manager, "summary").first ?? "" + XCTAssertTrue(line.contains(" failure=other "), line) + Self.assertContract(line) + } + } + /// Try again opens a new connection, which asks iPadOS again: once /// Local Network is on, the old denial isn't shown or remembered. func testTryAgainAfterLocalNetworkDeniedAsksIPadOSAgain() async throws { diff --git a/ios/EternalMonitorTests/StreamRecoveryTests.swift b/ios/EternalMonitorTests/StreamRecoveryTests.swift index 3610be20..1bb26bb0 100644 --- a/ios/EternalMonitorTests/StreamRecoveryTests.swift +++ b/ios/EternalMonitorTests/StreamRecoveryTests.swift @@ -87,6 +87,139 @@ final class StreamRecoveryTests: XCTestCase { } } + /// The lock screen or a permission prompt holds the picture on the PC. + /// Its heartbeats say so, and the iPad says so over the held picture, + /// still connected, until a heartbeat says capture works again. + func testALockedPCIsShownOverTheHeldPictureUntilItCapturesAgain() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + XCTAssertFalse(manager.pcLocked) + host.sendHeartbeat(notice: Heartbeat.noticePCLocked) + try await until("the PC says it is locked") { manager.pcLocked } + XCTAssertEqual(manager.state, .connected) + host.sendHeartbeat() + try await until("the PC captures again") { !manager.pcLocked } + // A notice this iPad doesn't know says nothing. + host.sendHeartbeat(notice: 0xC8) + try await Task.sleep(nanoseconds: 500_000_000) + XCTAssertFalse(manager.pcLocked) + host.sendHeartbeat(notice: Heartbeat.noticePCLocked) + try await until("locked again") { manager.pcLocked } + manager.cancel() + XCTAssertFalse(manager.pcLocked, "the next session starts without it") + } + + /// The PC restarts its stream and holds this session: once the picture + /// has been held a moment, the iPad says so, still connected, and the + /// first new frame clears it. + func testARestartingPCIsShownOnceThePictureIsHeldAWhile() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + let lastFrame = Date() + host.sendHeartbeat(notice: Heartbeat.noticeRestarting) + try await Task.sleep(nanoseconds: 700_000_000) + host.sendHeartbeat(notice: Heartbeat.noticeRestarting) + XCTAssertFalse(manager.pcRestarting, "shown before the picture was held a moment") + try await until("the PC says it is restarting") { manager.pcRestarting } + XCTAssertGreaterThanOrEqual(Date().timeIntervalSince(lastFrame), 1.3) + XCTAssertEqual(manager.state, .connected) + // The restart is over: the first new frame clears it. + host.sendHeartbeat(notice: Heartbeat.noticeRestarting) + let units = try twoGOPs() + host.sendFrame(units[4], seq: 5, isKeyframe: true) + try await until("the picture is back") { !manager.pcRestarting } + XCTAssertEqual(manager.state, .connected) + manager.cancel() + XCTAssertFalse(manager.pcRestarting, "the next session starts without it") + } + + /// A quick restart, whose new frame comes before the picture was held a + /// moment, shows nothing; nor does a notice that stops. + func testAQuickRestartShowsNothing() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + host.sendHeartbeat(notice: Heartbeat.noticeRestarting) + try await Task.sleep(nanoseconds: 400_000_000) + let units = try twoGOPs() + host.sendFrame(units[4], seq: 5, isKeyframe: true) + let deadline = Date().addingTimeInterval(2.5) + while Date() < deadline { + XCTAssertFalse(manager.pcRestarting, "a quick restart showed the pill") + try await Task.sleep(nanoseconds: 50_000_000) + } + host.sendHeartbeat(notice: Heartbeat.noticeRestarting) + host.sendHeartbeat() + try await Task.sleep(nanoseconds: 2_500_000_000) + XCTAssertFalse(manager.pcRestarting, "a notice that stopped still showed") + } + + /// A closed lid or screens that are off leave the PC nothing to + /// capture. Its heartbeats say so, and the iPad says so over the held + /// picture, still connected, until a display comes back. + func testAPCWithNoDisplayIsShownOverTheHeldPictureUntilOneComesBack() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + XCTAssertFalse(manager.pcNoDisplay) + host.sendHeartbeat(notice: Heartbeat.noticeNoDisplay) + try await until("the PC says it has no display") { manager.pcNoDisplay } + XCTAssertEqual(manager.state, .connected) + XCTAssertFalse(manager.pcLocked) + // Unlocking a PC that still has no display doesn't say it is locked. + host.sendHeartbeat(notice: Heartbeat.noticePCLocked) + try await until("locked instead") { manager.pcLocked && !manager.pcNoDisplay } + host.sendHeartbeat(notice: Heartbeat.noticeNoDisplay) + try await until("no display again") { manager.pcNoDisplay && !manager.pcLocked } + host.sendHeartbeat() + try await until("a display is back") { !manager.pcNoDisplay } + host.sendHeartbeat(notice: Heartbeat.noticeNoDisplay) + try await until("no display again") { manager.pcNoDisplay } + manager.cancel() + XCTAssertFalse(manager.pcNoDisplay, "the next session starts without it") + } + + /// The PC chose Second screen, but its Windows can't run it, so it + /// mirrors. Its heartbeats say why for a few seconds, and the iPad says + /// so over the working picture until they stop. + func testAPCThatNeedsWindows11ForSecondScreenSaysSoWhileItMirrors() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + XCTAssertFalse(manager.secondScreenNeedsWindows11) + host.sendHeartbeat(notice: Heartbeat.noticeSecondScreenNeedsWindows11) + try await until("the PC says Second screen needs Windows 11") { manager.secondScreenNeedsWindows11 } + XCTAssertEqual(manager.state, .connected) + XCTAssertFalse(manager.pcLocked || manager.pcNoDisplay) + host.sendHeartbeat() + try await until("the notice ends") { !manager.secondScreenNeedsWindows11 } + host.sendHeartbeat(notice: Heartbeat.noticeSecondScreenNeedsWindows11) + try await until("said again") { manager.secondScreenNeedsWindows11 } + manager.cancel() + XCTAssertFalse(manager.secondScreenNeedsWindows11, "the next session starts without it") + } + + /// Windows moved the PC's screen to the graphics card EternalMonitor + /// doesn't run on, mid-stream. Its heartbeats say so, which screen, and + /// the iPad says so over the held picture until the PC captures again. + /// A locked PC's notice still comes first. + func testAPCThatCantCaptureFromItsGraphicsCardIsShownOverTheHeldPicture() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + XCTAssertNil(manager.pcGraphicsCard) + host.sendHeartbeat(notice: Heartbeat.noticeUnsupportedGPU) + try await until("the PC says why") { manager.pcGraphicsCard == .screen } + XCTAssertEqual(manager.state, .connected) + XCTAssertFalse(manager.pcLocked || manager.pcNoDisplay) + host.sendHeartbeat(notice: Heartbeat.noticeSecondScreenUnsupportedGPU) + try await until("the second screen failed too") { manager.pcGraphicsCard == .secondScreen } + host.sendHeartbeat(notice: Heartbeat.noticePCLocked) + try await until("locked instead") { manager.pcLocked && manager.pcGraphicsCard == nil } + host.sendHeartbeat() + try await until("the PC captures again") { !manager.pcLocked && manager.pcGraphicsCard == nil } + host.sendHeartbeat(notice: Heartbeat.noticeUnsupportedGPU) + try await until("again") { manager.pcGraphicsCard == .screen } + manager.cancel() + XCTAssertNil(manager.pcGraphicsCard, "the next session starts without it") + } + func testContiguousNumberingRevealsALostReopenKeyframe() async throws { // The PC reopened its encoder and its new keyframe (seq 2) was lost; // the new encoder's first P-frame continues the codec's count. A PC @@ -217,4 +350,34 @@ final class StreamRecoveryTests: XCTestCase { let after = await Task.detached { host.keyframeRequests(within: 1) }.value XCTAssertEqual(after, []) } + + /// Video arrives but no frame ever completes, as when every full-size + /// datagram is lost: 2 s on, the iPad logs the missing fragments and + /// asks for a fresh keyframe with reason Startup, which the PC logs, + /// besides the requests the dropped frames make. + func testAFirstFrameThatNeverCompletesAsksForAFreshKeyframe() async throws { + let host = FakeUDPHost() + let manager = ConnectionManager() + addTeardownBlock { await MainActor.run { manager.cancel() } } + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + let fragments = Task.detached { + for seq in UInt32(1)...80 where !Task.isCancelled { + host.sendFirstFragment(Data([0, 0, 0, 1, 0x65]), seq: seq) + try? await Task.sleep(nanoseconds: 100_000_000) + } + } + defer { fragments.cancel() } + let reasons = await Task.detached { host.keyframeRequests(within: 8) { $0 == .startup } }.value + XCTAssertEqual(reasons.last, .startup, "\(reasons)") + try await until("logged") { + manager.diagnostics.contains { $0.category == "assembly" && $0.message.hasPrefix( + "Video arrives but no frame completed in 2 s; asked the PC for a fresh keyframe: seq=") } + } + let logged = manager.diagnostics.first { $0.message.hasPrefix("Video arrives") } + XCTAssertEqual(logged?.level, .warning) + XCTAssertTrue(logged?.message.contains(" dropped with 1/2 fragments, missing 1") == true, logged?.message ?? "") + XCTAssertEqual(manager.state, .connecting) + } } diff --git a/ios/EternalMonitorTests/USBLinkFailureTests.swift b/ios/EternalMonitorTests/USBLinkFailureTests.swift index a8ffaf14..146868fa 100644 --- a/ios/EternalMonitorTests/USBLinkFailureTests.swift +++ b/ios/EternalMonitorTests/USBLinkFailureTests.swift @@ -618,6 +618,174 @@ final class USBLinkFailureTests: XCTestCase { XCTAssertEqual(manager.reconnectAttempt, 0) } + // MARK: - A PC that turns the cable down + + /// Answers the iPad's HELLO2 on the cable's far end `fd` with `status`, + /// reporting protocol `version`. False after five quiet seconds. + nonisolated private static func refuseSession(on fd: Int32, status: HelloStatus, version: UInt8 = 2) -> Bool { + var timeout = timeval(tv_sec: 0, tv_usec: 300_000) + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, socklen_t(MemoryLayout.size)) + var framer = EMLinkFramer(expectsPreamble: false) + var buffer = [UInt8](repeating: 0, count: 4096) + let deadline = Date().addingTimeInterval(5) + while Date() < deadline { + let count = recv(fd, &buffer, buffer.count, 0) + guard count > 0, let datagrams = try? framer.append(Data(buffer.prefix(count))) else { continue } + for datagram in datagrams { + guard case .hello2(let hello) = Wire.parseControl(datagram)?.message, + let framed = try? EMLinkFramer.encode(FakeUDPHost.ack(nonce: hello.clientNonce, status: status, + acceptedVersion: version)) else { continue } + return framed.withUnsafeBytes { Darwin.send(fd, $0.baseAddress, $0.count, 0) } == framed.count + } + } + return false + } + + /// The newest cable's PC turns the iPad down with `status`. + private func refuse(_ status: HelloStatus, version: UInt8 = 2) async { + let fd = peer + let refused = await Task.detached { Self.refuseSession(on: fd, status: status, version: version) }.value + XCTAssertTrue(refused, "the iPad never sent HELLO2") + } + + /// The summary lines recorded so far. + private func summaries(_ manager: ConnectionManager) -> [String] { + manager.diagnostics.map(\.message).filter { $0.hasPrefix("session_summary ") } + } + + /// Offers the PC's dial every 100 ms until the iPad takes one. + private func dialUntilTaken(_ manager: ConnectionManager, cables: inout [FramedLink]) async throws { + let deadline = Date().addingTimeInterval(30) + while Date() < deadline { + cables.append(try await plugIn(manager)) + if manager.state == .connecting { return } + try await Task.sleep(nanoseconds: 100_000_000) + } + XCTFail("no dial was taken in 30 s") + } + + /// The wait after each Busy in a row: 30 s, doubling, at most 5 minutes. + func testTheBusyWaitDoublesUpToFiveMinutes() { + XCTAssertEqual((1...8).map(CableHold.busySeconds), [30, 60, 120, 240, 300, 300, 300, 300]) + XCTAssertEqual(CableHold.busySeconds(0), 30) + XCTAssertEqual(CableHold.busySeconds(.max), 300) + let at = Date(timeIntervalSince1970: 1_800_000_000) + let hold = CableHold(reason: .busy, until: at.addingTimeInterval(30)) + XCTAssertTrue(hold.holds(at: at.addingTimeInterval(29.9))) + XCTAssertFalse(hold.holds(at: at.addingTimeInterval(30))) + XCTAssertTrue(CableHold(reason: .version, until: nil).holds(at: .distantFuture)) + } + + /// A PC streaming to another device turns the cable down as Busy, then + /// dials it again a second later. The iPad shows why and doesn't take + /// the cable again until the wait ends, a longer wait after each Busy in + /// a row, back to the first after a session the PC accepted. Each + /// attempt's summary names the refusal. The waits run on a sleeper the + /// test ends, so the timing of this machine doesn't matter. + func testABusyPCOnTheCableIsAnsweredOnlyAfterAGrowingWait() async throws { + let previousUSB = UserDefaults.standard.object(forKey: "allowUSB") + defer { UserDefaults.standard.set(previousUSB, forKey: "allowUSB") } + // The end of a wait must not open the app's real USB listener port. + UserDefaults.standard.set(false, forKey: "allowUSB") + let sleeper = ManualSleeper() + var timing = answering + timing.cableHoldSleep = { await sleeper.sleep($0) } + let (manager, first) = try await connectOverCable(timing: timing) + var cables = [first] + defer { cables.forEach { $0.stop() }; manager.cancel() } + + await refuse(.busy) + try await until("the refusal shows") { manager.state == .disconnected && manager.problem == .pcBusy } + XCTAssertEqual(manager.usbStatus, "USB: disconnected") + XCTAssertTrue(summaries(manager).last?.contains(" stage=hello_sent outcome=error failure=pc_busy fallback=none end=error ") == true, + summaries(manager).joined(separator: "\n")) + try await until("the first wait starts") { sleeper.requested == [30] } + // The PC dials again and again within the wait: none is taken. + for _ in 1...5 { + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .disconnected, "the cable isn't taken during the wait") + } + XCTAssertEqual(manager.problem, .pcBusy, "the reason stays up") + XCTAssertEqual(summaries(manager).count, 1, "one attempt") + + // Once the wait is over the next dial is taken; the next Busy in a + // row waits twice as long. + sleeper.advance() + try await dialUntilTaken(manager, cables: &cables) + await refuse(.busy) + try await until("the second refusal") { manager.state == .disconnected } + try await until("the second wait starts") { sleeper.requested == [30, 60] } + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .disconnected, "the cable isn't taken during the second wait") + sleeper.advance() + try await dialUntilTaken(manager, cables: &cables) + + // The PC accepts a session, then ends it by closing the cable. Its + // next dial is taken as usual, and the next Busy waits the first + // wait again. + await answer() + try await until("the session") { manager.hostInfo != nil } + shutdown(peer, SHUT_RDWR) + try await until("the closed cable ends it") { manager.state == .disconnected } + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .connecting, "the next dial is taken") + await refuse(.busy) + try await until("the third refusal") { manager.state == .disconnected } + try await until("the third wait starts") { sleeper.requested.count == 3 } + XCTAssertEqual(sleeper.requested, [30, 60, 30]) + } + + /// Try again during the wait takes the PC's next dial at once. + func testTryAgainEndsTheBusyWait() async throws { + let previousUSB = UserDefaults.standard.object(forKey: "allowUSB") + defer { UserDefaults.standard.set(previousUSB, forKey: "allowUSB") } + UserDefaults.standard.set(false, forKey: "allowUSB") + let (manager, first) = try await connectOverCable(timing: answering) + var cables = [first] + defer { cables.forEach { $0.stop() }; manager.cancel() } + await refuse(.busy) + try await until("the refusal shows") { manager.problem == .pcBusy } + XCTAssertEqual(manager.problem?.actions, [.retry]) + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .disconnected) + manager.retry() + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .connecting, "Try again takes the cable at once") + } + + /// A PC too old for this app, or one so old it doesn't answer HELLO2, + /// can't connect until someone updates it: the iPad shows that, and + /// takes the cable again only after Try again or Connect USB, not when + /// it comes back to the foreground. Each names an old PC in its summary. + func testAnOldPCOnTheCableWaitsForTheUser() async throws { + let previousUSB = UserDefaults.standard.object(forKey: "allowUSB") + defer { UserDefaults.standard.set(previousUSB, forKey: "allowUSB") } + UserDefaults.standard.set(false, forKey: "allowUSB") + let (manager, first) = try await connectOverCable(timing: answering) + var cables = [first] + defer { cables.forEach { $0.stop() }; manager.cancel() } + await refuse(.versionUnsupported, version: 1) + try await until("the refusal shows") { manager.state == .disconnected && manager.versionMismatch == .pcOlder } + XCTAssertEqual(manager.connectionError, VersionMismatch.pcOlder.message) + XCTAssertEqual(manager.usbStatus, "USB: disconnected", "Connect USB is offered") + XCTAssertTrue(summaries(manager).last?.contains(" stage=hello_sent outcome=error failure=old_pc fallback=none end=error ") == true, + summaries(manager).joined(separator: "\n")) + try await Task.sleep(nanoseconds: 1_000_000_000) + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .disconnected, "no wait ends it") + manager.handleAppBackgrounded() + manager.handleAppForegrounded() + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .disconnected, "coming back to the app doesn't end it") + + manager.resumeUSBConnections() + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .connecting, "Connect USB takes the cable again") + // The PC was updated: it connects as any other. + await answer() + try await until("the session") { manager.hostInfo != nil } + } + /// The PC still holds this iPad's cable session and answers the Wi-Fi /// fallback with Busy, then frees up. func testBusyDuringTheFallbackRetriesUntilThePCIsFree() async throws { @@ -687,3 +855,22 @@ private final class UnresolvableLink: MediaLink { func send(_ data: Data) {} func stop() {} } + +/// Ends each wait when the test says so, and remembers how long each was. +@MainActor +private final class ManualSleeper { + private(set) var requested: [Double] = [] + private var waiting: [CheckedContinuation] = [] + + func sleep(_ seconds: Double) async { + requested.append(seconds) + await withCheckedContinuation { waiting.append($0) } + } + + /// Ends every wait that has started. + func advance() { + let ended = waiting + waiting = [] + ended.forEach { $0.resume() } + } +} diff --git a/ios/EternalMonitorTests/UpdateCheckerTests.swift b/ios/EternalMonitorTests/UpdateCheckerTests.swift index d838a807..568dd396 100644 --- a/ios/EternalMonitorTests/UpdateCheckerTests.swift +++ b/ios/EternalMonitorTests/UpdateCheckerTests.swift @@ -223,6 +223,47 @@ final class UpdateCheckerTests: XCTestCase { XCTAssertFalse(UpdateChecker(defaults: defaults, source: .disabled, installed: installed).automatic) } + func testManifestGivesTheWindowsVersionOnItsOwn() { + XCTAssertEqual(UpdateManifest.windowsVersion(manifest("0.3.4", 698)), "0.3.4") + XCTAssertEqual(UpdateManifest.windowsVersion(Data(#"{"schema":1,"windows":{"version":"0.3.5"}}"#.utf8)), "0.3.5") + XCTAssertNil(UpdateManifest.windowsVersion(manifest("0.3.4", 698, schema: 2))) + XCTAssertNil(UpdateManifest.windowsVersion(Data(#"{"schema":1,"windows":{"version":"0.3.5-beta.1"}}"#.utf8))) + XCTAssertNil(UpdateManifest.windowsVersion(Data(#"{"schema":1,"windows":{}}"#.utf8))) + XCTAssertNil(UpdateManifest.windowsVersion(Data(#"{"schema":1,"ipad":{"version":"0.3.4","build":698}}"#.utf8))) + // A broken Windows entry doesn't hide the iPad's update. + XCTAssertNotNil(UpdateManifest.parse(Data(#"{"schema":1,"windows":{"version":3},"ipad":{"version":"0.3.4","build":698}}"#.utf8))) + } + + /// A PC older than the manifest's Windows version gets the note, also + /// when this iPad is up to date and after a relaunch offline. Betas, + /// unknown versions and newer PCs don't. + @MainActor + func testAnOlderPCIsToldToUpdate() throws { + let suite = "UpdateCheckerTests-\(UUID().uuidString)" + let defaults = try XCTUnwrap(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + let current = InstalledApp(version: "0.3.4", build: 698) + + let checker = UpdateChecker(defaults: defaults, source: .injected(manifest("0.3.4", 698)), installed: current) + XCTAssertFalse(checker.pcNeedsUpdate("0.3.2"), "before any check") + checker.checkIfDue() + XCTAssertNil(checker.available) + XCTAssertEqual(checker.windowsVersion, "0.3.4") + for old in ["0.3.3", "0.3.2", "0.2.9"] { XCTAssertTrue(checker.pcNeedsUpdate(old), old) } + for fine in ["0.3.4", "0.3.5", "0.3.3-beta.1", "", "unknown", nil] as [String?] { + XCTAssertFalse(checker.pcNeedsUpdate(fine), fine ?? "nil") + } + + let relaunched = UpdateChecker(defaults: defaults, source: .disabled, installed: current) + XCTAssertTrue(relaunched.pcNeedsUpdate("0.3.3")) + + // A manifest without a Windows entry forgets it. + let without = UpdateChecker(defaults: defaults, source: .injected(Data(#"{"schema":1}"#.utf8)), installed: current) + without.checkIfDue() + XCTAssertNil(without.windowsVersion) + XCTAssertFalse(UpdateChecker(defaults: defaults, source: .disabled, installed: current).pcNeedsUpdate("0.3.3")) + } + @MainActor func testUntrustedTestFlightLinkFallsBackToTheInvite() throws { let suite = "UpdateCheckerTests-\(UUID().uuidString)" diff --git a/ios/EternalMonitorTests/VideoDecoderTests.swift b/ios/EternalMonitorTests/VideoDecoderTests.swift index 3e604ee3..f9340207 100644 --- a/ios/EternalMonitorTests/VideoDecoderTests.swift +++ b/ios/EternalMonitorTests/VideoDecoderTests.swift @@ -701,3 +701,169 @@ final class VideoDecoderTests: XCTestCase { XCTAssertNil(decoder.keyframeWaitUs) } } + +// MARK: - Why a hold started + +/// Each hold logs one line saying which check started it, beside the hold's +/// own lines, which keep their wording. In the field a still, lossless stream +/// (no drops, no NACKs) was held and reconnected over and over, and nothing +/// said what started the hold. +extension VideoDecoderTests { + private func holdLines(_ events: [String]) -> [String] { + events.filter { $0.hasPrefix("Picture hold started ") } + } + + /// The lines without their time since the last picture, which the real + /// clock and VideoToolbox's asynchronous output decide (see + /// `testTheLineSaysHowLongAgoThePictureWasDecoded`). + private func holdReasons(_ events: [String]) -> [String] { + holdLines(events).map { line in + line.range(of: " since_decoded_ms=").map { String(line[..<$0.lowerBound]) } ?? line + } + } + + func testAFrameNumberJumpOrPOCStepStartsOneHoldThatSaysSo() throws { + // seq 3 lost whole on a PC that doesn't number contiguously: only the + // codec's own count shows it. + let (h264, _) = try run("h264", [(1, 0), (2, 1), (4, 3), (5, 3), (6, 4), (7, 5)]) + XCTAssertEqual(holdReasons(h264.events), + ["Picture hold started reason=frame_num_jump expected_frame_num=2 frame_num=3 frame_seq=4"]) + let (hevc, _) = try run("h265", [(1, 0), (2, 1), (4, 3), (5, 3), (6, 4), (7, 5)]) + let line = try XCTUnwrap(holdLines(hevc.events).first) + XCTAssertEqual(holdLines(hevc.events).count, 1, "\(hevc.events)") + XCTAssertTrue(line.hasPrefix("Picture hold started reason=poc_step expected_step="), line) + XCTAssertTrue(line.contains(" frame_seq=4 since_decoded_ms="), line) + } + + func testAFrameNumberGapOrStreamRestartStartsOneHoldThatSaysSo() throws { + let numbered = VideoDecoder() + numbered.setFrameSeqContiguous(true) + let (gap, _) = try run("h264", [(1, 0), (3, 5), (4, 5), (5, 4), (6, 5)], decoder: numbered) + XCTAssertEqual(holdReasons(gap.events), + ["Picture hold started reason=frame_seq_gap previous_seq=1 frame_seq=3"]) + let (restart, _) = try run("h264", [(7, 0), (8, 1), (1, 1), (2, 2), (3, 4), (4, 5)]) + XCTAssertEqual(holdReasons(restart.events), + ["Picture hold started reason=stream_restarted previous_epoch=0 epoch=0 previous_seq=8 frame_seq=1"]) + let (epoch, _) = try run("h264", [(1, 0, 1), (2, 5, 2), (3, 4, 2), (4, 5, 2)]) + XCTAssertEqual(holdReasons(epoch.events), + ["Picture hold started reason=stream_restarted previous_epoch=1 epoch=2 previous_seq=1 frame_seq=2"]) + } + + func testEachHoldSaysWhyOnceAndALosslessStreamNever() throws { + // Two holds, each with frames held behind it: one line each. + let (twice, _) = try run("h264", [(1, 0), (2, 1), (4, 3), (5, 3), (6, 4), (7, 5), (9, 7), (10, 7), (11, 0)]) + XCTAssertEqual(holdLines(twice.events).count, 2, "\(twice.events)") + XCTAssertTrue(holdLines(twice.events).allSatisfy { $0.contains("reason=frame_num_jump") }) + let (clean, _) = try run("h264", (0..<8).map { (UInt32($0 + 1), $0) }) + XCTAssertEqual(holdLines(clean.events), []) + // The first session waits for its keyframe without a hold. + let (start, _) = try run("h264", [(1, 1), (2, 0), (3, 1)]) + XCTAssertEqual(holdLines(start.events), []) + } + + func testTheLineSaysHowLongAgoThePictureWasDecoded() throws { + let units = try twoGOPs("h264") + let clock = OSAllocatedUnfairLock(initialState: 5_000_000) + let decoder = VideoDecoder(nowUs: { clock.withLock { $0 } }, runsRecoveryTimer: false) + defer { decoder.shutdown() } + let events = OSAllocatedUnfairLock(initialState: [String]()) + decoder.onEvent = { message in events.withLock { $0.append(message) } } + let decoded = OSAllocatedUnfairLock(initialState: 0) + decoder.onFrameDecoded = { _, _, _ in decoded.withLock { $0 += 1 } } + decoder.decode(packet: packet(seq: 1, data: units[0], keyframe: true)) + decoder.decode(packet: packet(seq: 2, data: units[1], keyframe: false)) + let both = expectation(for: NSPredicate { _, _ in decoded.withLock { $0 } == 2 }, evaluatedWith: nil) + wait(for: [both], timeout: 10) + clock.withLock { $0 = 5_750_000 } + decoder.decode(packet: packet(seq: 4, data: units[3], keyframe: false)) + let held = expectation(for: NSPredicate { _, _ in decoder.keyframeWaitUs != nil }, evaluatedWith: nil) + wait(for: [held], timeout: 10) + XCTAssertEqual(holdLines(events.withLock { $0 }), + ["Picture hold started reason=frame_num_jump expected_frame_num=2 frame_num=3 frame_seq=4 since_decoded_ms=750"]) + } + + func testADecodeErrorStartsOneHoldAndAFailedKeyframeContinuesIt() throws { + let data = try fixtureData() + let decoder = VideoDecoder() + defer { decoder.shutdown() } + let events = OSAllocatedUnfairLock(initialState: [String]()) + decoder.onEvent = { message in events.withLock { $0.append(message) } } + let first = expectation(description: "first picture") + let recovered = expectation(description: "recovered") + decoder.onFrameDecoded = { _, timestamp, _ in + if timestamp == 16_667 { first.fulfill() } + if timestamp == 20 * 16_667 { recovered.fulfill() } + } + let failures = OSAllocatedUnfairLock(initialState: 0) + decoder.onFailure = { message in + if message.hasPrefix("Decoder callback") || message.hasPrefix("VTDecodeFrame") { failures.withLock { $0 += 1 } } + } + func failed(_ count: Int) { + let met = expectation(for: NSPredicate { _, _ in failures.withLock { $0 } == count }, evaluatedWith: nil) + wait(for: [met], timeout: 10) + } + decoder.decode(packet: packet(seq: 1, data: data, keyframe: true)) + wait(for: [first], timeout: 10) + // VideoToolbox reports the error asynchronously: each step waits for it. + let corrupt = Data([0, 0, 0, 1, 0x41, 0x80]) + decoder.decode(packet: packet(seq: 2, data: corrupt, keyframe: false)) + failed(1) + for seq in 3...5 { decoder.decode(packet: packet(seq: UInt32(seq), data: corrupt, keyframe: false)) } + // A keyframe VideoToolbox rejects continues the same hold. + decoder.decode(packet: packet(seq: 6, data: Data([0, 0, 0, 1, 0x65, 0x80]), keyframe: true)) + failed(2) + decoder.decode(packet: packet(seq: 20, data: data, keyframe: true)) + wait(for: [recovered], timeout: 10) + let lines = holdLines(events.withLock { $0 }) + XCTAssertEqual(lines.count, 1, "\(lines)") + let line = try XCTUnwrap(lines.first) + XCTAssertTrue(line.hasPrefix("Picture hold started reason=decode_error stage="), line) + XCTAssertTrue(line.contains(" keyframe=false frame_seq=2 since_decoded_ms="), line) + } + + func testANewDecoderSessionMidStreamStartsOneHoldThatSaysWhetherItOpened() throws { + let first = try gopFixture() + let bigger = try fixture("idr_96x64", "h264") + // It opened: the new size's keyframe is shown at once. + let replaced = VideoDecoder() + defer { replaced.shutdown() } + let replacedEvents = OSAllocatedUnfairLock(initialState: [String]()) + replaced.onEvent = { message in replacedEvents.withLock { $0.append(message) } } + let shownBigger = expectation(description: "new size shown") + replaced.onFrameDecoded = { buffer, _, _ in if CVPixelBufferGetWidth(buffer) == 96 { shownBigger.fulfill() } } + replaced.decode(packet: packet(seq: 1, data: first.idr, keyframe: true)) + replaced.decode(packet: packet(seq: 2, data: bigger, keyframe: true)) + wait(for: [shownBigger], timeout: 10) + let lines = holdLines(replacedEvents.withLock { $0 }) + XCTAssertEqual(lines.count, 1, "\(lines)") + XCTAssertTrue(lines.first?.hasPrefix("Picture hold started reason=decoder_session_replaced since_decoded_ms=") == true, + "\(lines)") + // It didn't open; the next keyframe's session does, in the same hold. + let attempts = OSAllocatedUnfairLock(initialState: 0) + let failing = VideoDecoder(makeSession: { format, specification, attributes in + let attempt = attempts.withLock { value -> Int in value += 1; return value } + if attempt == 2 { return (kVTVideoDecoderMalfunctionErr, nil) } + var session: VTDecompressionSession? + let status = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: format, + decoderSpecification: specification, imageBufferAttributes: attributes, + outputCallback: nil, decompressionSessionOut: &session) + return (status, session) + }) + defer { failing.shutdown() } + let failingEvents = OSAllocatedUnfairLock(initialState: [String]()) + failing.onEvent = { message in failingEvents.withLock { $0.append(message) } } + let retried = expectation(description: "picture after the retry") + failing.onFrameDecoded = { buffer, timestamp, _ in + if timestamp == 9 * 16_667, CVPixelBufferGetWidth(buffer) == 96 { retried.fulfill() } + } + failing.decode(packet: packet(seq: 1, data: first.idr, keyframe: true)) + failing.decode(packet: packet(seq: 2, data: bigger, keyframe: true)) + failing.decode(packet: packet(seq: 3, data: first.p, keyframe: false)) + failing.decode(packet: packet(seq: 9, data: bigger, keyframe: true)) + wait(for: [retried], timeout: 10) + let failed = try XCTUnwrap(holdLines(failingEvents.withLock { $0 }).first) + XCTAssertEqual(holdLines(failingEvents.withLock { $0 }).count, 1) + XCTAssertTrue(failed.hasPrefix("Picture hold started reason=decoder_session_failed status=\(kVTVideoDecoderMalfunctionErr) since_decoded_ms="), + failed) + } +} diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index 56b742d2..3fc98061 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,11 +35,38 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 29, + Self.vectors.count, 49, "golden vector count drifted — update both test suites together" ) } + /// The identify request and its answer: nonce A0..AF, port 9876, two + /// addresses, and the proof of pairing token 01..10 over them, which the + /// iPad checks as the PC's own. + func testIdentifyAndIdentityVectors() throws { + let nonce = Data(0xA0...0xAF) + let request = try vector("identify") + let (requestHeader, asked) = try XCTUnwrap(Wire.parseControl(request)) + XCTAssertEqual(asked, .identify(WireIdentify(version: 1, nonce: nonce))) + XCTAssertEqual(requestHeader.sessionId, 0) + XCTAssertEqual(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: asked), request) + + let answer = try vector("identity") + XCTAssertEqual(answer.count, request.count, "never longer than the request") + let (_, message) = try XCTUnwrap(Wire.parseControl(answer)) + guard case .identity(let identity) = message else { return XCTFail("wrong type") } + XCTAssertEqual(identity.nonce, nonce) + XCTAssertEqual(identity.port, 9876) + XCTAssertEqual(identity.addresses, [Data([192, 0, 2, 10]), Data([198, 51, 100, 10])]) + XCTAssertEqual(identity.proof, MovedPC.proof(token: Data(1...16), nonce: nonce, port: 9876, + addresses: identity.addresses)) + // More addresses than fit are refused, never read past. + var five = answer + five[16 + 1 + 16 + 2] = 5 + XCTAssertNil(Wire.parseControl(five)) + XCTAssertEqual(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: message), answer) + } + func testPairingRejectionVectors() throws { for (name, status): (String, HelloStatus) in [("hello_ack_unauthorized", .unauthorized), ("hello_ack_rate_limited", .rateLimited)] { let data = try vector(name) @@ -52,6 +79,53 @@ final class WireProtocolGoldenTests: XCTestCase { } } + /// 0.3.5: the PC answers while it isn't streaming, a Failed answer with + /// its reason token. + func testNotStreamingVectors() throws { + for (name, status, reason): (String, HelloStatus, String) in [ + ("hello_ack_stopped", .stopped, ""), ("hello_ack_restarting", .restarting, ""), + ("hello_ack_failed", .failed, "display_changing") + ] { + let data = try vector(name) + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .helloAck(let ack) = message else { return XCTFail("wrong type") } + XCTAssertEqual(ack.status, status) + XCTAssertTrue(ack.status.pcNotStreaming) + XCTAssertEqual(ack.reason, reason) + XCTAssertEqual(ack.sessionId, 0) + XCTAssertEqual(ack.hostCaps, HelloAck.hostCapNack) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + } + } + + /// An iPad that asks for standby answers pads its HELLO2 with zeros, as + /// the PC encodes it, so no answer is larger than the question. + func testAStandbyHelloIsPaddedToFitAnyAnswer() throws { + let data = try vector("hello2_standby_padded") + XCTAssertEqual(data.count, Hello2.standbyLength) + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .hello2(let hello) = message else { return XCTFail("wrong type") } + XCTAssertNotEqual(hello.featureCaps & Hello2.featureStandbyStatus, 0) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + // Without the bit, nothing is added. + var plain = hello + plain.featureCaps &= ~Hello2.featureStandbyStatus + XCTAssertLessThan(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .hello2(plain)).count, data.count) + } + + /// How 0.3.4 and older treat a status they don't know: the whole ACK is + /// dropped, as if it were lost. The PC sends 6-8 only to iPads that set + /// `featureStandbyStatus`. + func testAnUnknownStatusIsDroppedLikeALostAck() throws { + var data = try vector("hello_ack_busy") + data[16] = 9 + XCTAssertNil(Wire.parseControl(data)) + XCTAssertEqual(Hello2.featureStandbyStatus, 1 << 3) + for status: HelloStatus in [.ok, .busy, .versionUnsupported, .error, .unauthorized, .rateLimited] { + XCTAssertFalse(status.pcNotStreaming, "\(status)") + } + } + func testNativePenVectorsAndTruncatedTilt() throws { for (name, phase, buttons, pressure): (String, UInt8, UInt8, UInt16) in [ ("input_pen_down", 0, 1, 750), ("input_pen_hover", 1, 0, 0), ("input_pen_cancel", 3, 1, 0) @@ -243,6 +317,94 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(legacy.audioBufferMs, 0) } + // MARK: Pointer + + func testPointerCapabilityVectors() throws { + XCTAssertEqual(Hello2.featureDrawsPointer, 0x0020) + XCTAssertEqual(HelloAck.hostCapPointer, 0x0040) + let (_, hello) = try XCTUnwrap(Wire.parseControl(try vector("hello2_draws_pointer"))) + guard case .hello2(let h) = hello else { return XCTFail("wrong type") } + XCTAssertNotEqual(h.featureCaps & Hello2.featureDrawsPointer, 0) + let (_, ack) = try XCTUnwrap(Wire.parseControl(try vector("hello_ack_pointer"))) + guard case .helloAck(let a) = ack else { return XCTFail("wrong type") } + XCTAssertNotEqual(a.hostCaps & HelloAck.hostCapPointer, 0) + let (_, old) = try XCTUnwrap(Wire.parseControl(try vector("hello_ack_paired"))) + guard case .helloAck(let legacy) = old else { return XCTFail("wrong type") } + XCTAssertEqual(legacy.hostCaps & HelloAck.hostCapPointer, 0) + } + + func testPointerShapeVector() throws { + let data = try vector("pointer_shape") + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + XCTAssertEqual(header.packetType, .pointerShape) + guard case .pointerShape(let chunk) = message else { return XCTFail("wrong type") } + XCTAssertEqual(chunk.shapeId, 0x0A0B_0C0D) + XCTAssertEqual(chunk.sourceType, 1) + XCTAssertEqual(chunk.flags, PointerShapeChunk.invertFlag) + XCTAssertEqual([chunk.width, chunk.height, chunk.hotX, chunk.hotY], [3, 2, 1, 1]) + XCTAssertEqual(chunk.totalLen, 26) + XCTAssertEqual(chunk.offset, 0) + XCTAssertEqual(Array(chunk.data.suffix(2)), [0b0010_0000, 0b1000_0000]) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + let image = PointerOverlay.image(chunk, bytes: [UInt8](chunk.data)) + XCTAssertEqual([UInt8](try XCTUnwrap(image.invertMask)), [0, 0, 255, 255, 0, 0]) + XCTAssertEqual([UInt8](image.pixels.prefix(8)), [0, 0, 0, 255, 255, 255, 255, 255]) + } + + func testPointerStateVectors() throws { + for (name, visible, x, y) in [("pointer_state", true, Int32(-5), Int32(1234)), + ("pointer_hidden", false, 3000, 7)] { + let data = try vector(name) + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .pointerState(let state) = message else { return XCTFail(name) } + XCTAssertEqual(state.visible, visible, name) + XCTAssertEqual(state.x, x, name) + XCTAssertEqual(state.y, y, name) + XCTAssertEqual(state.shapeId, 0x0A0B_0C0D, name) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + } + } + + func testReceiverReportNamesThePointerShape() throws { + let data = try vector("receiver_report_pointer") + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .receiverReport(let report) = message else { return XCTFail("wrong type") } + XCTAssertEqual(report.pointerShapeId, 0x0A0B_0C0D) + XCTAssertEqual(report.fragsRepaired, 1234) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + let (_, old) = try XCTUnwrap(Wire.parseControl(try vector("receiver_report_v030"))) + guard case .receiverReport(let legacy) = old else { return XCTFail("wrong type") } + XCTAssertEqual(legacy.pointerShapeId, 0) + } + + func testPointerMessagesRejectWhatRustRejects() throws { + let good = try vector("pointer_shape") + // Body offsets: version 16, shape id 20, width 24, total length 32, + // chunk length 40. + let breaks: [(String, Range, [UInt8])] = [ + ("version", 16..<17, [2]), ("zero id", 20..<24, [0, 0, 0, 0]), ("zero width", 24..<26, [0, 0]), + ("wide", 24..<26, [0x01, 0x02]), ("total length", 32..<36, [99, 0, 0, 0]), + ("past the end", 36..<40, [1, 0, 0, 0]), ("empty chunk", 40..<42, [0, 0]), + ] + for (name, range, bytes) in breaks { + var bad = good + bad.replaceSubrange(range, with: bytes) + XCTAssertNil(Wire.parseControl(bad), name) + } + for length in 0..= 8 { + let body = UInt16(max(0, truncated.count - ControlHeader.size)) + truncated[6] = UInt8(body & 0xFF) + truncated[7] = UInt8(body >> 8) + } + XCTAssertNil(Wire.parseControl(truncated), "truncated to \(length)") + } + var state = try vector("pointer_state") + state[16] = 2 + XCTAssertNil(Wire.parseControl(state)) + } + // MARK: Control func testHello2Vector() throws { @@ -291,6 +453,21 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(ack.streamConfig, StreamConfig()) } + /// A refusal without the PC name and host caps, as a HELLO2 too small + /// for the whole answer gets. Every iPad reads only its status. + func testHelloAckBusyBareVector() throws { + let data = try vector("hello_ack_busy_bare") + XCTAssertEqual(data.count, 49) + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .helloAck(let ack) = message else { return XCTFail("wrong type") } + XCTAssertEqual(ack.status, .busy) + XCTAssertEqual(ack.acceptedVersion, 2) + XCTAssertEqual(ack.clientNonce, 0xDEAD_BEEF) + XCTAssertEqual(ack.hostName, "") + XCTAssertEqual(ack.hostCaps, 0) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + } + func testHeartbeatVector() throws { let data = try vector("heartbeat") let (header, message) = try XCTUnwrap(Wire.parseControl(data)) @@ -302,12 +479,93 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: 42, message: message), data) } + /// The PC says it is locked in one byte after the stream config. An + /// iPad older than the field reads the same heartbeat without it. + func testPCLockedHeartbeatVector() throws { + let data = try vector("heartbeat_pc_locked") + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .heartbeat(let hb) = message else { return XCTFail("wrong type") } + XCTAssertEqual(hb.hostTimeUs, 987_654_321) + XCTAssertEqual(hb.streamConfig, Self.sampleConfig) + XCTAssertEqual(hb.notice, Heartbeat.noticePCLocked) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: 42, message: message), data) + let plain = try vector("heartbeat") + XCTAssertEqual(data.count, plain.count + 1) + XCTAssertEqual(data.dropFirst(8).dropLast(), plain.dropFirst(8)) + guard case .heartbeat(let old) = try XCTUnwrap(Wire.parseControl(plain)).message else { return XCTFail("wrong type") } + XCTAssertEqual(old.notice, Heartbeat.noticeNone) + } + + /// A PC that takes smaller media datagrams says so, to an iPad that + /// asked; the request and the frame count are appended to the bodies + /// earlier versions read, so those read them as before. + func testMediaSizeVectors() throws { + XCTAssertEqual(HelloAck.hostCapMediaSize, 0x0080) + XCTAssertEqual(HelloAck.hostCapKeyframePerLoss, 0x0100) + XCTAssertEqual(Hello2.featureMediaSize, 0x0040) + XCTAssertEqual(KeyframeRequest.minMediaDgram, 576) + + let ackData = try vector("hello_ack_media_size") + let (ackHeader, ackMessage) = try XCTUnwrap(Wire.parseControl(ackData)) + guard case .helloAck(let ack) = ackMessage else { return XCTFail("wrong type") } + XCTAssertEqual(ack.hostCaps, HelloAck.hostCapNack | HelloAck.hostCapContiguousFrameSeq | HelloAck.hostCapMediaSize) + XCTAssertEqual(Wire.encodeControl(sessionId: ackHeader.sessionId, msgSeq: ackHeader.msgSeq, message: ackMessage), ackData) + + let request = try vector("keyframe_request_media_size") + let (requestHeader, requestMessage) = try XCTUnwrap(Wire.parseControl(request)) + XCTAssertEqual(requestHeader.msgSeq, 51) + XCTAssertEqual(requestMessage, .keyframeRequest(KeyframeRequest(streamEpoch: 3, lastCompleteSeq: 0, reason: .startup, + maxMediaDgram: 1200))) + XCTAssertEqual(Wire.encodeControl(sessionId: 0x1234_5678, msgSeq: 51, message: requestMessage), request) + XCTAssertEqual(request.count, try vector("keyframe_request").count + 2) + + let counted = try vector("heartbeat_frames_sent") + let (heartbeatHeader, heartbeatMessage) = try XCTUnwrap(Wire.parseControl(counted)) + guard case .heartbeat(let hb) = heartbeatMessage else { return XCTFail("wrong type") } + XCTAssertEqual(hb.framesSent, 1234) + XCTAssertEqual(hb.notice, Heartbeat.noticeNone) + XCTAssertEqual(hb.streamConfig, Self.sampleConfig) + XCTAssertEqual(Wire.encodeControl(sessionId: 0x1234_5678, msgSeq: heartbeatHeader.msgSeq, message: heartbeatMessage), + counted) + let plain = try vector("heartbeat") + XCTAssertEqual(counted.count, plain.count + 5) + XCTAssertEqual(counted.dropFirst(16).prefix(plain.count - 16), plain.dropFirst(16)) + guard case .heartbeat(let old) = try XCTUnwrap(Wire.parseControl(plain)).message else { return XCTFail("wrong type") } + XCTAssertNil(old.framesSent) + + // A notice and the count together: the notice byte, then the count. + let both = try vector("heartbeat_no_display_frames_sent") + let (bothHeader, bothMessage) = try XCTUnwrap(Wire.parseControl(both)) + guard case .heartbeat(let noDisplay) = bothMessage else { return XCTFail("wrong type") } + XCTAssertEqual(noDisplay.notice, Heartbeat.noticeNoDisplay) + XCTAssertEqual(noDisplay.framesSent, 77) + XCTAssertEqual(Array(both.dropFirst(plain.count)), [Heartbeat.noticeNoDisplay, 77, 0, 0, 0]) + XCTAssertEqual(Wire.encodeControl(sessionId: 0x1234_5678, msgSeq: bothHeader.msgSeq, message: bothMessage), both) + } + func testByeVector() throws { let data = try vector("bye_background") let (_, message) = try XCTUnwrap(Wire.parseControl(data)) XCTAssertEqual(message, .bye(.appBackground)) } + /// The goodbyes a PC sends as it goes to sleep or powers off, only to an + /// iPad that sets `featurePowerBye`. An app without them drops the whole + /// message, as 0.3.4 does. + func testPowerGoodbyeVectors() throws { + for (name, reason) in [("bye_host_sleeping", ByeReason.hostSleeping), + ("bye_host_powering_off", .hostPoweringOff)] { + let data = try vector(name) + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + XCTAssertEqual(message, .bye(reason)) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + } + var unknown = try vector("bye_host_powering_off") + unknown[unknown.count - 1] = 7 + XCTAssertNil(Wire.parseControl(unknown)) + XCTAssertEqual(Hello2.featurePowerBye, 1 << 7) + } + func testKeyframeRequestVector() throws { let data = try vector("keyframe_request") let (_, message) = try XCTUnwrap(Wire.parseControl(data)) diff --git a/ios/EternalMonitorUITests/ConnectScreenTests.swift b/ios/EternalMonitorUITests/ConnectScreenTests.swift index 8385b6d8..6d84fc24 100644 --- a/ios/EternalMonitorUITests/ConnectScreenTests.swift +++ b/ios/EternalMonitorUITests/ConnectScreenTests.swift @@ -61,6 +61,28 @@ final class ConnectScreenTests: UITestCase { XCTAssertTrue(gone(star, within: 5), "Close did not hide the card") } + /// A PC older than 0.3.4: a quiet card on the Connect screen, above the + /// connect controls, that names the PC and closes. When it shows is a + /// unit test (OldPCTests). + func testAnOldPCsCardNamesThePCAndCloses() { + let app = XCUIApplication() + app.launchArguments = ["-didSeeOnboarding", "YES", "-allowUSB", "NO", "-checkForUpdates", ""] + app.launchEnvironment["EM_UI_PREVIEW"] = "old-pc" + app.launchEnvironment["EM_USB_PORT"] = "0" + launchCleanly(app) + let card = app.descendants(matching: .any)["connect.oldPC"] + XCTAssertTrue(card.waitForExistence(timeout: 15), app.debugDescription) + XCTAssertTrue(card.label.contains("Your PC app is out of date"), card.label) + XCTAssertTrue(card.label.contains("Update EternalMonitor on Studio PC for faster, steadier streaming."), card.label) + XCTAssertTrue(card.label.contains("eternalmonitor.dev"), card.label) + XCTAssertLessThan(card.frame.maxY, app.buttons["connect.button"].frame.minY, "above the connect controls") + XCTAssertTrue(app.buttons["connect.button"].isEnabled, "connecting isn't blocked") + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = "old-pc-card"; shot.lifetime = .keepAlways; add(shot) + app.buttons["connect.oldPCClose"].tap() + XCTAssertTrue(gone(card, within: 5), "one tap on Close hides the card") + } + private func gone(_ element: XCUIElement, within timeout: TimeInterval) -> Bool { let gone = XCTNSPredicateExpectation(predicate: NSPredicate(format: "exists == false"), object: element) return XCTWaiter().wait(for: [gone], timeout: timeout) == .completed diff --git a/ios/EternalMonitorUITests/DisplayStateTests.swift b/ios/EternalMonitorUITests/DisplayStateTests.swift index fd82b984..66019d3f 100644 --- a/ios/EternalMonitorUITests/DisplayStateTests.swift +++ b/ios/EternalMonitorUITests/DisplayStateTests.swift @@ -26,4 +26,34 @@ final class DisplayStateTests: UITestCase { screenshot.lifetime = .keepAlways add(screenshot) } + + func testALockedPCSaysSoOverTheHeldPicture() { + let app = XCUIApplication() + app.launchArguments = ["-didSeeOnboarding", "YES", "-allowUSB", "NO"] + app.launchEnvironment["EM_UI_PREVIEW"] = "locked" + launchCleanly(app) + let note = app.otherElements["display.pcLocked"] + XCTAssertTrue(note.waitForExistence(timeout: 10), "a locked PC must not look like a frozen picture") + XCTAssertEqual(note.label, "Your PC is locked. Unlock it to continue.") + XCTAssertFalse(app.otherElements["display.restoringPicture"].exists) + let screenshot = XCTAttachment(screenshot: app.screenshot()) + screenshot.name = "pc-locked" + screenshot.lifetime = .keepAlways + add(screenshot) + } + + func testAPCWithNoDisplaySaysSoOverTheHeldPicture() { + let app = XCUIApplication() + app.launchArguments = ["-didSeeOnboarding", "YES", "-allowUSB", "NO"] + app.launchEnvironment["EM_UI_PREVIEW"] = "no-display" + launchCleanly(app) + let note = app.otherElements["display.pcNoDisplay"] + XCTAssertTrue(note.waitForExistence(timeout: 10), "a PC with no display on must not look like a frozen picture") + XCTAssertEqual(note.label, "Your PC has no active display. Turn on a screen to continue.") + XCTAssertFalse(app.otherElements["display.pcLocked"].exists) + let screenshot = XCTAttachment(screenshot: app.screenshot()) + screenshot.name = "pc-no-display" + screenshot.lifetime = .keepAlways + add(screenshot) + } } diff --git a/ios/EternalMonitorUITests/PairingFlowTests.swift b/ios/EternalMonitorUITests/PairingFlowTests.swift index 9d02ce09..aad0a8aa 100644 --- a/ios/EternalMonitorUITests/PairingFlowTests.swift +++ b/ios/EternalMonitorUITests/PairingFlowTests.swift @@ -8,7 +8,13 @@ final class PairingFlowTests: UITestCase { let host = try XCTUnwrap(environment["EM_PAIRING_HOST"]) XCTAssertEqual(code.count, 6) let app = XCUIApplication() - app.launchArguments = ["-didSeeOnboarding", "YES", "-allowUSB", "NO", "-controlPC", "NO", "-playPCaudio", "NO", "-pictureFit", "fit"] + // VoiceOver keeps the HUD up, so Disconnect is there from the first + // frame until the test acts. Otherwise the HUD hides five seconds + // after the stream opens, and reading the closed sheet's field back + // can take that long on a starved runner (6.6 s twice on CI). + // HUDHideTests covers the HUD hiding by itself. + app.launchArguments = ["-didSeeOnboarding", "YES", "-allowUSB", "NO", "-controlPC", "NO", "-playPCaudio", "NO", + "-pictureFit", "fit", "-assumeVoiceOver", "YES"] app.launchEnvironment["EM_AUTOCONNECT"] = host app.launchEnvironment["EM_E2E_LOG"] = "1" launchCleanly(app) diff --git a/proto/src/reassembly.rs b/proto/src/reassembly.rs index 4a1d7b8d..56ef19df 100644 --- a/proto/src/reassembly.rs +++ b/proto/src/reassembly.rs @@ -72,6 +72,8 @@ struct PendingFrame { deadline: Option, retry_at: Option, requested_at: Option, + /// The latest request for this frame (see `FrameAssembler.PendingFrame.askedAt`). + asked_at: Option, completed_at: Option, stall_used: Duration, stall_counted_until: Option, @@ -313,7 +315,7 @@ impl Reassembler { // once a complete frame waits. let latest = now + self.repair_window(); for frame in self.pending.values_mut() { - if !frame.hold_retry.is_zero() { + if !frame.hold_retry.is_zero() && !frame.header.is_keyframe { frame.drop_by = Some(frame.drop_by.map_or(latest, |by| by.min(latest))); } } @@ -331,6 +333,7 @@ impl Reassembler { deadline: None, retry_at: None, requested_at: None, + asked_at: None, completed_at: None, stall_used: Duration::ZERO, stall_counted_until: None, @@ -438,7 +441,10 @@ impl Reassembler { .is_some_and(|deadline| now >= deadline + self.stall(frame, now)) { if self.pending.first_key_value().map(|(&s, _)| s) == Some(seq) { - let blocking = self.pending.values().any(PendingFrame::complete); + // The frames behind a keyframe need it, so they never + // make it go (see `FrameAssembler.tick(at:)`). + let blocking = !frame.header.is_keyframe + && self.pending.values().any(PendingFrame::complete); if !blocking && frame.gap_at.is_some_and(|gap| now < gap + MAX_HOLD) { self.hold(seq, now); } else if blocking { @@ -474,12 +480,16 @@ impl Reassembler { { let through = frame.header.frag_count - 1; self.begin_gap(seq, through, now); - } else if !frame.retried && frame.retry_at.is_some_and(|retry| now >= retry) { - let through = frame.header.frag_count - 1; - if let Some(frame) = self.pending.get_mut(&seq) { - frame.retried = true; - } - self.request(seq, through, true, now); + } else if !frame.header.is_keyframe + && !frame.retried + && frame.retry_at.is_some_and(|retry| now >= retry) + { + self.retry(seq, now); + } + // A keyframe's requests follow their own schedule, apart + // from its hold (see `FrameAssembler.tick(at:)`). + if self.keyframe_retry_due(seq, now) { + self.retry(seq, now); } } else if now.saturating_duration_since(frame.last_fragment_at) >= STALE_FRAME_TIMEOUT { // Timed from the latest fragment: over a slow USB tunnel a @@ -511,6 +521,34 @@ impl Reassembler { self.request(seq, through, false, now); } + /// A keyframe asks again every RTT + 5 ms; after its first retry, only + /// once something arrived since it last asked (see + /// `FrameAssembler.keyframeRetryDue(_:at:)`). + fn keyframe_retry_due(&self, seq: u32, now: Instant) -> bool { + let Some(frame) = self.pending.get(&seq) else { + return false; + }; + frame.header.is_keyframe + && !frame.complete() + && frame.retry_at.is_some_and(|retry| now >= retry) + && (!frame.retried + || self.last_arrival.is_some_and(|arrival| { + arrival < now && frame.asked_at.is_some_and(|asked| arrival > asked) + })) + } + + /// Asks again for every missing fragment and schedules the next retry. + fn retry(&mut self, seq: u32, now: Instant) { + let next = now + self.rtt + Duration::from_millis(5); + let Some(frame) = self.pending.get_mut(&seq) else { + return; + }; + frame.retried = true; + frame.retry_at = Some(next); + let through = frame.header.frag_count - 1; + self.request(seq, through, true, now); + } + fn repair_window(&self) -> Duration { let window_us = (2 * self.rtt.as_micros() + self.frame_period.as_micros() * 3 / 2).clamp(8_000, 25_000); @@ -519,7 +557,7 @@ impl Reassembler { /// No complete frame waits behind this frame, so dropping it gains /// nothing (see `FrameAssembler.hold(_:now:)`): ask again, less often - /// each time. + /// each time. A keyframe's hold only moves its deadline. fn hold(&mut self, seq: u32, now: Instant) { let window = self.repair_window(); let newer_waiting = self.pending.len() > 1; @@ -533,10 +571,13 @@ impl Reassembler { }; frame.hold_retry = wait; frame.deadline = Some(now + wait); - if newer_waiting { + if newer_waiting && !frame.header.is_keyframe { frame.drop_by = Some(frame.drop_by.map_or(now, |by| by.min(now))); } frame.stall_used = MAX_STALL_ALLOWANCE; + if frame.header.is_keyframe { + return; + } frame.retried = true; let through = frame.header.frag_count - 1; self.request(seq, through, true, now); @@ -602,6 +643,7 @@ impl Reassembler { } frame.requested.extend(&missing); frame.requested_at.get_or_insert(now); + frame.asked_at = Some(now); let frag_count = frame.header.frag_count; for chunk in missing.chunks(64) { self.counters.nacks_sent += 1; diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index f3e9f069..7f1606a5 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -37,9 +37,26 @@ pub const CAP_DECODE_HEVC_10BIT: u16 = 1 << 2; pub const FEATURE_WANTS_INPUT: u16 = 1 << 0; pub const FEATURE_WANTS_AUDIO: u16 = 1 << 1; pub const FEATURE_SUPPORTS_NACK: u16 = 1 << 2; +/// The iPad understands [`HelloStatus::Stopped`], [`HelloStatus::Restarting`] +/// and [`HelloStatus::Failed`] (0.3.5 and later). Older iPads drop an ACK +/// whose status they don't know, so the host sends them only statuses 0-5. +pub const FEATURE_STANDBY_STATUS: u16 = 1 << 3; /// The iPad app checks for its own updates (0.3.2 and later), so a HELLO2 /// without it comes from 0.3.1 or older. Older hosts ignore it. pub const FEATURE_CHECKS_UPDATES: u16 = 1 << 4; +/// The iPad draws the pointer itself from POINTER_SHAPE and POINTER_STATE. +/// A host with [`HOSTCAP_POINTER`] then leaves the pointer out of the video. +/// Bit 3 is taken by another change in review. +pub const FEATURE_DRAWS_POINTER: u16 = 1 << 5; +/// The iPad may ask for smaller media datagrams in a +/// [`KeyframeRequest::max_media_dgram`], and wants [`Heartbeat::frames_sent`] +/// to tell a PC that sends nothing from video lost on the way (0.3.5 and +/// later). Bit 5 is [`FEATURE_DRAWS_POINTER`]. +pub const FEATURE_MEDIA_SIZE: u16 = 1 << 6; +/// The iPad understands [`ByeReason::HostSleeping`] and +/// [`ByeReason::HostPoweringOff`] (0.3.5 and later). Older iPads drop a +/// goodbye whose reason they don't know, so the host doesn't send them one. +pub const FEATURE_POWER_BYE: u16 = 1 << 7; pub const HOSTCAP_NACK: u16 = 1 << 0; pub const HOSTCAP_AUDIO: u16 = 1 << 1; pub const HOSTCAP_USB: u16 = 1 << 2; @@ -49,7 +66,27 @@ pub const HOSTCAP_PEN: u16 = 1 << 4; /// Media `frame_seq` counts the access units sent in a stream epoch without /// gaps, so a forward gap is a frame lost in transit. Without it, numbers /// may skip frames that were never sent. Older iPads ignore the bit. +/// Every 0.3.4 and later host sets it in every reply that accepts a +/// session, and iPads read an accepted reply without it as a PC to update, +/// so it stays set there. pub const HOSTCAP_CONTIGUOUS_FRAME_SEQ: u16 = 1 << 5; +/// The host sends POINTER_SHAPE and POINTER_STATE, and leaves the pointer +/// out of the video, to a client with [`FEATURE_DRAWS_POINTER`]. +pub const HOSTCAP_POINTER: u16 = 1 << 6; +/// The host honours [`KeyframeRequest::max_media_dgram`] for the rest of the +/// session and puts [`Heartbeat::frames_sent`] in its heartbeats. Set only +/// for an iPad whose HELLO2 has [`FEATURE_MEDIA_SIZE`], so an older iPad +/// gets the same bytes as before. Bit 6 is [`HOSTCAP_POINTER`]. +pub const HOSTCAP_MEDIA_SIZE: u16 = 1 << 7; +/// The host answers a keyframe request inside its 500 ms window when the +/// request names a frame from the first keyframe it sent since its last +/// grant on: a loss after that recovery. Older iPads ignore the bit; an iPad +/// asks early only a host that sets it, since an older one moves its window +/// for any grant. +pub const HOSTCAP_KEYFRAME_PER_LOSS: u16 = 1 << 8; +/// The smallest media datagram an iPad may ask for: the IPv4 minimum +/// reassembly size, also the smallest packet size the PC's own setting takes. +pub const MIN_MEDIA_DGRAM: u16 = 576; /// [`StreamConfig::flags`] bits. pub const STREAM_FLAG_SOFTWARE_ENCODER: u8 = 1 << 0; @@ -136,6 +173,13 @@ pub enum HelloStatus { Error = 3, Unauthorized = 4, RateLimited = 5, + /// The PC answered but isn't streaming: someone clicked Stop on it. + Stopped = 6, + /// The PC is restarting its stream; ask again shortly. + Restarting = 7, + /// The PC stopped streaming after repeated errors, until someone clicks + /// Start. [`HelloAck::reason`] carries a [`FailedReason`] token. + Failed = 8, } impl HelloStatus { @@ -147,6 +191,9 @@ impl HelloStatus { 3 => Self::Error, 4 => Self::Unauthorized, 5 => Self::RateLimited, + 6 => Self::Stopped, + 7 => Self::Restarting, + 8 => Self::Failed, _ => return None, }) } @@ -167,13 +214,116 @@ pub struct HelloAck { pub host_name: String, pub auth_token: [u8; 16], pub host_caps: u16, + /// Why the PC isn't streaming, for [`HelloStatus::Failed`]: a + /// [`FailedReason`] token, empty otherwise. Appended after `host_caps` + /// (0.3.5), at most [`MAX_REASON_LEN`] bytes; older parsers ignore it. + pub reason: String, } +/// Longest [`HelloAck::reason`] on the wire, in bytes. +pub const MAX_REASON_LEN: usize = 32; + +/// The longest HELLO_ACK datagram: the header, the fixed fields and stream +/// config, the longest host name, the token and host caps, and the longest +/// reason. +pub const MAX_HELLO_ACK_LEN: usize = + CONTROL_HEADER_SIZE + 16 + STREAM_CONFIG_SIZE + 1 + MAX_NAME_LEN + 16 + 2 + 1 + MAX_REASON_LEN; + +/// A HELLO2 that sets [`FEATURE_STANDBY_STATUS`] is padded with zeros to at +/// least this size, so any answer to it is no larger than the question. A +/// PC answers while it isn't streaming only within that bound. Older +/// parsers ignore the padding, as any trailing body bytes. +pub const STANDBY_HELLO2_LEN: usize = MAX_HELLO_ACK_LEN; + +/// Why a PC stopped streaming after repeated errors, as [`HelloAck::reason`] +/// carries it. A token, not text: the iPad says it in its own words, and an +/// iPad that doesn't know a token shows the general advice. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FailedReason { + Capture, + Encoder, + Transport, + SecureDesktop, + DisplayChanging, + NoDisplay, + UnsupportedOnThisGpu, + TooManyCaptureApps, + SessionDisconnected, + DriverNotResponding, +} + +impl FailedReason { + pub const ALL: [FailedReason; 10] = [ + Self::Capture, + Self::Encoder, + Self::Transport, + Self::SecureDesktop, + Self::DisplayChanging, + Self::NoDisplay, + Self::UnsupportedOnThisGpu, + Self::TooManyCaptureApps, + Self::SessionDisconnected, + Self::DriverNotResponding, + ]; + + pub fn token(self) -> &'static str { + match self { + Self::Capture => "capture", + Self::Encoder => "encoder", + Self::Transport => "transport", + Self::SecureDesktop => "secure_desktop", + Self::DisplayChanging => "display_changing", + Self::NoDisplay => "no_display", + Self::UnsupportedOnThisGpu => "unsupported_gpu", + Self::TooManyCaptureApps => "too_many_capture_apps", + Self::SessionDisconnected => "session_disconnected", + Self::DriverNotResponding => "driver_not_responding", + } + } + + pub fn from_token(token: &str) -> Option { + Self::ALL.into_iter().find(|reason| reason.token() == token) + } +} + +/// [`Heartbeat::notice`] values. A notice is appended to the heartbeat only +/// when it is not [`NOTICE_NONE`]: iPads older than the field ignore the byte, +/// and an iPad ignores a value it does not know. +pub const NOTICE_NONE: u8 = 0; +/// The PC is locked or shows a permission prompt, so Windows lets nothing +/// capture its screen and the picture stops until the user returns. +pub const NOTICE_PC_LOCKED: u8 = 1; +/// No display is attached to the PC's desktop (a closed lid, screens off or +/// unplugged), so there is nothing to capture until one comes back. +pub const NOTICE_NO_DISPLAY: u8 = 2; +/// Windows won't let EternalMonitor capture the screen from the graphics +/// card it runs on, as on a laptop whose screen is wired to its other card. +/// The PC's window says how to switch; nothing changes until someone does. +pub const NOTICE_UNSUPPORTED_GPU: u8 = 3; +/// [`NOTICE_UNSUPPORTED_GPU`] in Second screen mode, once the second screen +/// failed to start: the main screen it fell back to can't be captured either. +pub const NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU: u8 = 4; +/// The PC is restarting its stream and keeps the session meanwhile: the +/// picture comes back by itself. Sent only to iPads that set +/// [`FEATURE_STANDBY_STATUS`]. (3 and 4 are taken by the graphics card +/// notices.) +pub const NOTICE_RESTARTING: u8 = 5; +/// Second screen needs Windows 11, so the iPad shows the PC's main screen +/// instead. Sent for a few seconds after the session starts mirroring for +/// it, over a picture that works. +pub const NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11: u8 = 6; + /// Host → client, every heartbeat interval while a session is active. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Heartbeat { pub host_time_us: u64, pub stream_config: StreamConfig, + /// What the iPad should say over the picture, for as long as it lasts. + pub notice: u8, + /// Video frames sent to this session so far, for an iPad that set + /// [`FEATURE_MEDIA_SIZE`]; `None` for others. Appended after `notice`, + /// which is then always written. + pub frames_sent: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -184,6 +334,11 @@ pub enum ByeReason { HostShuttingDown = 2, Error = 3, Superseded = 4, + /// The PC is going to sleep. Only to an iPad with [`FEATURE_POWER_BYE`]. + HostSleeping = 5, + /// The PC is shutting down, restarting or signing out. Only to an iPad + /// with [`FEATURE_POWER_BYE`]. + HostPoweringOff = 6, } impl ByeReason { @@ -194,6 +349,8 @@ impl ByeReason { 2 => Self::HostShuttingDown, 3 => Self::Error, 4 => Self::Superseded, + 5 => Self::HostSleeping, + 6 => Self::HostPoweringOff, _ => return None, }) } @@ -226,6 +383,11 @@ pub struct KeyframeRequest { pub stream_epoch: u32, pub last_complete_seq: u32, pub reason: KeyframeReason, + /// The largest media datagram the client asks for from now on in this + /// session, at least [`MIN_MEDIA_DGRAM`]; 0 asks nothing. Appended + /// (0.3.5): sent only to a host with [`HOSTCAP_MEDIA_SIZE`], and older + /// hosts ignore it as any trailing body bytes. + pub max_media_dgram: u16, } /// Requests missing fragments from one frame. Indices are unique and sorted. @@ -273,6 +435,9 @@ pub struct ReceiverReport { pub nacks_sent: u32, pub audio_packets_lost: u32, pub audio_buffer_ms: u16, + /// The last pointer shape the client holds complete, 0 for none. The + /// host sends the current shape again when this differs from it. + pub pointer_shape_id: u32, } pub const RECEIVER_REPORT_SIZE: usize = 36; @@ -315,9 +480,154 @@ pub struct InputEvent { pub tilt_y: i16, } +/// [`Identify`] and [`Identity`] layout version. +pub const IDENTIFY_VERSION: u8 = 1; +/// What an [`Identity`] proof signs after the nonce. +pub const IDENTIFY_CONTEXT: &[u8] = b"EternalMonitor identify v1"; + +/// At most this many addresses in an [`Identity`]. +pub const IDENTITY_MAX_ADDRESSES: usize = 4; + +/// Client → host, session_id 0: before reconnecting to a PC it found at a new +/// address, the iPad checks that it is the same PC, at that address. The host +/// answers with an [`Identity`] whose proof only the PC holding the pairing +/// token can make. Reserved bytes keep the request as long as the answer. +/// Hosts before 0.3.5 don't know the type and drop it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Identify { + pub version: u8, + pub nonce: [u8; 16], +} + +/// Host → client, session_id 0: the port the host receives the stream on and +/// its home network IPv4 addresses (at most [`IDENTITY_MAX_ADDRESSES`]), and +/// `proof`, HMAC-SHA256 keyed with the pairing token over +/// [`identity_message`]. The iPad goes on only when the address it asked at +/// is one of them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Identity { + pub version: u8, + pub nonce: [u8; 16], + pub port: u16, + pub addresses: Vec<[u8; 4]>, + pub proof: [u8; 32], +} + +/// Both bodies: version, nonce, port, address count, the address slots +/// (unused ones zero) and the proof. +pub const IDENTIFY_BODY_SIZE: usize = 1 + 16 + 2 + 1 + 4 * IDENTITY_MAX_ADDRESSES + 32; + +/// What an [`Identity`] proof signs: `nonce ‖ IDENTIFY_CONTEXT ‖ port (LE) ‖ +/// count ‖ addresses`. Only the first [`IDENTITY_MAX_ADDRESSES`] count. +pub fn identity_message(nonce: &[u8; 16], port: u16, addresses: &[[u8; 4]]) -> Vec { + let addresses = &addresses[..addresses.len().min(IDENTITY_MAX_ADDRESSES)]; + let mut message = Vec::with_capacity(16 + IDENTIFY_CONTEXT.len() + 3 + 4 * addresses.len()); + message.extend_from_slice(nonce); + message.extend_from_slice(IDENTIFY_CONTEXT); + message.extend_from_slice(&port.to_le_bytes()); + message.push(addresses.len() as u8); + for address in addresses { + message.extend_from_slice(address); + } + message +} + pub const INPUT_EVENT_SIZE: usize = 30; pub const PEN_INPUT_EVENT_SIZE: usize = INPUT_EVENT_SIZE + 4; +/// The pointer messages' body layout version. +pub const POINTER_VERSION: u8 = 1; +/// [`PointerShapeChunk::flags`]: the image is followed by an invert mask. +pub const POINTER_SHAPE_INVERT: u8 = 1 << 0; +/// [`PointerState::flags`]: the pointer is on the captured screen and shown. +pub const POINTER_VISIBLE: u8 = 1 << 0; +/// Widest and tallest pointer image sent, in pixels. +pub const POINTER_MAX_SIDE: u16 = 512; +/// The fixed fields of a POINTER_SHAPE body, before its bytes. +pub const POINTER_SHAPE_FIXED_SIZE: usize = 26; +/// Image bytes in one POINTER_SHAPE datagram. +pub const POINTER_CHUNK_MAX: usize = + super::MAX_DGRAM_SIZE - CONTROL_HEADER_SIZE - POINTER_SHAPE_FIXED_SIZE; +pub const POINTER_STATE_SIZE: usize = 24; + +/// Bytes of a pointer image: `height` rows of `width` premultiplied BGRA +/// pixels, then, with [`POINTER_SHAPE_INVERT`], `height` rows of +/// `ceil(width / 8)` mask bytes, most significant bit first, where a set bit +/// inverts the picture under that pixel. +pub fn pointer_shape_len(width: u16, height: u16, flags: u8) -> usize { + let (w, h) = (usize::from(width), usize::from(height)); + let mask = if flags & POINTER_SHAPE_INVERT != 0 { + w.div_ceil(8) * h + } else { + 0 + }; + w * h * 4 + mask +} + +/// Host → client: one chunk of the pointer image with id `shape_id`. The +/// client keeps chunks until it holds all `total_len` bytes, then draws that +/// image at the position in [`PointerState`]. A new id replaces the image. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PointerShapeChunk { + /// Nonzero; changes whenever the image does. + pub shape_id: u32, + /// The Windows pointer type it came from (1 monochrome, 2 color, + /// 4 masked color). For diagnostics; the bytes are already converted. + pub source_type: u8, + pub flags: u8, + pub width: u16, + pub height: u16, + /// The click point inside the image. + pub hot_x: u16, + pub hot_y: u16, + pub total_len: u32, + pub offset: u32, + pub data: Vec, +} + +/// Splits one pointer image into the chunks that carry it. +#[allow(clippy::too_many_arguments)] +pub fn pointer_shape_chunks( + shape_id: u32, + source_type: u8, + flags: u8, + (width, height): (u16, u16), + (hot_x, hot_y): (u16, u16), + bytes: &[u8], +) -> Vec { + debug_assert_eq!(bytes.len(), pointer_shape_len(width, height, flags)); + bytes + .chunks(POINTER_CHUNK_MAX) + .enumerate() + .map(|(index, data)| PointerShapeChunk { + shape_id, + source_type, + flags, + width, + height, + hot_x, + hot_y, + total_len: bytes.len() as u32, + offset: (index * POINTER_CHUNK_MAX) as u32, + data: data.to_vec(), + }) + .collect() +} + +/// Host → client: where the pointer is, latest wins by `msg_seq`. `x`/`y` +/// place the image's top-left corner in video pixels, and may be negative +/// or past the edge when the pointer is partly off the screen. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct PointerState { + pub flags: u8, + /// The image to draw; 0 while the host has none. + pub shape_id: u32, + pub x: i32, + pub y: i32, + /// Host clock when the host read this position. + pub host_time_us: u64, +} + /// One parsed control message. #[derive(Debug, Clone, PartialEq)] pub enum ControlMessage { @@ -332,6 +642,10 @@ pub enum ControlMessage { Pong(Pong), StreamConfig(StreamConfig), InputEvent(InputEvent), + PointerShape(PointerShapeChunk), + PointerState(PointerState), + Identify(Identify), + Identity(Identity), } impl ControlMessage { @@ -348,6 +662,10 @@ impl ControlMessage { Self::Pong(_) => PacketType::Pong, Self::StreamConfig(_) => PacketType::StreamConfig, Self::InputEvent(_) => PacketType::InputEvent, + Self::PointerShape(_) => PacketType::PointerShape, + Self::PointerState(_) => PacketType::PointerState, + Self::Identify(_) => PacketType::Identify, + Self::Identity(_) => PacketType::Identity, } } } @@ -380,6 +698,9 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - body.extend_from_slice(&h.auth_token); body.extend_from_slice(&h.pairing_code.to_le_bytes()); } + if h.feature_caps & FEATURE_STANDBY_STATUS != 0 { + body.resize(body.len().max(STANDBY_HELLO2_LEN - CONTROL_HEADER_SIZE), 0); + } } ControlMessage::HelloAck(a) => { let name = truncated_name(&a.host_name); @@ -395,22 +716,36 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - body.extend_from_slice(&cfg); body.push(name.len() as u8); body.extend_from_slice(name.as_bytes()); - if a.host_caps != 0 || a.auth_token != [0; 16] { + let reason = truncated(&a.reason, MAX_REASON_LEN); + if a.host_caps != 0 || a.auth_token != [0; 16] || !reason.is_empty() { body.extend_from_slice(&a.auth_token); body.extend_from_slice(&a.host_caps.to_le_bytes()); } + if !reason.is_empty() { + body.push(reason.len() as u8); + body.extend_from_slice(reason.as_bytes()); + } } ControlMessage::Heartbeat(hb) => { body.extend_from_slice(&hb.host_time_us.to_le_bytes()); let mut cfg = [0u8; STREAM_CONFIG_SIZE]; hb.stream_config.encode_into(&mut cfg); body.extend_from_slice(&cfg); + if hb.notice != NOTICE_NONE || hb.frames_sent.is_some() { + body.push(hb.notice); + } + if let Some(frames) = hb.frames_sent { + body.extend_from_slice(&frames.to_le_bytes()); + } } ControlMessage::Bye(reason) => body.push(*reason as u8), ControlMessage::KeyframeRequest(k) => { body.extend_from_slice(&k.stream_epoch.to_le_bytes()); body.extend_from_slice(&k.last_complete_seq.to_le_bytes()); body.push(k.reason as u8); + if k.max_media_dgram != 0 { + body.extend_from_slice(&k.max_media_dgram.to_le_bytes()); + } } ControlMessage::Nack(n) => { n.validate().expect("invalid outgoing NACK"); @@ -439,12 +774,16 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - || r.nacks_sent != 0 || r.audio_packets_lost != 0 || r.audio_buffer_ms != 0 + || r.pointer_shape_id != 0 { body.extend_from_slice(&r.frags_repaired.to_le_bytes()); body.extend_from_slice(&r.nacks_sent.to_le_bytes()); body.extend_from_slice(&r.audio_packets_lost.to_le_bytes()); body.extend_from_slice(&r.audio_buffer_ms.to_le_bytes()); } + if r.pointer_shape_id != 0 { + body.extend_from_slice(&r.pointer_shape_id.to_le_bytes()); + } } ControlMessage::Ping(p) => body.extend_from_slice(&p.t1_us.to_le_bytes()), ControlMessage::Pong(p) => { @@ -457,6 +796,22 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - cfg.encode_into(&mut block); body.extend_from_slice(&block); } + ControlMessage::Identify(i) => { + body.push(i.version); + body.extend_from_slice(&i.nonce); + body.extend_from_slice(&[0; IDENTIFY_BODY_SIZE - 17]); + } + ControlMessage::Identity(i) => { + let count = i.addresses.len().min(IDENTITY_MAX_ADDRESSES); + body.push(i.version); + body.extend_from_slice(&i.nonce); + body.extend_from_slice(&i.port.to_le_bytes()); + body.push(count as u8); + for slot in 0..IDENTITY_MAX_ADDRESSES { + body.extend_from_slice(i.addresses.get(slot).unwrap_or(&[0; 4])); + } + body.extend_from_slice(&i.proof); + } ControlMessage::InputEvent(e) => { body.push(e.input_ver); body.push(e.kind); @@ -477,6 +832,25 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - body.extend_from_slice(&e.tilt_y.to_le_bytes()); } } + ControlMessage::PointerShape(c) => { + body.extend_from_slice(&[POINTER_VERSION, c.source_type, c.flags, 0]); + body.extend_from_slice(&c.shape_id.to_le_bytes()); + body.extend_from_slice(&c.width.to_le_bytes()); + body.extend_from_slice(&c.height.to_le_bytes()); + body.extend_from_slice(&c.hot_x.to_le_bytes()); + body.extend_from_slice(&c.hot_y.to_le_bytes()); + body.extend_from_slice(&c.total_len.to_le_bytes()); + body.extend_from_slice(&c.offset.to_le_bytes()); + body.extend_from_slice(&(c.data.len() as u16).to_le_bytes()); + body.extend_from_slice(&c.data); + } + ControlMessage::PointerState(p) => { + body.extend_from_slice(&[POINTER_VERSION, p.flags, 0, 0]); + body.extend_from_slice(&p.shape_id.to_le_bytes()); + body.extend_from_slice(&p.x.to_le_bytes()); + body.extend_from_slice(&p.y.to_le_bytes()); + body.extend_from_slice(&p.host_time_us.to_le_bytes()); + } } let mut datagram = vec![0u8; CONTROL_HEADER_SIZE + body.len()]; @@ -569,6 +943,16 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), } else { ([0; 16], 0) }; + let reason = if r.remaining() > 0 { + let len = usize::from(r.u8()?); + if len > MAX_REASON_LEN { + return Err(WireError::InvalidField("reason_len")); + } + String::from_utf8(r.take(len)?.to_vec()) + .map_err(|_| WireError::InvalidField("reason_utf8"))? + } else { + String::new() + }; if status == HelloStatus::Ok && session_id == 0 { return Err(WireError::InvalidField("session_id")); } @@ -584,11 +968,22 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), host_name, auth_token, host_caps, + reason, }) } PacketType::Heartbeat => ControlMessage::Heartbeat(Heartbeat { host_time_us: r.u64()?, stream_config: StreamConfig::decode(r.take(STREAM_CONFIG_SIZE)?)?, + notice: if r.remaining() >= 1 { + r.u8()? + } else { + NOTICE_NONE + }, + frames_sent: if r.remaining() >= 4 { + Some(r.u32()?) + } else { + None + }, }), PacketType::Bye => ControlMessage::Bye( ByeReason::from_u8(r.u8()?).ok_or(WireError::InvalidField("bye_reason"))?, @@ -598,6 +993,7 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), last_complete_seq: r.u32()?, reason: KeyframeReason::from_u8(r.u8()?) .ok_or(WireError::InvalidField("keyframe_reason"))?, + max_media_dgram: if r.remaining() >= 2 { r.u16()? } else { 0 }, }), PacketType::Nack => { let stream_epoch = r.u32()?; @@ -642,6 +1038,9 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), report.audio_packets_lost = r.u32()?; report.audio_buffer_ms = r.u16()?; } + if r.remaining() >= 4 { + report.pointer_shape_id = r.u32()?; + } ControlMessage::ReceiverReport(report) } PacketType::Ping => ControlMessage::Ping(Ping { t1_us: r.u64()? }), @@ -677,6 +1076,80 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), tilt_y: if input_ver == 2 { r.i16()? } else { 0 }, }) } + PacketType::PointerShape => { + pointer_version(r.u8()?)?; + let source_type = r.u8()?; + let flags = r.u8()?; + let _reserved = r.u8()?; + let chunk = PointerShapeChunk { + shape_id: r.u32()?, + source_type, + flags, + width: r.u16()?, + height: r.u16()?, + hot_x: r.u16()?, + hot_y: r.u16()?, + total_len: r.u32()?, + offset: r.u32()?, + data: { + let len = usize::from(r.u16()?); + r.take(len)?.to_vec() + }, + }; + if chunk.shape_id == 0 { + return Err(WireError::InvalidField("shape_id")); + } + if !(1..=POINTER_MAX_SIDE).contains(&chunk.width) + || !(1..=POINTER_MAX_SIDE).contains(&chunk.height) + { + return Err(WireError::InvalidField("pointer_size")); + } + if chunk.total_len as usize != pointer_shape_len(chunk.width, chunk.height, flags) { + return Err(WireError::InvalidField("total_len")); + } + if chunk.data.is_empty() + || chunk.offset as usize + chunk.data.len() > chunk.total_len as usize + { + return Err(WireError::InvalidField("offset")); + } + ControlMessage::PointerShape(chunk) + } + PacketType::PointerState => { + pointer_version(r.u8()?)?; + let flags = r.u8()?; + let _reserved = r.u16()?; + ControlMessage::PointerState(PointerState { + flags, + shape_id: r.u32()?, + x: r.i32()?, + y: r.i32()?, + host_time_us: r.u64()?, + }) + } + PacketType::Identify => { + let version = r.u8()?; + let nonce = r.take(16)?.try_into().unwrap(); + r.take(IDENTIFY_BODY_SIZE - 17)?; + ControlMessage::Identify(Identify { version, nonce }) + } + PacketType::Identity => { + let version = r.u8()?; + let nonce = r.take(16)?.try_into().unwrap(); + let port = r.u16()?; + let count = usize::from(r.u8()?); + if count > IDENTITY_MAX_ADDRESSES { + return Err(WireError::InvalidField("identity_addresses")); + } + let slots = r.take(4 * IDENTITY_MAX_ADDRESSES)?; + let addresses = slots.as_chunks::<4>().0[..count].to_vec(); + ControlMessage::Identity(Identity { + version, + nonce, + port, + addresses, + proof: r.take(32)?.try_into().unwrap(), + }) + } PacketType::Media | PacketType::MediaFec | PacketType::Audio | PacketType::Error => { return Err(WireError::InvalidField("packet_type")); } @@ -685,12 +1158,25 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), Ok((header, message)) } +/// A pointer body of another layout version is not one this parser knows. +fn pointer_version(version: u8) -> Result<(), WireError> { + if version == POINTER_VERSION { + Ok(()) + } else { + Err(WireError::InvalidField("pointer_version")) + } +} + fn truncated_name(name: &str) -> &str { - if name.len() <= MAX_NAME_LEN { + truncated(name, MAX_NAME_LEN) +} + +fn truncated(name: &str, max: usize) -> &str { + if name.len() <= max { return name; } // Truncate on a char boundary at or below the cap. - let mut end = MAX_NAME_LEN; + let mut end = max; while end > 0 && !name.is_char_boundary(end) { end -= 1; } @@ -741,6 +1227,10 @@ impl<'a> Reader<'a> { Ok(u32::from_le_bytes([b[0], b[1], b[2], b[3]])) } + fn i32(&mut self) -> Result { + Ok(self.u32()? as i32) + } + fn u64(&mut self) -> Result { let b = self.take(8)?; Ok(u64::from_le_bytes([ @@ -854,6 +1344,7 @@ mod tests { assert_eq!(report.nacks_sent, 0); assert_eq!(report.audio_packets_lost, 0); assert_eq!(report.audio_buffer_ms, 0); + assert_eq!(report.pointer_shape_id, 0); } _ => {} } @@ -904,16 +1395,34 @@ mod tests { host_name: "ALI-PC".to_string(), auth_token: [0; 16], host_caps: 0, + reason: String::new(), }), ControlMessage::Heartbeat(Heartbeat { host_time_us: 987_654_321, stream_config: sample_config(), + notice: NOTICE_NONE, + frames_sent: None, + }), + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: sample_config(), + notice: NOTICE_NONE, + frames_sent: Some(0), }), ControlMessage::Bye(ByeReason::AppBackground), + ControlMessage::Bye(ByeReason::HostSleeping), + ControlMessage::Bye(ByeReason::HostPoweringOff), ControlMessage::KeyframeRequest(KeyframeRequest { stream_epoch: 3, last_complete_seq: 4111, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, + }), + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 3, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram: 1200, }), ControlMessage::ReceiverReport(ReceiverReport { stream_epoch: 3, @@ -937,6 +1446,17 @@ mod tests { t3_us: 44, }), ControlMessage::StreamConfig(sample_config()), + ControlMessage::Identify(Identify { + version: IDENTIFY_VERSION, + nonce: [7; 16], + }), + ControlMessage::Identity(Identity { + version: IDENTIFY_VERSION, + nonce: [7; 16], + port: 9876, + addresses: vec![[192, 0, 2, 7], [198, 51, 100, 7]], + proof: [9; 32], + }), ControlMessage::InputEvent(InputEvent { input_ver: 1, kind: 0, @@ -954,9 +1474,132 @@ mod tests { tilt_x: 0, tilt_y: 0, }), + ControlMessage::PointerShape(PointerShapeChunk { + shape_id: 7, + source_type: 1, + flags: POINTER_SHAPE_INVERT, + width: 2, + height: 2, + hot_x: 1, + hot_y: 0, + total_len: 18, + offset: 16, + data: vec![0x80, 0x40], + }), + ControlMessage::PointerState(PointerState { + flags: POINTER_VISIBLE, + shape_id: 7, + x: -3, + y: 1439, + host_time_us: 5_000_001, + }), ] } + fn shape_chunk() -> PointerShapeChunk { + all_messages() + .into_iter() + .find_map(|message| match message { + ControlMessage::PointerShape(chunk) => Some(chunk), + _ => None, + }) + .unwrap() + } + + #[test] + fn pointer_shape_chunks_cover_the_image_in_order_within_a_datagram() { + let (width, height) = (64u16, 48u16); + let bytes: Vec = (0..pointer_shape_len(width, height, POINTER_SHAPE_INVERT)) + .map(|i| i as u8) + .collect(); + let chunks = + pointer_shape_chunks(9, 1, POINTER_SHAPE_INVERT, (width, height), (3, 4), &bytes); + assert_eq!(chunks.len(), bytes.len().div_ceil(POINTER_CHUNK_MAX)); + let mut joined = Vec::new(); + for chunk in &chunks { + assert_eq!(chunk.offset as usize, joined.len()); + let datagram = encode_control(1, 1, &ControlMessage::PointerShape(chunk.clone())); + assert!(datagram.len() <= super::super::MAX_DGRAM_SIZE); + let (_, parsed) = parse_control(&datagram).unwrap(); + assert_eq!(parsed, ControlMessage::PointerShape(chunk.clone())); + joined.extend_from_slice(&chunk.data); + } + assert_eq!(joined, bytes); + } + + #[test] + fn pointer_shapes_reject_a_zero_id_bad_sizes_lengths_and_versions() { + let mut cases: Vec<(PointerShapeChunk, &str)> = Vec::new(); + let mut zero = shape_chunk(); + zero.shape_id = 0; + cases.push((zero, "shape_id")); + for (w, h) in [(0, 2), (2, 0), (POINTER_MAX_SIDE + 1, 2)] { + let mut sized = shape_chunk(); + (sized.width, sized.height) = (w, h); + cases.push((sized, "pointer_size")); + } + let mut no_mask = shape_chunk(); + no_mask.flags = 0; + cases.push((no_mask, "total_len")); + let mut past_end = shape_chunk(); + past_end.offset = 17; + cases.push((past_end, "offset")); + let mut empty = shape_chunk(); + empty.data.clear(); + cases.push((empty, "offset")); + for (chunk, field) in cases { + let datagram = encode_control(1, 1, &ControlMessage::PointerShape(chunk)); + assert_eq!( + parse_control(&datagram), + Err(WireError::InvalidField(field)) + ); + } + for message in [ + ControlMessage::PointerShape(shape_chunk()), + ControlMessage::PointerState(PointerState::default()), + ] { + let mut datagram = encode_control(1, 1, &message); + datagram[CONTROL_HEADER_SIZE] = POINTER_VERSION + 1; + assert_eq!( + parse_control(&datagram), + Err(WireError::InvalidField("pointer_version")) + ); + } + } + + #[test] + fn a_report_names_the_pointer_shape_after_the_older_tail() { + let report = ReceiverReport { + stream_epoch: 3, + pointer_shape_id: 0x0102_0304, + ..Default::default() + }; + let datagram = encode_control(1, 1, &ControlMessage::ReceiverReport(report)); + assert_eq!( + datagram.len(), + CONTROL_HEADER_SIZE + RECEIVER_REPORT_SIZE + 14 + 4 + ); + assert_eq!( + parse_control(&datagram).unwrap().1, + ControlMessage::ReceiverReport(report) + ); + // A parser that stops after the older tail reads the same report + // without the id: the tail is appended, so nothing before it moved. + let mut older = datagram[..datagram.len() - 4].to_vec(); + let body_len = (older.len() - CONTROL_HEADER_SIZE) as u16; + older[6..8].copy_from_slice(&body_len.to_le_bytes()); + let ControlMessage::ReceiverReport(old) = parse_control(&older).unwrap().1 else { + unreachable!() + }; + assert_eq!( + old, + ReceiverReport { + pointer_shape_id: 0, + ..report + } + ); + } + #[test] fn every_message_round_trips() { for message in all_messages() { @@ -988,6 +1631,88 @@ mod tests { } } + #[test] + fn a_heartbeat_notice_is_appended_only_when_there_is_one() { + let heartbeat = |notice| { + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: sample_config(), + notice, + frames_sent: None, + }) + }; + // No notice: the bytes every earlier iPad already reads. + let plain = encode_control(7, 1, &heartbeat(NOTICE_NONE)); + assert_eq!(plain.len(), CONTROL_HEADER_SIZE + 8 + STREAM_CONFIG_SIZE); + // A notice is one byte more. Such a body, cut back to the old length + // as an older iPad reads it, is the plain heartbeat; an unknown value + // reaches the iPad as it is, for it to ignore. + for notice in [ + NOTICE_PC_LOCKED, + NOTICE_NO_DISPLAY, + NOTICE_UNSUPPORTED_GPU, + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, + NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11, + 0xC8, + ] { + let datagram = encode_control(7, 1, &heartbeat(notice)); + assert_eq!(datagram.len(), plain.len() + 1); + assert_eq!(datagram[..plain.len()][8..], plain[8..]); + assert_eq!(parse_control(&datagram).unwrap().1, heartbeat(notice)); + for len in 0..datagram.len() { + assert!(parse_control(&datagram[..len]).is_err()); + } + } + assert_eq!(parse_control(&plain).unwrap().1, heartbeat(NOTICE_NONE)); + } + + /// The media size fields are appended: without them the bytes are the + /// ones every earlier PC and iPad reads, and a body cut back to the old + /// length, as an older parser reads it, is the request without them. + #[test] + fn the_media_size_fields_are_appended_only_when_set() { + let request = |max_media_dgram| { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 3, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram, + }) + }; + let plain = encode_control(7, 1, &request(0)); + assert_eq!(plain.len(), CONTROL_HEADER_SIZE + 9); + let sized = encode_control(7, 1, &request(576)); + assert_eq!(sized.len(), plain.len() + 2); + assert_eq!(sized[8..plain.len()], plain[8..]); + assert_eq!(&sized[plain.len()..], &576u16.to_le_bytes()); + assert_eq!(parse_control(&sized).unwrap().1, request(576)); + assert_eq!(parse_control(&plain).unwrap().1, request(0)); + + let heartbeat = |notice, frames_sent| { + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: sample_config(), + notice, + frames_sent, + }) + }; + let plain = encode_control(7, 1, &heartbeat(NOTICE_NONE, None)); + // The count needs the notice byte before it, even with no notice. + let counted = encode_control(7, 1, &heartbeat(NOTICE_NONE, Some(0x0102_0304))); + assert_eq!(counted.len(), plain.len() + 5); + assert_eq!(counted[8..plain.len()], plain[8..]); + assert_eq!(&counted[plain.len()..], &[NOTICE_NONE, 4, 3, 2, 1]); + assert_eq!( + parse_control(&counted).unwrap().1, + heartbeat(NOTICE_NONE, Some(0x0102_0304)) + ); + let locked = encode_control(7, 1, &heartbeat(NOTICE_PC_LOCKED, Some(9))); + assert_eq!( + parse_control(&locked).unwrap().1, + heartbeat(NOTICE_PC_LOCKED, Some(9)) + ); + } + #[test] fn trailing_bytes_are_ignored_for_forward_compat() { let mut datagram = encode_control(1, 1, &ControlMessage::Ping(Ping { t1_us: 5 })); @@ -1022,6 +1747,7 @@ mod tests { host_name: String::new(), auth_token: [0; 16], host_caps: 0, + reason: String::new(), }); // encode_control would happily serialize it; the parser must reject. let datagram = encode_control(0, 1, &ack); @@ -1113,4 +1839,136 @@ mod tests { let _ = super::super::classify(&bytes); } } + + fn standby_ack(status: HelloStatus, reason: &str) -> HelloAck { + HelloAck { + status, + accepted_version: 2, + client_nonce: 9, + session_id: 0, + heartbeat_interval_ms: 1000, + report_interval_ms: 500, + liveness_timeout_ms: 3000, + stream_config: StreamConfig::default(), + host_name: "PC".to_string(), + auth_token: [0; 16], + host_caps: 0, + reason: reason.to_string(), + } + } + + #[test] + fn not_streaming_statuses_and_their_reason_round_trip() { + for status in [ + HelloStatus::Stopped, + HelloStatus::Restarting, + HelloStatus::Failed, + ] { + for reason in ["", "display_changing"] { + let ack = standby_ack(status, reason); + let bytes = encode_control(0, 1, &ControlMessage::HelloAck(ack.clone())); + assert_eq!( + parse_control(&bytes).unwrap().1, + ControlMessage::HelloAck(ack) + ); + } + } + // A reason past the cap is cut, so it always parses. + let long = "x".repeat(100); + let bytes = encode_control( + 0, + 1, + &ControlMessage::HelloAck(standby_ack(HelloStatus::Failed, &long)), + ); + let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&bytes) else { + panic!("a long reason must still parse"); + }; + assert_eq!(ack.reason.len(), MAX_REASON_LEN); + } + + /// 0.3.4 and older know statuses 0-5 only and drop the whole ACK for + /// any other, like a lost datagram: the host must not send them 6-8. + #[test] + fn a_status_past_the_known_ones_is_rejected() { + let mut bytes = encode_control( + 0, + 1, + &ControlMessage::HelloAck(standby_ack(HelloStatus::Error, "")), + ); + bytes[CONTROL_HEADER_SIZE] = 9; + assert_eq!( + parse_control(&bytes), + Err(WireError::InvalidField("status")) + ); + } + + #[test] + fn failed_reason_tokens_are_short_distinct_and_round_trip() { + let mut seen = std::collections::HashSet::new(); + for reason in FailedReason::ALL { + let token = reason.token(); + assert!(token.len() <= MAX_REASON_LEN && token.is_ascii(), "{token}"); + assert!(seen.insert(token), "{token} twice"); + assert_eq!(FailedReason::from_token(token), Some(reason)); + } + assert_eq!(FailedReason::from_token("something new"), None); + } + + fn standby_hello2(feature_caps: u16, device_name: &str) -> Vec { + encode_control( + 0, + 1, + &ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 7, + listen_port: 0, + decoder_caps: CAP_DECODE_H264, + feature_caps, + screen_px_w: 1, + screen_px_h: 1, + screen_pt_w: 1, + screen_pt_h: 1, + refresh_hz: 60, + device_name: device_name.into(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }), + ) + } + + /// Any answer fits in a HELLO2 that asks for standby answers: the + /// longest ACK there is, with a 64-byte host name, a token and the + /// longest reason, is no larger. The padding parses as the HELLO2 it was. + #[test] + fn a_hello2_that_asks_for_standby_answers_fits_any_ack() { + let mut longest = standby_ack(HelloStatus::Failed, &"r".repeat(MAX_REASON_LEN)); + longest.host_name = "h".repeat(MAX_NAME_LEN); + longest.auth_token = [9; 16]; + longest.host_caps = 0xFFFF; + let ack = encode_control(0, 1, &ControlMessage::HelloAck(longest)); + assert_eq!(ack.len(), MAX_HELLO_ACK_LEN); + for name in ["", "iPad", &"n".repeat(MAX_NAME_LEN)] { + let padded = standby_hello2(FEATURE_STANDBY_STATUS, name); + assert!( + padded.len() >= ack.len(), + "{} < {}", + padded.len(), + ack.len() + ); + let Ok((_, ControlMessage::Hello2(hello))) = parse_control(&padded) else { + panic!("a padded HELLO2 must parse"); + }; + assert_eq!(hello.device_name, name); + assert_eq!((hello.device_id, hello.auth_token), (0, [0; 16])); + } + assert_eq!( + standby_hello2(FEATURE_STANDBY_STATUS, "").len(), + STANDBY_HELLO2_LEN + ); + // Without the bit, as from 0.3.4, nothing is added. + assert_eq!(standby_hello2(0, "").len(), CONTROL_HEADER_SIZE + 22); + } } diff --git a/proto/src/v2/mod.rs b/proto/src/v2/mod.rs index 4b75ffe9..ec9b09b6 100644 --- a/proto/src/v2/mod.rs +++ b/proto/src/v2/mod.rs @@ -55,7 +55,16 @@ pub enum PacketType { Pong = 0x17, StreamConfig = 0x18, Nack = 0x19, + /// Host → client pointer image, in chunks. Sent only to clients with + /// [`control::FEATURE_DRAWS_POINTER`]. + PointerShape = 0x1C, + /// Host → client pointer position and visibility, as for `PointerShape`. + PointerState = 0x1D, InputEvent = 0x20, + /// Client → host: prove you are the PC this iPad paired with (0.3.5+). + Identify = 0x1A, + /// Host → client: the answer to [`PacketType::Identify`]. + Identity = 0x1B, /// Reserved. Not sent in v2.0. Error = 0x7F, } @@ -76,7 +85,11 @@ impl PacketType { 0x17 => Self::Pong, 0x18 => Self::StreamConfig, 0x19 => Self::Nack, + 0x1C => Self::PointerShape, + 0x1D => Self::PointerState, 0x20 => Self::InputEvent, + 0x1A => Self::Identify, + 0x1B => Self::Identity, 0x7F => Self::Error, _ => return None, }) diff --git a/proto/testdata/repair_vectors.txt b/proto/testdata/repair_vectors.txt index fd0c7faa..02d9c027 100644 --- a/proto/testdata/repair_vectors.txt +++ b/proto/testdata/repair_vectors.txt @@ -429,3 +429,114 @@ reset 16667 10000 add 0 1 1 0 1 0 1 add 1000 1 3 0 1 0 3 1 expect 2 0 2 0 0 0 0 1,3 0 - + +# Every frame up to the next keyframe refers to the keyframe, so a complete +# frame behind an incomplete keyframe is no reason to drop it: at its +# deadline the keyframe is held, and its repair then delivers both. On lossy +# WiFi, dropping it froze the picture for seconds, because each replacement +# keyframe arrived incomplete too. The hold only moves the deadline; the +# request at 40 ms is the keyframe's retry, due since 31.667 ms and sent +# because a resend arrived after its last request. +reset 16667 10000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +add 16667 1 2 0 1 0 4 +add 17000 1 1 0 3 1 1 1 +expect 0 0 3 0 0 2 1 - 0 1:1|1:1 +tick 40000 +expect 0 0 3 0 0 3 1 - 0 1:1|1:1|1:1 +add 45000 1 1 1 3 1 2 1 +expect 2 0 3 0 1 3 0 1,2 0 1:1|1:1|1:1 + +# With nothing arriving, a held keyframe asks no more: its request and the +# repair would wait in the same queue. It still goes a second after its gap, +# with the frames that need it, under one keyframe request. +reset 16667 10000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +add 16667 1 2 0 1 0 4 +add 17000 1 1 0 3 1 1 1 +tick 40000 +tick 65000 +tick 115000 +tick 215000 +tick 415000 +tick 615000 +tick 815000 +tick 1014999 +expect 0 0 3 0 0 3 1 - 0 1:1|1:1|1:1 +tick 1015000 +expect 0 2 3 1 0 3 0 - 1 1:1|1:1|1:1 + +# A keyframe has many fragments, and one round of requests and repairs can +# leave some missing, so it retries every RTT + 5 ms instead of once. After +# its first retry it asks only once something arrived since it last asked, +# and never on the tick that takes a datagram, which may be the missing one. +reset 16667 10000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +tick 16000 +expect 0 0 2 0 0 2 1 - 0 1:1|1:1 +tick 40000 +expect 0 0 2 0 0 2 1 - 0 1:1|1:1 +add 41000 1 1 0 3 1 1 1 +expect 0 0 2 0 0 2 1 - 0 1:1|1:1 +tick 41001 +expect 0 0 2 0 0 3 1 - 0 1:1|1:1|1:1 +tick 50000 +expect 0 0 2 0 0 3 1 - 0 1:1|1:1|1:1 +add 50000 1 1 1 3 1 2 1 +expect 1 0 2 0 1 3 0 1 0 1:1|1:1|1:1 + +# A keyframe's hold expiring on the tick its retry falls due sends one +# request, and the next waits a full RTT + 5 ms. +reset 16667 10000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +add 16667 1 2 0 1 0 4 +add 17000 1 1 0 3 1 1 1 +tick 40000 +expect 0 0 3 0 0 3 1 - 0 1:1|1:1|1:1 +add 50000 1 3 0 1 0 5 +tick 65000 +expect 0 0 4 0 0 4 1 - 0 1:1|1:1|1:1|1:1 +tick 79999 +expect 0 0 4 0 0 4 1 - 0 1:1|1:1|1:1|1:1 + +# At 1 ms RTT a keyframe retries every 6 ms, within the PC's 5 ms resend +# limit. Its hold expiring at 26 ms, 1 ms after a retry, asks nothing more. +reset 16667 1000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +add 6000 1 1 0 3 1 1 1 +tick 7000 +add 12000 1 1 0 3 1 1 1 +tick 13000 +add 18000 1 1 0 3 1 1 1 +tick 19000 +add 24000 1 1 0 3 1 1 1 +tick 25000 +expect 0 0 2 0 0 5 1 - 0 1:1|1:1|1:1|1:1|1:1 +tick 26000 +expect 0 0 2 0 0 5 1 - 0 1:1|1:1|1:1|1:1|1:1 +tick 30999 +expect 0 0 2 0 0 5 1 - 0 1:1|1:1|1:1|1:1|1:1 +add 31000 1 1 1 3 1 2 1 +expect 1 0 2 0 1 5 0 1 0 1:1|1:1|1:1|1:1|1:1 + +# At 200 ms RTT a repair can't return sooner than that, so the holds at 26 +# and 51 ms ask nothing; the retry follows at RTT + 5 ms. +reset 16667 200000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +add 20000 1 1 0 3 1 1 1 +tick 26000 +expect 0 0 2 0 0 1 1 - 0 1:1 +tick 51000 +expect 0 0 2 0 0 1 1 - 0 1:1 +tick 205999 +expect 0 0 2 0 0 1 1 - 0 1:1 +tick 206000 +expect 0 0 2 0 0 2 1 - 0 1:1|1:1 +add 210000 1 1 1 3 1 2 1 +expect 1 0 2 0 1 2 0 1 0 1:1|1:1 diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 6d072d05..d5ef023c 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -20,8 +20,14 @@ hello2 454d02100000240000000000010000000202efbeadde94260300010074098406ba0442037 hello_ack_ok 454d02110000270000000000010000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d5043 hello_ack_busy 454d02110000270000000000020000000102efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d5043 heartbeat 454d021200001800785634122a000000b168de3a0000000003000000000aa0053c000000c0e1e400 +heartbeat_pc_locked 454d021200001900785634122a000000b168de3a0000000003000000000aa0053c000000c0e1e40001 bye_background 454d021300000100785634122b00000001 +bye_host_sleeping 454d021300000100785634123700000005 +bye_host_powering_off 454d021300000100785634123800000006 keyframe_request 454d021400000900785634122c000000030000000f10000000 +keyframe_request_media_size 454d021400000b007856341233000000030000000000000002b004 +heartbeat_frames_sent 454d021200001d007856341234000000b168de3a0000000003000000000aa0053c000000c0e1e40000d2040000 +heartbeat_no_display_frames_sent 454d021200001d007856341235000000b168de3a0000000003000000000aa0053c000000c0e1e400024d000000 receiver_report 454d021500002400785634122d00000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00 ping 454d021600000800785634122e0000000807060504030201 pong 454d021700001800785634122e000000080706050403020110070605040302012007060504030201 @@ -31,4 +37,18 @@ nack 454d021900001100785634123100000003000000891300002c0103000011002b01 receiver_report_v030 454d021500003200785634123200000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00d204000037020000590000003c00 hello_ack_paired 454d02110000390000000000030000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f100100 hello_ack_contiguous_seq 454d02110000390000000000050000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f102100 +hello_ack_media_size 454d02110000390000000000060000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f10a100 hello2_v030 454d02100000410000000000040000000202efbeadde94260300010074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e20100 +hello2_standby_padded 454d02100000940000000000040000000202efbeadde94260300090074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e201000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +identify 454d021a00004400000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +identity 454d021b00004400000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf942602c000020ac633640a000000000000000018fb81d76bbb96e91101119775ad9766b0c332b2c49441e6f07ec33be1c6d94e +hello_ack_stopped 454d02110000390000000000050000000602efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d5043000000000000000000000000000000000100 +hello_ack_restarting 454d02110000390000000000050000000702efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d5043000000000000000000000000000000000100 +hello_ack_failed 454d021100004a0000000000050000000802efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d504300000000000000000000000000000000010010646973706c61795f6368616e67696e67 +hello_ack_busy_bare 454d02110000210000000000020000000102efbeadde00000000e803f401b80b0000000000000000000000000000000000 +hello2_draws_pointer 454d02100000410000000000060000000202efbeadde94260300210074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e20100 +pointer_shape 454d021c000034007856341233000000010101000d0c0b0a03000200010001001a000000000000001a00000000ffffffffff000000000000000010203080000000ff2080 +pointer_state 454d021d000018007856341234000000010100000d0c0b0afbffffffd2040000ab89674523010000 +pointer_hidden 454d021d000018007856341235000000010000000d0c0b0ab80b000007000000ac89674523010000 +receiver_report_pointer 454d021500003600785634123600000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00d204000037020000590000003c000d0c0b0a +hello_ack_pointer 454d02110000390000000000070000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f106100 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index e0e6c5ad..30af3809 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -111,6 +111,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { host_name: "ALI-PC".into(), auth_token: [0; 16], host_caps: 0, + reason: String::new(), }), ), )); @@ -218,6 +219,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { host_name: "ALI-PC".to_string(), auth_token: [0; 16], host_caps: 0, + reason: String::new(), }), ), ( @@ -236,6 +238,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { host_name: "ALI-PC".to_string(), auth_token: [0; 16], host_caps: 0, + reason: String::new(), }), ), ( @@ -245,6 +248,19 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { ControlMessage::Heartbeat(Heartbeat { host_time_us: 987_654_321, stream_config: stream_config(), + notice: NOTICE_NONE, + frames_sent: None, + }), + ), + ( + "heartbeat_pc_locked", + 0x1234_5678, + 42, + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: stream_config(), + notice: NOTICE_PC_LOCKED, + frames_sent: None, }), ), ( @@ -253,6 +269,18 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { 43, ControlMessage::Bye(ByeReason::AppBackground), ), + ( + "bye_host_sleeping", + 0x1234_5678, + 55, + ControlMessage::Bye(ByeReason::HostSleeping), + ), + ( + "bye_host_powering_off", + 0x1234_5678, + 56, + ControlMessage::Bye(ByeReason::HostPoweringOff), + ), ( "keyframe_request", 0x1234_5678, @@ -261,6 +289,40 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { stream_epoch: 3, last_complete_seq: 4111, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, + }), + ), + ( + "keyframe_request_media_size", + 0x1234_5678, + 51, + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 3, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram: 1200, + }), + ), + ( + "heartbeat_frames_sent", + 0x1234_5678, + 52, + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: stream_config(), + notice: NOTICE_NONE, + frames_sent: Some(1234), + }), + ), + ( + "heartbeat_no_display_frames_sent", + 0x1234_5678, + 53, + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: stream_config(), + notice: NOTICE_NO_DISPLAY, + frames_sent: Some(77), }), ), ( @@ -384,7 +446,12 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { ack.host_caps = HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ; vectors.push(( "hello_ack_contiguous_seq", - encode_control(0, 5, &ControlMessage::HelloAck(ack)), + encode_control(0, 5, &ControlMessage::HelloAck(ack.clone())), + )); + ack.host_caps = HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ | HOSTCAP_MEDIA_SIZE; + vectors.push(( + "hello_ack_media_size", + encode_control(0, 6, &ControlMessage::HelloAck(ack)), )); let (_, ControlMessage::Hello2(mut hello)) = parse_control( &vectors @@ -402,11 +469,214 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { hello.pairing_code = 123_456; vectors.push(( "hello2_v030", + encode_control(0, 4, &ControlMessage::Hello2(hello.clone())), + )); + // 0.3.5: an iPad that asks for standby answers pads its HELLO2, so an + // answer is never larger than the question. + hello.feature_caps |= FEATURE_STANDBY_STATUS; + vectors.push(( + "hello2_standby_padded", encode_control(0, 4, &ControlMessage::Hello2(hello)), )); + let nonce = std::array::from_fn(|index| 0xA0 + index as u8); + vectors.push(( + "identify", + encode_control( + 0, + 1, + &ControlMessage::Identify(Identify { + version: IDENTIFY_VERSION, + nonce, + }), + ), + )); + // The proof for pairing token 01..10, port 9876 and two addresses: the + // same known answer the host's pairing tests and the iPad's identity + // tests check. + vectors.push(( + "identity", + encode_control( + 0, + 1, + &ControlMessage::Identity(Identity { + version: IDENTIFY_VERSION, + nonce, + port: 9876, + addresses: IDENTITY_ADDRESSES.to_vec(), + proof: IDENTITY_PROOF, + }), + ), + )); + // 0.3.5: answers while the PC isn't streaming, a Failed one with its + // reason token appended. + for (name, status, reason) in [ + ("hello_ack_stopped", HelloStatus::Stopped, ""), + ("hello_ack_restarting", HelloStatus::Restarting, ""), + ( + "hello_ack_failed", + HelloStatus::Failed, + FailedReason::DisplayChanging.token(), + ), + ] { + vectors.push(( + name, + encode_control( + 0, + 5, + &ControlMessage::HelloAck(HelloAck { + status, + accepted_version: 2, + client_nonce: 0xDEAD_BEEF, + session_id: 0, + heartbeat_interval_ms: 1000, + report_interval_ms: 500, + liveness_timeout_ms: 3000, + stream_config: StreamConfig::default(), + host_name: "ALI-PC".into(), + auth_token: [0; 16], + host_caps: HOSTCAP_NACK, + reason: reason.into(), + }), + ), + )); + } + // A refusal without the PC name and host caps, as a HELLO2 too small + // for the whole answer gets: every iPad reads only its status. + vectors.push(( + "hello_ack_busy_bare", + encode_control( + 0, + 2, + &ControlMessage::HelloAck(HelloAck { + status: HelloStatus::Busy, + accepted_version: 2, + client_nonce: 0xDEAD_BEEF, + session_id: 0, + heartbeat_interval_ms: 1000, + report_interval_ms: 500, + liveness_timeout_ms: 3000, + stream_config: StreamConfig::default(), + host_name: String::new(), + auth_token: [0; 16], + host_caps: 0, + reason: String::new(), + }), + ), + )); + let (_, ControlMessage::Hello2(mut hello)) = parse_control( + &vectors + .iter() + .find(|(name, _)| *name == "hello2_v030") + .unwrap() + .1, + ) + .unwrap() else { + unreachable!() + }; + hello.feature_caps |= FEATURE_DRAWS_POINTER; + vectors.push(( + "hello2_draws_pointer", + encode_control(0, 6, &ControlMessage::Hello2(hello)), + )); + // A 3x2 image: opaque black, opaque white, two clear pixels the mask + // after the image inverts, a half-covered pixel and opaque black. + let image: Vec = [ + [0, 0, 0, 255], + [255, 255, 255, 255], + [0, 0, 0, 0], + [0, 0, 0, 0], + [0x10, 0x20, 0x30, 0x80], + [0, 0, 0, 255], + ] + .concat(); + let mut shape = image.clone(); + shape.extend_from_slice(&[0b0010_0000, 0b1000_0000]); + vectors.push(( + "pointer_shape", + encode_control( + 0x1234_5678, + 51, + &ControlMessage::PointerShape(PointerShapeChunk { + shape_id: 0x0A0B_0C0D, + source_type: 1, + flags: POINTER_SHAPE_INVERT, + width: 3, + height: 2, + hot_x: 1, + hot_y: 1, + total_len: shape.len() as u32, + offset: 0, + data: shape, + }), + ), + )); + vectors.push(( + "pointer_state", + encode_control( + 0x1234_5678, + 52, + &ControlMessage::PointerState(PointerState { + flags: POINTER_VISIBLE, + shape_id: 0x0A0B_0C0D, + x: -5, + y: 1_234, + host_time_us: 0x0000_0123_4567_89AB, + }), + ), + )); + vectors.push(( + "pointer_hidden", + encode_control( + 0x1234_5678, + 53, + &ControlMessage::PointerState(PointerState { + flags: 0, + shape_id: 0x0A0B_0C0D, + x: 3_000, + y: 7, + host_time_us: 0x0000_0123_4567_89AC, + }), + ), + )); + let (_, ControlMessage::ReceiverReport(mut report)) = parse_control( + &vectors + .iter() + .find(|(name, _)| *name == "receiver_report_v030") + .unwrap() + .1, + ) + .unwrap() else { + unreachable!() + }; + report.pointer_shape_id = 0x0A0B_0C0D; + vectors.push(( + "receiver_report_pointer", + encode_control(0x1234_5678, 54, &ControlMessage::ReceiverReport(report)), + )); + let (_, ControlMessage::HelloAck(mut ack)) = parse_control( + &vectors + .iter() + .find(|(name, _)| *name == "hello_ack_contiguous_seq") + .unwrap() + .1, + ) + .unwrap() else { + unreachable!() + }; + ack.host_caps |= HOSTCAP_POINTER; + vectors.push(( + "hello_ack_pointer", + encode_control(0, 7, &ControlMessage::HelloAck(ack)), + )); vectors } +const IDENTITY_ADDRESSES: [[u8; 4]; 2] = [[192, 0, 2, 10], [198, 51, 100, 10]]; +const IDENTITY_PROOF: [u8; 32] = [ + 0x18, 0xfb, 0x81, 0xd7, 0x6b, 0xbb, 0x96, 0xe9, 0x11, 0x01, 0x11, 0x97, 0x75, 0xad, 0x97, 0x66, + 0xb0, 0xc3, 0x32, 0xb2, 0xc4, 0x94, 0x41, 0xe6, 0xf0, 0x7e, 0xc3, 0x3b, 0xe1, 0xc6, 0xd9, 0x4e, +]; + fn render(vectors: &[(&'static str, Vec)]) -> String { let mut out = String::new(); out.push_str( @@ -537,7 +807,84 @@ fn packet_type_registry_is_stable() { assert_eq!(PacketType::Pong as u8, 0x17); assert_eq!(PacketType::StreamConfig as u8, 0x18); assert_eq!(PacketType::Nack as u8, 0x19); + assert_eq!(PacketType::PointerShape as u8, 0x1C); + assert_eq!(PacketType::PointerState as u8, 0x1D); assert_eq!(PacketType::InputEvent as u8, 0x20); + assert_eq!(PacketType::Identify as u8, 0x1A); + assert_eq!(PacketType::Identity as u8, 0x1B); +} + +/// The request is as long as the answer, so answering never sends more +/// bytes than arrived; a request cut short of its reserved bytes is refused. +#[test] +fn identify_request_is_as_long_as_its_answer() { + let vectors: BTreeMap<_, _> = canonical_vectors().into_iter().collect(); + assert_eq!(vectors["identify"].len(), vectors["identity"].len()); + assert_eq!(vectors["identify"].len(), 16 + IDENTIFY_BODY_SIZE); + match parse_control(&vectors["identity"]).unwrap().1 { + ControlMessage::Identity(identity) => { + assert_eq!(identity.proof, IDENTITY_PROOF); + assert_eq!(identity.port, 9876); + assert_eq!(identity.addresses, IDENTITY_ADDRESSES); + } + other => panic!("{other:?}"), + } + // Five addresses don't fit: the answer is refused, never longer. + let mut five = vectors["identity"].clone(); + five[16 + 1 + 16 + 2] = 5; + assert!(parse_control(&five).is_err()); + // Only the addresses counted are read and signed. + let mut one = vectors["identity"].clone(); + one[16 + 1 + 16 + 2] = 1; + match parse_control(&one).unwrap().1 { + ControlMessage::Identity(identity) => { + assert_eq!(identity.addresses, [IDENTITY_ADDRESSES[0]]) + } + other => panic!("{other:?}"), + } + assert_ne!( + identity_message(&[0xA0; 16], 9876, &IDENTITY_ADDRESSES), + identity_message(&[0xA0; 16], 9876, &IDENTITY_ADDRESSES[..1]) + ); +} + +/// The pointer bits, and the fields their vectors carry. +#[test] +fn pointer_vectors_round_trip_with_their_capabilities() { + let vectors: BTreeMap<_, _> = canonical_vectors().into_iter().collect(); + assert_eq!(FEATURE_DRAWS_POINTER, 0x0020); + assert_eq!(HOSTCAP_POINTER, 0x0040); + let ControlMessage::Hello2(hello) = parse_control(&vectors["hello2_draws_pointer"]).unwrap().1 + else { + panic!() + }; + assert_ne!(hello.feature_caps & FEATURE_DRAWS_POINTER, 0); + let ControlMessage::HelloAck(ack) = parse_control(&vectors["hello_ack_pointer"]).unwrap().1 + else { + panic!() + }; + assert_ne!(ack.host_caps & HOSTCAP_POINTER, 0); + let ControlMessage::PointerShape(shape) = parse_control(&vectors["pointer_shape"]).unwrap().1 + else { + panic!() + }; + assert_eq!((shape.width, shape.height, shape.total_len), (3, 2, 26)); + let ControlMessage::PointerState(state) = parse_control(&vectors["pointer_state"]).unwrap().1 + else { + panic!() + }; + assert_eq!( + (state.x, state.y, state.flags), + (-5, 1_234, POINTER_VISIBLE) + ); + let ControlMessage::ReceiverReport(report) = parse_control(&vectors["receiver_report_pointer"]) + .unwrap() + .1 + else { + panic!() + }; + assert_eq!(report.pointer_shape_id, 0x0A0B_0C0D); + assert_eq!(report.frags_repaired, 1234); } /// A host that numbers frames contiguously says so in HELLO_ACK; one that @@ -556,3 +903,51 @@ fn contiguous_frame_seq_capability_round_trips() { assert_eq!(caps("hello_ack_paired") & HOSTCAP_CONTIGUOUS_FRAME_SEQ, 0); assert_eq!(HOSTCAP_CONTIGUOUS_FRAME_SEQ, 0x0020); } + +/// A host that takes smaller media datagrams says so; the request and the +/// heartbeat count are appended to the bodies earlier versions read. +#[test] +fn media_size_vectors_round_trip_and_extend_the_old_bodies() { + let vectors: BTreeMap<_, _> = canonical_vectors().into_iter().collect(); + let message = |name: &str| parse_control(&vectors[name]).unwrap().1; + let ControlMessage::HelloAck(ack) = message("hello_ack_media_size") else { + panic!("hello_ack_media_size") + }; + assert_eq!( + ack.host_caps, + HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ | HOSTCAP_MEDIA_SIZE + ); + assert_eq!(HOSTCAP_MEDIA_SIZE, 0x0080); + assert_eq!(HOSTCAP_KEYFRAME_PER_LOSS, 0x0100); + assert_eq!(FEATURE_MEDIA_SIZE, 0x0040); + assert_eq!(MIN_MEDIA_DGRAM, 576); + + let ControlMessage::KeyframeRequest(request) = message("keyframe_request_media_size") else { + panic!("keyframe_request_media_size") + }; + assert_eq!(request.max_media_dgram, 1200); + assert_eq!(request.reason, KeyframeReason::Startup); + let sized = &vectors["keyframe_request_media_size"]; + let plain = &vectors["keyframe_request"]; + assert_eq!(sized.len(), plain.len() + 2); + + let ControlMessage::Heartbeat(heartbeat) = message("heartbeat_frames_sent") else { + panic!("heartbeat_frames_sent") + }; + assert_eq!(heartbeat.frames_sent, Some(1234)); + assert_eq!(heartbeat.notice, NOTICE_NONE); + let counted = &vectors["heartbeat_frames_sent"]; + let plain = &vectors["heartbeat"]; + assert_eq!(counted.len(), plain.len() + 5); + // Past the msg_seq, the old heartbeat is a prefix of the new one. + assert_eq!(counted[16..plain.len()], plain[16..]); + + // A notice and the count together: the notice byte, then the count. + let ControlMessage::Heartbeat(both) = message("heartbeat_no_display_frames_sent") else { + panic!("heartbeat_no_display_frames_sent") + }; + assert_eq!(both.notice, NOTICE_NO_DISPLAY); + assert_eq!(both.frames_sent, Some(77)); + let bytes = &vectors["heartbeat_no_display_frames_sent"]; + assert_eq!(&bytes[plain.len()..], &[NOTICE_NO_DISPLAY, 77, 0, 0, 0]); +} diff --git a/proto/tests/wifi_repair_sim.rs b/proto/tests/wifi_repair_sim.rs index e57a4709..d54da059 100644 --- a/proto/tests/wifi_repair_sim.rs +++ b/proto/tests/wifi_repair_sim.rs @@ -52,6 +52,13 @@ struct Outcome { nacks: u64, latency_p50_ms: f64, latency_p90_ms: f64, + /// Keyframes that reached the decoder whole. + keyframes_complete: u64, + /// The longest time the picture stood still between decodable frames. + longest_freeze_ms: f64, + /// Freezes the user notices: over 250 ms, and over a second. + freezes_over_250ms: u64, + freezes_over_1s: u64, } struct Rng(u64); @@ -181,6 +188,7 @@ fn simulate(s: Scenario) -> Outcome { let mut resends: HashMap<(u32, u16), u64> = HashMap::new(); let mut previous_sent: HashMap = HashMap::new(); let mut nacks_sent: HashMap = HashMap::new(); + let mut last_decoded = 0u64; while let Some(((now, _), event)) = events.pop_first() { if now > end { @@ -268,7 +276,13 @@ fn simulate(s: Scenario) -> Outcome { let decodes = frame.is_keyframe || decodes_from == previous_sent.get(&frame.seq).copied(); decodes_from = decodes.then_some(frame.seq); + outcome.keyframes_complete += u64::from(frame.is_keyframe); if decodes { + let freeze = (now - last_decoded) as f64 / 1000.0; + last_decoded = now; + outcome.longest_freeze_ms = outcome.longest_freeze_ms.max(freeze); + outcome.freezes_over_250ms += u64::from(freeze > 250.0); + outcome.freezes_over_1s += u64::from(freeze > 1000.0); outcome.decodable += 1; latencies.push((now - sent_at[&frame.seq].0) as f64 / 1000.0); } else { @@ -371,3 +385,65 @@ fn wifi_repair_table() { } } } + +/// A 1080p desktop at the bitrate ABR falls to on lossy WiFi: a keyframe is +/// about 200 KB, a delta about 16 KB. iPads in the field froze for 1-4 s at +/// 4-22% loss: every replacement keyframe arrived incomplete, and was dropped +/// as soon as the small frames behind it completed. +const LOSSY: Scenario = Scenario { + name: "lossy WiFi, 60 fps", + fps: 60, + delta_fragments: 12, + keyframe_fragments: 150, + loss: 0.10, + late_uplink: 0.3, + stall_every_s: 5, + seconds: 120, + capture_skips: false, +}; + +/// At 10% loss each way, recovery keyframes complete and the picture never +/// stands still for a second. 0.3.4's assembler completed 148 of 390 +/// keyframes here and froze 35 times for over a second, up to 3.4 s. +#[test] +fn keyframes_recover_the_picture_at_ten_percent_loss() { + let outcome = simulate(LOSSY); + eprintln!("WIFI_SIM {} {outcome:?}", LOSSY.name); + assert!( + outcome.keyframes_complete * 100 >= outcome.keyframes_sent * 95, + "{outcome:?}" + ); + assert_eq!(outcome.freezes_over_1s, 0, "{outcome:?}"); + assert!(outcome.longest_freeze_ms < 1_000.0, "{outcome:?}"); +} + +/// Prints the loss table used in the PR: 5-20% loss, keyframes of 80, 150 +/// and 300 fragments. +#[test] +fn lossy_wifi_table() { + for loss in [0.05, 0.10, 0.15, 0.20] { + for keyframe_fragments in [80, 150, 300] { + let outcome = simulate(Scenario { + loss, + keyframe_fragments, + ..LOSSY + }); + eprintln!( + "LOSSY loss={loss} kf={keyframe_fragments} keyframes={}/{} drops={} keyframe_requests={} \ + longest_freeze_ms={:.0} freezes_over_250ms={} freezes_over_1s={} decodable={}/{} \ + latency_p50_ms={:.1} latency_p90_ms={:.1}", + outcome.keyframes_complete, + outcome.keyframes_sent, + outcome.dropped, + outcome.keyframe_requests, + outcome.longest_freeze_ms, + outcome.freezes_over_250ms, + outcome.freezes_over_1s, + outcome.decodable, + outcome.frames, + outcome.latency_p50_ms, + outcome.latency_p90_ms + ); + } + } +} diff --git a/scripts/e2e_ios.sh b/scripts/e2e_ios.sh index 9bdad56c..2129c388 100755 --- a/scripts/e2e_ios.sh +++ b/scripts/e2e_ios.sh @@ -28,6 +28,9 @@ case "$TRANSPORT" in udp|usb|takeover) ;; *) echo "Invalid EM_TRANSPORT: $TRANSP SIZE="${EM_SIZE:-640x360}" SYNTH_W="${SIZE%x*}" SYNTH_H="${SIZE#*x}" +# An odd size streams without its last column or row. +VIDEO_W=$((SYNTH_W / 2 * 2)) +VIDEO_H=$((SYNTH_H / 2 * 2)) SCENARIO="${EM_SCENARIO:-$CODEC-$TRANSPORT}" OUT="${EM_OUTPUT_DIR:-$ROOT/build/e2e/$SCENARIO}" SHOT="${EM_SCREENSHOT:-$ROOT/build/screenshots/e2e-$SCENARIO.png}" @@ -325,8 +328,8 @@ if [ "$TRANSPORT" = takeover ] && ! grep -q 'superseding session in place' "$HOS echo "FAIL: host did not preserve the session during USB takeover"; exit 1 fi -if ! grep -q "w=$SYNTH_W h=$SYNTH_H" <<<"$last_stats"; then - echo "FAIL: decoded resolution mismatch: $last_stats (expected ${SYNTH_W}x${SYNTH_H})" +if ! grep -q "w=$VIDEO_W h=$VIDEO_H" <<<"$last_stats"; then + echo "FAIL: decoded resolution mismatch: $last_stats (expected ${VIDEO_W}x${VIDEO_H})" exit 1 fi @@ -349,7 +352,7 @@ PYBITRATE fi "$ROOT/scripts/screenshot.sh" "$UDID" "$SHOT" -"$ROOT/scripts/pixels.sh" "$SHOT" --video "$SIZE" --assert-pattern > "$OUT/pixels.json" +"$ROOT/scripts/pixels.sh" "$SHOT" --video "${VIDEO_W}x${VIDEO_H}" --assert-pattern > "$OUT/pixels.json" python3 "$ROOT/scripts/e2e_stats.py" "$MEASURED_LOG" --output "$OUT/result.json" \ --scenario "$SCENARIO" --screenshot "$SHOT" --elapsed "$((SECONDS - STARTED))" \ --min-frames "$WANT_DECODED" --duration "$MIN_SECONDS" --min-fps "$MIN_FPS" \ diff --git a/scripts/e2e_moved_pc.sh b/scripts/e2e_moved_pc.sh new file mode 100755 index 00000000..94441a9d --- /dev/null +++ b/scripts/e2e_moved_pc.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# A saved PC changes address, as after a new lease from the router or a move +# between its cable and Wi-Fi. The real app pairs with the real host at this +# Mac's home network address, connecting as a tap on the PC in Bonjour's list +# would. The host is then killed and started on another port, which it +# advertises with Bonjour under the same name, and a silent socket takes the +# old address. The app should check that it is the same PC and stream again +# from the new port by itself (EM_EXPECT=move, the default), or keep trying +# the saved address and never contact the new one (EM_EXPECT=stay): +# EM_PAIRING=off the PC has pairing off, so nothing proves it +# EM_MOVED_HOST_BIN= the PC comes back as a build before 0.3.5 +# +# On a shared Mac this is an iOS e2e run: hold the machine's e2e lock and use +# your own simulator (EM_SIM_UDID). The PC's two ports are picked free when the +# run starts; EM_PORT and EM_NEW_PORT choose them instead, and the run stops +# before anything starts if either is taken. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +export DEVELOPER_DIR="${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer}" +export PATH="$HOME/.cargo/bin:$PATH" +export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:-/opt/homebrew/opt/ffmpeg@7/lib/pkgconfig}" + +UDID="${EM_SIM_UDID:?set EM_SIM_UDID to your own simulator}" +EXPECT="${EM_EXPECT:-move}" +PAIRING="${EM_PAIRING:-on}" +TIMEOUT_SECS="${EM_TIMEOUT:-90}" +STAY_SECS="${EM_STAY_SECS:-45}" +OUT="${EM_OUTPUT_DIR:-$ROOT/build/e2e/moved-pc}" +HOST_BIN="${EM_HOST_BIN:-$ROOT/target/release/eternal-host}" +MOVED_HOST_BIN="${EM_MOVED_HOST_BIN:-$HOST_BIN}" +# The PC's name: its Bonjour service and HELLO_ACK both carry it. +PC_NAME="EMMovedPC$$" +# A test pairing token, written to the host's settings and handed to the +# app in a pairing link, as the PC's QR code would. +TOKEN="0123456789abcdef0123456789abcdef" +case "$EXPECT" in move|stay) ;; *) echo "Invalid EM_EXPECT: $EXPECT" >&2; exit 2;; esac +mkdir -p "$OUT" +rm -f "$OUT/result.json" "$OUT/old-address.count" +APP_LOG="$OUT/app.log" +HOST_LOG="$OUT/host.log" + +# This Mac's address on its home network, read now: the move needs one. +IFACE=$(route -n get default 2>/dev/null | awk '/interface:/ {print $2}') +LAN=$(ipconfig getifaddr "$IFACE" 2>/dev/null || true) +python3 - "$LAN" <<'PY' || { echo "FAIL: no home network address on the default interface" >&2; exit 1; } +import ipaddress,sys +a=ipaddress.ip_address(sys.argv[1]) +sys.exit(0 if a.version==4 and a.is_private and not a.is_loopback and a not in ipaddress.ip_network('100.64.0.0/10') else 1) +PY + +# The PC's port before and after the move: each free for UDP and TCP on every +# address, checked together so the two differ. +PORTS=$(python3 - "${EM_PORT:-0}" "${EM_NEW_PORT:-0}" <<'PY' +import socket,sys +held=[] +for wanted in map(int, sys.argv[1:3]): + try: + udp=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); udp.bind(('0.0.0.0',wanted)); held.append(udp) + port=udp.getsockname()[1] + tcp=socket.socket(socket.AF_INET,socket.SOCK_STREAM); tcp.bind(('0.0.0.0',port)); held.append(tcp) + except OSError: + print(f"port {wanted} is in use"); sys.exit(1) + print(port) +PY +) || { echo "FAIL: $PORTS" >&2; exit 1; } +PORT=$(echo "$PORTS" | sed -n 1p) +NEW_PORT=$(echo "$PORTS" | sed -n 2p) +[ "$PORT" != "$NEW_PORT" ] || { echo "FAIL: EM_PORT and EM_NEW_PORT must differ" >&2; exit 1; } + +HOST_PID="" +SILENT_PID="" +cleanup() { + status=$? + trap - EXIT + [ -n "$SILENT_PID" ] && kill "$SILENT_PID" 2>/dev/null || true + [ -n "$SILENT_PID" ] && wait "$SILENT_PID" 2>/dev/null || true + [ -n "$HOST_PID" ] && kill -TERM "$HOST_PID" 2>/dev/null || true + [ -n "$HOST_PID" ] && wait "$HOST_PID" 2>/dev/null || true + xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true + if [ -f "$APP_LOG" ]; then + cp "$APP_LOG" "$APP_LOG.snapshot" && mv "$APP_LOG.snapshot" "$APP_LOG" || status=1 + fi + python3 - "$OUT/result.json" "$status" <<'PY' +import json,pathlib,sys +p=pathlib.Path(sys.argv[1]) +r=json.loads(p.read_text()) if p.exists() else {} +r.setdefault('status', 'PASS' if sys.argv[2]=='0' else 'FAIL') +p.write_text(json.dumps(r, indent=2)+'\n') +PY + [ "$status" -eq 0 ] || echo "FAIL: evidence at $OUT" >&2 + exit "$status" +} +trap cleanup EXIT + +if [ "${EM_SKIP_BUILD:-0}" != 1 ]; then + echo "==> Building app" + (cd "$ROOT/ios" && xcodegen generate >/dev/null) + xcodebuild build -project "$ROOT/ios/EternalMonitor.xcodeproj" -scheme EternalMonitor -configuration Release \ + -destination "platform=iOS Simulator,id=$UDID" -derivedDataPath "$ROOT/ios/build/e2e" \ + CODE_SIGNING_ALLOWED=YES CODE_SIGN_IDENTITY=- -quiet + if [ -z "${EM_HOST_BIN:-}" ]; then + echo "==> Building host" + (cd "$ROOT" && cargo build -q --release -p eternal-host --locked) + fi +fi +APP="$ROOT/ios/build/e2e/Build/Products/Release-iphonesimulator/EternalMonitor.app" +[ -d "$APP" ] || { echo "FAIL: app bundle not found at $APP"; exit 1; } +[ -x "$HOST_BIN" ] && [ -x "$MOVED_HOST_BIN" ] || { echo "FAIL: host binary not found"; exit 1; } + +echo "==> Installing app with nothing saved" +xcrun simctl bootstatus "$UDID" -b >/dev/null +xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true +xcrun simctl uninstall "$UDID" com.eternal.monitor 2>/dev/null || true +# Pairings live in the simulator's Keychain and outlast the app. +xcrun simctl keychain "$UDID" reset +STAGE="$(mktemp -d /tmp/em-moved-pc-install.XXXXXX)" +ditto "$APP" "$STAGE/EternalMonitor.app" +xcrun simctl install "$UDID" "$STAGE/EternalMonitor.app" +rm -rf "$STAGE" +APP_DATA=$(xcrun simctl get_app_container "$UDID" com.eternal.monitor data) +SIM_LOG="$APP_DATA/tmp/eternal-e2e.log" +mkdir -p "$(dirname "$SIM_LOG")" +rm -f "$SIM_LOG" "$APP_LOG" +touch "$SIM_LOG" +ln "$SIM_LOG" "$APP_LOG" + +mkdir -p "$OUT/state/EternalMonitor" +REQUIRE=$([ "$PAIRING" = on ] && echo true || echo false) +echo "{\"bitrate_mbps\": 8, \"target_fps\": 30, \"start_on_boot\": false, \"stream_audio\": false, \"require_pairing\": $REQUIRE, \"auth_token_hex\": \"$TOKEN\"}" \ + > "$OUT/state/EternalMonitor/settings.json" +start_host() { + APPDATA="$OUT/state" COMPUTERNAME="$PC_NAME" ETERNAL_HEADLESS=1 ETERNAL_E2E_LOG=1 ETERNAL_USB_DIRECT= \ + ETERNAL_CAPTURE=synthetic ETERNAL_AUDIO=synthetic ETERNAL_SYNTH_SIZE=640x360 ETERNAL_ENCODER=libx264 ETERNAL_FPS=30 \ + "$1" "$2" >>"$HOST_LOG" 2>&1 & + HOST_PID=$! +} +frames_at_least() { [ "$(grep -c 'E2E_FIRST_FRAME' "$APP_LOG" || true)" -ge "$1" ]; } +wait_for() { + local what=$1 limit=$2 elapsed=0 + shift 2 + until "$@"; do + sleep 1 + elapsed=$((elapsed + 1)) + if [ "$elapsed" -ge "$limit" ]; then + echo "FAIL: $what after ${limit}s" + echo "----- app milestones -----"; grep -v E2E_STATS "$APP_LOG" | tail -40 + exit 1 + fi + done +} + +: > "$HOST_LOG" +echo "==> Starting host $PC_NAME on port $PORT (pairing $PAIRING)" +start_host "$HOST_BIN" "$PORT" +echo "==> Launching app: pairs and streams from the home network address, port $PORT" +LINK="$LAN:$PORT" +[ "$PAIRING" = on ] && LINK="eternaldisplay://$LINK?t=$TOKEN" +# The simulator reaches this Mac's own address over loopback; the Mac's +# network interface stands in for the iPad's Wi-Fi. +SIMCTL_CHILD_EM_AUTOCONNECT="$LINK" SIMCTL_CHILD_EM_AUTOCONNECT_FOUND=1 SIMCTL_CHILD_EM_E2E_LOG=1 SIMCTL_CHILD_EM_USB_PORT=0 \ + SIMCTL_CHILD_EM_ROUTE_INTERFACE="$IFACE" \ + xcrun simctl launch "$UDID" com.eternal.monitor -didSeeOnboarding YES -allowUSB NO -autoReconnect YES -playPCaudio NO >/dev/null +wait_for "no first stream" "$TIMEOUT_SECS" frames_at_least 1 + +echo "==> Moving the PC to port $NEW_PORT; the old address goes silent" +MOVED_AT=$(wc -l < "$APP_LOG") +kill -KILL "$HOST_PID" +wait "$HOST_PID" 2>/dev/null || true +python3 -u - "$LAN" "$PORT" "$OUT/old-address.count" <<'PY' & +import signal,socket,sys +s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) +s.bind((sys.argv[1],int(sys.argv[2]))) +count=0 +def done(*_): + open(sys.argv[3],'w').write(str(count)); sys.exit(0) +signal.signal(signal.SIGTERM,done) +while True: + s.recv(4096); count+=1 +PY +SILENT_PID=$! +start_host "$MOVED_HOST_BIN" "$NEW_PORT" + +if [ "$EXPECT" = move ]; then + echo "==> Waiting for the app to stream again by itself (timeout ${TIMEOUT_SECS}s)" + wait_for "no stream after the move" "$TIMEOUT_SECS" frames_at_least 2 + sleep 2 +else + echo "==> Watching for ${STAY_SECS}s: the app must not move" + sleep "$STAY_SECS" +fi +kill "$SILENT_PID"; wait "$SILENT_PID" 2>/dev/null || true; SILENT_PID="" + +python3 - "$APP_LOG" "$MOVED_AT" "$PORT" "$NEW_PORT" "$OUT/old-address.count" "$OUT/result.json" "$EXPECT" "$HOST_LOG" <<'PY' +import json,pathlib,re,sys +log,moved_at,port,new_port,count,result,expect,host_log=sys.argv[1:9] +lines=pathlib.Path(log).read_text(errors='replace').splitlines()[int(moved_at):] +starts=[m.group(1) for l in lines if (m:=re.search(r'Starting connection attempt=\S+ transport=WiFi to \S+:(\d+) ', l))] +frames=[i for i,l in enumerate(lines) if 'E2E_FIRST_FRAME' in l] +old=int(pathlib.Path(count).read_text()) +host=pathlib.Path(host_log).read_text(errors='replace') +errors=[] +if port not in starts: errors.append('never retried the saved address') +if old==0: errors.append('the saved address got no HELLO2') +if expect=='move': + before=[m.group(1) for l in lines[:frames[0]] if (m:=re.search(r'Starting connection attempt=\S+ transport=WiFi to \S+:(\d+) ', l))] if frames else [] + if not frames: errors.append('no stream after the move') + elif before[-1]!=new_port: errors.append('streamed again, but not from the new port') + if not any('It is the saved PC: connecting there' in l for l in lines): errors.append('no identity check before the move') +else: + if frames: errors.append('streamed again although it must not move') + if new_port in starts: errors.append('connected to the new port') +result_data=dict(scenario='moved-pc', expect=expect, status='FAIL' if errors else 'PASS', errors=errors, + attempts_after_move=len(starts), attempts_at_new_port=starts.count(new_port), + datagrams_at_old_address=old, + identity_checks=sum('Checking that the PC at' in l for l in lines), + lookups=sum('Looking for the saved PC by name' in l for l in lines)) +pathlib.Path(result).write_text(json.dumps(result_data, indent=2)+'\n') +print(json.dumps(result_data)) +sys.exit(1 if errors else 0) +PY +xcrun simctl io "$UDID" screenshot "$OUT/after-move.png" >/dev/null 2>&1 || true +echo "PASS ($EXPECT). Evidence: $OUT" diff --git a/scripts/e2e_pc_returns.sh b/scripts/e2e_pc_returns.sh new file mode 100755 index 00000000..16be95fa --- /dev/null +++ b/scripts/e2e_pc_returns.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# The PC app isn't open when the iPad connects, and opens afterwards. The real +# app streams once from the real host, so it knows the PC's name and pairing. +# The host is then stopped and the app opened again: it connects to the saved +# PC, which doesn't answer, and shows "didn't answer". The host then starts +# again and advertises with Bonjour, and the app should connect and stream by +# itself, with no tap. The app opens a second after the host stopped, while +# Bonjour may still list the PC unchanged: the listing is asked until the PC +# proves its pairing, so a return that changes nothing on Bonjour still counts. +# +# On a shared Mac this is an iOS e2e run: hold the machine's e2e lock and use +# your own simulator (EM_SIM_UDID). The PC's port is picked free when the run +# starts; EM_PORT chooses it instead, and the run stops before anything starts +# if it is taken. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +export DEVELOPER_DIR="${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer}" +export PATH="$HOME/.cargo/bin:$PATH" +export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:-/opt/homebrew/opt/ffmpeg@7/lib/pkgconfig}" + +UDID="${EM_SIM_UDID:?set EM_SIM_UDID to your own simulator}" +TIMEOUT_SECS="${EM_TIMEOUT:-90}" +# How long the card is up before the PC app opens. +CLOSED_SECS="${EM_CLOSED_SECS:-8}" +OUT="${EM_OUTPUT_DIR:-$ROOT/build/e2e/pc-returns}" +HOST_BIN="${EM_HOST_BIN:-$ROOT/target/release/eternal-host}" +# The PC's name: its Bonjour service and HELLO_ACK both carry it. A new one +# each run, so no earlier run's Bonjour record stands in for it. +PC_NAME="EMReturnPC$$" +# A test pairing token, written to the host's settings and handed to the +# app in a pairing link, as the PC's QR code would. +TOKEN="0123456789abcdef0123456789abcdef" +mkdir -p "$OUT" +rm -f "$OUT/result.json" +APP_LOG="$OUT/app.log" +HOST_LOG="$OUT/host.log" + +# This Mac's address on its home network, as an iPad on the same Wi-Fi dials it. +IFACE=$(route -n get default 2>/dev/null | awk '/interface:/ {print $2}') +LAN=$(ipconfig getifaddr "$IFACE" 2>/dev/null || true) +[ -n "$LAN" ] || { echo "FAIL: no address on the default interface" >&2; exit 1; } + +PORT=$(python3 - "${EM_PORT:-0}" <<'PY' +import socket,sys +wanted=int(sys.argv[1]) +try: + udp=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); udp.bind(('0.0.0.0',wanted)) + port=udp.getsockname()[1] + tcp=socket.socket(socket.AF_INET,socket.SOCK_STREAM); tcp.bind(('0.0.0.0',port)) +except OSError: + print(f"port {wanted} is in use"); sys.exit(1) +print(port) +PY +) || { echo "FAIL: $PORT" >&2; exit 1; } + +HOST_PID="" +cleanup() { + status=$? + trap - EXIT + [ -n "$HOST_PID" ] && kill -TERM "$HOST_PID" 2>/dev/null || true + [ -n "$HOST_PID" ] && wait "$HOST_PID" 2>/dev/null || true + xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true + python3 - "$OUT/result.json" "$status" <<'PY' +import json,pathlib,sys +p=pathlib.Path(sys.argv[1]) +r=json.loads(p.read_text()) if p.exists() else {} +r.setdefault('status', 'PASS' if sys.argv[2]=='0' else 'FAIL') +p.write_text(json.dumps(r, indent=2)+'\n') +PY + [ "$status" -eq 0 ] || echo "FAIL: evidence at $OUT" >&2 + exit "$status" +} +trap cleanup EXIT + +if [ "${EM_SKIP_BUILD:-0}" != 1 ]; then + echo "==> Building app" + (cd "$ROOT/ios" && xcodegen generate >/dev/null) + xcodebuild build -project "$ROOT/ios/EternalMonitor.xcodeproj" -scheme EternalMonitor -configuration Release \ + -destination "platform=iOS Simulator,id=$UDID" -derivedDataPath "$ROOT/ios/build/e2e" \ + CODE_SIGNING_ALLOWED=YES CODE_SIGN_IDENTITY=- -quiet + if [ -z "${EM_HOST_BIN:-}" ]; then + echo "==> Building host" + (cd "$ROOT" && cargo build -q --release -p eternal-host --locked) + fi +fi +APP="$ROOT/ios/build/e2e/Build/Products/Release-iphonesimulator/EternalMonitor.app" +[ -d "$APP" ] || { echo "FAIL: app bundle not found at $APP"; exit 1; } +[ -x "$HOST_BIN" ] || { echo "FAIL: host binary not found"; exit 1; } + +echo "==> Installing app with nothing saved" +xcrun simctl bootstatus "$UDID" -b >/dev/null +xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true +xcrun simctl uninstall "$UDID" com.eternal.monitor 2>/dev/null || true +# Pairings live in the simulator's Keychain and outlast the app. +xcrun simctl keychain "$UDID" reset +STAGE="$(mktemp -d /tmp/em-pc-returns-install.XXXXXX)" +ditto "$APP" "$STAGE/EternalMonitor.app" +xcrun simctl install "$UDID" "$STAGE/EternalMonitor.app" +rm -rf "$STAGE" +APP_DATA=$(xcrun simctl get_app_container "$UDID" com.eternal.monitor data) +SIM_LOG="$APP_DATA/tmp/eternal-e2e.log" +mkdir -p "$(dirname "$SIM_LOG")" + +mkdir -p "$OUT/state/EternalMonitor" +echo "{\"bitrate_mbps\": 8, \"target_fps\": 30, \"start_on_boot\": false, \"stream_audio\": false, \"require_pairing\": true, \"auth_token_hex\": \"$TOKEN\"}" \ + > "$OUT/state/EternalMonitor/settings.json" +start_host() { + APPDATA="$OUT/state" COMPUTERNAME="$PC_NAME" ETERNAL_HEADLESS=1 ETERNAL_E2E_LOG=1 ETERNAL_USB_DIRECT= \ + ETERNAL_CAPTURE=synthetic ETERNAL_AUDIO=synthetic ETERNAL_SYNTH_SIZE=640x360 ETERNAL_ENCODER=libx264 ETERNAL_FPS=30 \ + "$HOST_BIN" "$PORT" >>"$HOST_LOG" 2>&1 & + HOST_PID=$! +} +stop_host() { + # A stopped app says goodbye on Bonjour, as closing it on the PC does. + kill -TERM "$HOST_PID" + wait "$HOST_PID" 2>/dev/null || true + HOST_PID="" +} +# The app writes its log afresh at each launch. +launch_app() { + rm -f "$SIM_LOG" + touch "$SIM_LOG" + SIMCTL_CHILD_EM_AUTOCONNECT="$1" SIMCTL_CHILD_EM_E2E_LOG=1 SIMCTL_CHILD_EM_USB_PORT=0 \ + xcrun simctl launch "$UDID" com.eternal.monitor -didSeeOnboarding YES -allowUSB NO -autoReconnect NO \ + -autoResumeOnForeground NO -playPCaudio NO >/dev/null +} +seen() { grep -q "$1" "$SIM_LOG"; } +wait_for() { + local what=$1 limit=$2 elapsed=0 + shift 2 + until "$@"; do + sleep 1 + elapsed=$((elapsed + 1)) + if [ "$elapsed" -ge "$limit" ]; then + echo "FAIL: $what after ${limit}s" + echo "----- app milestones -----"; grep -v E2E_STATS "$SIM_LOG" | tail -40 + cp "$SIM_LOG" "$APP_LOG" 2>/dev/null || true + exit 1 + fi + done +} + +: > "$HOST_LOG" +echo "==> Pairing with $PC_NAME at port $PORT and streaming once" +start_host +launch_app "eternaldisplay://$LAN:$PORT?t=$TOKEN" +wait_for "no first stream" "$TIMEOUT_SECS" seen E2E_FIRST_FRAME +cp "$SIM_LOG" "$OUT/app-first.log" + +echo "==> Closing the PC app, then opening the iPad app on the saved PC" +stop_host +xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true +launch_app "$LAN:$PORT" +wait_for "no \"didn't answer\" card" "$TIMEOUT_SECS" seen "pc_return result=watching" +sleep "$CLOSED_SECS" +xcrun simctl io "$UDID" screenshot "$OUT/didnt-answer.png" >/dev/null 2>&1 || true +seen E2E_FIRST_FRAME && { echo "FAIL: streamed with the PC app closed"; exit 1; } + +echo "==> Opening the PC app; the iPad should connect by itself (timeout ${TIMEOUT_SECS}s)" +OPENED_AT=$(wc -l < "$SIM_LOG") +start_host +wait_for "no stream after the PC app opened" "$TIMEOUT_SECS" seen E2E_FIRST_FRAME +sleep 2 +xcrun simctl io "$UDID" screenshot "$OUT/streaming.png" >/dev/null 2>&1 || true +cp "$SIM_LOG" "$APP_LOG" + +python3 - "$APP_LOG" "$OPENED_AT" "$OUT/result.json" <<'PY' +import json,pathlib,sys +log,opened_at,result=sys.argv[1:4] +lines=pathlib.Path(log).read_text(errors='replace').splitlines() +before,after=lines[:int(opened_at)],lines[int(opened_at):] +attempt=lambda l: 'Starting connection attempt=' in l and 'transport=WiFi' in l +errors=[] +if not any('outcome=no_reply' in l for l in before): errors.append('no no_reply summary before the PC app opened') +if sum(map(attempt, before))!=1: errors.append('more than one attempt while the PC app was closed') +tries=sum('pc_return result=tried' in l for l in after) +if tries!=1: errors.append(f'{tries} automatic tries after the PC app opened, not 1') +if not any('E2E_FIRST_FRAME' in l for l in after): errors.append('no stream after the PC app opened') +data=dict(scenario='pc-returns', status='FAIL' if errors else 'PASS', errors=errors, + attempts_while_closed=sum(map(attempt, before)), automatic_tries=tries, + attempts_after_opening=sum(map(attempt, after))) +pathlib.Path(result).write_text(json.dumps(data, indent=2)+'\n') +print(json.dumps(data)) +sys.exit(1 if errors else 0) +PY +echo "PASS. Evidence: $OUT" diff --git a/scripts/e2e_pointer.sh b/scripts/e2e_pointer.sh new file mode 100755 index 00000000..4a90a513 --- /dev/null +++ b/scripts/e2e_pointer.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# End-to-end pointer test on a Mac (#183): the real host binary with the +# synthetic pointer scene (a still picture, the pointer moving every frame) +# streams over localhost UDP to the real iPad app in the Simulator. +# +# Two runs, counted from the app's E2E_POINTER lines over EM_WINDOW seconds: +# - draws: the app draws the pointer. It must move on the iPad (positions +# applied and drawn) while almost no video frames decode: only +# the host's idle keepalive. +# - legacy: EM_DRAWS_POINTER=0, as apps before 0.3.5. No pointer messages, +# and every move is a decoded video frame, as before. +# +# Requirements as for e2e_ios.sh. Uses EM_SIM_UDID (or EM_SIM_NAME), a +# private EM_PORT and a USB listener on a free port, so it shares no fixed +# port with other simulators. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +export DEVELOPER_DIR="${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer}" +export PATH="$HOME/.cargo/bin:$PATH" +export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:-/opt/homebrew/opt/ffmpeg@7/lib/pkgconfig}" + +SIM_NAME="${EM_SIM_NAME:-iPad Pro 11-inch (M4)}" +PORT="${EM_PORT:-19936}" +WINDOW="${EM_WINDOW:-10}" +FPS="${ETERNAL_FPS:-60}" +SIZE="${EM_SIZE:-1280x720}" +OUT="${EM_OUTPUT_DIR:-$ROOT/build/e2e/pointer}" +mkdir -p "$OUT" +rm -f "$OUT/result.json" + +UDID="${EM_SIM_UDID:-}" +if [ -z "$UDID" ]; then +UDID=$(xcrun simctl list -j devices available | /usr/bin/python3 -c ' +import json, sys +for devices in json.load(sys.stdin)["devices"].values(): + for device in devices: + if device["name"] == sys.argv[1]: + print(device["udid"]); sys.exit(0) +sys.exit(1) +' "$SIM_NAME") +fi + +HOST_PID="" +cleanup() { + [ -n "$HOST_PID" ] && kill "$HOST_PID" 2>/dev/null || true + xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true +} +trap cleanup EXIT + +if [ "${EM_SKIP_BUILD:-0}" != 1 ]; then + echo "==> Building app and host" + (cd "$ROOT/ios" && xcodegen generate >/dev/null) + xcodebuild build -project "$ROOT/ios/EternalMonitor.xcodeproj" -scheme EternalMonitor \ + -configuration Release -destination "platform=iOS Simulator,id=$UDID" \ + -derivedDataPath "$ROOT/ios/build/e2e" CODE_SIGNING_ALLOWED=YES CODE_SIGN_IDENTITY=- -quiet + (cd "$ROOT" && cargo build -q --release -p eternal-host --locked) +fi +APP="$ROOT/ios/build/e2e/Build/Products/Release-iphonesimulator/EternalMonitor.app" +[ -d "$APP" ] || { echo "FAIL: app bundle not found at $APP"; exit 1; } + +xcrun simctl boot "$UDID" 2>/dev/null || true +xcrun simctl bootstatus "$UDID" -b >/dev/null +INSTALL_DIR="$(mktemp -d /tmp/em-e2e-pointer.XXXXXX)" +ditto "$APP" "$INSTALL_DIR/EternalMonitor.app" +xcrun simctl install "$UDID" "$INSTALL_DIR/EternalMonitor.app" +rm -rf "$INSTALL_DIR" + +run_mode() { + local mode=$1 draws=$2 + local dir="$OUT/$mode" + mkdir -p "$dir/state/EternalMonitor" + python3 - "$dir/state/EternalMonitor/settings.json" "$FPS" <<'PY' +import json, sys +json.dump(dict(bitrate_mbps=15.0, target_fps=int(sys.argv[2]), start_on_boot=False, + stream_audio=False, require_pairing=False), open(sys.argv[1], 'w')) +PY + xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true + local data + data=$(xcrun simctl get_app_container "$UDID" com.eternal.monitor data) + local sim_log="$data/tmp/eternal-e2e.log" + rm -f "$sim_log" + + echo "==> [$mode] host on 127.0.0.1:$PORT (synthetic pointer, ${SIZE}, ${FPS} fps)" + APPDATA="$dir/state" ETERNAL_HEADLESS=1 ETERNAL_E2E_LOG=1 ETERNAL_CAPTURE=synthetic \ + ETERNAL_SYNTH_POINTER=1 ETERNAL_SYNTH_SIZE="$SIZE" ETERNAL_ENCODER=libx264 ETERNAL_FPS="$FPS" \ + "$ROOT/target/release/eternal-host" "$PORT" >"$dir/host.log" 2>&1 & + HOST_PID=$! + sleep 2 + + echo "==> [$mode] app, EM_DRAWS_POINTER=$draws" + SIMCTL_CHILD_EM_AUTOCONNECT="127.0.0.1:$PORT" SIMCTL_CHILD_EM_E2E_LOG=1 \ + SIMCTL_CHILD_EM_USB_PORT=0 SIMCTL_CHILD_EM_DRAWS_POINTER="$draws" \ + xcrun simctl launch "$UDID" com.eternal.monitor -didSeeOnboarding YES -allowUSB NO \ + -playPCaudio NO -targetFPS "$FPS" >/dev/null + + local waited=0 + until grep -q E2E_FIRST_FRAME "$sim_log" 2>/dev/null; do + sleep 1; waited=$((waited + 1)) + [ "$waited" -lt 60 ] || { echo "FAIL [$mode]: no first frame"; exit 1; } + done + # Let the first frame's keyframe settle, then count over the window. + sleep 2 + local start end + start=$(grep E2E_POINTER "$sim_log" | tail -1) + sleep "$WINDOW" + end=$(grep E2E_POINTER "$sim_log" | tail -1) + xcrun simctl io "$UDID" screenshot "$dir/screen.png" >/dev/null 2>&1 || true + cp "$sim_log" "$dir/app.log" + xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true + kill "$HOST_PID" 2>/dev/null || true + wait "$HOST_PID" 2>/dev/null || true + HOST_PID="" + + python3 - "$mode" "$WINDOW" "$start" "$end" "$dir/host.log" "$OUT/$mode.json" <<'PY' +import json, re, sys +mode, window, start, end, host_log, out = sys.argv[1], float(sys.argv[2]), *sys.argv[3:] +field = lambda line, name: int(re.search(rf"\b{name}=(\d+)", line).group(1)) +delta = {name: field(end, name) - field(start, name) for name in ("states", "drawn", "moves", "decoded", "shapes")} +sent = [int(m) for m in re.findall(r"pointer_sent=(\d+)", re.sub(r"\x1b\[[0-9;]*m", "", open(host_log).read()))] +result = dict(mode=mode, window_s=window, **delta, pointer_states_per_s=round(delta["states"] / window, 1), + moves_per_s=round(delta["moves"] / window, 1), frames_per_s=round(delta["decoded"] / window, 2), + host_pointer_sent_last_summary=sent[-1] if sent else None, shapes_total=field(end, "shapes")) +json.dump(result, open(out, "w"), indent=1) +print(json.dumps(result)) +problems = [] +if mode == "draws": + if delta["moves"] < 15 * window: problems.append("the pointer barely moved on the iPad") + if field(end, "shapes") < 1: problems.append("no pointer image arrived") + # The 750 ms keepalive, and nothing for the moves. + if delta["decoded"] > window / 0.75 + 3: problems.append("video frames decoded while only the pointer moved") +else: + if field(end, "states") or field(end, "drawn"): problems.append("an older app got pointer messages") + if delta["decoded"] < 15 * window: problems.append("pointer moves did not arrive as video frames") +if problems: + print(f"FAIL [{mode}]: " + "; ".join(problems)); sys.exit(1) +print(f"PASS [{mode}]") +PY +} + +echo "load: $(uptime | sed 's/.*load averages*: //')" | tee "$OUT/load.txt" +run_mode draws 1 +run_mode legacy 0 +echo "load: $(uptime | sed 's/.*load averages*: //')" | tee -a "$OUT/load.txt" +python3 -c 'import json,sys; json.dump([json.load(open(p)) for p in sys.argv[2:]], open(sys.argv[1], "w"), indent=1)' \ + "$OUT/result.json" "$OUT/draws.json" "$OUT/legacy.json" +echo "PASS: $OUT/result.json" diff --git a/scripts/fixtures/.gitattributes b/scripts/fixtures/.gitattributes new file mode 100644 index 00000000..5201f7da --- /dev/null +++ b/scripts/fixtures/.gitattributes @@ -0,0 +1,2 @@ +# Signatures are raw bytes; a CRLF checkout must not touch them. +*.sig binary diff --git a/scripts/fixtures/beta-manifest.test.sig b/scripts/fixtures/beta-manifest.test.sig new file mode 100644 index 00000000..c5970634 --- /dev/null +++ b/scripts/fixtures/beta-manifest.test.sig @@ -0,0 +1,2 @@ +}lqsr: /fܑ1I m_T6Ơʲ, +(lY[#gHGF*d \ No newline at end of file diff --git a/scripts/fixtures/beta-manifest.txt b/scripts/fixtures/beta-manifest.txt new file mode 100644 index 00000000..c06f8f1f --- /dev/null +++ b/scripts/fixtures/beta-manifest.txt @@ -0,0 +1,5 @@ +eternalmonitor-beta-manifest 1 +version=0.3.5-beta.20261107.26 +sha256=abababababababababababababababababababababababababababababababab +size=63301683 +asset=EternalMonitor-Setup-beta.exe diff --git a/scripts/fixtures/beta-release-body.md b/scripts/fixtures/beta-release-body.md new file mode 100644 index 00000000..e00fee1d --- /dev/null +++ b/scripts/fixtures/beta-release-body.md @@ -0,0 +1,13 @@ +Version 0.3.5-beta.20261107.26, from the Beta channel: the newest fixes, built every night. For testing; you can go back to Stable anytime. + +Built from `release/0.3.5-candidate` at [`62b9d9d`](https://github.com/sidebandstudio/EternalMonitor/commit/62b9d9d000000000000000000000000000000000). [Changes since v0.3.4](https://github.com/sidebandstudio/EternalMonitor/compare/v0.3.4...62b9d9d000000000000000000000000000000000). + +To go back to Stable, run the installer from https://eternalmonitor.dev/download.html over this one. The app offers the next Stable version when it comes out. iPad betas come through TestFlight: https://eternalmonitor.dev/download.html#ipad + +SHA-256 of EternalMonitor-Setup-beta.exe: + +``` +abababababababababababababababababababababababababababababababab +``` + + diff --git a/scripts/nightly.py b/scripts/nightly.py index df60c1f3..51547eac 100644 --- a/scripts/nightly.py +++ b/scripts/nightly.py @@ -20,12 +20,15 @@ Users see these builds as the Beta channel. Each is a semver pre-release, -beta.., which semver orders below and above a -hand-made -beta.1. The apps never offer a pre-release, and a Beta user -is offered the next stable release as usual (host/src/updates.rs). +hand-made -beta.1. A stable install is never offered one. A Beta +install reads the beta release's body (beta_body()) and is offered the newer +of that Beta and the next stable release (host/src/updates.rs), once +scripts/sign_beta_manifest.py has signed the build; this script never signs. """ import argparse import datetime import hashlib +import http.client import json import os import re @@ -121,6 +124,8 @@ def stamp(cargo_toml, cargo_lock, version): def notes(version, branch, sha, stable, sha256, repo=REPO): + """A Beta install reads the first line's "Version , from the Beta + channel" and the hash after "SHA-256 of :" (scripts/fixtures/beta-release-body.md).""" if not re.fullmatch(r"[0-9a-f]{40}", sha) or not re.fullmatch(r"[0-9A-Fa-f]{64}", sha256): raise ValueError("notes need the full commit and the installer's SHA-256") return ( @@ -159,6 +164,11 @@ def marker(sha): return f"" +def beta_body(notes, sha): + """The beta release's body once sha's build has finished publishing.""" + return f"{notes.rstrip()}\n\n{marker(sha)}\n" + + def published_commit(body): match = MARKER.search(body or "") return match[1] if match else "" @@ -232,13 +242,27 @@ def assets(release, name): return [a for a in (release or {}).get("assets", []) if a["name"] == name] -def verify(github, url, size, sha256=None, tries=6, wait=10, sleep=time.sleep): +def transient(status): + """An HTTP status GitHub's download servers give while briefly unwell, which says + nothing about the file.""" + return status == 429 or 500 <= status <= 599 + + +def verify(github, url, size, sha256=None, tries=6, wait=10, transient_tries=12, sleep=time.sleep): """url serves size bytes (with sha256, those exact bytes) to an anonymous visitor. - A new or renamed asset can take a few seconds to show, so it retries.""" - for attempt in range(tries): + + A new or renamed asset can take a few seconds to show, so a missing or different + file is checked again: tries times, wait seconds apart. A server error (HTTP 429 or + 5xx) or a dropped connection, also one cut off partway through a download, says + nothing about the file, so those have a longer window of their own: + transient_tries, each wait 5 s longer up to 30 s (about four minutes). Either way + the size and SHA-256 must match exactly.""" + misses = errors = 0 + while True: try: status, length = github.head(url) seen = f"HTTP {status}, {length} bytes" + flaky = transient(status) if status == 200 and length == size: if sha256 is None: return @@ -246,9 +270,23 @@ def verify(github, url, size, sha256=None, tries=6, wait=10, sleep=time.sleep): if got == sha256: return seen = f"SHA-256 {got}" - except OSError as error: - seen = str(error) - if attempt + 1 < tries: + except urllib.error.HTTPError as error: + seen = f"HTTP {error.code}" + flaky = transient(error.code) + except (OSError, http.client.HTTPException) as error: + # A dropped connection, also one cut off partway through the download + # (http.client.IncompleteRead, which is no OSError). + seen = f"{type(error).__name__}: {error}" + flaky = True + if flaky: + errors += 1 + if errors >= transient_tries: + break + sleep(min(5 * errors, 30)) + else: + misses += 1 + if misses >= tries: + break sleep(wait) raise PublishError(f"{url}: {seen}; expected {size} bytes" + (f", SHA-256 {sha256}" if sha256 else "")) @@ -344,7 +382,7 @@ def publish(github, tag, sha, title, notes, path, sleep=time.sleep): # 3. The tag, then the marker that says this commit is done. github.move_tag(MOVING_TAG, sha) - github.edit(MOVING_TAG, title, f"{notes.rstrip()}\n\n{marker(sha)}\n") + github.edit(MOVING_TAG, title, beta_body(notes, sha)) def git(*args): diff --git a/scripts/sign_beta_manifest.py b/scripts/sign_beta_manifest.py new file mode 100755 index 00000000..b71757ca --- /dev/null +++ b/scripts/sign_beta_manifest.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Sign the Beta channel's manifest, so Beta installs offer the newest Beta. + + sign_beta_manifest.py --sha256 HEX [--version V] < private-key.pem + Read the beta release, check that it names the installer HEX (and V), + write beta-manifest.txt from it, sign that with the Ed25519 private key + read from stdin, check the signature with the pinned public key, upload + both files to the release, then verify them as a Beta install does. + HEX is the "Installer SHA-256" in the Publish job's summary of the Beta + builds run that published the build: the hash the build computed, + which GitHub's release can't vouch for by itself. Signs only for + sidebandstudio/EternalMonitor. + sign_beta_manifest.py --verify-only [--repo R] [--public-key PEM] [--sha256 HEX] [--version V] + Only verify the published manifest and signature. + +A Beta install offers a Beta only when beta-manifest.sig is a valid Ed25519 +signature, by the key pinned in host/src/updates.rs, over the exact bytes of +beta-manifest.txt, and the manifest names the version, SHA-256 and size the +release does. Until a build is signed, Beta installs are offered no Beta. + +The private key goes from stdin to openssl through a pipe. It is never an +argument or a file. Needs gh (signed in, for the upload) and OpenSSL 3 or +later; OPENSSL picks the binary. +""" +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile + +REPO = "sidebandstudio/EternalMonitor" +TAG = "beta" +ASSET = "EternalMonitor-Setup-beta.exe" +MANIFEST = "beta-manifest.txt" +SIGNATURE = "beta-manifest.sig" +# The public half of the key host/src/updates.rs pins (BETA_PUBLIC_KEY). +PUBLIC_KEY = b"""-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAgsKvIcaF0sBCxR9yo0x4vbXQBLrYnnWoMGpDIk0W7Fo= +-----END PUBLIC KEY----- +""" +VERSION_LINE = re.compile(r"(?m)^Version (\S+), from the Beta channel") +HASH_BLOCK = re.compile(r"(?m)^SHA-256 of EternalMonitor-Setup-beta\.exe:\s*```\s*([0-9A-Fa-f]{64})\s*```") +# x.y.z-beta., without build metadata; numbers have no leading zeros. +NUMBER = r"(?:0|[1-9]\d*)" +BETA_VERSION = re.compile(rf"{NUMBER}\.{NUMBER}\.{NUMBER}-beta(?:\.(?:{NUMBER}|\d*[A-Za-z-][0-9A-Za-z-]*))+") + + +class SigningError(Exception): + pass + + +def manifest_text(version, sha256, size): + """The canonical manifest, version 1 (host/src/updates.rs BetaManifest::text).""" + return (f"eternalmonitor-beta-manifest 1\nversion={version}\nsha256={sha256}\n" + f"size={size}\nasset={ASSET}\n") + + +def only(pattern, text, what): + found = pattern.findall(text) + if len(found) != 1: + raise SigningError(f"the release body names {len(found)} {what}, not one") + return found[0] + + +def read_release(release): + """(version, sha256, size) of the beta release's installer, once its body, asset + and GitHub digest agree.""" + if release.get("tag_name") != TAG or release.get("draft") or release.get("prerelease") is not True: + raise SigningError("not the published beta pre-release") + body = release.get("body") or "" + version = only(VERSION_LINE, body, "versions") + if not BETA_VERSION.fullmatch(version): + raise SigningError(f"not a Beta version: {version!r}") + sha256 = only(HASH_BLOCK, body, "installer hashes").lower() + assets = [a for a in release.get("assets", []) if a.get("name") == ASSET] + if len(assets) != 1: + raise SigningError(f"the release has no {ASSET}") + size, digest = assets[0].get("size"), (assets[0].get("digest") or "").lower() + if digest != f"sha256:{sha256}": + raise SigningError(f"GitHub's digest {digest or '(none)'} differs from the body's {sha256}") + if not isinstance(size, int) or size <= 0: + raise SigningError(f"bad installer size: {size!r}") + return version, sha256, size + + +class GitHub: + def __init__(self, repo=REPO, run=subprocess.run): + self.repo, self.run = repo, run + + def gh(self, *args): + return self.run(["gh", *args], check=True, capture_output=True, text=True).stdout + + def release(self): + return json.loads(self.gh("api", f"repos/{self.repo}/releases/tags/{TAG}")) + + def upload(self, paths): + self.gh("release", "upload", TAG, *paths, "--repo", self.repo, "--clobber") + + def download(self, names, directory): + patterns = [arg for name in names for arg in ("--pattern", name)] + self.gh("release", "download", TAG, "--repo", self.repo, "--dir", directory, "--clobber", *patterns) + + +def openssl(): + return os.environ.get("OPENSSL", "openssl") + + +def sign(manifest, signature, private_key, run=subprocess.run): + """Signs the file manifest into signature with the private key PEM (bytes), + which reaches openssl on its stdin.""" + result = run([openssl(), "pkeyutl", "-sign", "-rawin", "-inkey", "/dev/stdin", + "-in", manifest, "-out", signature], input=private_key, capture_output=True) + if result.returncode != 0: + raise SigningError(f"openssl could not sign: {result.stderr.decode(errors='replace').strip()}") + if os.path.getsize(signature) != 64: + raise SigningError("the signature is not 64 bytes: is the key Ed25519?") + + +def verified(manifest, signature, public_key, run=subprocess.run): + """Whether signature is public_key's (PEM bytes) signature over manifest.""" + with tempfile.TemporaryDirectory() as tmp: + key = os.path.join(tmp, "public.pem") + with open(key, "wb") as f: + f.write(public_key) + result = run([openssl(), "pkeyutl", "-verify", "-rawin", "-pubin", "-inkey", key, + "-in", manifest, "-sigfile", signature], capture_output=True) + return result.returncode == 0 and os.path.getsize(signature) == 64 + + +def verify_published(github, public_key=PUBLIC_KEY, run=subprocess.run, sha256=None, version=None): + """Checks the published manifest and signature as a Beta install does, and + returns the version they vouch for. With sha256 or version, the release must + name that installer or version; each is checked on its own.""" + found = read_release(github.release()) + expect(found, sha256, version) + version, sha256, size = found + with tempfile.TemporaryDirectory() as tmp: + try: + github.download([MANIFEST, SIGNATURE], tmp) + except subprocess.CalledProcessError as error: + raise SigningError(f"could not download the manifest and signature: {error.stderr}") + manifest, signature = os.path.join(tmp, MANIFEST), os.path.join(tmp, SIGNATURE) + if not (os.path.isfile(manifest) and os.path.isfile(signature)): + raise SigningError("the release lacks the manifest or its signature") + if not verified(manifest, signature, public_key, run): + raise SigningError("the signature does not verify with the public key") + with open(manifest, "rb") as f: + if f.read() != manifest_text(version, sha256, size).encode(): + raise SigningError("the signed manifest does not match the release") + return version + + +def expect(found, sha256, version=None): + """Refuses a release that does not name the installer the nightly run built.""" + found_version, found_sha256, _ = found + if sha256 is not None and found_sha256 != sha256.lower(): + raise SigningError(f"the release names installer {found_sha256}, not {sha256.lower()} from the nightly run") + if version is not None and found_version != version: + raise SigningError(f"the release names {found_version}, not {version}") + + +def sign_and_publish(github, private_key, sha256, version=None, run=subprocess.run): + """Writes, signs and uploads the manifest for the release as it is now, once it + names the installer sha256 (and version), then verifies what was published + against the pinned key. Returns the version.""" + if not sha256: + raise SigningError("signing needs the installer SHA-256 from the nightly run") + found = read_release(github.release()) + expect(found, sha256, version) + version, sha256, size = found + with tempfile.TemporaryDirectory() as tmp: + manifest, signature = os.path.join(tmp, MANIFEST), os.path.join(tmp, SIGNATURE) + with open(manifest, "w", encoding="utf-8", newline="\n") as f: + f.write(manifest_text(version, sha256, size)) + sign(manifest, signature, private_key, run) + # A wrong key must not replace a good signature. + if not verified(manifest, signature, PUBLIC_KEY, run): + raise SigningError("the private key does not match the pinned public key") + github.upload([manifest, signature]) + published = verify_published(github, PUBLIC_KEY, run) + if published != version: + raise SigningError(f"the release moved to {published} while signing {version}; run again") + return version + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--repo", default=os.environ.get("GH_REPO", REPO)) + parser.add_argument("--public-key", help="--verify-only: PEM file to verify with instead of the pinned key") + parser.add_argument("--sha256", help="the installer's SHA-256 from the nightly run (required to sign)") + parser.add_argument("--version", help="the Beta version the nightly run published") + parser.add_argument("--verify-only", action="store_true") + args = parser.parse_args(argv) + if args.sha256 is not None and not re.fullmatch(r"[0-9A-Fa-f]{64}", args.sha256): + parser.error("--sha256 takes the installer's 64-digit hex SHA-256") + if not args.verify_only: + if args.sha256 is None: + parser.error("signing needs --sha256, from the nightly run's Publish summary") + if args.public_key: + parser.error("--public-key only goes with --verify-only; signing checks the pinned key") + if args.repo != REPO: + parser.error(f"signing is only for {REPO}, not {args.repo}") + try: + if args.verify_only: + public_key = PUBLIC_KEY + if args.public_key: + with open(args.public_key, "rb") as f: + public_key = f.read() + version = verify_published(GitHub(args.repo), public_key, sha256=args.sha256, version=args.version) + print(f"Verified the signed manifest for {version}") + else: + private_key = sys.stdin.buffer.read() + if b"PRIVATE KEY" not in private_key: + raise SigningError("pipe the Ed25519 private key PEM into stdin") + version = sign_and_publish(GitHub(args.repo), private_key, args.sha256, args.version) + print(f"Signed and verified the manifest for {version}") + except SigningError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_nightly.py b/scripts/test_nightly.py index 111d8016..a68865aa 100644 --- a/scripts/test_nightly.py +++ b/scripts/test_nightly.py @@ -1,6 +1,7 @@ """Beta channel builds must sort below the release they lead to and never touch other releases.""" import copy import hashlib +import http.client import itertools import os import pathlib @@ -9,6 +10,7 @@ import sys import tempfile import unittest +import urllib.error sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) import nightly # noqa: E402 @@ -140,6 +142,14 @@ def test_notes_name_the_build_and_the_way_back(self): self.assertIn("TestFlight", text) self.assertIn(f"```\n{HASH}\n```", text) + def test_the_beta_body_is_the_one_the_windows_app_reads(self): + # host/src/updates.rs reads the version and SHA-256 from this fixture too. + fixture = (ROOT / "scripts/fixtures/beta-release-body.md").read_text(encoding="utf-8") + text = nightly.beta_body( + nightly.notes("0.3.5-beta.20261107.26", "release/0.3.5-candidate", SHA, "0.3.4", HASH), SHA) + self.assertEqual(text, fixture.replace("\r\n", "\n")) + self.assertEqual(nightly.published_commit(text), SHA) + def test_notes_need_the_full_commit_and_hash(self): with self.assertRaises(ValueError): nightly.notes("v", "b", SHA[:7], "0.3.3", HASH) @@ -289,6 +299,110 @@ def sha256(self, url): return hashlib.sha256(data + (b"x" if self.corrupt else b"")).hexdigest() +class ScriptedDownloads: + """An asset URL that answers HEAD and the SHA-256 download from scripts, one + entry per call, then keeps answering with the last entry. An entry that is an + exception is raised.""" + + def __init__(self, heads, hashes=()): + self.heads, self.hashes = list(heads), list(hashes) + self.calls = [] + + @staticmethod + def answer(script): + entry = script.pop(0) if len(script) > 1 else script[0] + if isinstance(entry, BaseException): + raise entry + return entry + + def head(self, url): + self.calls.append("head") + return self.answer(self.heads) + + def sha256(self, url): + self.calls.append("sha256") + return self.answer(self.hashes) + + +def http_error(code): + return urllib.error.HTTPError(URL, code, f"HTTP {code}", {}, None) + + +URL = "https://github.com/o/r/releases/download/beta/EternalMonitor-Setup.exe" +SIZE = 1234 +OK = (200, SIZE) + + +class VerifyTest(unittest.TestCase): + """verify() rides out GitHub's 5xx spells but never accepts a different file.""" + + def verify(self, github, sha256=None): + waits = [] + try: + nightly.verify(github, URL, SIZE, sha256, sleep=waits.append) + except nightly.PublishError as error: + return waits, str(error) + return waits, None + + def test_a_5xx_spell_is_waited_out_with_growing_waits(self): + github = ScriptedDownloads([(504, -1)] * 11 + [OK], [HASH]) + waits, failed = self.verify(github, HASH) + self.assertIsNone(failed) + self.assertEqual(waits, [5, 10, 15, 20, 25, 30, 30, 30, 30, 30, 30]) + + def test_a_5xx_spell_that_goes_on_fails_after_twelve_tries(self): + github = ScriptedDownloads([(502, -1)]) + waits, failed = self.verify(github) + self.assertEqual(github.calls.count("head"), 12) + self.assertIn("HTTP 502", failed) + + def test_rate_limits_and_dropped_connections_are_waited_out_too(self): + github = ScriptedDownloads([(429, -1), urllib.error.URLError("connection reset"), + TimeoutError("timed out"), OK]) + self.assertEqual(self.verify(github), ([5, 10, 15], None)) + + def test_a_download_cut_off_partway_is_waited_out(self): + github = ScriptedDownloads([OK], [http.client.IncompleteRead(b"x" * 10, SIZE - 10), + http.client.RemoteDisconnected("closed"), HASH]) + self.assertEqual(self.verify(github, HASH), ([5, 10], None)) + + def test_downloads_cut_off_every_time_fail_after_twelve_tries(self): + github = ScriptedDownloads([OK], [http.client.IncompleteRead(b"", SIZE)]) + waits, failed = self.verify(github, HASH) + self.assertEqual(github.calls.count("sha256"), 12) + self.assertEqual(waits, [5, 10, 15, 20, 25, 30, 30, 30, 30, 30, 30]) + self.assertIn("IncompleteRead", failed) + + def test_a_5xx_while_hashing_is_waited_out(self): + github = ScriptedDownloads([OK], [http_error(504), http_error(503), HASH]) + self.assertEqual(self.verify(github, HASH), ([5, 10], None)) + + def test_a_missing_file_has_the_short_window_it_had(self): + # A new or renamed asset can take a few seconds to show. + github = ScriptedDownloads([(404, -1)] * 3 + [OK]) + self.assertEqual(self.verify(github), ([10, 10, 10], None)) + github = ScriptedDownloads([(404, -1)]) + waits, failed = self.verify(github) + self.assertEqual((len(waits), github.calls.count("head")), (5, 6)) + self.assertIn("HTTP 404", failed) + + def test_a_different_file_is_never_taken_for_a_server_error(self): + wrong_size = ScriptedDownloads([(200, SIZE + 1)]) + waits, failed = self.verify(wrong_size, HASH) + self.assertEqual(wrong_size.calls.count("head"), 6) + self.assertNotIn("sha256", wrong_size.calls) + self.assertIn(f"{SIZE + 1} bytes", failed) + wrong_bytes = ScriptedDownloads([OK], ["cd" * 32]) + waits, failed = self.verify(wrong_bytes, HASH) + self.assertEqual(wrong_bytes.calls.count("sha256"), 6) + self.assertIn("SHA-256 " + "cd" * 32, failed) + self.assertEqual(waits, [10] * 5) + + def test_server_errors_do_not_use_up_the_window_for_a_late_file(self): + github = ScriptedDownloads([(504, -1)] * 4 + [(404, -1)] * 5 + [OK]) + self.assertEqual(self.verify(github), ([5, 10, 15, 20] + [10] * 5, None)) + + OLD_SHA, NEW_SHA = "1" * 40, "2" * 40 WRITES = ("create", "edit", "upload", "rename", "delete", "move_tag") diff --git a/scripts/test_sign_beta_manifest.py b/scripts/test_sign_beta_manifest.py new file mode 100644 index 00000000..a8e5e46b --- /dev/null +++ b/scripts/test_sign_beta_manifest.py @@ -0,0 +1,272 @@ +"""The Beta manifest is written from a release whose parts agree, signed without the key +touching an argument or a file, and verified the way a Beta install verifies it.""" +import base64 +import copy +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import nightly # noqa: E402 +import sign_beta_manifest as signer # noqa: E402 + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SHA = "62b9d9d" + "0" * 33 +HASH = "ab" * 32 +VERSION = "0.3.5-beta.20261107.26" +SIZE = 63301683 +# A published, test-only seed: host/src/updates/tests.rs signs with it too. +TEST_SEED = bytes(range(32)) + + +def run_openssl(*args, data=None): + return subprocess.run([signer.openssl(), *args], input=data, capture_output=True, check=True).stdout + + +def ed25519_available(): + try: + key = run_openssl("genpkey", "-algorithm", "ed25519") + with tempfile.TemporaryDirectory() as tmp: + manifest, signature = os.path.join(tmp, "m"), os.path.join(tmp, "s") + pathlib.Path(manifest).write_bytes(b"x\n") + signer.sign(manifest, signature, key) + return True + except (OSError, subprocess.CalledProcessError, signer.SigningError): + return False + + +def private_from_seed(seed): + der = bytes.fromhex("302e020100300506032b657004220420") + seed + return b"-----BEGIN PRIVATE KEY-----\n" + base64.b64encode(der) + b"\n-----END PRIVATE KEY-----\n" + + +def new_key(): + """A throwaway Ed25519 key pair: (private PEM, public PEM).""" + private = run_openssl("genpkey", "-algorithm", "ed25519") + return private, run_openssl("pkey", "-pubout", data=private) + + +def release(body=None, digest=None, size=SIZE, **fields): + notes = nightly.notes(VERSION, "release/0.3.5-candidate", SHA, "0.3.4", HASH) + data = { + "tag_name": "beta", "draft": False, "prerelease": True, + "body": nightly.beta_body(notes, SHA) if body is None else body, + "assets": [{"name": signer.ASSET, "size": size, + "digest": f"sha256:{HASH}" if digest is None else digest}], + } + data.update(fields) + return data + + +class FakeGitHub: + """The beta release and its uploaded assets, in memory.""" + + def __init__(self, data=None): + self.data, self.assets, self.uploads = data or release(), {}, 0 + + def release(self): + return copy.deepcopy(self.data) + + def upload(self, paths): + self.uploads += 1 + for path in paths: + self.assets[os.path.basename(path)] = pathlib.Path(path).read_bytes() + + def download(self, names, directory): + for name in names: + if name in self.assets: + pathlib.Path(directory, name).write_bytes(self.assets[name]) + + +class Recorder: + """subprocess.run, recording each call's arguments and whether it had input.""" + + def __init__(self): + self.calls = [] + + def __call__(self, args, **kwargs): + self.calls.append((list(args), kwargs.get("input"))) + return subprocess.run(args, **kwargs) + + +class ManifestTest(unittest.TestCase): + def test_the_manifest_is_the_canonical_text_the_host_parses(self): + fixture = (ROOT / "scripts/fixtures/beta-manifest.txt").read_bytes().replace(b"\r\n", b"\n") + self.assertEqual(signer.manifest_text(VERSION, HASH, SIZE).encode(), fixture) + + def test_the_release_parts_must_agree(self): + self.assertEqual(signer.read_release(release()), (VERSION, HASH, SIZE)) + good = release()["body"] + bad = [ + release(digest=f"sha256:{'0' * 64}"), + release(digest=""), + release(body=good.replace(HASH, "")), + release(body=good.replace(VERSION, "0.3.5")), + release(body=good.replace(VERSION, "0.3.5-beta.01")), + release(body=good.replace(VERSION, "0.3.5-beta.1+ci")), + release(body=f"Version 0.3.5-beta.20261108.27, from the Beta channel\n{good}"), + release(size=0), + release(tag_name="v0.3.5"), + release(prerelease=False), + release(draft=True), + release(assets=[]), + ] + for data in bad: + with self.assertRaises(signer.SigningError, msg=str(data)[:200]): + signer.read_release(data) + + +def pinned(public): + """Signing checks the pinned key; tests pin a throwaway one.""" + return mock.patch.object(signer, "PUBLIC_KEY", public) + + +@unittest.skipUnless(ed25519_available(), "needs OpenSSL 3 with Ed25519") +class SigningTest(unittest.TestCase): + def test_signs_uploads_and_verifies(self): + private, public = new_key() + github, run = FakeGitHub(), Recorder() + with pinned(public): + self.assertEqual(signer.sign_and_publish(github, private, HASH.upper(), VERSION, run), VERSION) + self.assertEqual(github.assets[signer.MANIFEST], signer.manifest_text(VERSION, HASH, SIZE).encode()) + self.assertEqual(len(github.assets[signer.SIGNATURE]), 64) + self.assertEqual(signer.verify_published(github, public), VERSION) + self.assertEqual(signer.verify_published(github, public, sha256=HASH, version=VERSION), VERSION) + self.assertEqual(signer.verify_published(github, public, version=VERSION), VERSION) + with self.assertRaises(signer.SigningError): + signer.verify_published(github, public, version="0.3.5-beta.20990101.999") + # The key went to openssl's stdin only: never an argument, never uploaded. + with_key = [args for args, given in run.calls if given == private] + self.assertEqual(len(with_key), 1) + self.assertIn("/dev/stdin", with_key[0]) + for args, _ in run.calls: + self.assertFalse(any("PRIVATE KEY" in arg for arg in args), args) + for data in github.assets.values(): + self.assertNotIn(b"PRIVATE KEY", data) + + def test_a_release_that_is_not_the_nightly_runs_build_is_not_signed(self): + private, public = new_key() + run = Recorder() + for sha256, version in [("0" * 64, None), ("0" * 64, VERSION), (HASH, "0.3.5-beta.20261108.27")]: + github = FakeGitHub() + with pinned(public), self.assertRaises(signer.SigningError, msg=(sha256, version)): + signer.sign_and_publish(github, private, sha256, version, run) + self.assertEqual(github.uploads, 0) + self.assertEqual(run.calls, [], "refused before anything is signed") + github = FakeGitHub() + with pinned(public): + signer.sign_and_publish(github, private, HASH) + with self.assertRaises(signer.SigningError): + signer.verify_published(github, public, sha256="0" * 64) + + def test_a_key_that_is_not_the_pinned_one_uploads_nothing(self): + private, _ = new_key() + _, other = new_key() + github = FakeGitHub() + with pinned(other), self.assertRaises(signer.SigningError): + signer.sign_and_publish(github, private, HASH) + self.assertEqual(github.uploads, 0) + # Against the production key, a throwaway key fails too. + with self.assertRaises(signer.SigningError): + signer.sign_and_publish(github, private, HASH) + self.assertEqual(github.uploads, 0) + + def test_a_release_whose_parts_disagree_is_not_signed(self): + private, public = new_key() + github = FakeGitHub(release(digest=f"sha256:{'0' * 64}")) + with pinned(public), self.assertRaises(signer.SigningError): + signer.sign_and_publish(github, private, HASH) + self.assertEqual(github.uploads, 0) + + def test_verify_only_refuses_a_bad_or_missing_manifest(self): + private, public = new_key() + github = FakeGitHub() + with pinned(public): + signer.sign_and_publish(github, private, HASH) + _, other = new_key() + with self.assertRaises(signer.SigningError): + signer.verify_published(github, other) + good = dict(github.assets) + flipped = bytearray(good[signer.MANIFEST]) + flipped[40] ^= 1 + cases = { + "flipped byte": {**good, signer.MANIFEST: bytes(flipped)}, + "no signature": {signer.MANIFEST: good[signer.MANIFEST]}, + "no manifest": {signer.SIGNATURE: good[signer.SIGNATURE]}, + } + for name, assets in cases.items(): + github.assets = assets + with self.assertRaises(signer.SigningError, msg=name): + signer.verify_published(github, public) + # Signed, but the nightly has moved the release on since. + github.assets = good + github.data = release(body=github.data["body"].replace(VERSION, "0.3.5-beta.20261108.27")) + with self.assertRaises(signer.SigningError): + signer.verify_published(github, public) + + def test_openssl_signs_what_the_host_verifies(self): + # Ed25519 is deterministic: the host's test signs this fixture with ring and + # gets the same bytes. + fixture = ROOT / "scripts/fixtures/beta-manifest.test.sig" + with tempfile.TemporaryDirectory() as tmp: + manifest = os.path.join(tmp, signer.MANIFEST) + signature = os.path.join(tmp, signer.SIGNATURE) + shutil.copyfile(ROOT / "scripts/fixtures/beta-manifest.txt", manifest) + pathlib.Path(manifest).write_bytes(pathlib.Path(manifest).read_bytes().replace(b"\r\n", b"\n")) + signer.sign(manifest, signature, private_from_seed(TEST_SEED)) + self.assertEqual(pathlib.Path(signature).read_bytes(), fixture.read_bytes()) + + +class CommandTest(unittest.TestCase): + def main(self, argv, stdin=b"-----BEGIN PRIVATE KEY-----\n"): + """main(argv) with stdin; the exit status, or the usage error's. Nothing + reaches GitHub: every case here stops before it would.""" + class Stdin: + buffer = __import__("io").BytesIO(stdin) + + saved = sys.stdin + sys.stdin = Stdin() + try: + with mock.patch.object(signer, "GitHub", side_effect=AssertionError("no GitHub")), \ + mock.patch("sys.stderr"): + return signer.main(argv) + except SystemExit as exit: + return exit.code + finally: + sys.stdin = saved + + def test_signing_needs_the_nightly_runs_hash(self): + self.assertEqual(self.main([]), 2) + self.assertEqual(self.main(["--sha256", "abc"]), 2) + self.assertEqual(self.main(["--verify-only", "--sha256", "abc"]), 2) + + def test_signing_is_only_for_the_real_repo_and_the_pinned_key(self): + self.assertEqual(self.main(["--sha256", HASH, "--repo", "someone/fork"]), 2) + with mock.patch.dict(os.environ, {"GH_REPO": "someone/fork"}): + self.assertEqual(self.main(["--sha256", HASH]), 2) + self.assertEqual(self.main(["--sha256", HASH, "--public-key", "other.pem"]), 2) + + def test_signing_needs_a_private_key_on_stdin(self): + self.assertEqual(self.main(["--sha256", HASH], stdin=b"not a key"), 1) + + +class WorkflowTest(unittest.TestCase): + def test_the_nightly_run_shows_the_hash_and_version_to_sign(self): + text = (ROOT / ".github/workflows/nightly.yml").read_text(encoding="utf-8") + publish = text[text.index("\n publish:"):] + self.assertIn("SHA256: ${{ needs.build.outputs.sha256 }}", publish) + self.assertIn("VERSION: ${{ needs.plan.outputs.version }}", publish) + summary = publish[publish.index("- name: Summarize the build to sign"):] + summary = summary[:summary.index("\n - name:", 1)] + self.assertIn('echo "| Installer SHA-256 | \\`$SHA256\\` |"', summary) + self.assertIn('echo "| Version | $VERSION |"', summary) + self.assertIn('} >> "$GITHUB_STEP_SUMMARY"', summary) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/win/Test-VddTask.ps1 b/scripts/win/Test-VddTask.ps1 index 4d223a4a..1c3f8cef 100644 --- a/scripts/win/Test-VddTask.ps1 +++ b/scripts/win/Test-VddTask.ps1 @@ -6,6 +6,7 @@ $script = Get-Content -Raw (Join-Path $PSScriptRoot '..\..\host\src\vdd_task.ps1 $mock = @' # $EMBusy[name]: how many more state reads report that task as running. # $EMMissing: tasks that do not exist, as Task Scheduler reports them. +# $EMRunState: the state of the instance Run starts (3: it has finished). function New-Object { param([string]$ComObject, [Parameter(ValueFromRemainingArguments=$true)]$Rest) if ($ComObject -ne 'Schedule.Service') { throw "Unexpected COM object $ComObject" } @@ -29,7 +30,7 @@ function New-Object { param($arguments) $busy = ($global:EMBusy.Keys | Where-Object { $global:EMBusy[$_] -gt 0 }) -join ',' [Console]::Out.WriteLine("RUN $($this.Name) busy=[$busy]") - $running = [pscustomobject]@{ State = 3 } + $running = [pscustomobject]@{ State = $global:EMRunState } $running | Add-Member ScriptMethod Refresh {} $running } @@ -41,11 +42,11 @@ function New-Object { } '@ -function Invoke-Case([hashtable]$busy, [string]$arguments, [string[]]$missing = @()) { +function Invoke-Case([hashtable]$busy, [string]$arguments, [string[]]$missing = @(), [int]$runState = 3) { $file = Join-Path $env:TEMP ('em-vdd-task-test-' + [guid]::NewGuid().ToString('N') + '.ps1') $busyText = ($busy.GetEnumerator() | ForEach-Object { "'$($_.Key)' = $($_.Value)" }) -join '; ' $missingText = ($missing | ForEach-Object { "'$_'" }) -join ', ' - $body = "`$global:EMBusy = @{ $busyText }`n`$global:EMMissing = @($missingText)`n$mock`n& {`n$script`n} $arguments`n" + $body = "`$global:EMBusy = @{ $busyText }`n`$global:EMMissing = @($missingText)`n`$global:EMRunState = $runState`n$mock`n& {`n$script`n} $arguments`n" Set-Content -Path $file -Value $body -Encoding UTF8 # The child's stderr is expected output here, not a failure of this script. $ErrorActionPreference = 'Continue' @@ -70,13 +71,19 @@ if ($case.Code -ne 0 -or $case.Output -notmatch 'RUN Disable busy=\[\]') { $started = Get-Date $case = Invoke-Case @{ Enable = 1000000 } "-Task 'Disable' -After 'Enable'" $took = ((Get-Date) - $started).TotalSeconds -if ($case.Code -ne 1 -or $case.Output -notmatch 'Previous VDD task timed out: Enable' -or $case.Output -match 'RUN') { +if ($case.Code -ne 1 -or $case.Output -notmatch 'Previous VDD task timed out: Enable \(in progress\)' -or $case.Output -match 'RUN') { throw "A hung Enable was not reported as a timeout: $($case.Output)" } # The script's own wait is bounded at 10 s; a cold PowerShell start for the # child can take tens of seconds more on a busy PC. A wait that never ended # would take hours with this mock. if ($took -gt 120) { throw "The wait took $took s" } +# An instance Task Scheduler never starts reads as queued, so the host's log +# tells it apart from a toggle that hangs. +$case = Invoke-Case @{} "-Task 'Disable' -After 'Enable'" @() 2 +if ($case.Code -ne 1 -or $case.Output -notmatch 'VDD task timed out: Disable \(queued\)') { + throw "A queued instance was not reported as one: $($case.Output)" +} # Without -After, as older hosts call it, only this task is waited for. $case = Invoke-Case @{ Enable = 1000000 } "-Task 'Disable'" if ($case.Code -ne 0 -or $case.Output -notmatch 'RUN Disable') { @@ -92,7 +99,7 @@ $case = Invoke-Case @{} "-Task 'Disable' -After 'Enable'" @('Disable') if ($case.Code -ne 1 -or $case.Output -notmatch '0x80070002' -or $case.Output -match 'RUN') { throw "A missing requested task was not reported: $($case.Output)" } -Write-Output 'VDD task checks passed: waits for the other direction, earlier runs, hung Enable bound, no -After, missing tasks' +Write-Output 'VDD task checks passed: waits for the other direction, earlier runs, hung Enable bound, queued instance, no -After, missing tasks' # Some cases end their child with exit code 1 on purpose. Without this the # last one stays in $LASTEXITCODE, and a CI step that ends with it fails. # Every failure above throws, which exits 1. diff --git a/vendor/README.md b/vendor/README.md new file mode 100644 index 00000000..6653a502 --- /dev/null +++ b/vendor/README.md @@ -0,0 +1,82 @@ +# Vendored crates + +Cargo builds these in place of the crates.io release (`[patch.crates-io]` in +the workspace `Cargo.toml`). Each keeps its upstream license files. + +## egui-wgpu 0.36.2 + +Copied from the crates.io release of egui-wgpu 0.36.2 +(https://github.com/emilk/egui, MIT OR Apache-2.0), without its `Cargo.lock` +and `Cargo.toml.orig`. Every change is marked `EternalMonitor patch`. + +Why: the PC window must survive the loss of its graphics device, as in a +driver reset or a device removed while another program uses the GPU. The +release panics on the next frame ("Failed to create staging buffer for index +data"), because wgpu gives no staging buffer once the device is lost, and the +panic ends the app and the stream with it. 0.37.0 still does. + +The patch: +- `Renderer::update_buffers` skips a frame whose staging buffer can't be made, + and `Renderer::render` then draws nothing. +- The winit `Painter` watches each device's lost callback. A wgpu error + outside an error scope counts as a loss too, where wgpu's default handler + panics: a window whose device or surface fails is drawn on a new one. +- When the device is lost, or a frame's upload fails, the `Painter` drops the + device, its surfaces and egui's renderer. eframe calls `set_window` before + each paint, which makes new ones when `winit::Replacement` says: at once, + then after 0.5 s, doubling up to 30 s. After 10 failures in a row (devices + that could not be made, or new ones lost again before they drew for 10 s) + it gives up, says so once, and the window stays blank. Streaming runs on + its own devices and is not affected. The new device's first frame uploads + the font atlas again. Other textures are the app's to load again + (`host/src/gui/renderer.rs` counts devices for that). +- A replacement never blocks the window's thread. Two stages run on other + threads, which the window polls each frame: + - Listing adapters and making the device (`RenderState::new_device`). The + loss handlers go on the device before anything is made on it. + - egui's renderer (buffers, shaders, the pipeline), from the device, the + format and the options only. + - A driver can stall in either. Neither thread gets the window or a surface, + so a stuck one neither keeps the window alive (eframe ends its loop only + once the window is destroyed) nor touches it after it is gone. A late + answer after quitting is dropped with the closed channel. + - A stage that hasn't answered in 10 s, or that panicked, is a failed try, + and no other starts while one is out; a late answer is still taken. + - Between the stages, the window's thread makes the surface and checks the + new adapter can draw on it (`RenderState::surface_format`). After them it + configures the surface: DXGI sends the window messages while it makes the + swapchain. +- With no surface to check against, a replacement could keep choosing an + adapter that can't draw on the window, as on a hybrid PC. + `replacement_candidates` gives the host's adapter choice only the adapter + that last drew on the window, starting with the one from start, while it is + listed. An adapter that can't draw on the window is left out of later tries + until a replacement works. Adapters are told apart by `AdapterKey`: the + LUID (from DXGI on DX12, and `VkPhysicalDeviceIDProperties` on Vulkan 1.1+ + on Windows, through the `ash` dependency at wgpu-hal's version), else the + backend, PCI ids, type, name and PCI bus. wgpu's DX12 bus comes from the + PCI ids alone, so identical cards differ only by LUID. A driver update or reset can + give an adapter a new LUID, so an adapter is also found as the only listed + one that differs in the LUID alone; two such are no match (no preference, + nothing left out). The Vulkan LUID is read only when both the instance and + the device are Vulkan 1.1+. + The thread is named `winit::REPLACEMENT_THREAD`, so the host's panic hook + treats a panic there as one failed try: it doesn't turn the virtual display + off as it does for other background threads. +- `configure_surface` replaces the device, instead of panicking, when the + adapter supports no surface any more. +- `Renderer::upload_failed`, `winit::SKIPPED_FRAMES` and `winit::DEVICES_LOST` + let the app see skipped frames and dropped devices. The host counts them, so + a backend that only skips frames is never recorded as one that drew, and a + device lost before the window worked is recorded as its backend's failure + even if no replacement is ever made. +- A D3D12 device is only replaced once nothing holds the lost one, so the host + drops the copy eframe keeps in `Frame` (`release_render_state`). + +Drop this copy when an egui-wgpu release survives device loss, or carry the +patch to the version eframe moves to. When eframe moves on (0.37 uses +egui-wgpu 0.37) this patch stops applying, because `[patch.crates-io]` only +replaces 0.36.2: Cargo warns that the patch was not used. Then +`host/tests/window_device_loss.rs` is the guard. It fails to build without +the patch's `upload_failed` and `Replacement`, and on an unpatched release it +fails with the old panic. diff --git a/vendor/egui-wgpu/Cargo.toml b/vendor/egui-wgpu/Cargo.toml new file mode 100644 index 00000000..86d7f87d --- /dev/null +++ b/vendor/egui-wgpu/Cargo.toml @@ -0,0 +1,346 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.95" +name = "egui-wgpu" +version = "0.36.2" +authors = [ + "Nils Hasenbanck ", + "embotech ", + "Emil Ernerfeldt ", +] +build = false +include = [ + "../../LICENSE-APACHE", + "../../LICENSE-MIT", + "**/*.rs", + "**/*.wgsl", + "Cargo.toml", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Bindings for using egui natively using the wgpu library" +homepage = "https://github.com/emilk/egui/tree/main/crates/egui-wgpu" +readme = "README.md" +keywords = [ + "wgpu", + "egui", + "gui", + "gamedev", +] +categories = [ + "gui", + "game-development", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/emilk/egui/tree/main/crates/egui-wgpu" +resolver = "2" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--generate-link-to-definition"] + +[features] +capture = ["dep:egui"] +default = [ + "fragile-send-sync-non-atomic-wasm", + "macos-window-resize-jitter-fix", + "wgpu/default", + "wgpu/webgl", +] +fragile-send-sync-non-atomic-wasm = ["wgpu/fragile-send-sync-non-atomic-wasm"] +macos-window-resize-jitter-fix = ["wgpu/metal"] +wayland = ["winit?/wayland"] +winit = [ + "dep:winit", + "winit/rwh_06", + "dep:egui", + "capture", +] +x11 = ["winit?/x11"] + +[lib] +name = "egui_wgpu" +path = "src/lib.rs" + +[dependencies.ahash] +version = "0.8.12" +features = [ + "no-rng", + "std", +] +default-features = false + +[dependencies.bytemuck] +version = "1.25" + +[dependencies.document-features] +version = "0.2.12" + +[dependencies.egui] +version = "0.36.2" +optional = true +default-features = false + +[dependencies.epaint] +version = "0.36.2" +features = ["bytemuck"] +default-features = false + +[dependencies.log] +version = "0.4.33" +features = ["std"] + +[dependencies.profiling] +version = "1.0" +default-features = false + +[dependencies.thiserror] +version = "2.0" + +[dependencies.type-map] +version = "0.5.1" + +[dependencies.web-time] +version = "1.1" + +[dependencies.wgpu] +version = "30.0" +features = [ + "std", + "wgsl", +] +default-features = false + +[dependencies.winit] +version = "0.30.13" +optional = true +default-features = false + +[lints.clippy] +allow_attributes = "warn" +as_ptr_cast_mut = "warn" +assigning_clones = "allow" +bool_to_int_with_if = "warn" +branches_sharing_code = "warn" +cast_possible_truncation = "allow" +cast_possible_wrap = "allow" +checked_conversions = "warn" +clear_with_drain = "warn" +clone_on_ref_ptr = "warn" +cloned_instead_of_copied = "warn" +coerce_container_to_any = "warn" +comparison_chain = "allow" +dbg_macro = "warn" +debug_assert_with_mut_call = "warn" +decimal_bitwise_operands = "warn" +default_union_representation = "warn" +derive_partial_eq_without_eq = "warn" +disallowed_script_idents = "warn" +doc_broken_link = "warn" +doc_comment_double_space_linebreaks = "warn" +doc_include_without_cfg = "warn" +doc_link_with_quotes = "warn" +doc_markdown = "warn" +duration_suboptimal_units = "warn" +elidable_lifetime_names = "warn" +empty_enum_variants_with_brackets = "warn" +empty_enums = "warn" +enum_glob_use = "warn" +equatable_if_let = "warn" +exit = "warn" +expl_impl_clone_on_copy = "warn" +explicit_deref_methods = "warn" +explicit_into_iter_loop = "warn" +explicit_iter_loop = "warn" +fallible_impl_from = "warn" +filter_map_next = "warn" +flat_map_option = "warn" +float_cmp_const = "warn" +fn_params_excessive_bools = "warn" +fn_to_numeric_cast_any = "warn" +format_push_string = "warn" +from_iter_instead_of_collect = "warn" +get_unwrap = "warn" +ignore_without_reason = "warn" +ignored_unit_patterns = "warn" +implicit_clone = "warn" +imprecise_flops = "warn" +inconsistent_struct_constructor = "warn" +index_refutable_slice = "warn" +inefficient_to_string = "warn" +infinite_loop = "warn" +into_iter_without_iter = "warn" +invalid_upcast_comparisons = "warn" +ip_constant = "warn" +iter_filter_is_ok = "warn" +iter_filter_is_some = "warn" +iter_not_returning_iterator = "warn" +iter_on_empty_collections = "warn" +iter_on_single_items = "warn" +iter_over_hash_type = "warn" +iter_without_into_iter = "warn" +large_digit_groups = "warn" +large_futures = "warn" +large_include_file = "warn" +large_stack_arrays = "warn" +large_stack_frames = "warn" +large_types_passed_by_value = "warn" +let_underscore_must_use = "allow" +let_underscore_untyped = "allow" +linkedlist = "warn" +literal_string_with_formatting_args = "warn" +lossy_float_literal = "warn" +macro_use_imports = "warn" +manual_assert = "warn" +manual_ilog2 = "warn" +manual_instant_elapsed = "warn" +manual_is_power_of_two = "warn" +manual_is_variant_and = "warn" +manual_let_else = "warn" +manual_midpoint = "warn" +manual_range_contains = "allow" +manual_string_new = "warn" +map_err_ignore = "warn" +map_unwrap_or = "allow" +match_bool = "warn" +match_same_arms = "warn" +match_wild_err_arm = "warn" +match_wildcard_for_single_variants = "warn" +mem_forget = "warn" +mismatching_type_param_order = "warn" +missing_assert_message = "warn" +missing_errors_doc = "warn" +missing_fields_in_debug = "warn" +mut_mut = "warn" +mutex_integer = "warn" +needless_continue = "warn" +needless_for_each = "warn" +needless_pass_by_ref_mut = "warn" +needless_pass_by_value = "warn" +needless_raw_string_hashes = "warn" +needless_type_cast = "warn" +negative_feature_names = "warn" +non_std_lazy_statics = "warn" +non_zero_suggestions = "warn" +nonstandard_macro_braces = "warn" +option_as_ref_cloned = "warn" +option_option = "warn" +or_fun_call = "warn" +path_buf_push_overwrite = "warn" +pathbuf_init_then_push = "warn" +precedence_bits = "warn" +print_stderr = "warn" +print_stdout = "warn" +ptr_as_ptr = "warn" +ptr_cast_constness = "warn" +pub_underscore_fields = "warn" +pub_without_shorthand = "warn" +rc_mutex = "warn" +redundant_type_annotations = "warn" +ref_as_ptr = "warn" +ref_option = "warn" +ref_option_ref = "warn" +ref_patterns = "warn" +rest_pat_in_fully_bound_structs = "warn" +return_and_then = "warn" +same_functions_in_if_condition = "warn" +same_length_and_capacity = "warn" +self_named_module_files = "allow" +self_only_used_in_recursion = "warn" +semicolon_if_nothing_returned = "warn" +set_contains_or_insert = "warn" +should_panic_without_expect = "allow" +significant_drop_tightening = "allow" +single_char_pattern = "warn" +single_match_else = "warn" +single_option_map = "warn" +std_instead_of_core = "warn" +str_split_at_newline = "warn" +str_to_string = "warn" +string_add = "warn" +string_add_assign = "warn" +string_lit_as_bytes = "warn" +string_lit_chars_any = "warn" +suspicious_xor_used_as_pow = "warn" +todo = "warn" +too_long_first_doc_paragraph = "warn" +too_many_lines = "allow" +trailing_empty_array = "warn" +trait_duplication_in_bounds = "warn" +transmute_ptr_to_ptr = "warn" +tuple_array_conversions = "warn" +unchecked_time_subtraction = "warn" +undocumented_unsafe_blocks = "warn" +unimplemented = "warn" +uninhabited_references = "warn" +uninlined_format_args = "warn" +unnecessary_box_returns = "warn" +unnecessary_debug_formatting = "warn" +unnecessary_literal_bound = "warn" +unnecessary_safety_comment = "warn" +unnecessary_safety_doc = "warn" +unnecessary_self_imports = "warn" +unnecessary_semicolon = "warn" +unnecessary_struct_initialization = "warn" +unnecessary_trailing_comma = "warn" +unnecessary_wraps = "warn" +unnested_or_patterns = "warn" +unused_async = "warn" +unused_peekable = "warn" +unused_rounding = "warn" +unused_self = "warn" +unused_trait_names = "warn" +unwrap_used = "warn" +use_self = "warn" +useless_let_if_seq = "warn" +verbose_file_reads = "warn" +wildcard_dependencies = "warn" +wildcard_imports = "allow" +zero_sized_map_values = "warn" + +[lints.clippy.all] +level = "warn" +priority = -1 + +[lints.rust] +trivial_casts = "allow" +trivial_numeric_casts = "warn" +unsafe_code = "deny" +unused_import_braces = "warn" +unused_lifetimes = "warn" +unused_qualifications = "allow" + +[lints.rust.future_incompatible] +level = "warn" +priority = -1 + +[lints.rust.nonstandard_style] +level = "warn" +priority = -1 + +[lints.rust.rust_2018_idioms] +level = "warn" +priority = -1 + +[lints.rustdoc] +all = "warn" + +# EternalMonitor patch: an adapter's Vulkan LUID (`AdapterKey`), at wgpu-hal's version. +[target.'cfg(windows)'.dependencies.ash] +version = "0.38" +default-features = false + diff --git a/vendor/egui-wgpu/LICENSE-APACHE b/vendor/egui-wgpu/LICENSE-APACHE new file mode 100644 index 00000000..11069edd --- /dev/null +++ b/vendor/egui-wgpu/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/vendor/egui-wgpu/LICENSE-MIT b/vendor/egui-wgpu/LICENSE-MIT new file mode 100644 index 00000000..673ea5f0 --- /dev/null +++ b/vendor/egui-wgpu/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2018-2021 Emil Ernerfeldt + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/vendor/egui-wgpu/README.md b/vendor/egui-wgpu/README.md new file mode 100644 index 00000000..9aab3159 --- /dev/null +++ b/vendor/egui-wgpu/README.md @@ -0,0 +1,10 @@ +# egui-wgpu + +[![Latest version](https://img.shields.io/crates/v/egui-wgpu.svg)](https://crates.io/crates/egui-wgpu) +[![Documentation](https://docs.rs/egui-wgpu/badge.svg)](https://docs.rs/egui-wgpu) +![MIT](https://img.shields.io/badge/license-MIT-blue.svg) +![Apache](https://img.shields.io/badge/license-Apache-blue.svg) + +This crates provides bindings between [`egui`](https://github.com/emilk/egui) and [wgpu](https://crates.io/crates/wgpu). + +This was originally hosted at https://github.com/hasenbanck/egui_wgpu_backend diff --git a/vendor/egui-wgpu/src/capture.rs b/vendor/egui-wgpu/src/capture.rs new file mode 100644 index 00000000..54ddeb88 --- /dev/null +++ b/vendor/egui-wgpu/src/capture.rs @@ -0,0 +1,267 @@ +use egui::{UserData, ViewportId}; +use epaint::ColorImage; +use std::sync::{Arc, mpsc}; +use wgpu::{BindGroupLayout, MultisampleState, StoreOp}; + +/// A texture and a buffer for reading the rendered frame back to the cpu. +/// +/// The texture is required since [`wgpu::TextureUsages::COPY_SRC`] is not an allowed +/// flag for the surface texture on all platforms. This means that anytime we want to +/// capture the frame, we first render it to this texture, and then we can copy it to +/// both the surface texture (via a render pass) and the buffer (via a texture to buffer copy), +/// from where we can pull it back +/// to the cpu. +pub struct CaptureState { + padding: BufferPadding, + pub texture: wgpu::Texture, + pipeline: wgpu::RenderPipeline, + bind_group: wgpu::BindGroup, +} + +pub type CaptureReceiver = mpsc::Receiver<(ViewportId, Vec, ColorImage)>; +pub type CaptureSender = mpsc::Sender<(ViewportId, Vec, ColorImage)>; +pub use mpsc::channel as capture_channel; + +impl CaptureState { + pub fn new(device: &wgpu::Device, surface_texture: &wgpu::Texture) -> Self { + let shader = device.create_shader_module(wgpu::include_wgsl!("texture_copy.wgsl")); + + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("texture_copy"), + layout: None, + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(surface_texture.format().into())], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + ..Default::default() + }, + depth_stencil: None, + multisample: MultisampleState::default(), + multiview_mask: None, + cache: None, + }); + + let bind_group_layout = pipeline.get_bind_group_layout(0); + + let (texture, padding, bind_group) = + Self::create_texture(device, surface_texture, &bind_group_layout); + + Self { + padding, + texture, + pipeline, + bind_group, + } + } + + fn create_texture( + device: &wgpu::Device, + surface_texture: &wgpu::Texture, + layout: &BindGroupLayout, + ) -> (wgpu::Texture, BufferPadding, wgpu::BindGroup) { + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("egui_screen_capture_texture"), + size: surface_texture.size(), + mip_level_count: surface_texture.mip_level_count(), + sample_count: surface_texture.sample_count(), + dimension: surface_texture.dimension(), + format: surface_texture.format(), + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + + let padding = BufferPadding::new(surface_texture.width()); + + let view = texture.create_view(&Default::default()); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&view), + }], + label: None, + }); + + (texture, padding, bind_group) + } + + /// Updates the [`CaptureState`] if the size of the surface texture has changed + pub fn update(&mut self, device: &wgpu::Device, texture: &wgpu::Texture) { + if self.texture.size() != texture.size() { + let (new_texture, padding, bind_group) = + Self::create_texture(device, texture, &self.pipeline.get_bind_group_layout(0)); + self.texture = new_texture; + self.padding = padding; + self.bind_group = bind_group; + } + } + + /// Handles copying from the [`CaptureState`] texture to the surface texture and the buffer. + /// Pass the returned buffer to [`CaptureState::read_screen_rgba`] to read the data back to the cpu. + pub fn copy_textures( + &mut self, + device: &wgpu::Device, + output_frame: &wgpu::SurfaceTexture, + encoder: &mut wgpu::CommandEncoder, + ) -> wgpu::Buffer { + debug_assert_eq!( + self.texture.size(), + output_frame.texture.size(), + "Texture sizes must match, `CaptureState::update` was probably not called" + ); + + // It would be more efficient to reuse the Buffer, e.g. via some kind of ring buffer, but + // for most screenshot use cases this should be fine. When taking many screenshots (e.g. for a video) + // it might make sense to revisit this and implement a more efficient solution. + #[allow(clippy::allow_attributes, clippy::arc_with_non_send_sync)] // For wasm + let buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("egui_screen_capture_buffer"), + size: (self.padding.padded_bytes_per_row * self.texture.height()) as u64, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let padding = self.padding; + let tex = &mut self.texture; + + let tex_extent = tex.size(); + + encoder.copy_texture_to_buffer( + tex.as_image_copy(), + wgpu::TexelCopyBufferInfo { + buffer: &buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(padding.padded_bytes_per_row), + rows_per_image: None, + }, + }, + tex_extent, + ); + + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("texture_copy"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &output_frame.texture.create_view(&Default::default()), + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: StoreOp::Store, + }, + depth_slice: None, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + multiview_mask: None, + }); + + pass.set_pipeline(&self.pipeline); + pass.set_bind_group(0, &self.bind_group, &[]); + pass.draw(0..3, 0..1); + + buffer + } + + /// Handles copying from the [`CaptureState`] texture to the surface texture and the cpu + /// This function is non-blocking and will send the data to the given sender when it's ready. + /// Pass in the buffer returned from [`CaptureState::copy_textures`]. + /// Make sure to call this after the encoder has been submitted. + pub fn read_screen_rgba( + &self, + ctx: egui::Context, + buffer: wgpu::Buffer, + data: Vec, + tx: CaptureSender, + viewport_id: ViewportId, + ) { + #[allow(clippy::allow_attributes, clippy::arc_with_non_send_sync)] // For wasm + let buffer = Arc::new(buffer); + let buffer_clone = Arc::clone(&buffer); + let buffer_slice = buffer_clone.slice(..); + let format = self.texture.format(); + let tex_extent = self.texture.size(); + let padding = self.padding; + let to_rgba = match format { + wgpu::TextureFormat::Rgba8Unorm => [0, 1, 2, 3], + wgpu::TextureFormat::Bgra8Unorm => [2, 1, 0, 3], + _ => { + log::error!( + "Screen can't be captured unless the surface format is Rgba8Unorm or Bgra8Unorm. Current surface format is {format:?}" + ); + return; + } + }; + buffer_slice.map_async(wgpu::MapMode::Read, move |result| { + if let Err(err) = result { + log::error!("Failed to map buffer for reading: {err}"); + return; + } + let buffer_slice = buffer.slice(..); + + let mut pixels = Vec::with_capacity((tex_extent.width * tex_extent.height) as usize); + let mapped_range = match buffer_slice.get_mapped_range() { + Ok(range) => range, + Err(err) => { + log::error!("Failed to get mapped range for reading: {err}"); + return; + } + }; + for padded_row in mapped_range.chunks(padding.padded_bytes_per_row as usize) { + let row = &padded_row[..padding.unpadded_bytes_per_row as usize]; + for color in row.chunks(4) { + pixels.push(epaint::Color32::from_rgba_premultiplied( + color[to_rgba[0]], + color[to_rgba[1]], + color[to_rgba[2]], + color[to_rgba[3]], + )); + } + } + drop(mapped_range); + buffer.unmap(); + + tx.send(( + viewport_id, + data, + ColorImage::new( + [tex_extent.width as usize, tex_extent.height as usize], + pixels, + ), + )) + .ok(); + ctx.request_repaint(); + }); + } +} + +#[derive(Copy, Clone)] +struct BufferPadding { + unpadded_bytes_per_row: u32, + padded_bytes_per_row: u32, +} + +impl BufferPadding { + fn new(width: u32) -> Self { + let bytes_per_pixel = core::mem::size_of::() as u32; + let unpadded_bytes_per_row = width * bytes_per_pixel; + let padded_bytes_per_row = + wgpu::util::align_to(unpadded_bytes_per_row, wgpu::COPY_BYTES_PER_ROW_ALIGNMENT); + Self { + unpadded_bytes_per_row, + padded_bytes_per_row, + } + } +} diff --git a/vendor/egui-wgpu/src/egui.wgsl b/vendor/egui-wgpu/src/egui.wgsl new file mode 100644 index 00000000..39210841 --- /dev/null +++ b/vendor/egui-wgpu/src/egui.wgsl @@ -0,0 +1,162 @@ +// Vertex shader bindings + +struct VertexOutput { + @location(0) tex_coord: vec2, + @location(1) color: vec4, // gamma 0-1 + @builtin(position) position: vec4, +}; + +struct Locals { + screen_size: vec2, + + /// 1 if dithering is enabled, 0 otherwise + dithering: u32, + + /// 1 to do manual filtering for more predictable kittest snapshot images. + /// See also https://github.com/emilk/egui/issues/5295 + predictable_texture_filtering: u32, +}; +@group(0) @binding(0) var r_locals: Locals; + + +// ----------------------------------------------- +// Adapted from +// https://www.shadertoy.com/view/llVGzG +// Originally presented in: +// Jimenez 2014, "Next Generation Post-Processing in Call of Duty" +// +// A good overview can be found in +// https://blog.demofox.org/2022/01/01/interleaved-gradient-noise-a-different-kind-of-low-discrepancy-sequence/ +// via https://github.com/rerun-io/rerun/ +fn interleaved_gradient_noise(n: vec2) -> f32 { + let f = 0.06711056 * n.x + 0.00583715 * n.y; + return fract(52.9829189 * fract(f)); +} + +fn dither_interleaved(rgb: vec3, levels: f32, frag_coord: vec4) -> vec3 { + var noise = interleaved_gradient_noise(frag_coord.xy); + // scale down the noise slightly to ensure flat colors aren't getting dithered + noise = (noise - 0.5) * 0.95; + return rgb + noise / (levels - 1.0); +} + +// 0-1 linear from 0-1 sRGB gamma +fn linear_from_gamma_rgb(srgb: vec3) -> vec3 { + let cutoff = srgb < vec3(0.04045); + let lower = srgb / vec3(12.92); + let higher = pow((srgb + vec3(0.055)) / vec3(1.055), vec3(2.4)); + return select(higher, lower, cutoff); +} + +// 0-1 sRGB gamma from 0-1 linear +fn gamma_from_linear_rgb(rgb: vec3) -> vec3 { + let cutoff = rgb < vec3(0.0031308); + let lower = rgb * vec3(12.92); + let higher = vec3(1.055) * pow(rgb, vec3(1.0 / 2.4)) - vec3(0.055); + return select(higher, lower, cutoff); +} + +// 0-1 sRGBA gamma from 0-1 linear +fn gamma_from_linear_rgba(linear_rgba: vec4) -> vec4 { + return vec4(gamma_from_linear_rgb(linear_rgba.rgb), linear_rgba.a); +} + +// [u8; 4] SRGB as u32 -> [r, g, b, a] in 0.-1 +fn unpack_color(color: u32) -> vec4 { + return vec4( + f32(color & 255u), + f32((color >> 8u) & 255u), + f32((color >> 16u) & 255u), + f32((color >> 24u) & 255u), + ) / 255.0; +} + +fn position_from_screen(screen_pos: vec2) -> vec4 { + return vec4( + 2.0 * screen_pos.x / r_locals.screen_size.x - 1.0, + 1.0 - 2.0 * screen_pos.y / r_locals.screen_size.y, + 0.0, + 1.0, + ); +} + +@vertex +fn vs_main( + @location(0) a_pos: vec2, + @location(1) a_tex_coord: vec2, + @location(2) a_color: u32, +) -> VertexOutput { + var out: VertexOutput; + out.tex_coord = a_tex_coord; + out.color = unpack_color(a_color); + out.position = position_from_screen(a_pos); + return out; +} + +// Fragment shader bindings + +@group(1) @binding(0) var r_tex_color: texture_2d; +@group(1) @binding(1) var r_tex_sampler: sampler; + +fn sample_texture(in: VertexOutput) -> vec4 { + if r_locals.predictable_texture_filtering == 0 { + // Hardware filtering: fast, but varies across GPUs and drivers. + return textureSample(r_tex_color, r_tex_sampler, in.tex_coord); + } else { + // Manual bilinear filtering with four taps at pixel centers using textureLoad + let texture_size = vec2(textureDimensions(r_tex_color, 0)); + let texture_size_f = vec2(texture_size); + let pixel_coord = in.tex_coord * texture_size_f - 0.5; + let pixel_fract = fract(pixel_coord); + let pixel_floor = vec2(floor(pixel_coord)); + + // Manual texture clamping + let max_coord = texture_size - vec2(1, 1); + let p00 = clamp(pixel_floor + vec2(0, 0), vec2(0, 0), max_coord); + let p10 = clamp(pixel_floor + vec2(1, 0), vec2(0, 0), max_coord); + let p01 = clamp(pixel_floor + vec2(0, 1), vec2(0, 0), max_coord); + let p11 = clamp(pixel_floor + vec2(1, 1), vec2(0, 0), max_coord); + + // Load at pixel centers + let tl = textureLoad(r_tex_color, p00, 0); + let tr = textureLoad(r_tex_color, p10, 0); + let bl = textureLoad(r_tex_color, p01, 0); + let br = textureLoad(r_tex_color, p11, 0); + + // Manual bilinear interpolation + let top = mix(tl, tr, pixel_fract.x); + let bottom = mix(bl, br, pixel_fract.x); + return mix(top, bottom, pixel_fract.y); + } +} + +@fragment +fn fs_main_linear_framebuffer(in: VertexOutput) -> @location(0) vec4 { + // We expect "normal" textures that are NOT sRGB-aware. + let tex_gamma = sample_texture(in); + var out_color_gamma = in.color * tex_gamma; + // Dither the float color down to eight bits to reduce banding. + // This step is optional for egui backends. + // Note that dithering is performed on the gamma encoded values, + // because this function is used together with a srgb converting target. + if r_locals.dithering == 1 { + let out_color_gamma_rgb = dither_interleaved(out_color_gamma.rgb, 256.0, in.position); + out_color_gamma = vec4(out_color_gamma_rgb, out_color_gamma.a); + } + let out_color_linear = linear_from_gamma_rgb(out_color_gamma.rgb); + return vec4(out_color_linear, out_color_gamma.a); +} + +@fragment +fn fs_main_gamma_framebuffer(in: VertexOutput) -> @location(0) vec4 { + // We expect "normal" textures that are NOT sRGB-aware. + let tex_gamma = sample_texture(in); + var out_color_gamma = in.color * tex_gamma; + // Dither the float color down to eight bits to reduce banding. + // This step is optional for egui backends. + if r_locals.dithering == 1 { + let out_color_gamma_rgb = dither_interleaved(out_color_gamma.rgb, 256.0, in.position); + out_color_gamma = vec4(out_color_gamma_rgb, out_color_gamma.a); + } + return out_color_gamma; +} diff --git a/vendor/egui-wgpu/src/lib.rs b/vendor/egui-wgpu/src/lib.rs new file mode 100644 index 00000000..d172d709 --- /dev/null +++ b/vendor/egui-wgpu/src/lib.rs @@ -0,0 +1,798 @@ +//! This crates provides bindings between [`egui`](https://github.com/emilk/egui) and [wgpu](https://crates.io/crates/wgpu). +//! +//! If you're targeting WebGL you also need to turn on the +//! `webgl` feature of the `wgpu` crate: +//! +//! ```toml +//! # Enable both WebGL and WebGPU backends on web. +//! wgpu = { version = "*", features = ["webgpu", "webgl"] } +//! ``` +//! +//! You can control whether WebGL or WebGPU will be picked at runtime by configuring +//! [`WgpuConfiguration::wgpu_setup`]. +//! The default is to prefer WebGPU and fall back on WebGL. +//! +//! ## Feature flags +#![doc = document_features::document_features!()] +//! + +pub use wgpu; + +/// Low-level painting of [`egui`](https://github.com/emilk/egui) on [`wgpu`]. +mod renderer; + +mod setup; + +pub use renderer::*; +pub use setup::{ + EguiDisplayHandle, NativeAdapterSelectorMethod, WgpuSetup, WgpuSetupCreateNew, + WgpuSetupExisting, +}; + +/// Helpers for capturing screenshots of the UI. +#[cfg(feature = "capture")] +pub mod capture; + +/// Module for painting [`egui`](https://github.com/emilk/egui) with [`wgpu`] on [`winit`]. +#[cfg(feature = "winit")] +pub mod winit; + +use std::sync::Arc; + +use epaint::mutex::RwLock; + +/// An error produced by egui-wgpu. +#[derive(thiserror::Error, Debug)] +pub enum WgpuError { + #[error(transparent)] + RequestAdapterError(#[from] wgpu::RequestAdapterError), + + #[error("Adapter selection failed: {0}")] + CustomNativeAdapterSelectionError(String), + + #[error("There was no valid format for the surface at all.")] + NoSurfaceFormatsAvailable, + + #[error(transparent)] + RequestDeviceError(#[from] wgpu::RequestDeviceError), + + #[error(transparent)] + CreateSurfaceError(#[from] wgpu::CreateSurfaceError), + + #[cfg(feature = "winit")] + #[error(transparent)] + HandleError(#[from] ::winit::raw_window_handle::HandleError), +} + +/// Runtime-mutable subset of [`WgpuConfiguration`]. +/// +/// Edit any field to have the surface reconfigured on the next paint. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct SurfaceConfig { + /// Present mode used for the primary surface. + pub present_mode: wgpu::PresentMode, + + /// Desired maximum number of frames that the presentation engine should queue in advance. + /// + /// Use `1` for low-latency, and `2` for high-throughput. + /// + /// See [`wgpu::SurfaceConfiguration::desired_maximum_frame_latency`] for details. + /// + /// `None` => Let `wgpu` pick a default (currently `2`). + pub desired_maximum_frame_latency: Option, +} + +impl SurfaceConfig { + /// Good default for GUIs with very little (or no) extra GPU work. + pub const LOW_LATENCY: Self = Self { + present_mode: wgpu::PresentMode::AutoVsync, + + desired_maximum_frame_latency: if cfg!(target_os = "ios") { + None // The default is good on iOS, while `Some(1)` cuts FPS in half + } else { + Some(1) + }, + }; + + /// Good default for GUIs with a lot of extra GPU work, + /// or that want to prioritize smoothness over latency. + pub const HIGH_THROUGHPUT: Self = Self { + present_mode: wgpu::PresentMode::AutoVsync, + desired_maximum_frame_latency: Some(2), // High-throughput. + }; +} + +/// Access to the render state for egui. +#[derive(Clone)] +pub struct RenderState { + /// Wgpu adapter used for rendering. + pub adapter: wgpu::Adapter, + + /// All the available adapters. + /// + /// This is not available on web. + /// On web, we always select WebGPU is available, then fall back to WebGL if not. + #[cfg(not(target_arch = "wasm32"))] + pub available_adapters: Vec, + + /// Wgpu instance used for creating surfaces and adapters. + pub instance: wgpu::Instance, + + /// Wgpu device used for rendering, created from the adapter. + pub device: wgpu::Device, + + /// Wgpu queue used for rendering, created from the adapter. + pub queue: wgpu::Queue, + + /// The target texture format used for presenting to the window. + pub target_format: wgpu::TextureFormat, + + /// Egui renderer responsible for drawing the UI. + pub renderer: Arc>, + + /// Runtime-mutable subset of the wgpu configuration. + /// + /// Update this to have the surface reconfigured on the next paint. + pub surface_config: SurfaceConfig, +} + +async fn request_adapter( + instance: &wgpu::Instance, + power_preference: wgpu::PowerPreference, + compatible_surface: Option<&wgpu::Surface<'_>>, + available_adapters: &[wgpu::Adapter], +) -> Result { + profiling::function_scope!(); + + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions { + power_preference, + compatible_surface, + // We don't expose this as an option right now since it's fairly rarely useful: + // * only has an effect on native + // * fails if there's no software rasterizer available + // * can achieve the same with `native_adapter_selector` + force_fallback_adapter: false, + apply_limit_buckets: false, + }) + .await + .inspect_err(|_err| { + if cfg!(target_arch = "wasm32") { + // Nothing to add here + } else if available_adapters.is_empty() { + if std::env::var("DYLD_LIBRARY_PATH").is_ok() { + // DYLD_LIBRARY_PATH can sometimes lead to loading dylibs that cause + // us to find zero adapters. Very strange. + // I don't want to debug this again. + // See https://github.com/rerun-io/rerun/issues/11351 for more + log::warn!( + "No wgpu adapter found. This could be because DYLD_LIBRARY_PATH causes dylibs to be loaded that interfere with Metal device creation. Try restarting with DYLD_LIBRARY_PATH=''" + ); + } else { + log::info!("No wgpu adapter found"); + } + } else if available_adapters.len() == 1 { + log::info!( + "The only available wgpu adapter was not suitable: {}", + adapter_info_summary(&available_adapters[0].get_info()) + ); + } else { + log::info!( + "No suitable wgpu adapter found out of the {} available ones: {}", + available_adapters.len(), + describe_adapters(available_adapters) + ); + } + })?; + + if 1 < available_adapters.len() { + log::info!( + "There are {} available wgpu adapters: {}", + available_adapters.len(), + describe_adapters(available_adapters) + ); + } + + Ok(adapter) +} + +impl RenderState { + /// Creates a new [`RenderState`], containing everything needed for drawing egui with wgpu. + /// + /// # Errors + /// Wgpu initialization may fail due to incompatible hardware or driver for a given config. + pub async fn create( + config: &WgpuConfiguration, + instance: &wgpu::Instance, + compatible_surface: Option<&wgpu::Surface<'static>>, + options: RendererOptions, + ) -> Result { + profiling::scope!("RenderState::create"); // async yield give bad names using `profile_function` + + // This is always an empty list on web. + #[cfg(not(target_arch = "wasm32"))] + let available_adapters = { + let backends = if let WgpuSetup::CreateNew(create_new) = &config.wgpu_setup { + create_new.instance_descriptor.backends + } else { + wgpu::Backends::all() + }; + + instance.enumerate_adapters(backends).await + }; + + let (instance, adapter, device, queue) = match config.wgpu_setup.clone() { + WgpuSetup::CreateNew(WgpuSetupCreateNew { + instance_descriptor: _, + display_handle: _, + power_preference, + native_adapter_selector: _native_adapter_selector, + device_descriptor, + }) => { + let adapter = { + #[cfg(target_arch = "wasm32")] + { + request_adapter(instance, power_preference, compatible_surface, &[]).await + } + #[cfg(not(target_arch = "wasm32"))] + if let Some(native_adapter_selector) = _native_adapter_selector { + native_adapter_selector(&available_adapters, compatible_surface) + .map_err(WgpuError::CustomNativeAdapterSelectionError) + } else { + request_adapter( + instance, + power_preference, + compatible_surface, + &available_adapters, + ) + .await + } + }?; + + let (device, queue) = { + profiling::scope!("request_device"); + adapter + .request_device(&(*device_descriptor)(&adapter)) + .await? + }; + + (instance.clone(), adapter, device, queue) + } + WgpuSetup::Existing(WgpuSetupExisting { + instance, + adapter, + device, + queue, + }) => (instance, adapter, device, queue), + }; + + log_adapter_info(&adapter.get_info()); + + let surface_formats = { + profiling::scope!("get_capabilities"); + compatible_surface.map_or_else( + || vec![wgpu::TextureFormat::Rgba8Unorm], + |s| s.get_capabilities(&adapter).formats, + ) + }; + let target_format = crate::preferred_framebuffer_format(&surface_formats)?; + + let renderer = Renderer::new(&device, target_format, options); + + // On wasm, depending on feature flags, wgpu objects may or may not implement sync. + // It doesn't make sense to switch to Rc for that special usecase, so simply disable the lint. + #[allow(clippy::allow_attributes, clippy::arc_with_non_send_sync)] // For wasm + Ok(Self { + instance, + adapter, + #[cfg(not(target_arch = "wasm32"))] + available_adapters, + device, + queue, + target_format, + renderer: Arc::new(RwLock::new(renderer)), + surface_config: config.surface, + }) + } +} + +/// EternalMonitor patch: which adapter one is, across listings. Its LUID where the +/// backend gives one (DX12 always, Vulkan 1.1+ on Windows), which tells identical cards +/// apart. Without one, the backend, the PCI ids, the type, the name and the PCI bus: wgpu's +/// DX12 bus comes from the PCI ids alone, so identical cards there would share it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdapterKey { + pub luid: Option, + pub backend: wgpu::Backend, + pub vendor: u32, + pub device: u32, + pub device_type: wgpu::DeviceType, + pub name: String, + pub pci_bus: String, +} + +impl AdapterKey { + pub fn of(adapter: &wgpu::Adapter) -> Self { + let info = adapter.get_info(); + Self { + luid: luid(adapter), + backend: info.backend, + vendor: info.vendor, + device: info.device, + device_type: info.device_type, + name: info.name, + pci_bus: info.device_pci_bus_id, + } + } +} + +/// EternalMonitor patch: the adapter's LUID, from DXGI on DX12, or from +/// `VkPhysicalDeviceIDProperties` on Vulkan 1.1+. +#[cfg(windows)] +#[expect(unsafe_code)] // EternalMonitor patch: reads the adapter's backend objects. +fn luid(adapter: &wgpu::Adapter) -> Option { + // SAFETY: the DX12 adapter is only read, while `adapter` holds it. + let dx12 = unsafe { adapter.as_hal::() }; + if let Some(hal) = dx12 { + // SAFETY: `GetDesc1` only fills in a description of a live adapter. + let desc = unsafe { hal.raw_adapter().GetDesc1() }.ok()?; + let luid = desc.AdapterLuid; + return Some((u64::from(luid.HighPart.cast_unsigned()) << 32) | u64::from(luid.LowPart)); + } + // SAFETY: the Vulkan adapter is only read, while `adapter` holds it. + let vulkan = unsafe { adapter.as_hal::() }; + if let Some(hal) = vulkan { + let shared = hal.shared_instance(); + let device_version = hal.physical_device_capabilities().properties().api_version; + if shared.instance_api_version() < ash::vk::API_VERSION_1_1 + || device_version < ash::vk::API_VERSION_1_1 + { + return None; + } + let mut id = ash::vk::PhysicalDeviceIDProperties::default(); + { + let mut properties = ash::vk::PhysicalDeviceProperties2::default().push_next(&mut id); + // SAFETY: a query on the instance's own physical device, core in Vulkan 1.1. + unsafe { + shared + .raw_instance() + .get_physical_device_properties2(hal.raw_physical_device(), &mut properties); + } + } + return (id.device_luid_valid == ash::vk::TRUE) + .then_some(u64::from_le_bytes(id.device_luid)); + } + None +} + +#[cfg(not(windows))] +fn luid(_adapter: &wgpu::Adapter) -> Option { + None +} + +impl AdapterKey { + /// EternalMonitor patch: the same adapter as `other` but for the LUID, which a driver + /// update or reset can change. + fn same_but_luid(&self, other: &Self) -> bool { + ( + self.backend, + self.vendor, + self.device, + self.device_type, + &self.name, + &self.pci_bus, + ) == ( + other.backend, + other.vendor, + other.device, + other.device_type, + &other.name, + &other.pci_bus, + ) + } +} + +/// EternalMonitor patch: where `wanted` is in `keys`: itself, else the only one that +/// differs from it in the LUID alone. Two such (identical cards the key can't otherwise +/// tell apart) are no match. +fn find(keys: &[AdapterKey], wanted: &AdapterKey) -> Option { + keys.iter().position(|key| key == wanted).or_else(|| { + let mut alike = (0..keys.len()).filter(|&index| keys[index].same_but_luid(wanted)); + match (alike.next(), alike.next()) { + (Some(index), None) => Some(index), + _ => None, + } + }) +} + +/// EternalMonitor patch: the adapters, by index into `keys`, that a replacement may choose +/// from: only `preferred` (the last one that drew on the window) while it is listed and +/// not `excluded`, else every one not `excluded` (ones that couldn't draw on the window). +/// An adapter is found as [`find`] says, so one a driver update gave a new LUID is still +/// itself. +pub fn replacement_candidates( + keys: &[AdapterKey], + preferred: Option<&AdapterKey>, + excluded: &[AdapterKey], +) -> Vec { + let left_out: Vec = excluded.iter().filter_map(|key| find(keys, key)).collect(); + if let Some(index) = preferred.and_then(|preferred| find(keys, preferred)) + && !left_out.contains(&index) + { + return vec![index]; + } + (0..keys.len()) + .filter(|index| !left_out.contains(index)) + .collect() +} + +/// EternalMonitor patch: an adapter and device to replace a lost one, made without the +/// window's surface: the thread that makes them, where a driver can stall, never touches +/// the window. The window's thread then checks the surface ([`RenderState::surface_format`]). +#[cfg(not(target_arch = "wasm32"))] +pub struct NewDevice { + adapter: wgpu::Adapter, + available_adapters: Vec, + device: wgpu::Device, + queue: wgpu::Queue, +} + +#[cfg(not(target_arch = "wasm32"))] +impl NewDevice { + pub fn device(&self) -> &wgpu::Device { + &self.device + } + + pub fn adapter_key(&self) -> AdapterKey { + AdapterKey::of(&self.adapter) + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl RenderState { + /// EternalMonitor patch: list adapters, choose among [`replacement_candidates`] as + /// [`Self::create`] does but with no surface to check against, and make the device. + /// `watch` gets the device before anything is made on it. Takes no window or surface, + /// so it can run on a thread that may outlive the window. + pub async fn new_device( + config: &WgpuConfiguration, + instance: &wgpu::Instance, + preferred: Option<&AdapterKey>, + excluded: &[AdapterKey], + watch: impl FnOnce(&wgpu::Device), + ) -> Result { + let WgpuSetup::CreateNew(create_new) = &config.wgpu_setup else { + return Err(WgpuError::CustomNativeAdapterSelectionError( + "a device the app made can't be replaced".to_owned(), + )); + }; + let available_adapters = instance + .enumerate_adapters(create_new.instance_descriptor.backends) + .await; + let keys: Vec = available_adapters.iter().map(AdapterKey::of).collect(); + let candidates: Vec = replacement_candidates(&keys, preferred, excluded) + .into_iter() + .map(|index| available_adapters[index].clone()) + .collect(); + if candidates.is_empty() { + return Err(WgpuError::CustomNativeAdapterSelectionError( + "no adapter left that could draw on the window".to_owned(), + )); + } + let adapter = if let Some(selector) = &create_new.native_adapter_selector { + selector(&candidates, None).map_err(WgpuError::CustomNativeAdapterSelectionError)? + } else { + // wgpu chooses among all adapters, not these: keep its choice only if it is one. + let keys: Vec = candidates.iter().map(AdapterKey::of).collect(); + request_adapter(instance, create_new.power_preference, None, &candidates) + .await + .ok() + .filter(|adapter| keys.contains(&AdapterKey::of(adapter))) + .unwrap_or_else(|| candidates[0].clone()) + }; + let (device, queue) = adapter + .request_device(&(*create_new.device_descriptor)(&adapter)) + .await?; + watch(&device); + Ok(NewDevice { + adapter, + available_adapters, + device, + queue, + }) + } + + /// EternalMonitor patch: on the window's thread, the format to draw on `surface` with + /// `new`'s adapter, or an error when that adapter can't draw on it. + pub fn surface_format( + new: &NewDevice, + surface: &wgpu::Surface<'_>, + ) -> Result { + if !new.adapter.is_surface_supported(surface) { + return Err(WgpuError::CustomNativeAdapterSelectionError( + "the new adapter can't draw on the window".to_owned(), + )); + } + preferred_framebuffer_format(&surface.get_capabilities(&new.adapter).formats) + } + + /// EternalMonitor patch: the render state of a replacement, from its device and a + /// renderer made for `target_format`. + pub fn from_new_device( + new: NewDevice, + config: &WgpuConfiguration, + instance: &wgpu::Instance, + target_format: wgpu::TextureFormat, + renderer: Renderer, + ) -> Self { + log_adapter_info(&new.adapter.get_info()); + Self { + instance: instance.clone(), + adapter: new.adapter, + available_adapters: new.available_adapters, + device: new.device, + queue: new.queue, + target_format, + renderer: Arc::new(RwLock::new(renderer)), + surface_config: config.surface, + } + } +} + +fn describe_adapters(adapters: &[wgpu::Adapter]) -> String { + if adapters.is_empty() { + "(none)".to_owned() + } else if adapters.len() == 1 { + adapter_info_summary(&adapters[0].get_info()) + } else { + adapters + .iter() + .map(|a| format!("{{{}}}", adapter_info_summary(&a.get_info()))) + .collect::>() + .join(", ") + } +} + +/// Specifies which action should be taken as consequence of a surface error. +pub enum SurfaceErrorAction { + /// Do nothing and skip the current frame. + SkipFrame, + + /// Reconfigure the existing surface, then skip the current frame. + /// + /// Calls [`wgpu::Surface::configure`] on the current surface object. + /// Use for [`wgpu::CurrentSurfaceTexture::Outdated`]. + Reconfigure, + + /// Drop the surface, create a new one via [`wgpu::Instance::create_surface`], configure it, + /// then skip the current frame. + /// + /// Use for [`wgpu::CurrentSurfaceTexture::Lost`], where reconfiguring the same surface + /// object cannot recover. + RecreateSurface, +} + +/// Configuration for using wgpu with eframe or the egui-wgpu winit feature. +#[derive(Clone)] +pub struct WgpuConfiguration { + /// Runtime-mutable configuration for the surface (present mode, frame latency). + /// + /// These are the fields exposed via [`RenderState::surface_config`] for live + /// reconfiguration at runtime. + pub surface: SurfaceConfig, + + /// How to create the wgpu adapter & device + pub wgpu_setup: WgpuSetup, + + /// Callback for surface status changes. + /// + /// Called with the [`wgpu::CurrentSurfaceTexture`] result whenever acquiring a frame + /// does not return [`wgpu::CurrentSurfaceTexture::Success`]. For + /// [`wgpu::CurrentSurfaceTexture::Suboptimal`], egui uses the frame as-is and + /// defers surface reconfiguration to the next frame — the callback is not invoked + /// in that case either. + pub on_surface_status: + Arc SurfaceErrorAction + Send + Sync>, +} + +#[test] +fn wgpu_config_impl_send_sync() { + fn assert_send_sync() {} + assert_send_sync::(); +} + +impl core::fmt::Debug for WgpuConfiguration { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let Self { + surface, + wgpu_setup, + on_surface_status: _, + } = self; + f.debug_struct("WgpuConfiguration") + .field("surface", &surface) + .field("wgpu_setup", &wgpu_setup) + .finish_non_exhaustive() + } +} + +impl WgpuConfiguration { + #[inline] + pub fn with_surface_config(mut self, surface_config: SurfaceConfig) -> Self { + self.surface = surface_config; + self + } +} + +impl Default for WgpuConfiguration { + fn default() -> Self { + Self { + surface: SurfaceConfig::HIGH_THROUGHPUT, + + // No display handle available at this point — callers should replace this with + // `WgpuSetup::from_display_handle(...)` before creating the instance if one is available. + wgpu_setup: WgpuSetup::without_display_handle(), + on_surface_status: Arc::new(|status| match status { + wgpu::CurrentSurfaceTexture::Outdated => { + // The compositor changed the surface (resize, scale, output, …). wgpu + // requires us to reconfigure before the next acquire. Skipping would mean + // we are stuck in `Outdated` forever. + log::trace!("Dropped frame with error: {status:?}"); + SurfaceErrorAction::Reconfigure + } + wgpu::CurrentSurfaceTexture::Lost => { + // The underlying surface is gone and we need a fresh one from the `wgpu::Instance`. + log::debug!("Dropped frame with error: {status:?}"); + SurfaceErrorAction::RecreateSurface + } + wgpu::CurrentSurfaceTexture::Occluded => { + // App is hidden (minimized / behind another window). Skip silently. + log::trace!("Skipping frame due to occlusion."); + SurfaceErrorAction::SkipFrame + } + _ => { + log::warn!("Dropped frame with error: {status:?}"); + SurfaceErrorAction::SkipFrame + } + }), + } + } +} + +/// Find the framebuffer format that egui prefers +/// +/// # Errors +/// Returns [`WgpuError::NoSurfaceFormatsAvailable`] if the given list of formats is empty. +pub fn preferred_framebuffer_format( + formats: &[wgpu::TextureFormat], +) -> Result { + for &format in formats { + if matches!( + format, + wgpu::TextureFormat::Rgba8Unorm | wgpu::TextureFormat::Bgra8Unorm + ) { + return Ok(format); + } + } + + formats + .first() + .copied() + .ok_or(WgpuError::NoSurfaceFormatsAvailable) +} + +/// Take's epi's depth/stencil bits and returns the corresponding wgpu format. +pub fn depth_format_from_bits(depth_buffer: u8, stencil_buffer: u8) -> Option { + match (depth_buffer, stencil_buffer) { + (0, 8) => Some(wgpu::TextureFormat::Stencil8), + (16, 0) => Some(wgpu::TextureFormat::Depth16Unorm), + (24, 0) => Some(wgpu::TextureFormat::Depth24Plus), + (24, 8) => Some(wgpu::TextureFormat::Depth24PlusStencil8), + (32, 0) => Some(wgpu::TextureFormat::Depth32Float), + (32, 8) => Some(wgpu::TextureFormat::Depth32FloatStencil8), + _ => None, + } +} + +// --------------------------------------------------------------------------- + +fn log_adapter_info(info: &wgpu::AdapterInfo) { + let summary = adapter_info_summary(info); + + let is_test = cfg!(test); // Software rasterizers are expected (and preferred) during testing! + + if info.device_type == wgpu::DeviceType::Cpu && !is_test { + log::warn!("Software rasterizer detected - loss of performance expected. {summary}"); + } else { + log::debug!("wgpu adapter: {summary}"); + } +} + +/// A human-readable summary about an adapter +pub fn adapter_info_summary(info: &wgpu::AdapterInfo) -> String { + let wgpu::AdapterInfo { + name, + vendor, + device, + device_type, + driver, + driver_info, + backend, + device_pci_bus_id, + subgroup_min_size, + subgroup_max_size, + transient_saves_memory, + limit_bucket, + } = &info; + + // Example values: + // > name: "llvmpipe (LLVM 16.0.6, 256 bits)", device_type: Cpu, backend: Vulkan, driver: "llvmpipe", driver_info: "Mesa 23.1.6-arch1.4 (LLVM 16.0.6)" + // > name: "Apple M1 Pro", device_type: IntegratedGpu, backend: Metal, driver: "", driver_info: "" + // > name: "ANGLE (Apple, Apple M1 Pro, OpenGL 4.1)", device_type: IntegratedGpu, backend: Gl, driver: "", driver_info: "" + + use core::fmt::Write as _; + + let mut summary = format!("backend: {backend:?}, device_type: {device_type:?}"); + + if !name.is_empty() { + write!(summary, ", name: {name:?}").ok(); + } + if !driver.is_empty() { + write!(summary, ", driver: {driver:?}").ok(); + } + if !driver_info.is_empty() { + write!(summary, ", driver_info: {driver_info:?}").ok(); + } + if *vendor != 0 { + #[cfg(not(target_arch = "wasm32"))] + { + write!( + summary, + ", vendor: {} (0x{vendor:04X})", + parse_vendor_id(*vendor) + ) + .ok(); + } + #[cfg(target_arch = "wasm32")] + { + write!(summary, ", vendor: 0x{vendor:04X}").ok(); + } + } + if *device != 0 { + write!(summary, ", device: 0x{device:02X}").ok(); + } + if !device_pci_bus_id.is_empty() { + write!(summary, ", pci_bus_id: {device_pci_bus_id:?}").ok(); + } + if *subgroup_min_size != 0 || *subgroup_max_size != 0 { + write!( + summary, + ", subgroup_size: {subgroup_min_size}..={subgroup_max_size}" + ) + .ok(); + } + write!( + summary, + ", transient_saves_memory: {transient_saves_memory:?}" + ) + .ok(); + write!(summary, ", limit_bucket: {limit_bucket:?}").ok(); + + summary +} + +/// Tries to parse the adapter's vendor ID to a human-readable string. +#[cfg(not(target_arch = "wasm32"))] +pub fn parse_vendor_id(vendor_id: u32) -> &'static str { + match vendor_id { + wgpu::hal::auxil::db::amd::VENDOR => "AMD", + wgpu::hal::auxil::db::apple::VENDOR => "Apple", + wgpu::hal::auxil::db::arm::VENDOR => "ARM", + wgpu::hal::auxil::db::broadcom::VENDOR => "Broadcom", + wgpu::hal::auxil::db::imgtec::VENDOR => "Imagination Technologies", + wgpu::hal::auxil::db::intel::VENDOR => "Intel", + wgpu::hal::auxil::db::mesa::VENDOR => "Mesa", + wgpu::hal::auxil::db::nvidia::VENDOR => "NVIDIA", + wgpu::hal::auxil::db::qualcomm::VENDOR => "Qualcomm", + _ => "Unknown", + } +} diff --git a/vendor/egui-wgpu/src/renderer.rs b/vendor/egui-wgpu/src/renderer.rs new file mode 100644 index 00000000..43940ed3 --- /dev/null +++ b/vendor/egui-wgpu/src/renderer.rs @@ -0,0 +1,1203 @@ +use core::{num::NonZeroU64, ops::Range}; +use std::borrow::Cow; + +use ahash::HashMap; +use bytemuck::Zeroable as _; +use epaint::{PaintCallbackInfo, Primitive, Vertex, emath::NumExt as _}; + +use wgpu::util::DeviceExt as _; + +// Only implements Send + Sync on wasm32 in order to allow storing wgpu resources on the type map. +#[cfg(not(all( + target_arch = "wasm32", + not(feature = "fragile-send-sync-non-atomic-wasm"), +)))] +/// You can use this for storage when implementing [`CallbackTrait`]. +pub type CallbackResources = type_map::concurrent::TypeMap; +#[cfg(all( + target_arch = "wasm32", + not(feature = "fragile-send-sync-non-atomic-wasm"), +))] +/// You can use this for storage when implementing [`CallbackTrait`]. +pub type CallbackResources = type_map::TypeMap; + +/// You can use this to do custom [`wgpu`] rendering in an egui app. +/// +/// Implement [`CallbackTrait`] and call [`Callback::new_paint_callback`]. +/// +/// This can be turned into a [`epaint::PaintCallback`] and [`epaint::Shape`]. +pub struct Callback(Box); + +impl Callback { + /// Creates a new [`epaint::PaintCallback`] from a callback trait instance. + pub fn new_paint_callback( + rect: epaint::emath::Rect, + callback: impl CallbackTrait + 'static, + ) -> epaint::PaintCallback { + epaint::PaintCallback { + rect, + callback: std::sync::Arc::new(Self(Box::new(callback))), + } + } +} + +/// A callback trait that can be used to compose an [`epaint::PaintCallback`] via [`Callback`] +/// for custom WGPU rendering. +/// +/// Callbacks in [`Renderer`] are done in three steps: +/// * [`CallbackTrait::prepare`]: called for all registered callbacks before the main egui render pass. +/// * [`CallbackTrait::finish_prepare`]: called for all registered callbacks after all callbacks finished calling prepare. +/// * [`CallbackTrait::paint`]: called for all registered callbacks during the main egui render pass. +/// +/// Each callback has access to an instance of [`CallbackResources`] that is stored in the [`Renderer`]. +/// This can be used to store wgpu resources that need to be accessed during the [`CallbackTrait::paint`] step. +/// +/// The callbacks implementing [`CallbackTrait`] itself must always be Send + Sync, but resources stored in +/// [`Renderer::callback_resources`] are not required to implement Send + Sync when building for wasm. +/// (this is because wgpu stores references to the JS heap in most of its resources which can not be shared with other threads). +/// +/// +/// # Command submission +/// +/// ## Command Encoder +/// +/// The passed-in [`wgpu::CommandEncoder`] is egui's and can be used directly to register +/// wgpu commands for simple use cases. +/// This allows reusing the same [`wgpu::CommandEncoder`] for all callbacks and egui +/// rendering itself. +/// +/// ## Command Buffers +/// +/// For more complicated use cases, one can also return a list of arbitrary +/// [`wgpu::CommandBuffer`]s and have complete control over how they get created and fed. +/// In particular, this gives an opportunity to parallelize command registration and +/// prevents a faulty callback from poisoning the main wgpu pipeline. +/// +/// When using eframe, the main egui command buffer, as well as all user-defined +/// command buffers returned by this function, are guaranteed to all be submitted +/// at once in a single call. +/// +/// Command Buffers returned by [`CallbackTrait::finish_prepare`] will always be issued *after* +/// those returned by [`CallbackTrait::prepare`]. +/// Order within command buffers returned by [`CallbackTrait::prepare`] is dependent +/// on the order the respective [`epaint::Shape::Callback`]s were submitted in. +/// +/// # Example +/// +/// See the [`custom3d_wgpu`](https://github.com/emilk/egui/blob/main/crates/egui_demo_app/src/apps/custom3d_wgpu.rs) demo source for a detailed usage example. +pub trait CallbackTrait: Send + Sync { + fn prepare( + &self, + _device: &wgpu::Device, + _queue: &wgpu::Queue, + _screen_descriptor: &ScreenDescriptor, + _egui_encoder: &mut wgpu::CommandEncoder, + _callback_resources: &mut CallbackResources, + ) -> Vec { + Vec::new() + } + + /// Called after all [`CallbackTrait::prepare`] calls are done. + fn finish_prepare( + &self, + _device: &wgpu::Device, + _queue: &wgpu::Queue, + _egui_encoder: &mut wgpu::CommandEncoder, + _callback_resources: &mut CallbackResources, + ) -> Vec { + Vec::new() + } + + /// Called after all [`CallbackTrait::finish_prepare`] calls are done. + /// + /// It is given access to the [`wgpu::RenderPass`] so that it can issue draw commands + /// into the same [`wgpu::RenderPass`] that is used for all other egui elements. + fn paint( + &self, + info: PaintCallbackInfo, + render_pass: &mut wgpu::RenderPass<'static>, + callback_resources: &CallbackResources, + ); +} + +/// Information about the screen used for rendering. +pub struct ScreenDescriptor { + /// Size of the window in physical pixels. + pub size_in_pixels: [u32; 2], + + /// High-DPI scale factor (pixels per point). + pub pixels_per_point: f32, +} + +impl ScreenDescriptor { + /// size in "logical" points + fn screen_size_in_points(&self) -> [f32; 2] { + [ + self.size_in_pixels[0] as f32 / self.pixels_per_point, + self.size_in_pixels[1] as f32 / self.pixels_per_point, + ] + } +} + +/// Uniform buffer used when rendering. +#[derive(Clone, Copy, Debug, PartialEq, bytemuck::Pod, bytemuck::Zeroable)] +#[repr(C)] +struct UniformBuffer { + screen_size_in_points: [f32; 2], + dithering: u32, + + /// 1 to do manual filtering for more predictable kittest snapshot images. + /// + /// See also . + predictable_texture_filtering: u32, +} + +struct SlicedBuffer { + buffer: wgpu::Buffer, + slices: Vec>, + capacity: wgpu::BufferAddress, +} + +pub struct Texture { + /// The texture may be None if the `TextureId` is just a handle to a user-provided bind-group. + pub texture: Option, + + /// Bindgroup for the texture + sampler. + pub bind_group: wgpu::BindGroup, + + /// Options describing the sampler used in the bind group. This may be None if the `TextureId` + /// is just a handle to a user-provided bind-group. + pub options: Option, +} + +/// Ways to configure [`Renderer`] during creation. +#[derive(Clone, Copy, Debug)] +pub struct RendererOptions { + /// Set the level of the multisampling anti-aliasing (MSAA). + /// + /// Must be a power-of-two. Higher = more smooth 3D. + /// + /// A value of `0` or `1` turns it off (default). + /// + /// `egui` already performs anti-aliasing via "feathering" + /// (controlled by [`egui::epaint::TessellationOptions`]), + /// but if you are embedding 3D in egui you may want to turn on multisampling. + pub msaa_samples: u32, + + /// What format to use for the depth and stencil buffers, + /// e.g. [`wgpu::TextureFormat::Depth32FloatStencil8`]. + /// + /// egui doesn't need depth/stencil, so the default value is `None` (no depth or stancil buffers). + pub depth_stencil_format: Option, + + /// Controls whether to apply dithering to minimize banding artifacts. + /// + /// Dithering assumes an sRGB output and thus will apply noise to any input value that lies between + /// two 8bit values after applying the sRGB OETF function, i.e. if it's not a whole 8bit value in "gamma space". + /// This means that only inputs from texture interpolation and vertex colors should be affected in practice. + /// + /// Defaults to true. + pub dithering: bool, + + /// Perform texture filtering in software? + /// + /// This is useful when you want predictable rendering across + /// different hardware, e.g. for kittest snapshots. + /// + /// Default is `false`. + /// + /// See also . + pub predictable_texture_filtering: bool, +} + +impl RendererOptions { + /// Set options that produce the most predicatable output. + /// + /// Useful for image snapshot tests. + pub const PREDICTABLE: Self = Self { + msaa_samples: 1, + depth_stencil_format: None, + dithering: false, + predictable_texture_filtering: true, + }; +} + +impl Default for RendererOptions { + fn default() -> Self { + Self { + msaa_samples: 0, + depth_stencil_format: None, + dithering: true, + predictable_texture_filtering: false, + } + } +} + +/// Renderer for a egui based GUI. +pub struct Renderer { + pipeline: wgpu::RenderPipeline, + + index_buffer: SlicedBuffer, + vertex_buffer: SlicedBuffer, + + uniform_buffer: wgpu::Buffer, + previous_uniform_buffer_content: UniformBuffer, + uniform_bind_group: wgpu::BindGroup, + texture_bind_group_layout: wgpu::BindGroupLayout, + + /// Map of egui texture IDs to textures and their associated bindgroups (texture view + + /// sampler). The texture may be None if the `TextureId` is just a handle to a user-provided + /// sampler. + textures: HashMap, + next_user_texture_id: u64, + samplers: HashMap, + + options: RendererOptions, + + /// Storage for resources shared with all invocations of [`CallbackTrait`]'s methods. + /// + /// See also [`CallbackTrait`]. + pub callback_resources: CallbackResources, + + /// EternalMonitor patch: the last [`Self::update_buffers`] could not upload its meshes, + /// as happens once the device is lost, so [`Self::render`] draws nothing. + upload_failed: bool, +} + +impl Renderer { + /// EternalMonitor patch: the last [`Self::update_buffers`] could not upload its meshes, + /// as happens once the device is lost; [`Self::render`] then draws nothing. + pub fn upload_failed(&self) -> bool { + self.upload_failed + } + + /// Creates a renderer for a egui UI. + /// + /// `output_color_format` should preferably be [`wgpu::TextureFormat::Rgba8Unorm`] or + /// [`wgpu::TextureFormat::Bgra8Unorm`], i.e. in gamma-space. + pub fn new( + device: &wgpu::Device, + output_color_format: wgpu::TextureFormat, + options: RendererOptions, + ) -> Self { + profiling::function_scope!(); + + let shader = wgpu::ShaderModuleDescriptor { + label: Some("egui"), + source: wgpu::ShaderSource::Wgsl(Cow::Borrowed(include_str!("egui.wgsl"))), + }; + let module = { + profiling::scope!("create_shader_module"); + device.create_shader_module(shader) + }; + + let uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("egui_uniform_buffer"), + contents: bytemuck::cast_slice(&[UniformBuffer { + screen_size_in_points: [0.0, 0.0], + dithering: u32::from(options.dithering), + predictable_texture_filtering: u32::from(options.predictable_texture_filtering), + }]), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + + let uniform_bind_group_layout = { + profiling::scope!("create_bind_group_layout"); + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("egui_uniform_bind_group_layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: NonZeroU64::new( + core::mem::size_of::() as _ + ), + ty: wgpu::BufferBindingType::Uniform, + }, + count: None, + }], + }) + }; + + let uniform_bind_group = { + profiling::scope!("create_bind_group"); + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("egui_uniform_bind_group"), + layout: &uniform_bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding { + buffer: &uniform_buffer, + offset: 0, + size: None, + }), + }], + }) + }; + + let texture_bind_group_layout = { + profiling::scope!("create_bind_group_layout"); + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("egui_texture_bind_group_layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + multisampled: false, + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }) + }; + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("egui_pipeline_layout"), + bind_group_layouts: &[ + Some(&uniform_bind_group_layout), + Some(&texture_bind_group_layout), + ], + immediate_size: 0, + }); + + let depth_stencil = options + .depth_stencil_format + .map(|format| wgpu::DepthStencilState { + format, + depth_write_enabled: Some(false), + depth_compare: Some(wgpu::CompareFunction::Always), + stencil: wgpu::StencilState::default(), + bias: wgpu::DepthBiasState::default(), + }); + + let pipeline = { + profiling::scope!("create_render_pipeline"); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("egui_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + entry_point: Some("vs_main"), + module: &module, + buffers: &[Some(wgpu::VertexBufferLayout { + array_stride: 5 * 4, + step_mode: wgpu::VertexStepMode::Vertex, + // 0: vec2 position + // 1: vec2 texture coordinates + // 2: uint color + attributes: &wgpu::vertex_attr_array![0 => Float32x2, 1 => Float32x2, 2 => Uint32], + })], + compilation_options: wgpu::PipelineCompilationOptions::default() + }, + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + unclipped_depth: false, + conservative: false, + cull_mode: None, + front_face: wgpu::FrontFace::default(), + polygon_mode: wgpu::PolygonMode::default(), + strip_index_format: None, + }, + depth_stencil, + multisample: wgpu::MultisampleState { + alpha_to_coverage_enabled: false, + count: options.msaa_samples.max(1), + mask: !0, + }, + + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some(if output_color_format.is_srgb() { + log::warn!("Detected a linear (sRGBA aware) framebuffer {output_color_format:?}. egui prefers Rgba8Unorm or Bgra8Unorm"); + "fs_main_linear_framebuffer" + } else { + "fs_main_gamma_framebuffer" // this is what we prefer + }), + targets: &[Some(wgpu::ColorTargetState { + format: output_color_format, + blend: Some(wgpu::BlendState { + color: wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::One, + dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha, + operation: wgpu::BlendOperation::Add, + }, + alpha: wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::OneMinusDstAlpha, + dst_factor: wgpu::BlendFactor::One, + operation: wgpu::BlendOperation::Add, + }, + }), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default() + }), + multiview_mask: None, + cache: None, + } + ) + }; + + const VERTEX_BUFFER_START_CAPACITY: wgpu::BufferAddress = + (core::mem::size_of::() * 1024) as _; + const INDEX_BUFFER_START_CAPACITY: wgpu::BufferAddress = + (core::mem::size_of::() * 1024 * 3) as _; + + Self { + pipeline, + vertex_buffer: SlicedBuffer { + buffer: create_vertex_buffer(device, VERTEX_BUFFER_START_CAPACITY), + slices: Vec::with_capacity(64), + capacity: VERTEX_BUFFER_START_CAPACITY, + }, + index_buffer: SlicedBuffer { + buffer: create_index_buffer(device, INDEX_BUFFER_START_CAPACITY), + slices: Vec::with_capacity(64), + capacity: INDEX_BUFFER_START_CAPACITY, + }, + uniform_buffer, + // Buffers on wgpu are zero initialized, so this is indeed its current state! + previous_uniform_buffer_content: UniformBuffer::zeroed(), + uniform_bind_group, + texture_bind_group_layout, + textures: HashMap::default(), + next_user_texture_id: 0, + samplers: HashMap::default(), + options, + callback_resources: CallbackResources::default(), + upload_failed: false, + } + } + + /// Executes the egui renderer onto an existing wgpu renderpass. + /// + /// Note that the lifetime of `render_pass` is `'static` which requires a call to [`wgpu::RenderPass::forget_lifetime`]. + /// This allows users to pass resources that live outside of the callback resources to the render pass. + /// The render pass internally keeps all referenced resources alive as long as necessary. + /// The only consequence of `forget_lifetime` is that any operation on the parent encoder will cause a runtime error + /// instead of a compile time error. + /// + /// # Panic + /// Always ensure that [`Renderer::update_buffers`] has been called otherwise calling [`Renderer::render`] will panic! + pub fn render( + &self, + render_pass: &mut wgpu::RenderPass<'static>, + paint_jobs: &[epaint::ClippedPrimitive], + screen_descriptor: &ScreenDescriptor, + ) { + profiling::function_scope!(); + + if self.upload_failed { + return; + } + + let pixels_per_point = screen_descriptor.pixels_per_point; + let size_in_pixels = screen_descriptor.size_in_pixels; + + // Whether or not we need to reset the render pass because a paint callback has just + // run. + let mut needs_reset = true; + + let mut index_buffer_slices = self.index_buffer.slices.iter(); + let mut vertex_buffer_slices = self.vertex_buffer.slices.iter(); + + for epaint::ClippedPrimitive { + clip_rect, + primitive, + } in paint_jobs + { + if needs_reset { + render_pass.set_viewport( + 0.0, + 0.0, + size_in_pixels[0] as f32, + size_in_pixels[1] as f32, + 0.0, + 1.0, + ); + render_pass.set_pipeline(&self.pipeline); + render_pass.set_bind_group(0, &self.uniform_bind_group, &[]); + needs_reset = false; + } + + { + let rect = ScissorRect::new(clip_rect, pixels_per_point, size_in_pixels); + + if rect.width == 0 || rect.height == 0 { + // Skip rendering zero-sized clip areas. + if let Primitive::Mesh(_) = primitive { + // If this is a mesh, we need to advance the index and vertex buffer iterators: + index_buffer_slices + .next() + .expect("You must call .update_buffers() before .render()"); + vertex_buffer_slices + .next() + .expect("You must call .update_buffers() before .render()"); + } + continue; + } + + render_pass.set_scissor_rect(rect.x, rect.y, rect.width, rect.height); + } + + match primitive { + Primitive::Mesh(mesh) => { + let index_buffer_slice = index_buffer_slices + .next() + .expect("You must call .update_buffers() before .render()"); + let vertex_buffer_slice = vertex_buffer_slices + .next() + .expect("You must call .update_buffers() before .render()"); + + if let Some(Texture { bind_group, .. }) = self.textures.get(&mesh.texture_id) { + render_pass.set_bind_group(1, bind_group, &[]); + render_pass.set_index_buffer( + self.index_buffer.buffer.slice( + index_buffer_slice.start as u64..index_buffer_slice.end as u64, + ), + wgpu::IndexFormat::Uint32, + ); + render_pass.set_vertex_buffer( + 0, + self.vertex_buffer.buffer.slice( + vertex_buffer_slice.start as u64..vertex_buffer_slice.end as u64, + ), + ); + render_pass.draw_indexed(0..mesh.indices.len() as u32, 0, 0..1); + } else { + log::warn!("Missing texture: {:?}", mesh.texture_id); + } + } + Primitive::Callback(callback) => { + let Some(cbfn) = callback.callback.downcast_ref::() else { + // We already warned in the `prepare` callback + continue; + }; + + let info = PaintCallbackInfo { + viewport: callback.rect, + clip_rect: *clip_rect, + pixels_per_point, + screen_size_px: size_in_pixels, + }; + + let viewport_px = info.viewport_in_pixels(); + if viewport_px.width_px > 0 && viewport_px.height_px > 0 { + profiling::scope!("callback"); + + needs_reset = true; + + // We're setting a default viewport for the render pass as a + // courtesy for the user, so that they don't have to think about + // it in the simple case where they just want to fill the whole + // paint area. + // + // The user still has the possibility of setting their own custom + // viewport during the paint callback, effectively overriding this + // one. + render_pass.set_viewport( + viewport_px.left_px as f32, + viewport_px.top_px as f32, + viewport_px.width_px as f32, + viewport_px.height_px as f32, + 0.0, + 1.0, + ); + + cbfn.0.paint(info, render_pass, &self.callback_resources); + } + } + } + } + + render_pass.set_scissor_rect(0, 0, size_in_pixels[0], size_in_pixels[1]); + } + + /// Should be called before [`Self::render`]. + pub fn update_texture( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + id: epaint::TextureId, + image_delta: &epaint::ImageDelta, + ) { + profiling::function_scope!(); + + let width = image_delta.image.width() as u32; + let height = image_delta.image.height() as u32; + + let size = wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }; + + let data_color32 = match &image_delta.image { + epaint::ImageData::Color(image) => { + assert_eq!( + width as usize * height as usize, + image.pixels.len(), + "Mismatch between texture size and texel count" + ); + Cow::Borrowed(&image.pixels) + } + }; + let data_bytes: &[u8] = bytemuck::cast_slice(data_color32.as_slice()); + + let queue_write_data_to_texture = |texture, origin| { + profiling::scope!("write_texture"); + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture, + mip_level: 0, + origin, + aspect: wgpu::TextureAspect::All, + }, + data_bytes, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(4 * width), + rows_per_image: Some(height), + }, + size, + ); + }; + + // Use same label for all resources associated with this texture id (no point in retyping the type) + let label_str = format!("egui_texid_{id:?}"); + let label = Some(label_str.as_str()); + + let (texture, origin, bind_group) = if let Some(pos) = image_delta.pos { + // update the existing texture + let Texture { + texture, + bind_group, + options, + } = self + .textures + .remove(&id) + .expect("Tried to update a texture that has not been allocated yet."); + let texture = texture.expect("Tried to update user texture."); + let options = options.expect("Tried to update user texture."); + let origin = wgpu::Origin3d { + x: pos[0] as u32, + y: pos[1] as u32, + z: 0, + }; + + ( + texture, + origin, + // If the TextureOptions are the same as the previous ones, we can reuse the bind group. Otherwise we + // have to recreate it. + if image_delta.options == options { + Some(bind_group) + } else { + None + }, + ) + } else { + // allocate a new texture + let texture = { + profiling::scope!("create_texture"); + device.create_texture(&wgpu::TextureDescriptor { + label, + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[wgpu::TextureFormat::Rgba8Unorm], + }) + }; + let origin = wgpu::Origin3d::ZERO; + (texture, origin, None) + }; + + let bind_group = bind_group.unwrap_or_else(|| { + let sampler = self + .samplers + .entry(image_delta.options) + .or_insert_with(|| create_sampler(image_delta.options, device)); + device.create_bind_group(&wgpu::BindGroupDescriptor { + label, + layout: &self.texture_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView( + &texture.create_view(&wgpu::TextureViewDescriptor::default()), + ), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(sampler), + }, + ], + }) + }); + + queue_write_data_to_texture(&texture, origin); + + // A full update must (re)create the texture at exactly the delta's size, + // or glyph UVs (normalized by the CPU atlas size) will sample the wrong rows. + debug_assert!( + image_delta.pos.is_some() || [texture.width(), texture.height()] == [width, height], + "egui texture {id:?}: GPU texture is {}x{} but full delta is {width}x{height}", + texture.width(), + texture.height(), + ); + + self.textures.insert( + id, + Texture { + texture: Some(texture), + bind_group, + options: Some(image_delta.options), + }, + ); + } + + pub fn free_texture(&mut self, id: &epaint::TextureId) { + if let Some(texture) = self.textures.remove(id).and_then(|t| t.texture) { + texture.destroy(); + } + } + + /// Get the WGPU texture and bind group associated to a texture that has been allocated by egui. + /// + /// This could be used by custom paint hooks to render images that have been added through + /// [`epaint::Context::load_texture`](https://docs.rs/egui/latest/egui/struct.Context.html#method.load_texture). + pub fn texture(&self, id: &epaint::TextureId) -> Option<&Texture> { + self.textures.get(id) + } + + /// Registers a [`wgpu::Texture`] with a [`epaint::TextureId`]. + /// + /// This enables the application to reference the texture inside an image ui element. + /// This effectively enables off-screen rendering inside the egui UI. Texture must have + /// the texture format [`wgpu::TextureFormat::Rgba8Unorm`]. + pub fn register_native_texture( + &mut self, + device: &wgpu::Device, + texture: &wgpu::TextureView, + texture_filter: wgpu::FilterMode, + ) -> epaint::TextureId { + self.register_native_texture_with_sampler_options( + device, + texture, + wgpu::SamplerDescriptor { + label: Some(format!("egui_user_image_{}", self.next_user_texture_id).as_str()), + mag_filter: texture_filter, + min_filter: texture_filter, + ..Default::default() + }, + ) + } + + /// Registers a [`wgpu::Texture`] with an existing [`epaint::TextureId`]. + /// + /// This enables applications to reuse [`epaint::TextureId`]s. + pub fn update_egui_texture_from_wgpu_texture( + &mut self, + device: &wgpu::Device, + texture: &wgpu::TextureView, + texture_filter: wgpu::FilterMode, + id: epaint::TextureId, + ) { + self.update_egui_texture_from_wgpu_texture_with_sampler_options( + device, + texture, + wgpu::SamplerDescriptor { + label: Some(format!("egui_user_image_{}", self.next_user_texture_id).as_str()), + mag_filter: texture_filter, + min_filter: texture_filter, + ..Default::default() + }, + id, + ); + } + + /// Registers a [`wgpu::Texture`] with a [`epaint::TextureId`] while also accepting custom + /// [`wgpu::SamplerDescriptor`] options. + /// + /// This allows applications to specify individual minification/magnification filters as well as + /// custom mipmap and tiling options. + /// + /// The texture must have the format [`wgpu::TextureFormat::Rgba8Unorm`]. + /// Any compare function supplied in the [`wgpu::SamplerDescriptor`] will be ignored. + #[expect(clippy::needless_pass_by_value)] // false positive + pub fn register_native_texture_with_sampler_options( + &mut self, + device: &wgpu::Device, + texture: &wgpu::TextureView, + sampler_descriptor: wgpu::SamplerDescriptor<'_>, + ) -> epaint::TextureId { + profiling::function_scope!(); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + compare: None, + ..sampler_descriptor + }); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some(format!("egui_user_image_{}", self.next_user_texture_id).as_str()), + layout: &self.texture_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(texture), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&sampler), + }, + ], + }); + + let id = epaint::TextureId::User(self.next_user_texture_id); + self.textures.insert( + id, + Texture { + texture: None, + bind_group, + options: None, + }, + ); + self.next_user_texture_id += 1; + + id + } + + /// Registers a [`wgpu::Texture`] with an existing [`epaint::TextureId`] while also accepting custom + /// [`wgpu::SamplerDescriptor`] options. + /// + /// This allows applications to reuse [`epaint::TextureId`]s created with custom sampler options. + #[expect(clippy::needless_pass_by_value)] // false positive + pub fn update_egui_texture_from_wgpu_texture_with_sampler_options( + &mut self, + device: &wgpu::Device, + texture: &wgpu::TextureView, + sampler_descriptor: wgpu::SamplerDescriptor<'_>, + id: epaint::TextureId, + ) { + profiling::function_scope!(); + + let Texture { + bind_group: user_texture_binding, + .. + } = self + .textures + .get_mut(&id) + .expect("Tried to update a texture that has not been allocated yet."); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + compare: None, + ..sampler_descriptor + }); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some(format!("egui_user_image_{}", self.next_user_texture_id).as_str()), + layout: &self.texture_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(texture), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&sampler), + }, + ], + }); + + *user_texture_binding = bind_group; + } + + /// Uploads the uniform, vertex and index data used by the renderer. + /// Should be called before [`Self::render`]. + /// + /// Returns all user-defined command buffers gathered from [`CallbackTrait::prepare`] & [`CallbackTrait::finish_prepare`] callbacks. + pub fn update_buffers( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + encoder: &mut wgpu::CommandEncoder, + paint_jobs: &[epaint::ClippedPrimitive], + screen_descriptor: &ScreenDescriptor, + ) -> Vec { + profiling::function_scope!(); + + let screen_size_in_points = screen_descriptor.screen_size_in_points(); + + let uniform_buffer_content = UniformBuffer { + screen_size_in_points, + dithering: u32::from(self.options.dithering), + predictable_texture_filtering: u32::from(self.options.predictable_texture_filtering), + }; + if uniform_buffer_content != self.previous_uniform_buffer_content { + profiling::scope!("update uniforms"); + queue.write_buffer( + &self.uniform_buffer, + 0, + bytemuck::cast_slice(&[uniform_buffer_content]), + ); + self.previous_uniform_buffer_content = uniform_buffer_content; + } + + // Determine how many vertices & indices need to be rendered, and gather prepare callbacks + let mut callbacks = Vec::new(); + let (vertex_count, index_count) = { + profiling::scope!("count_vertices_indices"); + paint_jobs.iter().fold((0, 0), |acc, clipped_primitive| { + match &clipped_primitive.primitive { + Primitive::Mesh(mesh) => { + (acc.0 + mesh.vertices.len(), acc.1 + mesh.indices.len()) + } + Primitive::Callback(callback) => { + if let Some(c) = callback.callback.downcast_ref::() { + callbacks.push(c.0.as_ref()); + } else { + log::warn!("Unknown paint callback: expected `egui_wgpu::Callback`"); + } + acc + } + } + }) + }; + + self.upload_failed = false; + + if index_count > 0 { + profiling::scope!("indices", index_count.to_string().as_str()); + + self.index_buffer.slices.clear(); + + let required_index_buffer_size = (core::mem::size_of::() * index_count) as u64; + if self.index_buffer.capacity < required_index_buffer_size { + // Resize index buffer if needed. + self.index_buffer.capacity = + (self.index_buffer.capacity * 2).at_least(required_index_buffer_size); + self.index_buffer.buffer = create_index_buffer(device, self.index_buffer.capacity); + } + + let index_buffer_staging = queue.write_buffer_with( + &self.index_buffer.buffer, + 0, + #[expect(clippy::unwrap_used)] // Checked above + NonZeroU64::new(required_index_buffer_size).unwrap(), + ); + + // EternalMonitor patch: wgpu gives no staging buffer once the device is lost (or out + // of memory). Skip this frame instead of panicking; the painter replaces the device. + let Some(mut index_buffer_staging) = index_buffer_staging else { + log::warn!( + "Failed to create staging buffer for index data. Index count: {index_count}. Required index buffer size: {required_index_buffer_size}. Actual size {} and capacity: {} (bytes)", + self.index_buffer.buffer.size(), + self.index_buffer.capacity + ); + self.upload_failed = true; + return Vec::new(); + }; + + let mut index_offset = 0; + for epaint::ClippedPrimitive { primitive, .. } in paint_jobs { + match primitive { + Primitive::Mesh(mesh) => { + let size = mesh.indices.len() * core::mem::size_of::(); + let slice = index_offset..(size + index_offset); + index_buffer_staging + .slice(slice.clone()) + .copy_from_slice(bytemuck::cast_slice(&mesh.indices)); + self.index_buffer.slices.push(slice); + index_offset += size; + } + Primitive::Callback(_) => {} + } + } + } + if vertex_count > 0 { + profiling::scope!("vertices", vertex_count.to_string().as_str()); + + self.vertex_buffer.slices.clear(); + + let required_vertex_buffer_size = + (core::mem::size_of::() * vertex_count) as u64; + if self.vertex_buffer.capacity < required_vertex_buffer_size { + // Resize vertex buffer if needed. + self.vertex_buffer.capacity = + (self.vertex_buffer.capacity * 2).at_least(required_vertex_buffer_size); + self.vertex_buffer.buffer = + create_vertex_buffer(device, self.vertex_buffer.capacity); + } + + let vertex_buffer_staging = queue.write_buffer_with( + &self.vertex_buffer.buffer, + 0, + #[expect(clippy::unwrap_used)] // Checked above + NonZeroU64::new(required_vertex_buffer_size).unwrap(), + ); + + // EternalMonitor patch: wgpu gives no staging buffer once the device is lost (or out + // of memory). Skip this frame instead of panicking; the painter replaces the device. + let Some(mut vertex_buffer_staging) = vertex_buffer_staging else { + log::warn!( + "Failed to create staging buffer for vertex data. Vertex count: {vertex_count}. Required vertex buffer size: {required_vertex_buffer_size}. Actual size {} and capacity: {} (bytes)", + self.vertex_buffer.buffer.size(), + self.vertex_buffer.capacity + ); + self.upload_failed = true; + return Vec::new(); + }; + + let mut vertex_offset = 0; + for epaint::ClippedPrimitive { primitive, .. } in paint_jobs { + match primitive { + Primitive::Mesh(mesh) => { + let size = mesh.vertices.len() * core::mem::size_of::(); + let slice = vertex_offset..(size + vertex_offset); + vertex_buffer_staging + .slice(slice.clone()) + .copy_from_slice(bytemuck::cast_slice(&mesh.vertices)); + self.vertex_buffer.slices.push(slice); + vertex_offset += size; + } + Primitive::Callback(_) => {} + } + } + } + + let mut user_cmd_bufs = Vec::new(); + { + profiling::scope!("prepare callbacks"); + for callback in &callbacks { + user_cmd_bufs.extend(callback.prepare( + device, + queue, + screen_descriptor, + encoder, + &mut self.callback_resources, + )); + } + } + { + profiling::scope!("finish prepare callbacks"); + for callback in &callbacks { + user_cmd_bufs.extend(callback.finish_prepare( + device, + queue, + encoder, + &mut self.callback_resources, + )); + } + } + + user_cmd_bufs + } +} + +fn create_sampler( + options: epaint::textures::TextureOptions, + device: &wgpu::Device, +) -> wgpu::Sampler { + let mag_filter = match options.magnification { + epaint::textures::TextureFilter::Nearest => wgpu::FilterMode::Nearest, + epaint::textures::TextureFilter::Linear => wgpu::FilterMode::Linear, + }; + let min_filter = match options.minification { + epaint::textures::TextureFilter::Nearest => wgpu::FilterMode::Nearest, + epaint::textures::TextureFilter::Linear => wgpu::FilterMode::Linear, + }; + let address_mode = match options.wrap_mode { + epaint::textures::TextureWrapMode::ClampToEdge => wgpu::AddressMode::ClampToEdge, + epaint::textures::TextureWrapMode::Repeat => wgpu::AddressMode::Repeat, + epaint::textures::TextureWrapMode::MirroredRepeat => wgpu::AddressMode::MirrorRepeat, + }; + device.create_sampler(&wgpu::SamplerDescriptor { + label: Some(&format!( + "egui sampler (mag: {mag_filter:?}, min {min_filter:?})" + )), + mag_filter, + min_filter, + address_mode_u: address_mode, + address_mode_v: address_mode, + ..Default::default() + }) +} + +fn create_vertex_buffer(device: &wgpu::Device, size: u64) -> wgpu::Buffer { + profiling::function_scope!(); + device.create_buffer(&wgpu::BufferDescriptor { + label: Some("egui_vertex_buffer"), + usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, + size, + mapped_at_creation: false, + }) +} + +fn create_index_buffer(device: &wgpu::Device, size: u64) -> wgpu::Buffer { + profiling::function_scope!(); + device.create_buffer(&wgpu::BufferDescriptor { + label: Some("egui_index_buffer"), + usage: wgpu::BufferUsages::INDEX | wgpu::BufferUsages::COPY_DST, + size, + mapped_at_creation: false, + }) +} + +/// A Rect in physical pixel space, used for setting clipping rectangles. +struct ScissorRect { + x: u32, + y: u32, + width: u32, + height: u32, +} + +impl ScissorRect { + fn new(clip_rect: &epaint::Rect, pixels_per_point: f32, target_size: [u32; 2]) -> Self { + // Transform clip rect to physical pixels: + let clip_min_x = pixels_per_point * clip_rect.min.x; + let clip_min_y = pixels_per_point * clip_rect.min.y; + let clip_max_x = pixels_per_point * clip_rect.max.x; + let clip_max_y = pixels_per_point * clip_rect.max.y; + + // Round to integer: + let clip_min_x = clip_min_x.round() as u32; + let clip_min_y = clip_min_y.round() as u32; + let clip_max_x = clip_max_x.round() as u32; + let clip_max_y = clip_max_y.round() as u32; + + // Clamp: + let clip_min_x = clip_min_x.clamp(0, target_size[0]); + let clip_min_y = clip_min_y.clamp(0, target_size[1]); + let clip_max_x = clip_max_x.clamp(clip_min_x, target_size[0]); + let clip_max_y = clip_max_y.clamp(clip_min_y, target_size[1]); + + Self { + x: clip_min_x, + y: clip_min_y, + width: clip_max_x - clip_min_x, + height: clip_max_y - clip_min_y, + } + } +} + +// Look at the feature flag for an explanation. +#[cfg(not(all( + target_arch = "wasm32", + not(feature = "fragile-send-sync-non-atomic-wasm"), +)))] +#[test] +fn renderer_impl_send_sync() { + fn assert_send_sync() {} + assert_send_sync::(); +} diff --git a/vendor/egui-wgpu/src/setup.rs b/vendor/egui-wgpu/src/setup.rs new file mode 100644 index 00000000..3c9ceae5 --- /dev/null +++ b/vendor/egui-wgpu/src/setup.rs @@ -0,0 +1,331 @@ +use std::sync::Arc; + +/// A cloneable display handle for use with [`wgpu::InstanceDescriptor`]. +/// +/// [`wgpu::InstanceDescriptor`] stores its display handle as a non-cloneable +/// `Box`. This trait wraps it so it can be cloned +/// alongside the rest of the egui wgpu configuration. +/// +/// Automatically implemented for all types that satisfy the bounds +/// (including [`winit::event_loop::OwnedDisplayHandle`]). +pub trait EguiDisplayHandle: + wgpu::rwh::HasDisplayHandle + core::fmt::Debug + Send + Sync + 'static +{ + /// Clone into a `Box` for [`wgpu::InstanceDescriptor::display`]. + fn clone_for_wgpu(&self) -> Box; + + /// Clone into a new `Box`. + fn clone_display_handle(&self) -> Box; +} + +impl Clone for Box { + fn clone(&self) -> Self { + // We need to deref here, otherwise this causes infinite recursion stack overflow. + (**self).clone_display_handle() + } +} + +impl EguiDisplayHandle for T +where + T: wgpu::rwh::HasDisplayHandle + Clone + core::fmt::Debug + Send + Sync + 'static, +{ + fn clone_for_wgpu(&self) -> Box { + Box::new(self.clone()) + } + + fn clone_display_handle(&self) -> Box { + Box::new(self.clone()) + } +} + +#[derive(Clone)] +#[expect(clippy::large_enum_variant)] +pub enum WgpuSetup { + /// Construct a wgpu setup using some predefined settings & heuristics. + /// This is the default option. You can customize most behaviours overriding the + /// supported backends, power preferences, and device description. + /// + /// By default can also be configured with various environment variables: + /// * `WGPU_BACKEND`: `vulkan`, `dx12`, `metal`, `opengl`, `webgpu` + /// * `WGPU_POWER_PREF`: `low`, `high` or `none` + /// * `WGPU_TRACE`: Path to a file to output a wgpu trace file. + /// + /// Each instance flag also comes with an environment variable (for details see [`wgpu::InstanceFlags`]): + /// * `WGPU_VALIDATION`: Enables validation (enabled by default in debug builds). + /// * `WGPU_DEBUG`: Generate debug information in shaders and objects (enabled by default in debug builds). + /// * `WGPU_ALLOW_UNDERLYING_NONCOMPLIANT_ADAPTER`: Whether wgpu should expose adapters that run on top of non-compliant adapters. + /// * `WGPU_GPU_BASED_VALIDATION`: Enable GPU-based validation. + CreateNew(WgpuSetupCreateNew), + + /// Run on an existing wgpu setup. + Existing(WgpuSetupExisting), +} + +impl WgpuSetup { + /// Creates a new [`WgpuSetup::CreateNew`] with the given display handle. + /// + /// See [`WgpuSetupCreateNew::from_display_handle`] for details. + pub fn from_display_handle(display_handle: impl EguiDisplayHandle) -> Self { + Self::CreateNew(WgpuSetupCreateNew::from_display_handle(display_handle)) + } + + /// Creates a new [`WgpuSetup::CreateNew`] without a display handle. + /// + /// See [`WgpuSetupCreateNew::without_display_handle`] for details. + pub fn without_display_handle() -> Self { + Self::CreateNew(WgpuSetupCreateNew::without_display_handle()) + } +} + +impl core::fmt::Debug for WgpuSetup { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::CreateNew(create_new) => f + .debug_tuple("WgpuSetup::CreateNew") + .field(create_new) + .finish(), + Self::Existing { .. } => f.debug_tuple("WgpuSetup::Existing").finish(), + } + } +} + +impl WgpuSetup { + /// Creates a new [`wgpu::Instance`] or clones the existing one. + /// + /// Does *not* store the wgpu instance, so calling this repeatedly may + /// create a new instance every time! + pub async fn new_instance(&self) -> wgpu::Instance { + match self { + Self::CreateNew(create_new) => { + #[allow(clippy::allow_attributes, unused_mut)] + let mut backends = create_new.instance_descriptor.backends; + + // Don't try WebGPU if we're not in a secure context. + // Emscripten is excluded: wgpu gates both its `webgpu` backend and + // its `web_sys` re-export on `not(Emscripten)` (see the cfg aliases + // in `wgpu/build.rs`). + #[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))] + if backends.contains(wgpu::Backends::BROWSER_WEBGPU) { + let is_secure_context = + wgpu::web_sys::window().is_some_and(|w| w.is_secure_context()); + if !is_secure_context { + log::info!( + "WebGPU is only available in secure contexts, i.e. on HTTPS and on localhost." + ); + backends.remove(wgpu::Backends::BROWSER_WEBGPU); + } + } + + log::debug!("Creating wgpu instance with backends {backends:?}"); + let desc = &create_new.instance_descriptor; + let descriptor = wgpu::InstanceDescriptor { + backends: desc.backends, + flags: desc.flags, + backend_options: desc.backend_options.clone(), + memory_budget_thresholds: desc.memory_budget_thresholds, + display: create_new + .display_handle + .as_ref() + .map(|handle| handle.clone_for_wgpu()), + }; + wgpu::util::new_instance_with_webgpu_detection(descriptor).await + } + Self::Existing(existing) => existing.instance.clone(), + } + } +} + +impl From for WgpuSetup { + fn from(create_new: WgpuSetupCreateNew) -> Self { + Self::CreateNew(create_new) + } +} + +impl From for WgpuSetup { + fn from(existing: WgpuSetupExisting) -> Self { + Self::Existing(existing) + } +} + +/// Method for selecting an adapter on native. +/// +/// This can be used for fully custom adapter selection. +/// If available, `wgpu::Surface` is passed to allow checking for surface compatibility. +pub type NativeAdapterSelectorMethod = Arc< + dyn Fn(&[wgpu::Adapter], Option<&wgpu::Surface<'_>>) -> Result + + Send + + Sync, +>; + +/// Configuration for creating a new wgpu setup. +/// +/// Used for [`WgpuSetup::CreateNew`]. +/// +/// Prefer [`Self::from_display_handle`] when you have a display handle available. +/// Most platforms work without one, but some (e.g. Wayland with GLES, or WebGL) +/// require it, so providing one ensures maximum compatibility. +/// With winit, pass [`EventLoop::owned_display_handle`](winit::event_loop::EventLoop::owned_display_handle). +/// +/// Note: The display handle is stored in [`Self::display_handle`] rather than in +/// [`Self::instance_descriptor`] so the config can be cloned +/// ([`wgpu::InstanceDescriptor`] is not `Clone`). It is injected at instance creation time. +pub struct WgpuSetupCreateNew { + /// Descriptor for the wgpu instance. + /// + /// Leave [`wgpu::InstanceDescriptor::display`] as `None` — use [`Self::display_handle`] + /// instead (injected at instance creation time). + /// + /// The most important field is [`wgpu::InstanceDescriptor::backends`], which controls + /// which backends are supported (wgpu will pick one of these). For example, set it to + /// [`wgpu::Backends::GL`] to use only WebGL. By default on web, WebGPU is preferred + /// with WebGL as a fallback (requires the `webgl` feature of crate `wgpu`). + pub instance_descriptor: wgpu::InstanceDescriptor, + + /// Display handle passed to wgpu at instance creation time. + /// + /// Required on some platforms (e.g. Wayland with GLES, WebGL); optional elsewhere. + /// With winit, use [`winit::event_loop::OwnedDisplayHandle`]. + /// + /// `eframe` 's winit & web integrations will attempt to fill the display handle automatically if it is left empty. + pub display_handle: Option>, + + /// Power preference for the adapter if [`Self::native_adapter_selector`] is not set or targeting web. + pub power_preference: wgpu::PowerPreference, + + /// Optional selector for native adapters. + /// + /// This field has no effect when targeting web! + /// Otherwise, if set [`Self::power_preference`] is ignored and the adapter is instead selected by this method. + /// Note that [`Self::instance_descriptor`]'s [`wgpu::InstanceDescriptor::backends`] + /// are still used to filter the adapter enumeration in the first place. + /// + /// Defaults to `None`. + pub native_adapter_selector: Option, + + /// Configuration passed on device request, given an adapter + pub device_descriptor: + Arc wgpu::DeviceDescriptor<'static> + Send + Sync>, +} + +impl WgpuSetupCreateNew { + /// Creates a new configuration with the given display handle. + /// + /// This is the recommended constructor. Most platforms (Windows, macOS/iOS, Android, web) + /// work fine without a display handle, but some (e.g. Wayland on Linux with GLES) require + /// one. Providing it unconditionally ensures your app works everywhere. + /// + /// If you don't have a display handle available, use [`Self::without_display_handle`] + /// instead — it will still work on the majority of platforms. + /// + /// With winit, pass [`EventLoop::owned_display_handle`](winit::event_loop::EventLoop::owned_display_handle). + pub fn from_display_handle(display_handle: impl EguiDisplayHandle) -> Self { + Self { + display_handle: Some(Box::new(display_handle)), + ..Self::without_display_handle() + } + } + + /// Creates a new configuration without a display handle. + /// + /// A display handle is not required for headless operation (offscreen rendering, tests, + /// compute-only workloads). It also isn't needed on most platforms even when presenting + /// to a window — only some configurations (e.g. Wayland on Linux with GLES) require one. + /// + /// If you do have a display handle available, prefer [`Self::from_display_handle`] for + /// maximum compatibility. + /// + /// With winit you can obtain one via [`EventLoop::owned_display_handle`](winit::event_loop::EventLoop::owned_display_handle). + /// + /// `eframe` 's winit & web integrations will attempt to fill the display handle automatically if it is left empty. + pub fn without_display_handle() -> Self { + Self { + instance_descriptor: wgpu::InstanceDescriptor { + // Add GL backend, primarily because WebGPU is not stable enough yet. + // (note however, that the GL backend needs to be opted-in via the wgpu feature flag "webgl") + backends: wgpu::Backends::from_env() + .unwrap_or(wgpu::Backends::PRIMARY | wgpu::Backends::GL), + flags: wgpu::InstanceFlags::from_build_config().with_env(), + backend_options: wgpu::BackendOptions::from_env_or_default(), + memory_budget_thresholds: wgpu::MemoryBudgetThresholds::default(), + display: None, + }, + + display_handle: None, + + power_preference: wgpu::PowerPreference::from_env() + .unwrap_or(wgpu::PowerPreference::HighPerformance), + + native_adapter_selector: None, + + device_descriptor: Arc::new(|adapter| { + let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl { + wgpu::Limits::downlevel_webgl2_defaults() + } else { + wgpu::Limits::default() + }; + + wgpu::DeviceDescriptor { + label: Some("egui wgpu device"), + required_limits: wgpu::Limits { + // When using a depth buffer, we have to be able to create a texture + // large enough for the entire surface, and we want to support 4k+ displays. + max_texture_dimension_2d: 8192, + ..base_limits + }, + ..Default::default() + } + }), + } + } +} + +impl Clone for WgpuSetupCreateNew { + fn clone(&self) -> Self { + let desc = &self.instance_descriptor; + Self { + instance_descriptor: wgpu::InstanceDescriptor { + backends: desc.backends, + flags: desc.flags, + backend_options: desc.backend_options.clone(), + memory_budget_thresholds: desc.memory_budget_thresholds, + display: None, + }, + display_handle: self.display_handle.clone(), + power_preference: self.power_preference, + native_adapter_selector: self.native_adapter_selector.clone(), + device_descriptor: Arc::clone(&self.device_descriptor), + } + } +} + +impl core::fmt::Debug for WgpuSetupCreateNew { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let Self { + instance_descriptor, + display_handle, + power_preference, + native_adapter_selector, + device_descriptor: _, + } = self; + f.debug_struct("WgpuSetupCreateNew") + .field("instance_descriptor", instance_descriptor) + .field("display_handle", display_handle) + .field("power_preference", power_preference) + .field( + "native_adapter_selector", + &native_adapter_selector.is_some(), + ) + .finish_non_exhaustive() + } +} + +/// Configuration for using an existing wgpu setup. +/// +/// Used for [`WgpuSetup::Existing`]. +#[derive(Clone)] +pub struct WgpuSetupExisting { + pub instance: wgpu::Instance, + pub adapter: wgpu::Adapter, + pub device: wgpu::Device, + pub queue: wgpu::Queue, +} diff --git a/vendor/egui-wgpu/src/texture_copy.wgsl b/vendor/egui-wgpu/src/texture_copy.wgsl new file mode 100644 index 00000000..4096d164 --- /dev/null +++ b/vendor/egui-wgpu/src/texture_copy.wgsl @@ -0,0 +1,43 @@ +struct VertexOutput { + @builtin(position) position: vec4, +}; + +var positions: array = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f(3.0, 1.0) +); + +// meant to be called with 3 vertex indices: 0, 1, 2 +// draws one large triangle over the clip space like this: +// (the asterisks represent the clip space bounds) +//-1,1 1,1 +// --------------------------------- +// | * . +// | * . +// | * . +// | * . +// | * . +// | * . +// |*************** +// | . 1,-1 +// | . +// | . +// | . +// | . +// |. +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var result: VertexOutput; + result.position = vec4f(positions[vertex_index], 0.0, 1.0); + return result; +} + +@group(0) +@binding(0) +var r_color: texture_2d; + +@fragment +fn fs_main(vertex: VertexOutput) -> @location(0) vec4 { + return textureLoad(r_color, vec2i(vertex.position.xy), 0); +} diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs new file mode 100644 index 00000000..707321fb --- /dev/null +++ b/vendor/egui-wgpu/src/winit.rs @@ -0,0 +1,1319 @@ +#![expect(clippy::missing_errors_doc)] +#![expect(clippy::undocumented_unsafe_blocks)] +#![expect(clippy::unwrap_used)] // TODO(emilk): avoid unwraps +#![expect(unsafe_code)] + +use crate::{RenderState, SurfaceConfig, SurfaceErrorAction, WgpuConfiguration, renderer}; +use crate::{ + RendererOptions, + capture::{CaptureReceiver, CaptureSender, CaptureState, capture_channel}, +}; +use core::num::NonZeroU32; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use core::time::Duration; +use egui::{Context, Event, UserData, ViewportId, ViewportIdMap, ViewportIdSet}; +use std::sync::Arc; +use web_time::Instant; + +/// EternalMonitor patch: frames the painter skipped because its device was lost or was +/// being replaced. An app that tells a working window by its frames counts these as +/// failures. +pub static SKIPPED_FRAMES: AtomicU64 = AtomicU64::new(0); + +/// EternalMonitor patch: devices the painter dropped because they were lost, whether or +/// not a replacement was ever made. +pub static DEVICES_LOST: AtomicU64 = AtomicU64::new(0); + +/// EternalMonitor patch: how long a replacement device may take before the try counts as +/// failed. A driver call can stall for much longer, so the window never waits on it. +pub const REPLACEMENT_TIMEOUT: Duration = Duration::from_secs(10); + +/// EternalMonitor patch: the name of the thread that makes a replacement device. A panic +/// there ends only that try, so an app's panic hook can tell it from others. +pub const REPLACEMENT_THREAD: &str = "wgpu device replacement"; + +/// EternalMonitor patch: how often the window looks for a replacement device being made. +const REPLACEMENT_POLL: Duration = Duration::from_millis(100); + +/// EternalMonitor patch: what [`Answer::poll`] found. +#[derive(Debug)] +pub enum Asked { + Answered(T), + Waiting, + /// No answer by the deadline. The next deadline is a timeout later; a late answer + /// still arrives. + TimedOut, + /// The thread ended without an answer, as by a panic. + Ended, +} + +/// EternalMonitor patch: work done on its own thread, which the window polls without ever +/// waiting for it. +pub struct Answer { + receiver: std::sync::mpsc::Receiver, + deadline: Instant, + timeout: Duration, +} + +impl Answer { + pub fn ask( + name: &str, + now: Instant, + timeout: Duration, + work: impl FnOnce() -> T + Send + 'static, + ) -> std::io::Result { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + std::thread::Builder::new() + .name(name.to_owned()) + .spawn(move || { + let _ = sender.send(work()); + })?; + Ok(Self { + receiver, + deadline: now + timeout, + timeout, + }) + } + + /// Never blocks. + pub fn poll(&mut self, now: Instant) -> Asked { + match self.receiver.try_recv() { + Ok(answer) => Asked::Answered(answer), + Err(std::sync::mpsc::TryRecvError::Disconnected) => Asked::Ended, + Err(std::sync::mpsc::TryRecvError::Empty) if now >= self.deadline => { + self.deadline = now + self.timeout; + Asked::TimedOut + } + Err(std::sync::mpsc::TryRecvError::Empty) => Asked::Waiting, + } + } +} + +/// EternalMonitor patch: run `future` on this thread. wgpu's native futures are ready at +/// once; this parks until woken otherwise. +fn block_on(future: F) -> F::Output { + struct Unpark(std::thread::Thread); + impl std::task::Wake for Unpark { + fn wake(self: Arc) { + self.0.unpark(); + } + } + let waker = std::task::Waker::from(Arc::new(Unpark(std::thread::current()))); + let mut cx = std::task::Context::from_waker(&waker); + let mut future = core::pin::pin!(future); + loop { + if let core::task::Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + std::thread::park(); + } +} + +/// EternalMonitor patch: a replacement being made on other threads, in two stages, each +/// polled and timed out on its own. Neither thread gets the window or a surface. +enum Replacing { + /// The adapter and the device, with the flag its handlers set when it is lost. + Device { + answer: Answer>, + lost: Arc, + }, + /// egui's renderer for the device, while the window's thread keeps the device and the + /// surface it checked it against. + Renderer { + answer: Answer, + new: crate::NewDevice, + surface: wgpu::Surface<'static>, + format: wgpu::TextureFormat, + lost: Arc, + }, +} + +/// EternalMonitor patch: flag the loss of `device` in `lost`. An error wgpu reports outside +/// an error scope flags it too, instead of panicking as wgpu's default handler does: a +/// window whose device or surface fails is drawn again on a new one. +pub fn watch(device: &wgpu::Device, lost: &Arc) { + let flag = Arc::clone(lost); + device.set_device_lost_callback(move |reason, message| { + log::warn!("The wgpu device was lost ({reason:?}): {message}"); + flag.store(true, Ordering::SeqCst); + }); + let flag = Arc::clone(lost); + device.on_uncaptured_error(Arc::new(move |error| { + if flag.swap(true, Ordering::SeqCst) { + log::debug!("wgpu error on the lost device: {error}"); + } else { + log::error!("wgpu error; replacing the window's device: {error}"); + } + })); +} + +/// EternalMonitor patch: when the painter tries to replace a lost device, and when it +/// stops trying. +/// +/// The first try is at once. Each failure, a device that could not be made or a new one +/// lost before it drew for [`Self::STABLE_AFTER`], waits twice as long as the last, from +/// 0.5 s up to 30 s. After [`Self::MAX_FAILURES`] the painter gives up and the window +/// stays blank. A device that drew for [`Self::STABLE_AFTER`] starts the count again. +#[derive(Debug, Default)] +pub struct Replacement { + failures: u32, + next_try: Option, + drawing_since: Option, + gave_up: bool, +} + +impl Replacement { + pub const MAX_FAILURES: u32 = 10; + pub const STABLE_AFTER: Duration = Duration::from_secs(10); + const FIRST_WAIT: Duration = Duration::from_millis(500); + const LONGEST_WAIT: Duration = Duration::from_secs(30); + + /// The device was lost at `now`. True when the painter just gave up. + pub fn lost(&mut self, now: Instant) -> bool { + let drew = self + .drawing_since + .is_some_and(|since| now.duration_since(since) >= Self::STABLE_AFTER); + self.drawing_since = None; + if drew { + self.failures = 0; + self.next_try = Some(now); + return false; + } + if self.next_try.is_none() { + // The device the window started with: its replacement is the first try. + self.next_try = Some(now); + return false; + } + self.failed(now) + } + + /// A replacement could not be made at `now`. True when the painter just gave up. + pub fn failed(&mut self, now: Instant) -> bool { + if self.gave_up { + return false; + } + self.failures += 1; + if self.failures >= Self::MAX_FAILURES { + self.gave_up = true; + return true; + } + let wait = Self::FIRST_WAIT.saturating_mul(1 << (self.failures - 1).min(16)); + self.next_try = Some(now + wait.min(Self::LONGEST_WAIT)); + false + } + + /// The device presented a frame at `now`. + pub fn drew(&mut self, now: Instant) { + self.drawing_since.get_or_insert(now); + } + + /// How long until the next try, or `None` once the painter gave up. + pub fn wait(&self, now: Instant) -> Option { + (!self.gave_up).then(|| { + self.next_try + .map_or(Duration::ZERO, |at| at.saturating_duration_since(now)) + }) + } +} + +struct SurfaceState { + surface: wgpu::Surface<'static>, + alpha_mode: wgpu::CompositeAlphaMode, + width: u32, + height: u32, + resizing: bool, + needs_reconfigure: bool, + needs_recreate: bool, +} + +/// Everything you need to paint egui with [`wgpu`] on [`winit`]. +/// +/// Alternatively you can use [`crate::Renderer`] directly. +/// +/// NOTE: all egui viewports share the same painter. +pub struct Painter { + context: Context, + config: WgpuConfiguration, + options: RendererOptions, + support_transparent_backbuffer: bool, + screen_capture_state: Option, + + instance: wgpu::Instance, + render_state: Option, + + // Per viewport/window: + depth_texture_view: ViewportIdMap, + msaa_texture_view: ViewportIdMap, + surfaces: ViewportIdMap, + capture_tx: CaptureSender, + capture_rx: CaptureReceiver, + + // EternalMonitor patch: device-loss recovery. + /// Set when the current device is lost: by its lost callback, or when a frame's upload + /// fails. The next paint drops the device and everything made on it. + device_lost: Arc, + /// The lost device was dropped; [`Self::set_window`] builds a new one when + /// [`Self::replacement`] says, and a failure is retried instead of returned. + rebuilding: bool, + /// The new device's first paint uploads the font atlas again. + restore_fonts: bool, + /// When to try replacing a lost device, and when to stop. + replacement: Replacement, + /// The replacement being made, if any. Never more than one: a stuck one is waited + /// for, not joined by another. + replacing: Option, + /// The adapter that last drew on the window, which a replacement prefers. + preferred: Option, + /// Adapters that couldn't draw on the window since the last replacement that could. + excluded: Vec, +} + +impl Painter { + /// Manages [`wgpu`] state, including surface state, required to render egui. + /// + /// Only the [`wgpu::Instance`] is initialized here. Device selection and the initialization + /// of render + surface state is deferred until the painter is given its first window target + /// via [`set_window()`](Self::set_window). (Ensuring that a device that's compatible with the + /// native window is chosen) + /// + /// Before calling [`paint_and_update_textures()`](Self::paint_and_update_textures) a + /// [`wgpu::Surface`] must be initialized (and corresponding render state) by calling + /// [`set_window()`](Self::set_window) once you have + /// a [`winit::window::Window`] with a valid `.raw_window_handle()` + /// associated. + pub async fn new( + context: Context, + config: WgpuConfiguration, + support_transparent_backbuffer: bool, + options: RendererOptions, + ) -> Self { + let (capture_tx, capture_rx) = capture_channel(); + let instance = config.wgpu_setup.new_instance().await; + + Self { + context, + config, + options, + support_transparent_backbuffer, + screen_capture_state: None, + + instance, + render_state: None, + + depth_texture_view: Default::default(), + surfaces: Default::default(), + msaa_texture_view: Default::default(), + + capture_tx, + capture_rx, + + device_lost: Arc::new(AtomicBool::new(false)), + rebuilding: false, + restore_fonts: false, + replacement: Replacement::default(), + replacing: None, + preferred: None, + excluded: Vec::new(), + } + } + + /// Get the [`RenderState`]. + /// + /// Will return [`None`] if the render state has not been initialized yet. + pub fn render_state(&self) -> Option { + self.render_state.clone() + } + + fn configure_surface( + surface_state: &SurfaceState, + render_state: &RenderState, + config: &SurfaceConfig, + device_lost: &AtomicBool, + ) { + profiling::function_scope!(); + + let SurfaceConfig { + present_mode, + desired_maximum_frame_latency, + } = *config; + + // Transaction presentation can hold a drawable during AppKit live resize. Keep the + // configured low-latency path normally, but use three Metal drawables while resizing. + #[cfg(all(target_os = "macos", feature = "macos-window-resize-jitter-fix"))] + let desired_maximum_frame_latency = if surface_state.resizing { + Some(desired_maximum_frame_latency.unwrap_or(2).max(2)) + } else { + desired_maximum_frame_latency + }; + + let width = surface_state.width; + let height = surface_state.height; + + // EternalMonitor patch: a lost device's adapter may support no surface before its + // lost callback runs. Replace the device instead of panicking. + let Some(default_config) = + surface_state + .surface + .get_default_config(&render_state.adapter, width, height) + else { + log::warn!("The surface isn't supported by this adapter; replacing the device"); + device_lost.store(true, Ordering::SeqCst); + return; + }; + let mut surf_config = wgpu::SurfaceConfiguration { + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + format: render_state.target_format, + present_mode, + alpha_mode: surface_state.alpha_mode, + view_formats: vec![render_state.target_format], + ..default_config + }; + + if let Some(desired_maximum_frame_latency) = desired_maximum_frame_latency { + surf_config.desired_maximum_frame_latency = desired_maximum_frame_latency; + } + + surface_state + .surface + .configure(&render_state.device, &surf_config); + } + + /// Drop the existing [`wgpu::Surface`] for `viewport_id` and create a fresh one for the + /// given window via [`wgpu::Instance::create_surface`], then configure it. + /// + /// Used to recover from [`wgpu::CurrentSurfaceTexture::Lost`], where reconfiguring the + /// existing surface object cannot recover. + fn recreate_surface( + &mut self, + viewport_id: ViewportId, + window: &Arc, + ) -> Result<(), crate::WgpuError> { + profiling::function_scope!(); + + let Some(old_state) = self.surfaces.remove(&viewport_id) else { + return Ok(()); + }; + + let surface = self.instance.create_surface(Arc::clone(window))?; + self.install_surface( + surface, + viewport_id, + old_state.width, + old_state.height, + old_state.resizing, + ); + Ok(()) + } + + /// Updates (or clears) the [`winit::window::Window`] associated with the [`Painter`] + /// + /// This creates a [`wgpu::Surface`] for the given Window (as well as initializing render + /// state if needed) that is used for egui rendering. + /// + /// This must be called before trying to render via + /// [`paint_and_update_textures`](Self::paint_and_update_textures) + /// + /// # Portability + /// + /// _In particular it's important to note that on Android a it's only possible to create + /// a window surface between `Resumed` and `Paused` lifecycle events, and Winit will panic on + /// attempts to query the raw window handle while paused._ + /// + /// On Android [`set_window`](Self::set_window) should be called with `Some(window)` for each + /// `Resumed` event and `None` for each `Paused` event. Currently, on all other platforms + /// [`set_window`](Self::set_window) may be called with `Some(window)` as soon as you have a + /// valid [`winit::window::Window`]. + /// + /// # Errors + /// If the provided wgpu configuration does not match an available device. + pub async fn set_window( + &mut self, + viewport_id: ViewportId, + window: Option>, + ) -> Result<(), crate::WgpuError> { + profiling::scope!("Painter::set_window"); // profile_function gives bad names for async functions + + if let Some(window) = window { + let size = window.inner_size(); + if !self.surfaces.contains_key(&viewport_id) { + // EternalMonitor patch: a lost device is replaced off this thread. + if self.rebuilding { + self.replace_device(viewport_id, window); + return Ok(()); + } + let surface = self.instance.create_surface(window)?; + self.add_surface(surface, viewport_id, size).await?; + } + } else { + log::warn!("No window - clearing all surfaces"); + self.surfaces.clear(); + } + Ok(()) + } + + /// Updates (or clears) the [`winit::window::Window`] associated with the [`Painter`] without taking ownership of the window. + /// + /// Like [`set_window`](Self::set_window) except: + /// + /// # Safety + /// The user is responsible for ensuring that the window is alive for as long as it is set. + pub async unsafe fn set_window_unsafe( + &mut self, + viewport_id: ViewportId, + window: Option<&winit::window::Window>, + ) -> Result<(), crate::WgpuError> { + profiling::scope!("Painter::set_window_unsafe"); // profile_function gives bad names for async functions + + if let Some(window) = window { + let size = window.inner_size(); + if !self.surfaces.contains_key(&viewport_id) { + let surface = unsafe { + self.instance + .create_surface_unsafe(wgpu::SurfaceTargetUnsafe::from_window(&window)?)? + }; + self.add_surface(surface, viewport_id, size).await?; + } + } else { + log::warn!("No window - clearing all surfaces"); + self.surfaces.clear(); + } + Ok(()) + } + + async fn add_surface( + &mut self, + surface: wgpu::Surface<'static>, + viewport_id: ViewportId, + size: winit::dpi::PhysicalSize, + ) -> Result<(), crate::WgpuError> { + if self.render_state.is_none() { + let render_state = + RenderState::create(&self.config, &self.instance, Some(&surface), self.options) + .await?; + let lost = Arc::new(AtomicBool::new(false)); + watch(&render_state.device, &lost); + self.device_lost = lost; + self.preferred = Some(crate::AdapterKey::of(&render_state.adapter)); + self.render_state = Some(render_state); + } + self.install_surface(surface, viewport_id, size.width, size.height, false); + Ok(()) + } + + /// EternalMonitor patch: forget a lost device and everything made on it. eframe calls + /// [`Self::set_window`] before each paint, which builds a new device and surfaces. + fn drop_lost_device(&mut self) { + self.render_state = None; + self.surfaces.clear(); + self.depth_texture_view.clear(); + self.msaa_texture_view.clear(); + self.screen_capture_state = None; + self.rebuilding = true; + self.restore_fonts = false; + DEVICES_LOST.fetch_add(1, Ordering::SeqCst); + let now = Instant::now(); + if self.replacement.lost(now) { + Self::log_gave_up(&"the new device was lost too"); + } else { + log::warn!("Rebuilding the window renderer after its wgpu device was lost"); + if let Some(wait) = self.replacement.wait(now) { + self.context.request_repaint_after(wait); + } + } + } + + /// EternalMonitor patch: work towards a new device for a lost one, without blocking. + /// + /// Two stages run on other threads, which this polls: listing adapters and making the + /// device ([`RenderState::new_device`], which installs the loss handlers before anything + /// is made on the device), then egui's renderer (buffers, shaders, the pipeline). A + /// driver can stall in either. Neither thread gets the window or a surface, so one stuck + /// in a driver neither keeps the window alive (eframe ends its loop only once the window + /// is destroyed) nor touches it after it is gone. A stage that hasn't answered in + /// [`REPLACEMENT_TIMEOUT`], or that panicked, is a failed try, and no other starts while + /// one is out; a late answer is still taken. Between the stages, this thread makes the + /// window's surface and checks the new adapter can draw on it (if not, the adapter is + /// left out of later tries); after them it configures the surface, as DXGI sends the + /// window messages while it makes the swapchain. + fn replace_device(&mut self, viewport_id: ViewportId, window: Arc) { + let now = Instant::now(); + match self.replacing.take() { + None => self.start_replacement(now), + Some(Replacing::Device { mut answer, lost }) => match answer.poll(now) { + Asked::Waiting => self.replacing = Some(Replacing::Device { answer, lost }), + Asked::TimedOut => { + self.replacing = Some(Replacing::Device { answer, lost }); + self.replacement_failed(now, &"no new device within 10 s"); + } + Asked::Ended => self.replacement_failed(now, &"the thread making the device ended"), + Asked::Answered(Err(err)) => self.replacement_failed(now, &err), + Asked::Answered(Ok(new)) => self.make_renderer(new, lost, &window, now), + }, + Some(Replacing::Renderer { + mut answer, + new, + surface, + format, + lost, + }) => match answer.poll(now) { + Asked::Answered(renderer) => { + self.adopt(new, surface, format, renderer, lost, viewport_id, &window); + return; + } + Asked::Waiting => { + self.replacing = Some(Replacing::Renderer { + answer, + new, + surface, + format, + lost, + }); + } + Asked::TimedOut => { + self.replacing = Some(Replacing::Renderer { + answer, + new, + surface, + format, + lost, + }); + self.replacement_failed(now, &"no renderer within 10 s"); + } + Asked::Ended => { + self.replacement_failed(now, &"the thread making the renderer ended"); + } + }, + } + if let Some(wait) = self.next_check(now) { + self.context.request_repaint_after(wait); + } + } + + /// EternalMonitor patch: start the first stage of a replacement when it is due. + fn start_replacement(&mut self, now: Instant) { + if self.replacement.wait(now) != Some(Duration::ZERO) { + return; + } + let lost = Arc::new(AtomicBool::new(false)); + let flag = Arc::clone(&lost); + let (config, instance) = (self.config.clone(), self.instance.clone()); + let (preferred, excluded) = (self.preferred.clone(), self.excluded.clone()); + let asked = Answer::ask(REPLACEMENT_THREAD, now, REPLACEMENT_TIMEOUT, move || { + block_on(RenderState::new_device( + &config, + &instance, + preferred.as_ref(), + &excluded, + |device| watch(device, &flag), + )) + }); + match asked { + Ok(answer) => self.replacing = Some(Replacing::Device { answer, lost }), + Err(err) => self.replacement_failed(now, &err), + } + } + + /// EternalMonitor patch: on the window's thread, make the window's surface and check the + /// new adapter can draw on it, then start the renderer stage with the device, the format + /// and the options only. + fn make_renderer( + &mut self, + new: crate::NewDevice, + lost: Arc, + window: &Arc, + now: Instant, + ) { + let surface = match self.instance.create_surface(Arc::clone(window)) { + Ok(surface) => surface, + Err(err) => return self.replacement_failed(now, &err), + }; + let format = match RenderState::surface_format(&new, &surface) { + Ok(format) => format, + Err(err) => { + let key = new.adapter_key(); + log::warn!( + "Leaving out {} ({:?}) for the window: {err}", + key.name, + key.backend + ); + self.excluded.push(key); + return self.replacement_failed(now, &err); + } + }; + let (device, options) = (new.device().clone(), self.options); + let asked = Answer::ask(REPLACEMENT_THREAD, now, REPLACEMENT_TIMEOUT, move || { + renderer::Renderer::new(&device, format, options) + }); + match asked { + Ok(answer) => { + self.replacing = Some(Replacing::Renderer { + answer, + new, + surface, + format, + lost, + }); + } + Err(err) => self.replacement_failed(now, &err), + } + } + + /// EternalMonitor patch: draw with a finished replacement. + #[expect(clippy::too_many_arguments)] + fn adopt( + &mut self, + new: crate::NewDevice, + surface: wgpu::Surface<'static>, + format: wgpu::TextureFormat, + renderer: renderer::Renderer, + lost: Arc, + viewport_id: ViewportId, + window: &Arc, + ) { + self.preferred = Some(new.adapter_key()); + self.excluded.clear(); + let render_state = + RenderState::from_new_device(new, &self.config, &self.instance, format, renderer); + self.device_lost = lost; + self.render_state = Some(render_state); + let size = window.inner_size(); + self.install_surface(surface, viewport_id, size.width, size.height, false); + self.rebuilding = false; + self.restore_fonts = true; + log::info!("Replaced the lost wgpu device"); + } + + /// EternalMonitor patch: a try at a new device failed at `now`. + fn replacement_failed(&mut self, now: Instant, why: &dyn core::fmt::Display) { + if self.replacement.failed(now) { + Self::log_gave_up(why); + // A stuck thread may still answer; nothing waits for it. + self.replacing = None; + } else { + log::warn!("Could not replace the lost wgpu device yet: {why}"); + } + } + + /// EternalMonitor patch: when the window should look again at a device being replaced, + /// or `None` once the painter gave up. + fn next_check(&self, now: Instant) -> Option { + if self.replacing.is_some() { + Some(REPLACEMENT_POLL) + } else { + self.replacement.wait(now) + } + } + + /// EternalMonitor patch: said once, when [`Replacement`] gives up. + fn log_gave_up(last: &dyn core::fmt::Display) { + log::error!( + "Gave up replacing the window's lost wgpu device after {} failures ({last}); the window stays blank", + Replacement::MAX_FAILURES + ); + } + + /// Inserts a freshly created surface into [`Self::surfaces`] and configures it. + /// + /// Render state must already be initialised before calling this. + // NOTE: The same assumption is already required by `resize_and_generate_depth_texture_view_and_msaa_view`. + fn install_surface( + &mut self, + surface: wgpu::Surface<'static>, + viewport_id: ViewportId, + width: u32, + height: u32, + resizing: bool, + ) { + let alpha_mode = { + // Panic: We use the same failure mode as `resize_and_generate_depth_texture_view_and_msaa_view` + let render_state = self + .render_state + .as_ref() + .expect("install_surface called before render_state initialization"); + if self.support_transparent_backbuffer { + let supported_alpha_modes = + surface.get_capabilities(&render_state.adapter).alpha_modes; + // Prefer pre multiplied over post multiplied! + if supported_alpha_modes.contains(&wgpu::CompositeAlphaMode::PreMultiplied) { + wgpu::CompositeAlphaMode::PreMultiplied + } else if supported_alpha_modes.contains(&wgpu::CompositeAlphaMode::PostMultiplied) + { + wgpu::CompositeAlphaMode::PostMultiplied + } else { + log::warn!( + "Transparent window was requested, but the active wgpu surface does not support a `CompositeAlphaMode` with transparency." + ); + wgpu::CompositeAlphaMode::Auto + } + } else { + wgpu::CompositeAlphaMode::Auto + } + }; + self.surfaces.insert( + viewport_id, + SurfaceState { + surface, + width, + height, + alpha_mode, + resizing, + needs_reconfigure: false, + needs_recreate: false, + }, + ); + let Some(width) = NonZeroU32::new(width) else { + log::debug!("The window width was zero; skipping generate textures"); + return; + }; + let Some(height) = NonZeroU32::new(height) else { + log::debug!("The window height was zero; skipping generate textures"); + return; + }; + self.resize_and_generate_depth_texture_view_and_msaa_view(viewport_id, width, height); + } + + /// Returns the maximum texture dimension supported if known + /// + /// This API will only return a known dimension after `set_window()` has been called + /// at least once, since the underlying device and render state are initialized lazily + /// once we have a window (that may determine the choice of adapter/device). + pub fn max_texture_side(&self) -> Option { + self.render_state + .as_ref() + .map(|rs| rs.device.limits().max_texture_dimension_2d as usize) + } + + fn resize_and_generate_depth_texture_view_and_msaa_view( + &mut self, + viewport_id: ViewportId, + width_in_pixels: NonZeroU32, + height_in_pixels: NonZeroU32, + ) { + profiling::function_scope!(); + + let width = width_in_pixels.get(); + let height = height_in_pixels.get(); + + let render_state = self.render_state.as_ref().unwrap(); + let surface_state = self.surfaces.get_mut(&viewport_id).unwrap(); + + surface_state.width = width; + surface_state.height = height; + + Self::configure_surface( + surface_state, + render_state, + &self.config.surface, + &self.device_lost, + ); + + if let Some(depth_format) = self.options.depth_stencil_format { + self.depth_texture_view.insert( + viewport_id, + render_state + .device + .create_texture(&wgpu::TextureDescriptor { + label: Some("egui_depth_texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: self.options.msaa_samples.max(1), + dimension: wgpu::TextureDimension::D2, + format: depth_format, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[depth_format], + }) + .create_view(&wgpu::TextureViewDescriptor::default()), + ); + } + + if let Some(render_state) = (self.options.msaa_samples > 1) + .then_some(self.render_state.as_ref()) + .flatten() + { + let texture_format = render_state.target_format; + self.msaa_texture_view.insert( + viewport_id, + render_state + .device + .create_texture(&wgpu::TextureDescriptor { + label: Some("egui_msaa_texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: self.options.msaa_samples.max(1), + dimension: wgpu::TextureDimension::D2, + format: texture_format, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + view_formats: &[texture_format], + }) + .create_view(&wgpu::TextureViewDescriptor::default()), + ); + } + } + + /// Handles changes of the resizing state. + /// + /// Should be called prior to the first [`Painter::on_window_resized`] call and after the last in + /// the chain. Used to apply platform-specific logic, e.g. OSX Metal window resize jitter fix. + pub fn on_window_resize_state_change(&mut self, viewport_id: ViewportId, resizing: bool) { + profiling::function_scope!(); + + let Some(state) = self.surfaces.get_mut(&viewport_id) else { + return; + }; + if state.resizing == resizing { + if resizing { + log::debug!( + "Painter::on_window_resize_state_change() redundant call while resizing" + ); + } else { + log::debug!( + "Painter::on_window_resize_state_change() redundant call after resizing" + ); + } + return; + } + + // Set before reconfiguring so macOS live resize uses the temporary latency bump above. + state.resizing = resizing; + + // Resizing is a bit tricky on macOS. + // It requires enabling ["present_with_transaction"](https://developer.apple.com/documentation/quartzcore/cametallayer/presentswithtransaction) + // flag to avoid jittering during the resize. Even though resize jittering on macOS + // is common across rendering backends, the solution for wgpu/metal is known. + // + // See https://github.com/emilk/egui/issues/903 + #[cfg(all(target_os = "macos", feature = "macos-window-resize-jitter-fix"))] + { + // SAFETY: `as_hal::()` returns `None` unless this surface is backed by wgpu's + // Metal backend. + unsafe { + if let (Some(render_state), Some(hal_surface)) = ( + self.render_state.as_ref(), + state.surface.as_hal::(), + ) { + hal_surface + .render_layer() + .lock() + .setPresentsWithTransaction(resizing); + + Self::configure_surface( + state, + render_state, + &self.config.surface, + &self.device_lost, + ); + } + } + } + } + + pub fn on_window_resized( + &mut self, + viewport_id: ViewportId, + width_in_pixels: NonZeroU32, + height_in_pixels: NonZeroU32, + ) { + profiling::function_scope!(); + + if self.surfaces.contains_key(&viewport_id) { + self.resize_and_generate_depth_texture_view_and_msaa_view( + viewport_id, + width_in_pixels, + height_in_pixels, + ); + } else { + log::warn!( + "Ignoring window resize notification with no surface created via Painter::set_window()" + ); + } + } + + /// Returns two things: + /// + /// The approximate number of seconds spent on vsync-waiting (if any), + /// and the captures captured screenshot if it was requested. + /// + /// If `capture_data` isn't empty, a screenshot will be captured. + #[expect(clippy::too_many_arguments)] + pub fn paint_and_update_textures( + &mut self, + viewport_id: ViewportId, + pixels_per_point: f32, + clear_color: [f32; 4], + clipped_primitives: &[epaint::ClippedPrimitive], + textures_delta: &mut epaint::textures::TexturesDelta, + capture_data: Vec, + window: &Arc, + ) -> f32 { + profiling::function_scope!(); + + /// Guard to ensure that commands are always submitted to the renderer queue + /// so that calls to [`write_buffer()`](https://docs.rs/wgpu/latest/wgpu/struct.Queue.html#method.write_buffer) + /// are completed even if we take a codepath which doesn't submit commands and avoids + /// internal buffers growing indefinitely. + /// + /// This may happen, for example, if no output frame is resolved. + /// See for full context. + struct RendererQueueGuard<'q> { + queue: &'q wgpu::Queue, + commands_submitted: bool, + } + + impl Drop for RendererQueueGuard<'_> { + fn drop(&mut self) { + // Only submit an empty command buffer array if no commands were + // explicitly submitted. + if !self.commands_submitted { + self.queue.submit([]); + } + } + } + + let capture = !capture_data.is_empty(); + let mut vsync_sec = 0.0; + + // EternalMonitor patch: while a lost device is replaced there is nothing to draw + // with; set_window makes the next one when `replacement` says. A device lost since + // the last frame draws nothing again, so it goes. + if self.rebuilding { + SKIPPED_FRAMES.fetch_add(1, Ordering::SeqCst); + if let Some(wait) = self.next_check(Instant::now()) { + self.context.request_repaint_after(wait); + } + return vsync_sec; + } + if self.device_lost.load(Ordering::SeqCst) { + SKIPPED_FRAMES.fetch_add(1, Ordering::SeqCst); + self.drop_lost_device(); + return vsync_sec; + } + + // If the previous frame produced `CurrentSurfaceTexture::Lost`, the action match + // below set `needs_recreate`. Recreate the surface now, before re-borrowing + // `self.render_state` / `self.surfaces` for the rest of the paint. + if self + .surfaces + .get(&viewport_id) + .is_some_and(|s| s.needs_recreate) + && let Err(err) = self.recreate_surface(viewport_id, window) + { + log::error!("Failed to recreate surface for {viewport_id:?}: {err}"); + return vsync_sec; + } + + // Apply any runtime changes requested via `RenderState::surface_config`. + // We diff against the already-applied values in `self.config.surface` + // and, if anything differs, mark every surface as needing reconfiguration so + // the existing `needs_reconfigure` pathway below picks them up. + if let Some(render_state) = self.render_state.as_ref() + && render_state.surface_config != self.config.surface + { + self.config.surface = render_state.surface_config; + #[expect(clippy::iter_over_hash_type)] + for surface in self.surfaces.values_mut() { + surface.needs_reconfigure = true; + } + } + + let Some(render_state) = self.render_state.as_mut() else { + return vsync_sec; + }; + + let mut render_queue_guard = RendererQueueGuard { + queue: &render_state.queue, + commands_submitted: false, + }; + + let Some(surface_state) = self.surfaces.get_mut(&viewport_id) else { + return vsync_sec; + }; + + let mut encoder = + render_state + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("encoder"), + }); + + // Upload all resources for the GPU. + let screen_descriptor = renderer::ScreenDescriptor { + size_in_pixels: [surface_state.width, surface_state.height], + pixels_per_point, + }; + + let user_cmd_bufs = { + let mut renderer = render_state.renderer.write(); + // EternalMonitor patch: the font atlas went with the lost device. Other textures + // are the app's to load again. + if core::mem::take(&mut self.restore_fonts) { + let atlas = self.context.fonts(|fonts| fonts.image()); + renderer.update_texture( + &render_state.device, + &render_state.queue, + epaint::TextureId::default(), + &epaint::ImageDelta::full(atlas, epaint::TextureAtlas::texture_options()), + ); + } + #[expect(clippy::iter_over_hash_type)] // Order doesn't matter here + for (id, image_deltas) in textures_delta.set.drain() { + for image_delta in image_deltas { + renderer.update_texture( + &render_state.device, + &render_state.queue, + id, + &image_delta, + ); + } + } + + renderer.update_buffers( + &render_state.device, + &render_state.queue, + &mut encoder, + clipped_primitives, + &screen_descriptor, + ) + }; + + // EternalMonitor patch: skip a frame whose upload failed; the next one replaces the + // device. + if render_state.renderer.read().upload_failed() || self.device_lost.load(Ordering::SeqCst) { + SKIPPED_FRAMES.fetch_add(1, Ordering::SeqCst); + self.device_lost.store(true, Ordering::SeqCst); + self.context.request_repaint_of(viewport_id); + return vsync_sec; + } + + if surface_state.needs_reconfigure { + Self::configure_surface( + surface_state, + render_state, + &self.config.surface, + &self.device_lost, + ); + surface_state.needs_reconfigure = false; + } + + let output_frame = { + profiling::scope!("get_current_texture"); + // This is what vsync-waiting happens on my Mac. + let start = web_time::Instant::now(); + let output_frame = surface_state.surface.get_current_texture(); + vsync_sec += start.elapsed().as_secs_f32(); + output_frame + }; + + let output_frame = match output_frame { + wgpu::CurrentSurfaceTexture::Success(frame) => frame, + wgpu::CurrentSurfaceTexture::Suboptimal(frame) => { + surface_state.needs_reconfigure = true; + frame + } + other => { + match (*self.config.on_surface_status)(&other) { + SurfaceErrorAction::Reconfigure => { + Self::configure_surface( + surface_state, + render_state, + &self.config.surface, + &self.device_lost, + ); + self.context.request_repaint_of(viewport_id); + } + SurfaceErrorAction::RecreateSurface => { + // Because of ownership, I could not find an easy way to do a full recovery here, + // as that would involve dropping the old surface and creating a new one. + // For now, we defer the recreation to the beginning of the next frame (which + // we ensure to arrive via `request_repaint_of`). A cleaner solution would be + // to untangle the ownership of `RenderState`. + surface_state.needs_recreate = true; + self.context.request_repaint_of(viewport_id); + } + SurfaceErrorAction::SkipFrame => {} + } + return vsync_sec; + } + }; + + let mut capture_buffer = None; + { + let renderer = render_state.renderer.read(); + + let target_texture = if capture { + let capture_state = self.screen_capture_state.get_or_insert_with(|| { + CaptureState::new(&render_state.device, &output_frame.texture) + }); + capture_state.update(&render_state.device, &output_frame.texture); + + &capture_state.texture + } else { + &output_frame.texture + }; + let target_view = target_texture.create_view(&wgpu::TextureViewDescriptor::default()); + + let (view, resolve_target) = (self.options.msaa_samples > 1) + .then_some(self.msaa_texture_view.get(&viewport_id)) + .flatten() + .map_or((&target_view, None), |texture_view| { + (texture_view, Some(&target_view)) + }); + + let render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("egui_render"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view, + resolve_target, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: clear_color[0] as f64, + g: clear_color[1] as f64, + b: clear_color[2] as f64, + a: clear_color[3] as f64, + }), + store: wgpu::StoreOp::Store, + }, + depth_slice: None, + })], + depth_stencil_attachment: self.depth_texture_view.get(&viewport_id).map(|view| { + wgpu::RenderPassDepthStencilAttachment { + view, + depth_ops: self + .options + .depth_stencil_format + .is_some_and(|depth_stencil_format| { + depth_stencil_format.has_depth_aspect() + }) + .then_some(wgpu::Operations { + load: wgpu::LoadOp::Clear(1.0), + // It is very unlikely that the depth buffer is needed after egui finished rendering + // so no need to store it. (this can improve performance on tiling GPUs like mobile chips or Apple Silicon) + store: wgpu::StoreOp::Discard, + }), + stencil_ops: self + .options + .depth_stencil_format + .is_some_and(|depth_stencil_format| { + depth_stencil_format.has_stencil_aspect() + }) + .then_some(wgpu::Operations { + load: wgpu::LoadOp::Clear(0), + store: wgpu::StoreOp::Discard, + }), + } + }), + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + + // Forgetting the pass' lifetime means that we are no longer compile-time protected from + // runtime errors caused by accessing the parent encoder before the render pass is dropped. + // Since we don't pass it on to the renderer, we should be perfectly safe against this mistake here! + renderer.render( + &mut render_pass.forget_lifetime(), + clipped_primitives, + &screen_descriptor, + ); + + if capture && let Some(capture_state) = &mut self.screen_capture_state { + capture_buffer = Some(capture_state.copy_textures( + &render_state.device, + &output_frame, + &mut encoder, + )); + } + } + + let encoded = { + profiling::scope!("CommandEncoder::finish"); + encoder.finish() + }; + + // Submit the commands: both the main buffer and user-defined ones. + { + profiling::scope!("Queue::submit"); + // wgpu doesn't document where vsync can happen. Maybe here? + let start = web_time::Instant::now(); + render_state + .queue + .submit(core::iter::chain(user_cmd_bufs, [encoded])); + vsync_sec += start.elapsed().as_secs_f32(); + }; + + // Ensure that the queue guard does not do unnecessary work when dropped + render_queue_guard.commands_submitted = true; + + // Free textures marked for destruction **after** queue submit since they might still be used in the current frame. + // Calling `wgpu::Texture::destroy` on a texture that is still in use would invalidate the command buffer(s) it is used in. + // However, once we called `wgpu::Queue::submit`, it is up for wgpu to determine how long the underlying gpu resource has to live. + { + let mut renderer = render_state.renderer.write(); + #[expect(clippy::iter_over_hash_type)] // Order doesn't matter here + for id in textures_delta.free.drain() { + renderer.free_texture(&id); + } + } + + if let Some(capture_buffer) = capture_buffer + && let Some(screen_capture_state) = &mut self.screen_capture_state + { + screen_capture_state.read_screen_rgba( + self.context.clone(), + capture_buffer, + capture_data, + self.capture_tx.clone(), + viewport_id, + ); + } + + window.pre_present_notify(); + + { + profiling::scope!("present"); + // wgpu doesn't document where vsync can happen. Maybe here? + let start = web_time::Instant::now(); + render_state.queue.present(output_frame); + vsync_sec += start.elapsed().as_secs_f32(); + } + // EternalMonitor patch: a device that keeps drawing starts the replacement count again. + self.replacement.drew(Instant::now()); + + vsync_sec + } + + /// Call this at the beginning of each frame to receive the requested screenshots. + pub fn handle_screenshots(&self, events: &mut Vec) { + for (viewport_id, user_data, screenshot) in self.capture_rx.try_iter() { + let screenshot = Arc::new(screenshot); + for data in user_data { + events.push(Event::Screenshot { + viewport_id, + user_data: data, + image: Arc::clone(&screenshot), + }); + } + } + } + + pub fn gc_viewports(&mut self, active_viewports: &ViewportIdSet) { + self.surfaces.retain(|id, _| active_viewports.contains(id)); + self.depth_texture_view + .retain(|id, _| active_viewports.contains(id)); + self.msaa_texture_view + .retain(|id, _| active_viewports.contains(id)); + } + + #[expect(clippy::needless_pass_by_ref_mut, clippy::unused_self)] + pub fn destroy(&mut self) { + // TODO(emilk): something here? + } +}