From 0b29273b1bb7cc608ac930fa48dd55ebb8876d39 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Tue, 3 Nov 2026 17:36:55 -0500 Subject: [PATCH 001/226] Stream screens with an odd width or height libx264 refuses an odd width or height (4:2:0 needs even dimensions), so a 1680x945 screen with no usable hardware encoder never streamed: the encoder failed to open on every restart until the pipeline gave up. The encoder now opens at the even size below the capture and drops the odd last column or row, on every input path (swscale to YUV420P/NV12, Accelerate on macOS, the BGRA copy NVENC reads). The crop reads the capture's top-left region in place, so no pixel is resampled. The size check compares encoded sizes, so an odd capture does not reopen the encoder every frame. The stream config, GUI and stats report the encoded size; one info line per open names both sizes. Input still maps onto the whole capture: each pixel of the cropped picture lands on itself and the picture's far edge reaches the row or column left out. --- host/src/encoder/input.rs | 73 ++++++---- host/src/encoder/mod.rs | 275 +++++++++++++++++++++++++++++------ host/src/input/mod.rs | 42 ++++++ host/src/stats.rs | 5 +- host/src/telemetry/redact.rs | 7 + host/tests/e2e_synthetic.rs | 181 +++++++++++++++++++++++ scripts/e2e_ios.sh | 9 +- 7 files changed, 519 insertions(+), 73 deletions(-) diff --git a/host/src/encoder/input.rs b/host/src/encoder/input.rs index de34d4a2..ecc3d429 100644 --- a/host/src/encoder/input.rs +++ b/host/src/encoder/input.rs @@ -116,10 +116,12 @@ pub(super) fn conversion_context( .map(Some) } -/// Converts capture's tightly packed BGRA rows into `output` where they lie: -/// swscale reads the capture buffer itself, with no copy into a BGRA frame -/// first. The call returns when the conversion is done, so the buffer can -/// go back to capture straight after. +/// Converts capture's BGRA rows, `row_bytes` apart, into `output` where they +/// lie: swscale reads the capture buffer itself, with no copy into a BGRA +/// frame first. Rows wider than the scaler's input, or more of them, are a +/// capture with an odd last column or row, which is left out. The call +/// returns when the conversion is done, so the buffer can go back to +/// capture straight after. /// /// swscale can read up to [`FRAME_PADDING`] bytes past the last row, so /// `rows` without that many after the picture, as capture's @@ -128,12 +130,14 @@ pub(super) fn conversion_context( pub(super) fn convert_packed_bgra( scaler: &mut ffmpeg_next::software::scaling::Context, rows: &[u8], + row_bytes: usize, output: &mut ffmpeg_next::frame::Video, ) -> Result<(), ffmpeg_next::Error> { let input = *scaler.input(); - let row_bytes = input.width as usize * 4; - let frame_bytes = row_bytes * input.height as usize; - if input.format != Pixel::BGRA || rows.len() < frame_bytes { + let picture_bytes = input.width as usize * 4; + // The last row ends after its picture's pixels. + let frame_bytes = row_bytes * (input.height as usize).saturating_sub(1) + picture_bytes; + if input.format != Pixel::BGRA || row_bytes < picture_bytes || rows.len() < frame_bytes { return Err(ffmpeg_next::Error::InputChanged); } let wanted = scaler.output(); @@ -144,7 +148,7 @@ pub(super) fn convert_packed_bgra( return Err(ffmpeg_next::Error::OutputChanged); } if rows.len() < frame_bytes + FRAME_PADDING { - let copied = copied_frame(&rows[..frame_bytes], input.width, input.height); + let copied = copied_frame(&rows[..frame_bytes], row_bytes, input.width, input.height); return scaler.run(&copied, output); } let planes = [ @@ -154,9 +158,9 @@ pub(super) fn convert_packed_bgra( std::ptr::null(), ]; let strides = [row_bytes as libc::c_int, 0, 0, 0]; - // swscale reads `input.height` rows of `row_bytes` from `rows`, and at - // most FRAME_PADDING more, checked above, and writes the planes `output` - // allocated at the output size. + // swscale reads `input.height` rows of `picture_bytes`, `row_bytes` + // apart, from `rows`, and at most FRAME_PADDING more, checked above, and + // writes the planes `output` allocated at the output size. let result = unsafe { ffmpeg_next::ffi::sws_scale( scaler.as_mut_ptr(), @@ -174,19 +178,25 @@ pub(super) fn convert_packed_bgra( Ok(()) } -/// `rows` copied into a BGRA frame FFmpeg allocated. At odd widths swscale -/// reads one pixel past each row for the last chroma sample, which lands in -/// the frame's unset stride padding, so each row's last pixel is repeated -/// there. -fn copied_frame(rows: &[u8], width: u32, height: u32) -> ffmpeg_next::frame::Video { - let row_bytes = width as usize * 4; +/// The top-left `width`x`height` of `rows`, `row_bytes` apart, copied into a +/// BGRA frame FFmpeg allocated. At odd widths swscale reads one pixel past +/// each row for the last chroma sample, which lands in the frame's unset +/// stride padding, so each row's last pixel is repeated there. +fn copied_frame( + rows: &[u8], + row_bytes: usize, + width: u32, + height: u32, +) -> ffmpeg_next::frame::Video { + let picture_bytes = width as usize * 4; let mut copied = ffmpeg_next::frame::Video::new(Pixel::BGRA, width, height); let stride = copied.stride(0); - for (y, row) in rows.chunks_exact(row_bytes).enumerate() { + for (y, row) in rows.chunks(row_bytes).take(height as usize).enumerate() { + let row = &row[..picture_bytes]; let line = &mut copied.data_mut(0)[y * stride..(y + 1) * stride]; - line[..row_bytes].copy_from_slice(row); - if let Some(next) = line.get_mut(row_bytes..row_bytes + 4) { - next.copy_from_slice(&row[row_bytes - 4..]); + line[..picture_bytes].copy_from_slice(row); + if let Some(next) = line.get_mut(picture_bytes..picture_bytes + 4) { + next.copy_from_slice(&row[picture_bytes - 4..]); } } copied @@ -394,7 +404,7 @@ mod tests { height: u32, ) -> ffmpeg_next::frame::Video { let frame_bytes = width as usize * 4 * height as usize; - let copied = copied_frame(&rows[..frame_bytes], width, height); + let copied = copied_frame(&rows[..frame_bytes], width as usize * 4, width, height); let mut converted = ffmpeg_next::frame::Video::new(format, width, height); scaler.run(&copied, &mut converted).unwrap(); converted @@ -437,16 +447,25 @@ mod tests { let mut scaler = conversion_context(format, width, height).unwrap().unwrap(); let before = converted_from_copy(&mut scaler, format, &rows, width, height); let mut after = ffmpeg_next::frame::Video::new(format, width, height); - convert_packed_bgra(&mut scaler, &rows, &mut after).unwrap(); + convert_packed_bgra(&mut scaler, &rows, row_bytes, &mut after).unwrap(); assert_same_picture(&before, &after, &format!("{width}x{height} {format:?}")); // A short buffer or another size is refused, never read past. assert_eq!( - convert_packed_bgra(&mut scaler, &rows[..frame_bytes - 1], &mut after), + convert_packed_bgra( + &mut scaler, + &rows[..frame_bytes - 1], + row_bytes, + &mut after + ), + Err(ffmpeg_next::Error::InputChanged) + ); + assert_eq!( + convert_packed_bgra(&mut scaler, &rows, row_bytes - 4, &mut after), Err(ffmpeg_next::Error::InputChanged) ); let mut other = ffmpeg_next::frame::Video::new(format, width + 2, height); assert_eq!( - convert_packed_bgra(&mut scaler, &rows, &mut other), + convert_packed_bgra(&mut scaler, &rows, row_bytes, &mut other), Err(ffmpeg_next::Error::OutputChanged) ); } @@ -491,7 +510,7 @@ mod tests { }; fill_pattern(rows, row_bytes); // The pixel swscale reads after each odd-width row is set. - let copied = copied_frame(rows, width, height); + let copied = copied_frame(rows, row_bytes, width, height); let stride = copied.stride(0); for y in 0..height as usize { let line = &copied.data(0)[y * stride..y * stride + row_bytes + 4]; @@ -500,7 +519,7 @@ mod tests { for format in [Pixel::YUV420P, Pixel::NV12] { let mut scaler = conversion_context(format, width, height).unwrap().unwrap(); let mut after = ffmpeg_next::frame::Video::new(format, width, height); - convert_packed_bgra(&mut scaler, rows, &mut after).unwrap(); + convert_packed_bgra(&mut scaler, rows, row_bytes, &mut after).unwrap(); let before = converted_from_copy(&mut scaler, format, rows, width, height); assert_same_picture(&before, &after, &format!("{width}x{height} {format:?}")); } diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index e606b783..a865edf5 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -29,6 +29,16 @@ const AMF_CAPTURE_PACKET_LIMIT: u64 = 120; const AMF_IDR_WARNING_PACKET: u64 = 60; const AMF_FORCED_INTRA_PERIOD: u64 = 30; +/// The size a capture of `width`x`height` streams at. 4:2:0 needs even +/// dimensions, which libx264 enforces and hardware encoders may too, so a +/// screen such as 1680x945 encodes without its odd last row (or column): +/// cropped, not scaled, so every other pixel stays exact. Input still maps +/// onto the whole capture (see [`crate::input::norm_to_output_pixel`]), so +/// the cropped row stays reachable at the picture's edge. +pub fn encoded_size(width: u32, height: u32) -> (u32, u32) { + (width & !1, height & !1) +} + /// Encoded H.264 NAL unit ready for transport. #[derive(Debug, Clone)] pub struct NALUnit { @@ -404,8 +414,8 @@ fn run_encode_loop( from = previous, to = desired_bitrate, encoder = %desired_encoder, - width = raw_frame.width, - height = raw_frame.height, + width = state.frame.width(), + height = state.frame.height(), size_changed, keyframe = keyframe_reopen, reason, @@ -479,44 +489,8 @@ fn run_encode_loop( let current_bitrate = encoder.opened_bitrate; let opened_is_hevc = is_hevc_encoder(&encoder.opened_encoder); - let src_row_bytes = (raw_frame.width * 4) as usize; let convert_start = Instant::now(); - #[cfg(target_os = "macos")] - let converted = match encoder.native_converter.as_ref().map(|converter| { - converter.convert( - &raw_frame.data, - raw_frame.width, - raw_frame.height, - src_row_bytes, - &mut encoder.frame, - ) - }) { - Some(Ok(())) => true, - Some(Err(error)) => { - warn!(%error, "Using swscale after native color conversion failed"); - encoder.native_converter = None; - false - } - None => false, - }; - #[cfg(not(target_os = "macos"))] - let converted = false; - if !converted { - if let Some(scaler) = encoder.scaler.as_mut() { - input::convert_packed_bgra(scaler, &raw_frame.data, &mut encoder.frame)?; - } else { - // BGRA input: the encoder converts. Its frame takes a copy, - // so capture's buffer is free before the encoder can block. - let stride = encoder.frame.stride(0); - let bgra_plane = encoder.frame.data_mut(0); - for y in 0..raw_frame.height as usize { - let src_offset = y * src_row_bytes; - let dst_offset = y * stride; - bgra_plane[dst_offset..dst_offset + src_row_bytes] - .copy_from_slice(&raw_frame.data[src_offset..src_offset + src_row_bytes]); - } - } - } + encoder.convert(&raw_frame.data, (raw_frame.width * 4) as usize)?; // Conversion finished synchronously. Recycle capture's buffer before // the codec or sender can block, including on the direct macOS path. drop(raw_frame.data); @@ -725,19 +699,29 @@ impl EncoderState { true } + /// Opens a session for captures of `capture_width`x`capture_height`, + /// at their [`encoded_size`]. #[allow(clippy::too_many_arguments)] fn new( encoder_name: &str, - width: u32, - height: u32, + capture_width: u32, + capture_height: u32, bitrate_bps: u32, target_fps: u32, input_mode: &mut EncoderInput, opening: Opening, progress: &mut dyn FnMut(), ) -> Result> { + let (width, height) = encoded_size(capture_width, capture_height); info!(encoder = encoder_name, width, height, bitrate_bps, target_fps, input = ?input_mode, "Opening encoder session"); + if (width, height) != (capture_width, capture_height) { + info!( + capture_size = %format_args!("{capture_width}x{capture_height}"), + encoded_size = %format_args!("{width}x{height}"), + "Encoding without the capture's odd last column or row" + ); + } let codec = ffmpeg_next::encoder::find_by_name(encoder_name) .ok_or_else(|| format!("{} codec not found in FFmpeg", encoder_name))?; // AVCodec owns this terminated static list. A null list means that @@ -902,8 +886,38 @@ impl EncoderState { .unwrap_or(just_submitted) } + /// Converts a capture whose rows lie `row_bytes` apart into `self.frame`: + /// its top-left `self.frame`-sized part, pixel for pixel, which leaves an + /// odd last column or row out (see [`encoded_size`]). + fn convert(&mut self, rows: &[u8], row_bytes: usize) -> Result<(), ffmpeg_next::Error> { + let (width, height) = (self.frame.width(), self.frame.height()); + #[cfg(target_os = "macos")] + if let Some(converter) = self.native_converter.as_ref() { + match converter.convert(rows, width, height, row_bytes, &mut self.frame) { + Ok(()) => return Ok(()), + Err(error) => { + warn!(%error, "Using swscale after native color conversion failed"); + self.native_converter = None; + } + } + } + if let Some(scaler) = self.scaler.as_mut() { + return input::convert_packed_bgra(scaler, rows, row_bytes, &mut self.frame); + } + // BGRA input: the encoder converts. Its frame takes a copy, + // so capture's buffer is free before the encoder can block. + let picture_bytes = width as usize * 4; + let stride = self.frame.stride(0); + let bgra_plane = self.frame.data_mut(0); + for y in 0..height as usize { + bgra_plane[y * stride..][..picture_bytes] + .copy_from_slice(&rows[y * row_bytes..][..picture_bytes]); + } + Ok(()) + } + fn accepts_frame(&self, width: u32, height: u32) -> bool { - self.frame.width() == width && self.frame.height() == height + encoded_size(width, height) == (self.frame.width(), self.frame.height()) } fn observe_amf_diagnostics( @@ -2323,6 +2337,183 @@ mod tests { assert_eq!(labels, (480..490).map(|n| (at(n), n)).collect::>()); } + fn libx264_at(width: u32, height: u32) -> EncoderState { + EncoderState::new( + "libx264", + width, + height, + 8_000_000, + 60, + &mut EncoderInput::Yuv420, + Opening::First, + &mut || {}, + ) + .unwrap_or_else(|error| panic!("{width}x{height}: {error}")) + } + + /// A synthetic capture, in a buffer as capture hands it over. + fn synthetic_capture(width: u32, height: u32, frame_number: u64) -> Vec { + let frame_bytes = width as usize * 4 * height as usize; + let mut rows = crate::capture::frame_buffer(frame_bytes); + crate::capture::synthetic::render_synthetic_frame( + &mut rows[..frame_bytes], + width, + height, + frame_number, + ); + rows + } + + /// A screen with an odd height (1680x945, from a report) or width + /// streams: libx264 refused to open at those sizes, and the pipeline + /// gave up after six restarts. The encoder opens at the even size below, + /// takes the capture's frames, and they decode at that size with the + /// picture intact. + #[test] + fn odd_screen_sizes_encode_at_the_even_size_below() { + ffmpeg_next::init().unwrap(); + for ((width, height), encoded) in [ + ((1680, 945), (1680, 944)), + ((1365, 768), (1364, 768)), + ((641, 361), (640, 360)), + ((1366, 768), (1366, 768)), + ] { + let mut state = libx264_at(width, height); + assert_eq!((state.frame.width(), state.frame.height()), encoded); + // The same capture size never reopens the session; another does. + assert!(state.accepts_frame(width, height)); + assert!(!state.accepts_frame(width + 2, height)); + assert!(!state.accepts_frame(width, height + 2)); + let codec = ffmpeg_next::decoder::find(ffmpeg_next::codec::Id::H264).unwrap(); + let mut decoder = ffmpeg_next::codec::Context::new_with_codec(codec) + .decoder() + .video() + .unwrap(); + let mut decoded = ffmpeg_next::frame::Video::empty(); + let mut counters = Vec::new(); + let start = Instant::now(); + for n in 1..=10u64 { + let rows = synthetic_capture(width, height, n); + state.convert(&rows, width as usize * 4).unwrap(); + let captured = start + Duration::from_micros(n * 16_667); + let pts = state.next_pts(captured, n); + state.frame.set_pts(Some(pts)); + state.send_frame(pts, captured, n).unwrap(); + while state.encoder.receive_packet(&mut state.packet).is_ok() { + if state.h264.needs_parameter_sets() { + state + .h264 + .refresh_parameter_sets_from_extradata(encoder_extradata( + &state.encoder, + )); + } + // What the transport sends: Annex B with parameter sets. + let (annex_b, _) = normalize_h264_payload_with_info( + state.packet.data().unwrap_or_default(), + state.packet.is_key(), + &mut state.h264, + &state.opened_encoder, + ); + decoder + .send_packet(&ffmpeg_next::Packet::copy(&annex_b)) + .unwrap(); + while decoder.receive_frame(&mut decoded).is_ok() { + assert_eq!((decoded.width(), decoded.height()), encoded); + counters.push(crate::capture::synthetic::decode_frame_counter_from_luma( + decoded.data(0), + decoded.stride(0), + encoded.0, + )); + } + } + } + assert_eq!( + counters, + (1..=counters.len() as u64).collect::>(), + "{width}x{height}" + ); + assert!(counters.len() >= 5, "{width}x{height}: {counters:?}"); + } + } + + /// The rows of each of `frame`'s planes that hold the picture. + fn picture(frame: &ffmpeg_next::frame::Video) -> Vec> { + use ffmpeg_next::format::Pixel; + let mut rows = Vec::new(); + for plane in 0..frame.planes() { + let bytes = match (frame.format(), plane) { + (Pixel::BGRA | Pixel::BGRZ, _) => frame.width() as usize * 4, + (Pixel::NV12, 1) => frame.plane_width(1) as usize * 2, + _ => frame.plane_width(plane) as usize, + }; + for y in 0..frame.plane_height(plane) as usize { + rows.push(frame.data(plane)[y * frame.stride(plane)..][..bytes].to_vec()); + } + } + rows + } + + /// A capture with an odd last column or row encodes exactly what the + /// rest of it, alone, does: the crop leaves that column and row out and + /// resamples nothing, on every input path (swscale to YUV420P and NV12, + /// Accelerate on macOS, the BGRA copy NVENC reads), with or without + /// padding after the capture's last row. + #[test] + fn cropping_changes_no_pixel_of_the_picture() { + use ffmpeg_next::format::Pixel; + ffmpeg_next::init().unwrap(); + for (width, height) in [(1365u32, 945u32), (1365, 944), (1366, 945)] { + let (even_width, even_height) = encoded_size(width, height); + let capture = synthetic_capture(width, height, 77); + let row_bytes = width as usize * 4; + let unpadded = &capture[..row_bytes * height as usize]; + // The capture's top-left even_width x even_height, alone. + let even_bytes = even_width as usize * 4; + let mut alone = crate::capture::frame_buffer(even_bytes * even_height as usize); + for y in 0..even_height as usize { + alone[y * even_bytes..][..even_bytes] + .copy_from_slice(&capture[y * row_bytes..][..even_bytes]); + } + let mut paths = vec!["YUV420P", "NV12", "BGRA"]; + if cfg!(target_os = "macos") { + paths.push("Accelerate"); + } + for path in paths { + let label = format!("{width}x{height} {path}"); + let mut state = libx264_at(width, height); + // The frame and conversion an encoder that takes `path` has. + #[cfg(target_os = "macos")] + if path != "Accelerate" { + state.native_converter = None; + } + let format = match path { + "NV12" => Pixel::NV12, + "BGRA" => Pixel::BGRA, + _ => Pixel::YUV420P, + }; + state.frame = ffmpeg_next::frame::Video::new(format, even_width, even_height); + state.scaler = input::conversion_context(format, even_width, even_height).unwrap(); + assert_eq!(state.scaler.is_some(), format != Pixel::BGRA); + state.convert(&alone, even_bytes).unwrap(); + let expected = picture(&state.frame); + for rows in [&capture[..], unpadded] { + state.frame = ffmpeg_next::frame::Video::new(format, even_width, even_height); + state.convert(rows, row_bytes).unwrap(); + assert_eq!(picture(&state.frame), expected, "{label}"); + } + // Accelerate converted each, with no fallback to swscale. + #[cfg(target_os = "macos")] + assert_eq!(state.native_converter.is_some(), path == "Accelerate"); + if format == Pixel::BGRA { + let rows = capture.chunks(row_bytes).take(even_height as usize); + for (y, row) in rows.enumerate() { + assert_eq!(expected[y], row[..even_bytes], "{label} row {y}"); + } + } + } + } + } + #[test] fn nvenc_and_qsv_make_a_requested_keyframe_an_idr() { for (name, option) in [ diff --git a/host/src/input/mod.rs b/host/src/input/mod.rs index eb451ac3..d1edea10 100644 --- a/host/src/input/mod.rs +++ b/host/src/input/mod.rs @@ -136,6 +136,12 @@ pub struct PenSample { /// Map a wire-normalized point (0..65535 over the displayed video, which the /// client letterbox-corrects before sending) to a desktop pixel on the /// captured output. +/// +/// `output` is the whole capture, also when the video leaves out its odd +/// last column or row ([`crate::encoder::encoded_size`]): the video's `n` +/// pixels then span `n` steps of this mapping, so each one's centre lands +/// on its own desktop pixel, and the picture's far edge on the pixel left +/// out, which stays reachable. pub fn norm_to_output_pixel(x_norm: u16, y_norm: u16, output: CaptureGeometry) -> (i32, i32) { let x = output.left + ((u64::from(x_norm) * u64::from(output.width.saturating_sub(1))) / 65_535) as i32; @@ -1019,6 +1025,42 @@ mod tests { assert!((cy - 540).abs() <= 1, "center y {cy}"); } + /// A 1365x945 screen streams 1364x944. A touch on the centre of any + /// pixel of that picture, normalized as the iPad does, lands on that + /// pixel, and the picture's far edge on the column and row left out. + #[test] + fn a_cropped_picture_maps_each_pixel_onto_itself() { + let output = CaptureGeometry { + left: -1365, + top: 0, + width: 1365, + height: 945, + }; + let (width, height) = crate::encoder::encoded_size(output.width, output.height); + assert_eq!((width, height), (1364, 944)); + // TouchRelay.swift: the touch's fraction of the picture, rounded. + let norm = |pixel: u32, pixels: u32| { + ((f64::from(pixel) + 0.5) / f64::from(pixels) * 65_535.0).round() as u16 + }; + for x in 0..width { + for y in [0, height / 2, height - 1] { + assert_eq!( + norm_to_output_pixel(norm(x, width), norm(y, height), output), + (output.left + x as i32, y as i32), + "pixel {x},{y}" + ); + } + } + for y in 0..height { + assert_eq!( + norm_to_output_pixel(norm(width - 1, width), norm(y, height), output), + (-2, y as i32), + "row {y}" + ); + } + assert_eq!(norm_to_output_pixel(65_535, 65_535, output), (-1, 944)); + } + #[test] fn extended_display_offset_is_applied() { // Virtual display sits LEFT of the primary at -1920. diff --git a/host/src/stats.rs b/host/src/stats.rs index f74f03ea..98c9b58a 100644 --- a/host/src/stats.rs +++ b/host/src/stats.rs @@ -18,6 +18,9 @@ pub struct PipelineStats { // Capture pub capture_fps: f64, + /// The size the stream encodes at: capture's, without an odd last + /// column or row (see [`crate::encoder::encoded_size`]). The iPad's + /// stream config, the GUI and the stats summary report it. pub capture_resolution: (u32, u32), pub capture_frame_count: u64, pub gpu_name: String, @@ -207,7 +210,7 @@ impl PipelineStats { pub fn record_capture(&mut self, width: u32, height: u32) { let now = Instant::now(); self.capture_frame_count += 1; - self.capture_resolution = (width, height); + self.capture_resolution = crate::encoder::encoded_size(width, height); self.capture_timestamps.push_back(now); self.capture_fps = Self::calc_fps(&mut self.capture_timestamps, now); } diff --git a/host/src/telemetry/redact.rs b/host/src/telemetry/redact.rs index 92a17d50..f5988f98 100644 --- a/host/src/telemetry/redact.rs +++ b/host/src/telemetry/redact.rs @@ -124,6 +124,8 @@ const DIAGNOSTIC_KEYS: &[&str] = &[ "avoided", "attached", "advertised_formats", + "capture_size", + "encoded_size", "usb_state", "wire_stream", "running", @@ -1940,6 +1942,11 @@ mod tests { "modes", "INFO Second screen mode mode=2732x2048 requested=the driver's modes", ), + ( + "1680x945", + "INFO Encoding without the capture's odd last column or row \ + capture_size=1680x945 encoded_size=1680x944", + ), ("arm64", "INFO EternalMonitor host 0.3.4 revision=x | OS=Windows arch=arm64 logical_cpus=8"), ( "Client", diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index 4a889c10..76039d1e 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -885,6 +885,8 @@ fn free_udp_port() -> u16 { struct H264TestDecoder { decoder: ffmpeg_next::codec::decoder::Video, frame: ffmpeg_next::frame::Video, + /// Width and height of the last picture that came out. + size: (u32, u32), } impl H264TestDecoder { @@ -899,6 +901,7 @@ impl H264TestDecoder { Self { decoder, frame: ffmpeg_next::frame::Video::empty(), + size: (0, 0), } } @@ -911,6 +914,7 @@ impl H264TestDecoder { } let mut counters = Vec::new(); while self.decoder.receive_frame(&mut self.frame).is_ok() { + self.size = (self.frame.width(), self.frame.height()); let stride = self.frame.stride(0); let luma = self.frame.data(0); counters.push(decode_frame_counter_from_luma(luma, stride, SYNTH_W)); @@ -2700,6 +2704,183 @@ fn input_relay_maps_touches_end_to_end() { .expect("supervisor must shut down within 5s"); } +/// A screen with an odd size streams at the even size below it: libx264 +/// refused 1680x945 and the pipeline gave up after six restarts. The iPad +/// is told the size it decodes, the picture is intact, and a touch on its +/// last pixel lands there while its far edge reaches the row and column left +/// out. +#[test] +fn an_odd_screen_streams_cropped_and_input_still_reaches_its_edge() { + use eternal_host::input::{ + desktop_pixel_to_abs, recorder, Injection, VirtualScreen, KIND_TOUCH, + }; + use eternal_wire::v2::control::{InputEvent, FEATURE_WANTS_INPUT}; + const ODD_W: u32 = SYNTH_W + 1; + const ODD_H: u32 = SYNTH_H + 1; + + let _env = test_env(); + init_test_tracing(); + let _ = ffmpeg_next::init(); + + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{ODD_W}x{ODD_H}")); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::remove_var("ETERNAL_DROP"); + + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".to_string()); + let gpu_info = GpuInfo::software_fallback(); + + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor_shared = shared.clone(); + let supervisor_tx_clone = supervisor_tx.clone(); + let supervisor = std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + supervisor_shared, + gpu_info, + supervisor_tx_clone, + supervisor_rx, + ); + }); + + let mut receiver = FakeReceiver::connect_with_caps(listen_port, FEATURE_WANTS_INPUT); + let deadline = Instant::now() + DEADLINE; + let mut reassembler = Reassembler::new(); + let mut decoder = H264TestDecoder::new(); + let mut datagram = [0u8; 2048]; + let mut decoded_counters: Vec = Vec::new(); + let mut heartbeats_seen = 0u32; + let mut highest_seq = 0u32; + while decoded_counters.len() < WANT_DECODED_FRAMES || heartbeats_seen == 0 { + assert!( + Instant::now() < deadline, + "timed out: {} decoded frames, heartbeats={heartbeats_seen}", + decoded_counters.len() + ); + receiver.maybe_report(highest_seq, decoded_counters.len() as u32); + let Ok((len, _)) = receiver.socket.recv_from(&mut datagram) else { + continue; + }; + let bytes = &datagram[..len]; + match classify(bytes) { + Classified::Control(_) => { + // Heartbeats before the first capture carry no size. + if let Ok((_, ControlMessage::Heartbeat(hb))) = parse_control(bytes) { + let size = (hb.stream_config.width, hb.stream_config.height); + if size != (0, 0) { + heartbeats_seen += 1; + assert_eq!(size, (SYNTH_W as u16, SYNTH_H as u16)); + } + } + } + Classified::Media { .. } => { + let (header, payload) = MediaHeader::decode(bytes).expect("valid media datagram"); + highest_seq = highest_seq.max(header.frame_seq); + let AddOutcome::Completed(frame_bytes) = reassembler.add_fragment( + header.frame_seq, + header.frag_index, + header.frag_count, + header.stream_epoch, + payload, + Instant::now(), + ) else { + continue; + }; + let counters = decoder.decode(&frame_bytes); + if !counters.is_empty() { + assert_eq!(decoder.size, (SYNTH_W, SYNTH_H)); + } + decoded_counters.extend(counters); + } + _ => {} + } + } + // Real pictures: the counter in the top-left corner reads back and rises. + for pair in decoded_counters.windows(2) { + assert!(pair[1] > pair[0], "decoded counters must rise: {pair:?}"); + } + + let _ = recorder::take(); + let screen = VirtualScreen { + left: 0, + top: 0, + width: ODD_W, + height: ODD_H, + }; + // TouchRelay.swift: the touch's fraction of the 640x360 picture. + let centre = |pixel: u32, pixels: u32| { + ((f64::from(pixel) + 0.5) / f64::from(pixels) * 65_535.0).round() as u16 + }; + let tap = InputEvent { + input_ver: 1, + kind: KIND_TOUCH, + phase: 0, + buttons: 1, + event_id: 1, + x_norm: centre(SYNTH_W - 1, SYNTH_W), + y_norm: centre(SYNTH_H - 1, SYNTH_H), + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: 0, + modifiers: 0, + client_time_us: 0, + tilt_x: 0, + tilt_y: 0, + }; + for event in [ + tap, + InputEvent { + phase: 1, + event_id: 2, + x_norm: 65_535, + y_norm: 65_535, + ..tap + }, + InputEvent { + phase: 2, + event_id: 3, + x_norm: 65_535, + y_norm: 65_535, + ..tap + }, + ] { + receiver.send(&ControlMessage::InputEvent(event)); + } + let inject_deadline = Instant::now() + Duration::from_secs(5); + while recorder::peek().len() < 5 && Instant::now() < inject_deadline { + std::thread::sleep(Duration::from_millis(50)); + } + std::thread::sleep(Duration::from_millis(150)); + let (lx, ly) = desktop_pixel_to_abs(SYNTH_W as i32 - 1, SYNTH_H as i32 - 1, screen); + let (ex, ey) = desktop_pixel_to_abs(ODD_W as i32 - 1, ODD_H as i32 - 1, screen); + assert_eq!( + recorder::take(), + [ + Injection::MoveAbs { x: lx, y: ly }, + Injection::LeftDown { x: lx, y: ly }, + Injection::MoveAbs { x: ex, y: ey }, + Injection::MoveAbs { x: ex, y: ey }, + Injection::LeftUp { x: ex, y: ey }, + ] + ); + + shared.stop(); + supervisor_tx + .send(SupervisorCommand::Shutdown) + .expect("send shutdown"); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = supervisor.join(); + let _ = done_tx.send(()); + }); + done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("supervisor must shut down within 5s"); +} + /// An encoder crash mid-stream must be contained by the supervisor: automatic /// restart with backoff, a fresh stream epoch, and the SAME session resuming /// (the client never re-handshakes; its assembler resets on the epoch bump). diff --git a/scripts/e2e_ios.sh b/scripts/e2e_ios.sh index 9bdad56c..2129c388 100755 --- a/scripts/e2e_ios.sh +++ b/scripts/e2e_ios.sh @@ -28,6 +28,9 @@ case "$TRANSPORT" in udp|usb|takeover) ;; *) echo "Invalid EM_TRANSPORT: $TRANSP SIZE="${EM_SIZE:-640x360}" SYNTH_W="${SIZE%x*}" SYNTH_H="${SIZE#*x}" +# An odd size streams without its last column or row. +VIDEO_W=$((SYNTH_W / 2 * 2)) +VIDEO_H=$((SYNTH_H / 2 * 2)) SCENARIO="${EM_SCENARIO:-$CODEC-$TRANSPORT}" OUT="${EM_OUTPUT_DIR:-$ROOT/build/e2e/$SCENARIO}" SHOT="${EM_SCREENSHOT:-$ROOT/build/screenshots/e2e-$SCENARIO.png}" @@ -325,8 +328,8 @@ if [ "$TRANSPORT" = takeover ] && ! grep -q 'superseding session in place' "$HOS echo "FAIL: host did not preserve the session during USB takeover"; exit 1 fi -if ! grep -q "w=$SYNTH_W h=$SYNTH_H" <<<"$last_stats"; then - echo "FAIL: decoded resolution mismatch: $last_stats (expected ${SYNTH_W}x${SYNTH_H})" +if ! grep -q "w=$VIDEO_W h=$VIDEO_H" <<<"$last_stats"; then + echo "FAIL: decoded resolution mismatch: $last_stats (expected ${VIDEO_W}x${VIDEO_H})" exit 1 fi @@ -349,7 +352,7 @@ PYBITRATE fi "$ROOT/scripts/screenshot.sh" "$UDID" "$SHOT" -"$ROOT/scripts/pixels.sh" "$SHOT" --video "$SIZE" --assert-pattern > "$OUT/pixels.json" +"$ROOT/scripts/pixels.sh" "$SHOT" --video "${VIDEO_W}x${VIDEO_H}" --assert-pattern > "$OUT/pixels.json" python3 "$ROOT/scripts/e2e_stats.py" "$MEASURED_LOG" --output "$OUT/result.json" \ --scenario "$SCENARIO" --screenshot "$SHOT" --elapsed "$((SECONDS - STARTED))" \ --min-frames "$WANT_DECODED" --duration "$MIN_SECONDS" --min-fps "$MIN_FPS" \ From f88d43b2e3b5fa977007cba6e153e85657c06fc0 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Tue, 3 Nov 2026 17:40:38 -0500 Subject: [PATCH 002/226] Run the odd-screen end-to-end test off Windows, like the other input tests --- host/tests/e2e_synthetic.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index 76039d1e..fee915b7 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -2708,7 +2708,8 @@ fn input_relay_maps_touches_end_to_end() { /// refused 1680x945 and the pipeline gave up after six restarts. The iPad /// is told the size it decodes, the picture is intact, and a touch on its /// last pixel lands there while its far edge reaches the row and column left -/// out. +/// out. Off Windows only: the input recorder stands in for real injection. +#[cfg(not(windows))] #[test] fn an_odd_screen_streams_cropped_and_input_still_reaches_its_edge() { use eternal_host::input::{ From 49753c0dd00b0f0006609faaff945822328f4571 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Thu, 5 Nov 2026 19:20:38 -0500 Subject: [PATCH 003/226] Keep the laptop panel when the virtual display turns on alone On a hybrid laptop, Windows can put the desktop on the virtual display alone when it turns on. The host only logged that, waited 30 s for an attach that could not happen, then turned the display off, after which Windows reported no screen at all and refused every layout until a restart. The display guard now acts while the virtual display is still on. A turn on that leaves it alone, where the PC had a real screen before, switches Windows to Extend, else re-applies the saved layout, else PC screen only. A turn off that finds it alone puts a real screen back first, and does not run if nothing does. A start whose display was left alone falls back to the mirror at once with second_screen_fallback vdd_only, remembered like an attach timeout. --- ARCHITECTURE.md | 5 +- host/src/capture/second_screen.rs | 120 +++++- host/src/diagnostics.rs | 5 +- host/src/display_guard.rs | 684 ++++++++++++++++++++++++++---- host/src/vdd.rs | 3 + 5 files changed, 710 insertions(+), 107 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index eb9f289f..26ad1d4b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -196,7 +196,10 @@ connected client, writes the requested mode first in VDD XML, and reports actual task failures. Setup and the host log the active display paths around every driver change; when a change leaves the PC without any real screen it had, they re-apply the saved layout once, else switch to "PC screen only" if the built-in -panel was on (`display_guard.rs`, `display-guard.ps1`). It also ships a UDP +panel was on (`display_guard.rs`, `display-guard.ps1`). At runtime the host also +acts while the virtual display is still on: a turn on that leaves it alone +switches Windows to "Extend" (else the saved layout or "PC screen only"), and a +turn off never runs while it is the only screen. It also ships a UDP firewall rule. Runtime upgrade, task-ACL and uninstall checks remain hardware gates. diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 785d4fdd..b92c697d 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -1296,6 +1296,12 @@ pub(crate) trait Driver { fn settle(&self, chosen: OutputInfo) -> OutputInfo { chosen } + /// After an Enable that worked: Windows put the desktop on the display + /// alone, so it will not attach beside the PC's screens. The display + /// guard has already tried to bring them back. + fn alone(&self) -> bool { + false + } /// Say why the display did not appear or showed no picture, while it is /// still on. fn report(&self, _failure: Failure) {} @@ -1407,13 +1413,19 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura return; } let waiting_since = driver.now(); - let result = wait_for_virtual_output( - &before, - timeout, - || selected().then(|| driver.outputs()), - || driver.now(), - |duration| driver.sleep(duration), - ); + // No wait for a display that can't attach. + let alone = driver.alone(); + let result = if alone { + AttachResult::TimedOut + } else { + wait_for_virtual_output( + &before, + timeout, + || selected().then(|| driver.outputs()), + || driver.now(), + |duration| driver.sleep(duration), + ) + }; #[cfg(windows)] let result = match result { AttachResult::Attached(_) if super::simulated_attach_timeout() => AttachResult::TimedOut, @@ -1454,11 +1466,19 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura } } AttachResult::TimedOut => { - warn!( - second_screen_fallback = "attach_timeout", - timeout_ms = timeout.as_millis() as u64, - "Virtual display did not attach in time — capturing the primary display instead" - ); + if alone { + warn!( + second_screen_fallback = "vdd_only", + "Windows put the desktop on the virtual display alone — capturing the \ + primary display instead" + ); + } else { + warn!( + second_screen_fallback = "attach_timeout", + timeout_ms = timeout.as_millis() as u64, + "Virtual display did not attach in time — capturing the primary display instead" + ); + } driver.report(Failure::AttachTimeout); // Enable succeeded but nothing attached: turn it back off so no // half-enabled device lingers as a phantom monitor. @@ -1469,6 +1489,8 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura let mut status = shared.vdd_status.lock(); *status = VddStatus::Failed; let mut state = shared.second_screen.lock(); + // Alone too: the same notice and wait, in a record every + // version reads. state.failure = Some("attach_timeout"); state.remember("attach_timeout", now, false, blocked); if blocked && copied { @@ -1595,6 +1617,10 @@ impl Driver for WindowsDriver<'_> { chosen } + fn alone(&self) -> bool { + crate::display_guard::last_turn_on_alone() + } + fn report(&self, failure: Failure) { let request = self.request.borrow(); let device = match failure { @@ -1719,7 +1745,8 @@ mod tests { /// display attaches `attach_after` later, or never. `on_sleep` sees the /// clock each time the host waits, as the transport's ticks would. /// With `enable_fails`, the Enable task reports that instead. Setting - /// the mode after the attach takes `settle_takes`. + /// the mode after the attach takes `settle_takes`. With `alone`, the + /// Enable left the display as the only screen. struct FakeDriver<'a> { start: Instant, elapsed: Cell, @@ -1730,6 +1757,7 @@ mod tests { toggles: RefCell>, on_sleep: &'a dyn Fn(Duration), enable_fails: Option, + alone: bool, calls: RefCell>, } @@ -1749,6 +1777,7 @@ mod tests { toggles: RefCell::new(Vec::new()), on_sleep, enable_fails: None, + alone: false, calls: RefCell::new(Vec::new()), } } @@ -1796,6 +1825,10 @@ mod tests { chosen } + fn alone(&self) -> bool { + self.alone + } + fn report(&self, failure: Failure) { self.calls.borrow_mut().push(Call::Report(failure)); } @@ -2978,6 +3011,67 @@ mod tests { } } + /// The hybrid laptop: the Enable left the virtual display as the only + /// screen, and the display guard gave the panel back without it. The + /// start falls back to the mirror at once, with its own reason, instead + /// of waiting 30 s for a display that cannot attach; the failure is + /// remembered as a timeout's, with the same notice and wait. + #[test] + fn a_display_windows_left_alone_falls_back_at_once() { + use std::sync::Arc; + struct Sink(Arc>>); + impl std::io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let shared = second_screen_selected(); + connect_viewer(&shared); + assert_eq!(next_step(&shared, Instant::now()), Step::Attach); + let on_sleep = |_| {}; + // It would attach later, if it were waited for. + let driver = FakeDriver { + alone: true, + ..FakeDriver::new(secs(7.0), Some(secs(1.0)), &on_sleep) + }; + let out = Arc::new(parking_lot::Mutex::new(Vec::::new())); + let writer = Arc::clone(&out); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || Sink(Arc::clone(&writer))) + .finish(); + tracing::subscriber::with_default(subscriber, || attach(&shared, &driver, ATTACH_TIMEOUT)); + let lines = String::from_utf8(out.lock().clone()).unwrap(); + assert!( + lines.contains( + "Windows put the desktop on the virtual display alone — capturing the primary \ + display instead second_screen_fallback=\"vdd_only\"" + ), + "{lines}" + ); + assert!(!lines.contains("did not attach in time"), "{lines}"); + assert_eq!( + driver.calls(), + [ + Call::Prepare, + Call::Enable, + Call::Report(Failure::AttachTimeout), + Call::Disable + ] + ); + // Turned off after the gap, not after the attach wait. + let toggles = driver.toggles.borrow().clone(); + assert_eq!(toggles[1].0 - toggles[0].0, MIN_TOGGLE_GAP, "{toggles:?}"); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + assert_eq!(shared.vdd_output.lock().as_deref(), None); + assert_eq!(remembered_failure(&shared), Some("attach_timeout")); + assert_eq!(next_step(&shared, driver.now()), Step::Nothing); + } + /// "Windows added the second screen but kept your desktop off it": the /// iPad keeps the mirror, so after Try again the start waits 30 s for /// the user to choose Extend, as long as the usual wait and never added diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 02ccf0d6..251ec553 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -285,7 +285,9 @@ fn take_fallback(session_id: u32) -> Option<&'static str> { } /// A `second_screen_fallback` reason as the summary names it. `remembered` -/// is a failure saved by an earlier app run. +/// is a failure saved by an earlier app run. The backend's check of the +/// summary lists each token, so a newer reason, `vdd_only`, goes as `other` +/// until it lists that one too. fn fallback_token(reason: &str) -> &'static str { match reason { "attach_timeout" => "attach_timeout", @@ -883,6 +885,7 @@ pub(crate) mod tests { ("no_frames", "no_frames"), ("duplication_failed", "duplication_failed"), ("remembered", "remembered"), + ("vdd_only", "other"), ("C:\\Users\\someone", "other"), ] { assert_eq!(fallback_token(reason), token); diff --git a/host/src/display_guard.rs b/host/src/display_guard.rs index 3bdb16a6..d3189e24 100644 --- a/host/src/display_guard.rs +++ b/host/src/display_guard.rs @@ -13,11 +13,20 @@ //! turned on, the guard re-applies that layout, or else switches Windows to //! "PC screen only" when the built-in panel was on, and logs the result. //! Nothing else ever changes the layout, which stays the user's: a PC that -//! keeps any real screen sees no change at all. A turn on is only logged: -//! a layout with the virtual display alone cannot attach, so the attempt -//! times out and its turn off brings the guard in. One attempt per change. -//! Right before "PC screen only" the guard reads the screens again, and -//! stops if one came back or they can't be read. +//! keeps any real screen sees no change at all. +//! +//! Turning the display off while it is the only screen on is what loses the +//! panel for good: on a hybrid laptop, Windows then stops reporting any +//! screen, and refuses every layout until the PC restarts. So the guard acts +//! while the display is still on. A turn on that leaves the virtual display +//! alone, where the PC had a real screen before, switches Windows to +//! "Extend", which keeps the second screen, or else puts the saved layout +//! back, or "PC screen only". A turn off that finds the virtual display +//! alone first puts the saved layout back, "PC screen only", or "Extend", +//! and if none brings a real screen back it leaves the display on: the +//! desktop stays on it, which the iPad and Windows+P can still reach. +//! One attempt per change. Between two calls the guard reads the screens +//! again, and stops if one came back or they can't be read. //! //! The layout from before a turn on is also kept on disk, so the failure //! reporter, a new process, compares with it when the host that turned @@ -131,6 +140,8 @@ pub trait Displays { fn reapply(&mut self, saved: &Self::Saved) -> Result<(), i32>; /// `SetDisplayConfig` with "PC screen only". fn internal_only(&mut self) -> Result<(), i32>; + /// `SetDisplayConfig` with "Extend". + fn extend(&mut self) -> Result<(), i32>; fn sleep(&mut self, duration: Duration); fn log(&mut self, warn: bool, text: &str); /// The host whose layout from before its turn on is kept on disk: this @@ -152,11 +163,15 @@ pub trait Saved: Sized { fn decode(text: &str) -> Option; } -/// How a turn off ended. +/// How a change ended. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Outcome { /// Nothing to do, or nothing known to compare with. Left, + /// A turn on left the virtual display alone, and "Extend" put the PC's + /// screens back beside it. + Extended, + /// The PC's screens are back, without the virtual display. Restored { internal: bool, }, @@ -265,6 +280,9 @@ pub fn from_hex(text: &str) -> Option> { /// next turn off checks against it. pub struct Guard { reference: Option>, + /// The last turn on left the virtual display alone, and "Extend" did + /// not put the PC's screens beside it: it will not attach. + alone: bool, } impl Default for Guard { @@ -275,22 +293,65 @@ impl Default for Guard { impl Guard { pub const fn new() -> Self { - Self { reference: None } + Self { + reference: None, + alone: false, + } + } + + /// Whether the last turn on left the virtual display alone, with no + /// real screen beside it. + pub fn alone(&self) -> bool { + self.alone } } +/// A `SetDisplayConfig` call that can bring a real screen back while the +/// virtual display is on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Step { + /// "Extend": the PC's screens beside the virtual display. + Extend, + /// The layout from before the turn on, without the virtual display. + SavedLayout, + /// "PC screen only". + Internal, +} + +impl Step { + const fn name(self) -> &'static str { + match self { + Self::Extend => "extend", + Self::SavedLayout => "saved_layout", + Self::Internal => "internal", + } + } +} + +/// How [`take_back`] ended. +enum TakeBack { + /// This call brought a real screen back. + Applied(Step, Option>), + /// A real screen came back between two calls. + CameBack(Option>), + /// None did: each call's result, and the screens now. + Failed(Vec, Option>), +} + impl Guard { /// Before a driver change: log the screens. A turn on keeps them to /// compare with, also on disk for the failure reporter. A turn off /// with nothing kept, as at startup or in the reporter, compares with /// the record its host kept, else with the screens just before it. - /// `display_off`: the driver tasks know the display is off. + /// `display_off`: the driver tasks know the display is off. False when + /// the turn off must not run: the virtual display is the only screen + /// on, the PC had a real one, and nothing brought it back. pub fn before( &mut self, enabled: bool, display_off: bool, displays: &mut impl Displays, - ) { + ) -> bool { let layout = displays.read(); let text = format!( "Screens before the virtual display turns {}: {}", @@ -298,6 +359,7 @@ impl Guard { line(layout.as_ref().map(|layout| &layout.screens[..])) ); displays.log(false, &text); + let alone = layout.as_ref().is_some_and(|layout| alone(&layout.screens)); if enabled { // A turn on can run while the display is still on: after an // Enable that finished once the host had given up on it, or a @@ -332,11 +394,69 @@ impl Guard { } self.reference = kept.or(layout); } + if !enabled && alone { + return self.take_back_before_off(displays); + } + true } - /// After a driver change: log the screens, and after a turn off, - /// restore them if every real screen is gone. A turn off that `done` - /// deletes the kept record once checked. + /// The virtual display is the only screen on, and is about to turn off: + /// put a real screen back first. False when none came back and the PC + /// had one before, so the turn off must not run. + fn take_back_before_off(&mut self, displays: &mut impl Displays) -> bool { + let reference = self.reference.as_ref(); + let had_real = reference.is_some_and(|layout| real(&layout.screens).next().is_some()); + let internal = reference.is_some_and(|layout| real(&layout.screens).any(|s| s.internal)); + displays.log( + true, + &format!( + "The virtual display is the only screen on; bringing the PC's screens back \ + before turning it off. Before: {}", + line(reference.map(|layout| &layout.screens[..])) + ), + ); + let mut steps = Vec::new(); + if had_real { + steps.push(Step::SavedLayout); + } + if internal { + steps.push(Step::Internal); + } + steps.push(Step::Extend); + match take_back(displays, &steps, reference) { + TakeBack::Applied(step, now) => { + restored(displays, step.name(), &now); + true + } + TakeBack::CameBack(now) => { + came_back(displays, &now); + true + } + TakeBack::Failed(tried, now) if had_real => { + let text = format!( + "Leaving the virtual display on: turning it off now would leave the PC with \ + no screen outcome=kept_on {}{}; Windows+P then \"PC screen only\" brings \ + the built-in screen back", + tried + .iter() + .map(|step| format!("{step} ")) + .collect::(), + line(now.as_ref().map(|layout| &layout.screens[..])) + ); + displays.log(true, &text); + false + } + TakeBack::Failed(tried, now) => { + failed(displays, &tried, &now); + true + } + } + } + + /// After a driver change: log the screens. After a turn on that left + /// the virtual display alone, put the PC's screens back; after a turn + /// off, restore them if every real screen is gone. A turn off that + /// `done` deletes the kept record once checked. pub fn after( &mut self, enabled: bool, @@ -344,13 +464,9 @@ impl Guard { displays: &mut impl Displays, ) -> Outcome { if enabled { - let layout = displays.read(); - let text = format!( - "Screens after the virtual display turned on: {}", - line(layout.as_ref().map(|layout| &layout.screens[..])) - ); - displays.log(false, &text); - return Outcome::Left; + let outcome = self.check_turn_on(displays); + self.alone = matches!(outcome, Outcome::Restored { .. } | Outcome::Failed); + return outcome; } let outcome = self.check(displays); if done { @@ -359,6 +475,55 @@ impl Guard { outcome } + /// Windows may have put the desktop on the virtual display alone. A + /// turn off then would lose the PC's screens, so bring them back now, + /// while Windows still reports them: beside it with "Extend", which + /// keeps the second screen, else without it. + fn check_turn_on(&mut self, displays: &mut impl Displays) -> Outcome { + let reference = self.reference.as_ref(); + let before = reference.map_or(&[][..], |layout| &layout.screens[..]); + let layout = settle(displays, before); + let text = format!( + "Screens after the virtual display turned on: {}", + line(layout.as_ref().map(|layout| &layout.screens[..])) + ); + displays.log(false, &text); + let (Some(reference), Some(after)) = (reference, layout) else { + return Outcome::Left; + }; + let Plan::Restore { internal } = plan(&reference.screens, &after.screens) else { + return Outcome::Left; + }; + displays.log( + true, + &format!( + "Windows put the desktop on the virtual display alone; bringing the PC's \ + screens back while it is on. Before: {}", + line(Some(&reference.screens)) + ), + ); + let mut steps = vec![Step::Extend, Step::SavedLayout]; + if internal { + steps.push(Step::Internal); + } + match take_back(displays, &steps, Some(reference)) { + TakeBack::Applied(step, now) + if now.as_ref().is_some_and(|now| has_vdd(&now.screens)) => + { + let text = format!( + "Screens back beside the virtual display outcome=extended applied={} {}", + step.name(), + line(now.as_ref().map(|layout| &layout.screens[..])) + ); + displays.log(false, &text); + Outcome::Extended + } + TakeBack::Applied(step, now) => restored(displays, step.name(), &now), + TakeBack::CameBack(now) => came_back(displays, &now), + TakeBack::Failed(tried, now) => failed(displays, &tried, &now), + } + } + fn check(&mut self, displays: &mut impl Displays) -> Outcome { let reference = self.reference.take(); let before = reference @@ -384,52 +549,61 @@ impl Guard { line(Some(&reference.screens)) ), ); - // Each step's result, for the log when nothing worked. - let mut tried = Vec::new(); - match displays.reapply(&reference.saved) { + let mut steps = vec![Step::SavedLayout]; + // A switch to "PC screen only" would turn off an external monitor + // the user had on alone. + if internal { + steps.push(Step::Internal); + } + match take_back(displays, &steps, Some(&reference)) { + TakeBack::Applied(step, now) => restored(displays, step.name(), &now), + TakeBack::CameBack(now) => came_back(displays, &now), + TakeBack::Failed(tried, now) => failed(displays, &tried, &now), + } + } +} + +/// Make each call of `steps` in turn until a real screen is on, waiting up +/// to [`SETTLE`] after each one Windows takes. `reference` is the layout +/// from before the turn on, for [`Step::SavedLayout`]. +fn take_back( + displays: &mut D, + steps: &[Step], + reference: Option<&Layout>, +) -> TakeBack { + // Each call's result, for the log when nothing worked. + let mut tried = Vec::new(); + let mut now = None; + for &step in steps { + let result = match (step, reference) { + (Step::Extend, _) => displays.extend(), + (Step::SavedLayout, Some(reference)) => displays.reapply(&reference.saved), + (Step::SavedLayout, None) => continue, + (Step::Internal, _) => displays.internal_only(), + }; + match result { Ok(()) => { - let now = settle(displays, &reference.screens); + now = wait_for_real(displays); if has_real(&now) { - return restored(displays, "saved_layout", &now); + return TakeBack::Applied(step, now); } - tried.push("reason=saved_layout_no_screen".to_string()); + tried.push(format!("reason={}_no_screen", step.name())); } - Err(code) => tried.push(format!("reason=saved_layout_refused code={code}")), + Err(code) => tried.push(format!("reason={}_refused code={code}", step.name())), } // Windows may have changed the screens during that call. Look again // right before changing anything more: a screen that came back // stays as Windows set it. - let now = displays.read(); + now = displays.read(); if has_real(&now) { - let text = format!( - "Screens came back without a change outcome=restored applied=none {}", - line(now.as_ref().map(|layout| &layout.screens[..])) - ); - displays.log(false, &text); - return Outcome::Recovered; + return TakeBack::CameBack(now); } if now.is_none() { tried.push("reason=screens_unknown".into()); - return failed(displays, &tried, &now); - } - if !internal { - return failed(displays, &tried, &now); - } - match displays.internal_only() { - Ok(()) => { - let now = settle(displays, &reference.screens); - if has_real(&now) { - return restored(displays, "internal", &now); - } - tried.push("reason=internal_no_screen".into()); - failed(displays, &tried, &now) - } - Err(code) => { - tried.push(format!("reason=internal_refused code={code}")); - failed(displays, &tried, &now) - } + break; } } + TakeBack::Failed(tried, now) } /// The layout in the record this host kept, if there is one. @@ -453,6 +627,15 @@ fn restored( } } +fn came_back(displays: &mut D, now: &Option>) -> Outcome { + let text = format!( + "Screens came back without a change outcome=restored applied=none {}", + line(now.as_ref().map(|layout| &layout.screens[..])) + ); + displays.log(false, &text); + Outcome::Recovered +} + fn failed( displays: &mut D, tried: &[String], @@ -477,15 +660,33 @@ fn has_real(layout: &Option>) -> bool { .is_some_and(|layout| real(&layout.screens).next().is_some()) } +fn has_vdd(screens: &[Screen]) -> bool { + screens.iter().any(|screen| screen.virtual_display) +} + +/// The virtual display is on, and no real screen is. +fn alone(screens: &[Screen]) -> bool { + has_vdd(screens) && real(screens).next().is_none() +} + /// The screens once a real one is back, or as they are after [`SETTLE`]. A /// PC that has one returns at once; only one that lost them all waits. fn settle(displays: &mut D, before: &[Screen]) -> Option> { + if real(before).next().is_none() { + return displays.read(); + } + wait_for_real(displays) +} + +/// The screens once a real one is on, or as they are after [`SETTLE`], or +/// once they can't be read. +fn wait_for_real(displays: &mut D) -> Option> { let mut waited = Duration::ZERO; loop { let layout = displays.read(); let lost = layout .as_ref() - .is_some_and(|layout| plan(before, &layout.screens) != Plan::Leave); + .is_some_and(|layout| real(&layout.screens).next().is_none()); if !lost || waited >= SETTLE { return layout; } @@ -500,7 +701,8 @@ static GUARD: parking_lot::Mutex> = /// Run a driver change for `owner`'s display with the guard around it. /// `log` writes one line; `display_off` says the driver tasks know the -/// display is off; `done` tells a change that succeeded. +/// display is off; `done` tells a change that succeeded. A turn off that +/// would leave the PC with no screen does not run: `kept_on` is its result. #[cfg(windows)] pub(crate) fn around( enabled: bool, @@ -509,15 +711,25 @@ pub(crate) fn around( display_off: bool, change: impl FnOnce() -> T, done: impl FnOnce(&T) -> bool, + kept_on: impl FnOnce() -> T, ) -> T { let mut displays = windows_impl::Windows::new(owner, log); - GUARD.lock().before(enabled, display_off, &mut displays); + if !GUARD.lock().before(enabled, display_off, &mut displays) { + return kept_on(); + } let result = change(); let done = done(&result); GUARD.lock().after(enabled, done, &mut displays); result } +/// Whether the last turn on left the virtual display alone, with no real +/// screen beside it: then it does not attach as a second screen. +#[cfg(windows)] +pub(crate) fn last_turn_on_alone() -> bool { + GUARD.lock().alone() +} + #[cfg(windows)] mod windows_impl { use super::{Layout, Screen}; @@ -527,8 +739,8 @@ mod windows_impl { DISPLAYCONFIG_OUTPUT_TECHNOLOGY_DISPLAYPORT_EMBEDDED, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_INTERNAL, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_LVDS, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_UDI_EMBEDDED, DISPLAYCONFIG_PATH_INFO, - QDC_ONLY_ACTIVE_PATHS, SDC_ALLOW_CHANGES, SDC_APPLY, SDC_TOPOLOGY_INTERNAL, - SDC_USE_SUPPLIED_DISPLAY_CONFIG, SDC_VALIDATE, + QDC_ONLY_ACTIVE_PATHS, SDC_ALLOW_CHANGES, SDC_APPLY, SDC_TOPOLOGY_EXTEND, + SDC_TOPOLOGY_INTERNAL, SDC_USE_SUPPLIED_DISPLAY_CONFIG, SDC_VALIDATE, }; pub type Saved = (Vec, Vec); @@ -656,6 +868,13 @@ mod windows_impl { } } + fn extend(&mut self) -> Result<(), i32> { + match unsafe { SetDisplayConfig(None, None, SDC_APPLY | SDC_TOPOLOGY_EXTEND) } { + 0 => Ok(()), + code => Err(code), + } + } + fn sleep(&mut self, duration: Duration) { std::thread::sleep(duration); } @@ -756,13 +975,16 @@ mod tests { /// After an accepted call, reads return this. reapplied: Option>, internal: Option>, + extended: Option>, refuse_reapply: bool, refuse_internal: bool, + refuse_extend: bool, /// What reads return after a refused call: Windows changed the /// screens meanwhile. `Some(None)`: they can't be read. after_refused: Option>>, reapply_calls: Vec, internal_calls: usize, + extend_calls: usize, slept: Duration, logs: Vec<(bool, String)>, owner: Option, @@ -789,7 +1011,7 @@ mod tests { } fn set_display_config_calls(&self) -> usize { - self.reapply_calls.len() + self.internal_calls + self.reapply_calls.len() + self.internal_calls + self.extend_calls } fn text(&self) -> String { @@ -840,6 +1062,18 @@ mod tests { Ok(()) } + fn extend(&mut self) -> Result<(), i32> { + self.extend_calls += 1; + if self.refuse_extend { + return Err(1610); + } + if let Some(screens) = self.extended.clone() { + self.reads.clear(); + self.last = Some(screens); + } + Ok(()) + } + fn sleep(&mut self, duration: Duration) { self.slept += duration; } @@ -901,6 +1135,215 @@ mod tests { assert_eq!(plan(&[], &[]), Plan::Leave); } + /// The hybrid laptop where the panel went black: its panel, on the + /// integrated card, was the only screen before the turn on. + fn laptop_panel() -> Screen { + Screen { + adapter: 0x1298b, + target: 512, + internal: true, + virtual_display: false, + mode: Some((1920, 1200, 165_000)), + } + } + + /// After its turn on, Windows showed the desktop on the virtual display + /// alone, at 800x600. + fn lone_vdd() -> Screen { + Screen { + mode: Some((800, 600, 60_000)), + ..vdd() + } + } + + /// The guard's check before a turn off: whether it may run. + fn turn_off_allowed(guard: &mut Guard, fake: &mut Fake) -> bool { + guard.before(false, false, fake) + } + + /// The laptop's log: `paths=1 [internal …] vdd=0` before the turn on, + /// `paths=0 [] vdd=1` after it. The 0.3.4 guard only logged that, the + /// attempt timed out, and its turn off left Windows reporting no screen + /// at all (`paths=unknown`), which nothing but a restart repaired. Now + /// "Extend" puts the panel back beside the virtual display at once, + /// and the second screen can still attach. + #[test] + fn a_turn_on_that_leaves_the_virtual_display_alone_extends_at_once() { + let mut fake = Fake::reading([vec![laptop_panel()], vec![lone_vdd()]]); + fake.extended = Some(vec![laptop_panel(), lone_vdd()]); + let mut guard = Guard::new(); + assert!(guard.before(true, false, &mut fake)); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Extended); + assert_eq!( + ( + fake.extend_calls, + fake.reapply_calls.len(), + fake.internal_calls + ), + (1, 0, 0) + ); + assert!(!guard.alone()); + let text = fake.text(); + assert!( + text.contains( + "Screens after the virtual display turned on: paths=0 [] vdd=1\n\ + Windows put the desktop on the virtual display alone; bringing the PC's \ + screens back while it is on. Before: paths=1 [internal tgt=1298b/512 \ + 1920x1200@165Hz] vdd=0\n\ + Screens back beside the virtual display outcome=extended applied=extend \ + paths=1 [internal tgt=1298b/512 1920x1200@165Hz] vdd=1" + ), + "{text}" + ); + // The session's turn off then finds the panel on, as on any PC. + assert!(turn_off_allowed(&mut guard, &mut fake)); + fake.reads.push_back(Some(vec![laptop_panel()])); + assert_eq!(guard.after(false, true, &mut fake), Outcome::Left); + assert_eq!(fake.set_display_config_calls(), 1); + } + + /// Where "Extend" does not bring the panel back, the layout from before + /// the turn on does, without the virtual display: the start falls back + /// to the mirror, and its turn off finds the panel on. + #[test] + fn a_lone_virtual_display_that_extend_cannot_fix_gives_the_panel_back() { + let mut fake = Fake::reading([vec![laptop_panel()], vec![lone_vdd()]]); + fake.refuse_extend = true; + fake.reapplied = Some(vec![laptop_panel()]); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!( + guard.after(true, true, &mut fake), + Outcome::Restored { internal: false } + ); + assert_eq!((fake.extend_calls, &fake.reapply_calls[..]), (1, &[1][..])); + assert!(guard.alone(), "the display will not attach"); + assert!( + fake.text().ends_with( + "Screens restored outcome=restored applied=saved_layout paths=1 \ + [internal tgt=1298b/512 1920x1200@165Hz] vdd=0" + ), + "{}", + fake.text() + ); + assert!(turn_off_allowed(&mut guard, &mut fake)); + // Even a turn off after which Windows reports nothing, as on the + // laptop, changes nothing more: the panel was on before it. + fake.reads.push_back(None); + assert_eq!(guard.after(false, true, &mut fake), Outcome::Left); + assert_eq!(fake.set_display_config_calls(), 2); + + // Nothing works: the display stays alone and will not attach. + let mut fake = Fake::reading([vec![laptop_panel()], vec![lone_vdd()]]); + (fake.refuse_extend, fake.refuse_reapply) = (true, true); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); + assert_eq!((fake.extend_calls, fake.reapply_calls.len()), (1, 1)); + // "PC screen only" too: the panel is the built-in one. + assert_eq!(fake.internal_calls, 1); + assert!(guard.alone()); + // The next turn on that extends clears it. + let mut fake = Fake::reading([vec![laptop_panel()], vec![laptop_panel(), lone_vdd()]]); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); + assert!(!guard.alone()); + } + + /// The virtual display ended up alone during the session, Windows+P + /// "Second screen only" say. Its turn off would leave Windows with no + /// screen to report, so the guard puts the panel back first. + #[test] + fn a_turn_off_with_the_virtual_display_alone_puts_a_real_screen_back_first() { + let mut fake = Fake::reading([ + vec![laptop_panel()], + vec![laptop_panel(), lone_vdd()], + vec![lone_vdd()], + ]); + fake.reapplied = Some(vec![laptop_panel()]); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); + assert!(turn_off_allowed(&mut guard, &mut fake)); + // The layout from before the turn on, before the turn off runs. + assert_eq!(fake.reapply_calls, [1]); + let text = fake.text(); + assert!( + text.ends_with( + "Screens before the virtual display turns off: paths=0 [] vdd=1\n\ + The virtual display is the only screen on; bringing the PC's screens back \ + before turning it off. Before: paths=1 [internal tgt=1298b/512 \ + 1920x1200@165Hz] vdd=0\n\ + Screens restored outcome=restored applied=saved_layout paths=1 \ + [internal tgt=1298b/512 1920x1200@165Hz] vdd=0" + ), + "{text}" + ); + assert_eq!(guard.after(false, true, &mut fake), Outcome::Left); + assert_eq!(fake.set_display_config_calls(), 1); + } + + /// When nothing brings a real screen back, the turn off does not run: + /// the desktop stays on the virtual display, where the iPad and + /// Windows+P still reach it, rather than on no screen at all. + #[test] + fn the_virtual_display_is_never_turned_off_while_it_is_the_only_screen() { + let mut fake = Fake::reading([ + vec![laptop_panel()], + vec![laptop_panel(), lone_vdd()], + vec![lone_vdd()], + ]); + fake.owner = Some(HOST.into()); + (fake.refuse_reapply, fake.refuse_extend) = (true, true); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + guard.after(true, true, &mut fake); + assert!(!turn_off_allowed(&mut guard, &mut fake)); + // Every call, once: the layout, "PC screen only", "Extend". + assert_eq!( + ( + fake.reapply_calls.len(), + fake.internal_calls, + fake.extend_calls + ), + (1, 1, 1) + ); + let (warn, text) = fake.logs.last().unwrap(); + assert!(*warn); + assert_eq!( + text, + "Leaving the virtual display on: turning it off now would leave the PC with no \ + screen outcome=kept_on reason=saved_layout_refused code=87 \ + reason=internal_no_screen reason=extend_refused code=1610 paths=0 [] vdd=1; \ + Windows+P then \"PC screen only\" brings the built-in screen back" + ); + // The layout stays kept, in memory and for the failure reporter. + assert!(fake.kept.is_some()); + // The next turn off, the quit's say, tries again with it. + fake.refuse_reapply = false; + fake.reapplied = Some(vec![laptop_panel()]); + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.reapply_calls, [1, 1]); + guard.after(false, true, &mut fake); + assert_eq!(fake.kept, None); + + // A PC that had no real screen before turns it off as it did. + let mut fake = Fake::reading([vec![], vec![lone_vdd()], vec![lone_vdd()]]); + fake.refuse_extend = true; + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!( + ( + fake.reapply_calls.len(), + fake.internal_calls, + fake.extend_calls + ), + (0, 0, 1) + ); + } + #[test] fn a_healthy_pc_sees_no_set_display_config_call() { // Panel and monitor stay through the session; the monitor as the @@ -925,13 +1368,6 @@ mod tests { ]); assert_eq!(session(&mut fake).1, Outcome::Left); assert_eq!(fake.set_display_config_calls(), 0); - // A turn on that leaves the virtual display alone is only logged: - // the layout stays the user's until the turn off. - let mut fake = Fake::reading([vec![panel()], vec![vdd()]]); - let mut guard = Guard::new(); - guard.before(true, false, &mut fake); - assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); - assert_eq!(fake.set_display_config_calls(), 0); } #[test] @@ -975,8 +1411,13 @@ mod tests { #[test] fn every_real_screen_lost_reapplies_the_layout_from_before_the_turn_on() { - // The desktop moved to the virtual display alone, then nothing. - let mut fake = Fake::reading([vec![panel(), monitor()], vec![vdd()], vec![vdd()], vec![]]); + // Both screens went with the virtual display's turn off. + let mut fake = Fake::reading([ + vec![panel(), monitor()], + vec![panel(), monitor(), vdd()], + vec![panel(), monitor(), vdd()], + vec![], + ]); fake.reapplied = Some(vec![panel(), monitor()]); let (_, outcome) = session(&mut fake); assert_eq!(outcome, Outcome::Restored { internal: false }); @@ -1006,7 +1447,13 @@ mod tests { #[test] fn a_screen_windows_brings_back_by_itself_is_left_alone() { - let mut fake = Fake::reading([vec![panel()], vec![vdd()], vec![vdd()], vec![], vec![]]); + let mut fake = Fake::reading([ + vec![panel()], + vec![panel(), vdd()], + vec![panel(), vdd()], + vec![], + vec![], + ]); fake.reads.push_back(Some(vec![panel()])); assert_eq!(session(&mut fake).1, Outcome::Left); assert_eq!(fake.set_display_config_calls(), 0); @@ -1015,7 +1462,12 @@ mod tests { #[test] fn a_refused_layout_falls_back_to_pc_screen_only_when_the_panel_was_on() { - let mut fake = Fake::reading([vec![panel(), monitor()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel(), monitor()], + vec![panel(), monitor(), vdd()], + vec![panel(), monitor(), vdd()], + vec![], + ]); fake.refuse_reapply = true; fake.internal = Some(vec![panel()]); assert_eq!(session(&mut fake).1, Outcome::Restored { internal: true }); @@ -1029,7 +1481,12 @@ mod tests { fake.text() ); // A layout Windows took that brought no screen back counts as failed too. - let mut fake = Fake::reading([vec![panel()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel()], + vec![panel(), vdd()], + vec![panel(), vdd()], + vec![], + ]); fake.internal = Some(vec![panel()]); assert_eq!(session(&mut fake).1, Outcome::Restored { internal: true }); assert_eq!((fake.reapply_calls.len(), fake.internal_calls), (1, 1)); @@ -1038,7 +1495,12 @@ mod tests { #[test] fn without_the_panel_before_there_is_no_switch_to_it() { // A closed laptop on an external monitor that went dark. - let mut fake = Fake::reading([vec![monitor()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![monitor()], + vec![monitor(), vdd()], + vec![monitor(), vdd()], + vec![], + ]); fake.refuse_reapply = true; assert_eq!(session(&mut fake).1, Outcome::Failed); assert_eq!((fake.reapply_calls.len(), fake.internal_calls), (1, 0)); @@ -1054,7 +1516,12 @@ mod tests { #[test] fn one_attempt_per_change() { - let mut fake = Fake::reading([vec![panel()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel()], + vec![panel(), vdd()], + vec![panel(), vdd()], + vec![], + ]); (fake.refuse_reapply, fake.refuse_internal) = (true, true); let (mut guard, outcome) = session(&mut fake); assert_eq!(outcome, Outcome::Failed); @@ -1104,7 +1571,7 @@ mod tests { fake.text() ); // Known before, unknown after. - let mut fake = Fake::reading([vec![panel()], vec![vdd()], vec![vdd()]]); + let mut fake = Fake::reading([vec![panel()], vec![panel(), vdd()], vec![panel(), vdd()]]); fake.reads.push_back(None); assert_eq!(session(&mut fake).1, Outcome::Left); assert_eq!(fake.set_display_config_calls(), 0); @@ -1113,7 +1580,12 @@ mod tests { #[test] fn the_fallback_looks_again_and_leaves_screens_windows_brought_back() { // Windows put both screens back while it refused the saved layout. - let mut fake = Fake::reading([vec![panel(), monitor()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel(), monitor()], + vec![panel(), monitor(), vdd()], + vec![panel(), monitor(), vdd()], + vec![], + ]); fake.refuse_reapply = true; fake.after_refused = Some(Some(vec![panel(), monitor()])); fake.internal = Some(vec![panel()]); @@ -1130,7 +1602,12 @@ mod tests { fake.text() ); // The same after a layout Windows took that showed nothing at first. - let mut fake = Fake::reading([vec![panel(), monitor()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel(), monitor()], + vec![panel(), monitor(), vdd()], + vec![panel(), monitor(), vdd()], + vec![], + ]); fake.internal = Some(vec![panel()]); let mut guard = Guard::new(); guard.before(true, false, &mut fake); @@ -1163,6 +1640,9 @@ mod tests { fn internal_only(&mut self) -> Result<(), i32> { self.0.internal_only() } + fn extend(&mut self) -> Result<(), i32> { + self.0.extend() + } fn sleep(&mut self, duration: Duration) { self.0.sleep(duration) } @@ -1185,7 +1665,12 @@ mod tests { #[test] fn the_fallback_stops_when_the_screens_cannot_be_read() { - let mut fake = Fake::reading([vec![panel()], vec![vdd()], vec![vdd()], vec![]]); + let mut fake = Fake::reading([ + vec![panel()], + vec![panel(), vdd()], + vec![panel(), vdd()], + vec![], + ]); fake.refuse_reapply = true; fake.after_refused = Some(None); fake.internal = Some(vec![panel()]); @@ -1203,19 +1688,19 @@ mod tests { const HOST: &str = "4242-133456789012345678"; - /// The host turned the display on, which left the virtual display alone, - /// then crashed. Its failure reporter, a new process with nothing in - /// memory, turns the display off and the PC has no screen. + /// The host turned the display on, then crashed. Its failure reporter, + /// a new process with nothing in memory, turns the display off, which + /// takes the PC's screens with it. #[test] fn the_failure_reporter_restores_the_layout_its_host_kept() { - let mut host = Fake::reading([vec![panel(), monitor()], vec![vdd()]]); + let mut host = Fake::reading([vec![panel(), monitor()], vec![panel(), monitor(), vdd()]]); host.owner = Some(HOST.into()); let mut guard = Guard::new(); guard.before(true, false, &mut host); guard.after(true, true, &mut host); let kept = host.kept.clone().expect("the host kept its screens"); - let mut reporter = Fake::reading([vec![vdd()], vec![]]); + let mut reporter = Fake::reading([vec![panel(), monitor(), vdd()], vec![]]); reporter.read_count = 100; reporter.owner = Some(HOST.into()); reporter.kept = Some(kept.clone()); @@ -1239,16 +1724,31 @@ mod tests { // Checked after a turn off that worked: deleted. assert_eq!(reporter.kept, None); - // Without the record, the reporter only sees the virtual display before: - // nothing to compare with, the screen stays black. This is what the - // record fixes. + // The host's display was left on alone: the reporter puts that + // layout back before its turn off. + let mut reporter = Fake::reading([vec![vdd()]]); + reporter.read_count = 100; + reporter.owner = Some(HOST.into()); + reporter.kept = Some(kept); + reporter.reapplied = Some(vec![panel(), monitor()]); + let mut guard = Guard::new(); + assert!(guard.before(false, false, &mut reporter)); + assert_eq!(reporter.reapply_calls, [1]); + assert_eq!(guard.after(false, true, &mut reporter), Outcome::Left); + + // Without the record, the reporter only sees the virtual display + // before: no layout to put back, only "Extend", which found no + // screen here. This is what the record fixes. let mut reporter = Fake::reading([vec![vdd()], vec![]]); reporter.owner = Some(HOST.into()); reporter.reapplied = Some(vec![panel()]); let mut guard = Guard::new(); - guard.before(false, false, &mut reporter); + assert!(guard.before(false, false, &mut reporter)); assert_eq!(guard.after(false, true, &mut reporter), Outcome::Left); - assert_eq!(reporter.set_display_config_calls(), 0); + assert_eq!( + (reporter.reapply_calls.len(), reporter.extend_calls), + (0, 1) + ); } #[test] @@ -1282,7 +1782,9 @@ mod tests { Outcome::Left, "{record}" ); - assert_eq!(reporter.set_display_config_calls(), 0, "{record}"); + // Only "Extend", which needs no layout. + assert!(reporter.reapply_calls.is_empty(), "{record}"); + assert_eq!(reporter.internal_calls, 0, "{record}"); // Another host's record stays, for its own reporter. if owner != Some(HOST) { assert_eq!(reporter.kept.as_deref(), Some(record.as_str())); @@ -1391,14 +1893,12 @@ mod tests { // The record on disk still holds the first read, for the reporter. let kept = parse::(HOST, fake.kept.as_deref().unwrap()).unwrap(); assert_eq!((kept.screens.len(), kept.saved), (2, 1)); - guard.after(true, true, &mut fake); - guard.before(false, false, &mut fake); - fake.reads.push_back(Some(vec![])); + // "Extend" brings nothing back here; the first read's layout does. assert_eq!( - guard.after(false, true, &mut fake), + guard.after(true, true, &mut fake), Outcome::Restored { internal: false } ); - assert_eq!(fake.reapply_calls, [1]); + assert_eq!((fake.extend_calls, &fake.reapply_calls[..]), (1, &[1][..])); // The same retry in a host whose layout is only on disk, as after a // failed Disable that left nothing in memory. @@ -1436,7 +1936,7 @@ mod tests { let mut host = Fake::reading([vec![panel()]]); host.owner = Some(HOST.into()); Guard::new().before(true, false, &mut host); - let mut reporter = Fake::reading([vec![vdd()], vec![]]); + let mut reporter = Fake::reading([vec![panel(), vdd()], vec![]]); reporter.owner = Some(HOST.into()); reporter.kept = host.kept.clone(); reporter.reapplied = Some(vec![panel()]); diff --git a/host/src/vdd.rs b/host/src/vdd.rs index 23e3cb38..ffd1ed03 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -472,6 +472,8 @@ fn request_on_turn( display_off, || run_task(if enabled { TASK_ENABLE } else { TASK_DISABLE }, progress), |outcome| *outcome == TaskOutcome::Done, + // Left on: the next turn off tries again. + || TaskOutcome::Failed(None), ); outcome == TaskOutcome::Done }); @@ -539,6 +541,7 @@ pub(crate) fn disable_for_ended( false, || run_task(TASK_DISABLE, &mut || {}), |outcome| *outcome == TaskOutcome::Done, + || TaskOutcome::Failed(None), ) == TaskOutcome::Done }, ) From 53a66844d1862349792f2658426f4a4841c8436b Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Thu, 5 Nov 2026 19:21:41 -0500 Subject: [PATCH 004/226] Say when Windows Firewall keeps iPads out, and let Setup remove the block An iPad whose HELLO never reaches the PC says "Your PC didn't answer" while the PC says it is ready, and neither side could say why. - Setup now deletes every inbound rule for EternalMonitor-host.exe before it adds its Allow rule. Windows adds inbound Block rules for an app when its "allow access" prompt is dismissed, and a Block rule beats an Allow rule, so running Setup again, the fix the iPad suggests, left the block in place. - The host reads the firewall policy at startup and every minute and logs the profiles in use, the Allow and Block rules for its own exe and a verdict. The Stream page warns while a Block rule or "Blocks all incoming connections" keeps iPads out. - The network line names the adapter kinds that hold a home network address, which tells a PC on a cable from one on Wi-Fi. --- host/Cargo.toml | 2 + host/src/firewall.rs | 649 +++++++++++++++++++++++++++++++++++ host/src/gui/page_stream.rs | 30 ++ host/src/gui/tests.rs | 50 ++- host/src/lib.rs | 1 + host/src/main.rs | 1 + host/src/network.rs | 39 ++- host/src/telemetry/redact.rs | 1 + installer/EternalMonitor.iss | 4 + 9 files changed, 766 insertions(+), 11 deletions(-) create mode 100644 host/src/firewall.rs diff --git a/host/Cargo.toml b/host/Cargo.toml index 165e0bca..d8ece94d 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -86,6 +86,7 @@ features = [ "Win32_System_Kernel", "Win32_System_LibraryLoader", "Win32_System_Com_StructuredStorage", + "Win32_System_Ole", "Win32_System_Threading", "Win32_System_LibraryLoader", "Win32_System_Performance", @@ -107,6 +108,7 @@ features = [ "Win32_System_IO", "Win32_NetworkManagement_IpHelper", "Win32_NetworkManagement_Ndis", + "Win32_NetworkManagement_WindowsFirewall", "Win32_Networking_WinSock", "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_Input_Pointer", diff --git a/host/src/firewall.rs b/host/src/firewall.rs new file mode 100644 index 00000000..2e45f3c5 --- /dev/null +++ b/host/src/firewall.rs @@ -0,0 +1,649 @@ +//! Whether Windows Firewall lets iPads reach this copy of the host. +//! +//! An iPad whose HELLO never arrives says "Your PC didn't answer" while the +//! PC says it is ready, and neither side could tell why. The usual blocks are +//! on the PC: an inbound Block rule for this exe, which Windows adds when its +//! "allow access" prompt is dismissed and which beats every Allow rule, or +//! "Blocks all incoming connections" on the network's profile. A background +//! thread reads the firewall policy at startup and every [`RECHECK`], logs +//! the result when it changes, and the Stream page warns while it blocks. +//! +//! The log line names profiles and counts rules. It holds no path, rule name +//! or address. + +use std::path::Path; +use std::sync::Mutex; +use std::time::Duration; + +#[cfg(windows)] +use tracing::{info, warn}; + +/// How often the policy is read again: a network can turn Public, and a rule +/// can be added, while the app runs. +pub const RECHECK: Duration = Duration::from_secs(60); + +/// The Windows Firewall profiles, as `NET_FW_PROFILE_TYPE2` bits. +pub const DOMAIN: u32 = 1; +pub const PRIVATE: u32 = 2; +pub const PUBLIC: u32 = 4; + +/// One profile in use on a connected network. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Profile { + pub bit: u32, + pub on: bool, + /// "Blocks all incoming connections, including those in the list of + /// allowed apps". + pub block_all_inbound: bool, +} + +/// An inbound rule, as far as the verdict needs it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Rule { + /// The program path as stored, which may use `%ProgramFiles%` and the like. + pub program: String, + pub allow: bool, + pub enabled: bool, + /// UDP or any protocol. + pub udp: bool, + pub profiles: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Verdict { + /// An Allow rule for this exe, and nothing that beats it. + Allowed, + /// An enabled Block rule for this exe in a profile in use. + Blocked, + /// A profile in use blocks all incoming connections. + BlocksAll, + /// Neither rule exists: Windows asks, or blocks quietly if it may not. + NoRule, + /// Windows Firewall is off on every network: another firewall may be. + Off, +} + +impl Verdict { + fn label(self) -> &'static str { + match self { + Verdict::Allowed => "allowed", + Verdict::Blocked => "blocked", + Verdict::BlocksAll => "blocks_all", + Verdict::NoRule => "no_rule", + Verdict::Off => "off", + } + } +} + +/// What the firewall does to this exe's inbound UDP on the networks in use. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Report { + pub profiles: Vec, + pub allow_rules: usize, + pub block_rules: usize, + pub verdict: Verdict, +} + +impl Report { + /// The profiles that block all incoming connections, by name. + pub fn blocking_profiles(&self) -> Vec<&'static str> { + self.profiles + .iter() + .filter(|p| p.on && p.block_all_inbound) + .map(|p| profile_name(p.bit)) + .collect() + } + + /// The log line: profile names, switches and counts only. + pub fn summary(&self) -> String { + let profiles: Vec = self + .profiles + .iter() + .map(|p| { + let state = match (p.on, p.block_all_inbound) { + (false, _) => "off", + (true, true) => "blocks_all", + (true, false) => "on", + }; + format!("{}:{state}", profile_name(p.bit)) + }) + .collect(); + format!( + "Windows Firewall: profiles=[{}] allow_rules={} block_rules={} verdict={}", + profiles.join(" "), + self.allow_rules, + self.block_rules, + self.verdict.label(), + ) + } +} + +pub fn profile_name(bit: u32) -> &'static str { + match bit { + DOMAIN => "domain", + PRIVATE => "private", + PUBLIC => "public", + _ => "other", + } +} + +/// `%NAME%` replaced with `lookup(NAME)`, as Windows stores a rule made in +/// its own window (`%ProgramFiles%\...`). An unknown name stays as written. +pub fn expand(path: &str, lookup: impl Fn(&str) -> Option) -> String { + let mut out = String::new(); + let mut rest = path; + while let Some(start) = rest.find('%') { + let Some(len) = rest[start + 1..].find('%') else { + break; + }; + let name = &rest[start + 1..start + 1 + len]; + out.push_str(&rest[..start]); + match lookup(name).filter(|_| !name.is_empty()) { + Some(value) => out.push_str(&value), + None => out.push_str(&rest[start..start + len + 2]), + } + rest = &rest[start + len + 2..]; + } + out.push_str(rest); + out +} + +/// Whether a rule's program is `exe`. Windows compares paths without case, +/// and its prompt stores them in lower case. +pub fn same_program(program: &str, exe: &Path, lookup: impl Fn(&str) -> Option) -> bool { + let plain = |path: &str| { + path.trim() + .trim_matches('"') + .replace('/', "\\") + .to_lowercase() + }; + let exe = plain(&exe.to_string_lossy()); + let exe = exe.strip_prefix(r"\\?\").unwrap_or(&exe); + !exe.is_empty() && plain(&expand(program, lookup)) == exe +} + +/// The verdict for inbound UDP to `exe` on the profiles in use. Windows lets +/// "block all" beat every rule and a Block rule beat every Allow rule. +pub fn judge( + profiles: &[Profile], + rules: &[Rule], + exe: &Path, + lookup: impl Fn(&str) -> Option, +) -> Report { + let on: u32 = profiles.iter().filter(|p| p.on).map(|p| p.bit).sum(); + let counted = |allow: bool| { + rules + .iter() + .filter(|r| { + r.allow == allow + && r.enabled + && r.udp + && r.profiles & on != 0 + && same_program(&r.program, exe, &lookup) + }) + .count() + }; + let (allow_rules, block_rules) = (counted(true), counted(false)); + let verdict = if on == 0 { + Verdict::Off + } else if profiles.iter().any(|p| p.on && p.block_all_inbound) { + Verdict::BlocksAll + } else if block_rules > 0 { + Verdict::Blocked + } else if allow_rules > 0 { + Verdict::Allowed + } else { + Verdict::NoRule + }; + Report { + profiles: profiles.to_vec(), + allow_rules, + block_rules, + verdict, + } +} + +static LATEST: Mutex> = Mutex::new(None); + +/// The last reading, once there is one. `None` off Windows. +pub fn latest() -> Option { + LATEST.lock().ok()?.clone() +} + +/// Stores `report` and says whether it differs from the one before. +#[cfg(windows)] +fn store(report: Report) -> bool { + let Ok(mut latest) = LATEST.lock() else { + return false; + }; + let changed = latest.as_ref() != Some(&report); + *latest = Some(report); + changed +} + +/// Logs a new reading: a warning while it blocks. +#[cfg(windows)] +fn note(report: Report) { + let line = report.summary(); + let blocks = matches!(report.verdict, Verdict::Blocked | Verdict::BlocksAll); + if store(report) { + if blocks { + warn!("{line}"); + } else { + info!("{line}"); + } + } +} + +/// Reads the policy now and every [`RECHECK`] on its own thread. +pub fn watch() { + #[cfg(windows)] + { + let Ok(exe) = std::env::current_exe() else { + return; + }; + let _ = std::thread::Builder::new() + .name("firewall".into()) + .spawn(move || { + let mut failed = false; + loop { + match windows_policy::read(&exe) { + Ok(report) => { + failed = false; + note(report); + } + Err(error) if !failed => { + failed = true; + warn!(%error, "Could not read the Windows Firewall policy"); + } + Err(_) => {} + } + std::thread::sleep(RECHECK); + } + }); + } +} + +#[cfg(windows)] +mod windows_policy { + use std::path::Path; + + use windows::core::{IUnknown, Interface, VARIANT}; + use windows::Win32::NetworkManagement::WindowsFirewall::{ + INetFwPolicy2, INetFwRule, NetFwPolicy2, NET_FW_ACTION_ALLOW, NET_FW_IP_PROTOCOL_ANY, + NET_FW_IP_PROTOCOL_UDP, NET_FW_PROFILE_TYPE2, NET_FW_RULE_DIR_IN, + }; + use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_MULTITHREADED, + }; + use windows::Win32::System::Ole::IEnumVARIANT; + + use super::{judge, Profile, Report, Rule, DOMAIN, PRIVATE, PUBLIC}; + + /// VT_DISPATCH: what the rule enumerator hands out. + const VT_DISPATCH: u16 = 9; + + struct Apartment; + + impl Apartment { + fn new() -> windows::core::Result { + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED).ok()? }; + Ok(Self) + } + } + + impl Drop for Apartment { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } + } + + pub fn read(exe: &Path) -> windows::core::Result { + let _apartment = Apartment::new()?; + let policy: INetFwPolicy2 = + unsafe { CoCreateInstance(&NetFwPolicy2, None, CLSCTX_INPROC_SERVER)? }; + let current = unsafe { policy.CurrentProfileTypes()? } as u32; + let mut profiles = Vec::new(); + for bit in [DOMAIN, PRIVATE, PUBLIC] { + if current & bit == 0 { + continue; + } + let profile = NET_FW_PROFILE_TYPE2(bit as i32); + profiles.push(Profile { + bit, + on: unsafe { policy.get_FirewallEnabled(profile)? }.as_bool(), + block_all_inbound: unsafe { policy.get_BlockAllInboundTraffic(profile)? }.as_bool(), + }); + } + let rules = inbound_rules(&policy)?; + Ok(judge(&profiles, &rules, exe, |name| { + std::env::var(name).ok() + })) + } + + /// Every inbound rule that names a program. + fn inbound_rules(policy: &INetFwPolicy2) -> windows::core::Result> { + let all = unsafe { policy.Rules()? }; + let each: IEnumVARIANT = unsafe { all._NewEnum()? }.cast()?; + let mut rules = Vec::new(); + loop { + let mut item = [VARIANT::default()]; + let mut fetched = 0; + unsafe { each.Next(&mut item, &mut fetched).ok()? }; + if fetched == 0 { + break; + } + let raw = item[0].as_raw(); + let (vt, pointer) = unsafe { + ( + raw.Anonymous.Anonymous.vt, + raw.Anonymous.Anonymous.Anonymous.pdispVal, + ) + }; + if vt != VT_DISPATCH { + continue; + } + // IDispatch is an IUnknown; the VARIANT keeps its reference. + let Some(unknown) = (unsafe { IUnknown::from_raw_borrowed(&pointer) }) else { + continue; + }; + let Ok(rule) = unknown.cast::() else { + continue; + }; + if unsafe { rule.Direction()? } != NET_FW_RULE_DIR_IN { + continue; + } + let program = unsafe { rule.ApplicationName() } + .map(|name| name.to_string()) + .unwrap_or_default(); + if program.is_empty() { + continue; + } + let protocol = unsafe { rule.Protocol()? }; + rules.push(Rule { + program, + allow: unsafe { rule.Action()? } == NET_FW_ACTION_ALLOW, + enabled: unsafe { rule.Enabled()? }.as_bool(), + udp: protocol == NET_FW_IP_PROTOCOL_UDP.0 || protocol == NET_FW_IP_PROTOCOL_ANY.0, + profiles: unsafe { rule.Profiles()? } as u32, + }); + } + Ok(rules) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + const EXE: &str = r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"; + + fn env(name: &str) -> Option { + match name.to_lowercase().as_str() { + "programfiles" => Some(r"C:\Program Files".into()), + _ => None, + } + } + + fn on(bit: u32) -> Profile { + Profile { + bit, + on: true, + block_all_inbound: false, + } + } + + /// The installer's rule: all profiles, UDP. + fn installer_rule() -> Rule { + Rule { + program: EXE.into(), + allow: true, + enabled: true, + udp: true, + profiles: 0x7fff_ffff, + } + } + + /// What Windows adds when its prompt is dismissed: lower case, the + /// profiles in use then. + fn prompt_block(profiles: u32) -> Rule { + Rule { + program: EXE.to_lowercase(), + allow: false, + enabled: true, + udp: true, + profiles, + } + } + + fn verdict(profiles: &[Profile], rules: &[Rule]) -> Verdict { + judge(profiles, rules, &PathBuf::from(EXE), env).verdict + } + + #[test] + fn the_installer_rule_allows_on_every_network() { + for bit in [DOMAIN, PRIVATE, PUBLIC] { + assert_eq!(verdict(&[on(bit)], &[installer_rule()]), Verdict::Allowed); + } + } + + #[test] + fn a_dismissed_prompt_beats_the_installer_rule_on_its_networks_only() { + let rules = [installer_rule(), prompt_block(PUBLIC)]; + assert_eq!(verdict(&[on(PUBLIC)], &rules), Verdict::Blocked); + assert_eq!(verdict(&[on(PRIVATE)], &rules), Verdict::Allowed); + // Ethernet on a private network, Wi-Fi on a public one. + assert_eq!( + verdict(&[on(PRIVATE), on(PUBLIC)], &rules), + Verdict::Blocked + ); + } + + #[test] + fn blocking_all_incoming_connections_beats_every_rule() { + let shields = Profile { + block_all_inbound: true, + ..on(PUBLIC) + }; + assert_eq!(verdict(&[shields], &[installer_rule()]), Verdict::BlocksAll); + let report = judge( + &[on(PRIVATE), shields], + &[installer_rule()], + &PathBuf::from(EXE), + env, + ); + assert_eq!(report.blocking_profiles(), ["public"]); + } + + #[test] + fn rules_that_do_not_apply_are_not_counted() { + let tcp = Rule { + udp: false, + ..prompt_block(PUBLIC) + }; + let disabled = Rule { + enabled: false, + ..prompt_block(PUBLIC) + }; + let other_exe = Rule { + program: r"C:\Tools\EternalMonitor-host.exe".into(), + ..prompt_block(PUBLIC) + }; + let report = judge( + &[on(PUBLIC)], + &[ + installer_rule(), + tcp, + disabled, + other_exe, + prompt_block(PRIVATE), + ], + &PathBuf::from(EXE), + env, + ); + assert_eq!((report.allow_rules, report.block_rules), (1, 0)); + assert_eq!(report.verdict, Verdict::Allowed); + } + + #[test] + fn no_rule_and_firewall_off_are_told_apart() { + assert_eq!(verdict(&[on(PRIVATE)], &[]), Verdict::NoRule); + let off = Profile { + on: false, + ..on(PRIVATE) + }; + assert_eq!(verdict(&[off], &[prompt_block(PRIVATE)]), Verdict::Off); + // A rule for a profile whose firewall is off doesn't count. + assert_eq!( + verdict(&[off, on(PUBLIC)], &[prompt_block(PRIVATE)]), + Verdict::NoRule + ); + } + + #[test] + fn a_rule_made_in_windows_security_matches_through_its_variables() { + let ui_rule = Rule { + program: r"%ProgramFiles%\EternalMonitor\EternalMonitor-host.exe".into(), + ..installer_rule() + }; + assert_eq!(verdict(&[on(PRIVATE)], &[ui_rule]), Verdict::Allowed); + assert!(same_program( + "\"c:/program files/eternalmonitor/eternalmonitor-host.exe\"", + Path::new(EXE), + env + )); + assert!(same_program(EXE, Path::new(&format!(r"\\?\{EXE}")), env)); + assert!(!same_program("", Path::new(EXE), env)); + } + + #[test] + fn expansion_leaves_unknown_and_unpaired_names() { + assert_eq!(expand(r"%nope%\a%", env), r"%nope%\a%"); + assert_eq!(expand(r"%%\%ProgramFiles%", env), r"%%\C:\Program Files"); + } + + /// `key=value` or `key="value with spaces"` from a netsh command line. + fn netsh_arg(params: &str, key: &str) -> Option { + let mut rest = params; + while let Some(start) = rest.find(&format!(" {key}=")) { + let value = &rest[start + key.len() + 2..]; + if let Some(quoted) = value.strip_prefix('"') { + return quoted.split('"').next().map(str::to_string); + } + let value = value.split(' ').next().unwrap_or_default(); + if !value.is_empty() { + return Some(value.to_string()); + } + rest = &rest[start + 1..]; + } + None + } + + /// What the firewall holds after Setup runs the netsh lines of its + /// [Run] section, in order, into `app`. Every rule here is inbound. + fn after_setup(mut rules: Vec<(String, Rule)>, app: &str) -> Vec { + let iss = include_str!("../../installer/EternalMonitor.iss"); + let run = iss + .split("\n[Run]") + .nth(1) + .and_then(|section| section.split("\n[").next()) + .expect("EternalMonitor.iss has a [Run] section"); + for line in run.lines().filter(|line| line.contains(r"\netsh.exe")) { + let params = line + .split("Parameters: \"") + .nth(1) + .and_then(|rest| rest.split("\"; ").next()) + .expect("a netsh line has Parameters") + .replace("\"\"", "\"") + .replace("{app}", app); + let arg = |key| netsh_arg(¶ms, key); + let name = arg("name").expect("a netsh rule line names its rule"); + if params.starts_with("advfirewall firewall delete rule ") { + match arg("program") { + Some(program) if name == "all" => { + assert_ne!(arg("dir").as_deref(), Some("out")); + rules.retain(|(_, rule)| { + !same_program(&rule.program, Path::new(&program), env) + }); + } + _ => rules.retain(|(named, _)| *named != name), + } + } else if params.starts_with("advfirewall firewall add rule ") { + assert_eq!(arg("dir").as_deref(), Some("in"), "{params}"); + let profiles = arg("profile").unwrap_or_default(); + let bits = [("domain", DOMAIN), ("private", PRIVATE), ("public", PUBLIC)]; + rules.push(( + name, + Rule { + program: arg("program").unwrap_or_default(), + allow: arg("action").as_deref() == Some("allow"), + enabled: arg("enable").as_deref() != Some("no"), + udp: arg("protocol").as_deref() == Some("UDP"), + profiles: bits + .iter() + .filter(|(word, _)| profiles.split(',').any(|p| p == *word)) + .map(|(_, bit)| bit) + .sum(), + }, + )); + } else { + panic!("unexpected netsh line in [Run]: {params}"); + } + } + rules.into_iter().map(|(_, rule)| rule).collect() + } + + #[test] + fn running_setup_again_removes_a_dismissed_prompts_block() { + let app = r"C:\Program Files\EternalMonitor"; + let other = Rule { + program: r"C:\Tools\Other.exe".into(), + ..prompt_block(PUBLIC) + }; + let before = vec![ + ("EternalMonitor Host UDP".to_string(), installer_rule()), + ( + format!("UDP Query User{{0-1-2-3-4}}{}", EXE.to_lowercase()), + prompt_block(PRIVATE | PUBLIC), + ), + ("Other".to_string(), other.clone()), + ]; + let rules: Vec = before.iter().map(|(_, rule)| rule.clone()).collect(); + assert_eq!(verdict(&[on(PUBLIC)], &rules), Verdict::Blocked); + + let after = after_setup(before, app); + assert_eq!( + verdict(&[on(PUBLIC)], &after), + Verdict::Allowed, + "{after:?}" + ); + assert_eq!(verdict(&[on(PRIVATE)], &after), Verdict::Allowed); + assert!( + after.contains(&other), + "Setup removed another program's rule" + ); + // A first install: nothing to remove, the Allow rule still comes. + assert_eq!( + verdict(&[on(DOMAIN)], &after_setup(Vec::new(), app)), + Verdict::Allowed + ); + } + + #[test] + fn the_log_line_names_no_path_or_rule() { + let report = judge( + &[on(PRIVATE), on(PUBLIC)], + &[installer_rule(), prompt_block(PUBLIC)], + &PathBuf::from(EXE), + env, + ); + assert_eq!( + report.summary(), + "Windows Firewall: profiles=[private:on public:on] allow_rules=1 block_rules=1 verdict=blocked" + ); + let line = format!("2026-11-05T12:00:00.000000Z WARN {}", report.summary()); + assert_eq!(crate::telemetry::uploaded(&line), line); + } +} diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index 679e95e1..2addf59b 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -56,6 +56,14 @@ impl AnalyzerApp { /// Update, restart and fault notices, most urgent first. fn notices(&mut self, ui: &mut Ui, snap: &StatsSnapshot, client: &ClientSnapshot) { self.port_notice(ui); + let on_wifi = client.peer.is_some() && !client.on_usb(); + if let Some((title, body)) = crate::firewall::latest() + .filter(|_| !on_wifi) + .and_then(|report| firewall_notice(&report)) + { + widgets::banner(ui, WARNING, Icon::Warning, &title, &body, 0.0, |_| {}); + ui.add_space(GAP - 10.0); + } if let Some((title, body)) = usb_cable_notice( snap.usb_service_reachable, snap.usb_cabled_devices, @@ -1352,6 +1360,28 @@ pub(super) fn usb_cable_notice( }) } +/// What the Stream page says while Windows Firewall keeps iPads out. Only +/// for blocks it is sure of: a missing rule may be made up for by a rule it +/// doesn't read, such as one for the port. +pub(super) fn firewall_notice(report: &crate::firewall::Report) -> Option<(String, String)> { + use crate::firewall::Verdict; + let usb = "A USB cable still works."; + match report.verdict { + Verdict::Blocked => Some(( + "Windows Firewall is blocking EternalMonitor".into(), + format!("iPads on Wi-Fi can't reach this PC, so they say it didn't answer. Run the EternalMonitor installer again: it removes the block. {usb}"), + )), + Verdict::BlocksAll => { + let network = report.blocking_profiles().join(" and "); + Some(( + "Windows Firewall blocks every incoming connection".into(), + format!("iPads on Wi-Fi can't reach this PC, so they say it didn't answer. In Windows Security, open Firewall & network protection, choose the {network} network and turn off Blocks all incoming connections. {usb}"), + )) + } + Verdict::Allowed | Verdict::NoRule | Verdict::Off => None, + } +} + /// What the USB row says. The Apple Devices hint appears only when Apple's /// device service cannot be reached. pub(super) fn usb_summary( diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index d7f2b148..8a410307 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -1,9 +1,55 @@ use super::page_stream::{ - audio_summary, capture_block_notice, second_screen_notice, software_fallback_body, - usb_cable_notice, usb_summary, + audio_summary, capture_block_notice, firewall_notice, second_screen_notice, + software_fallback_body, usb_cable_notice, usb_summary, }; use super::*; +#[test] +fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { + use crate::firewall::{judge, Profile, Rule, PRIVATE, PUBLIC}; + let exe = std::path::Path::new(r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"); + let profile = |bit, block_all_inbound| Profile { + bit, + on: true, + block_all_inbound, + }; + let rule = |allow| Rule { + program: exe.to_string_lossy().to_lowercase(), + allow, + enabled: true, + udp: true, + profiles: PUBLIC, + }; + let notice = |profiles: &[Profile], rules: &[Rule]| { + firewall_notice(&judge(profiles, rules, exe, |_| None)) + }; + + let (title, body) = notice(&[profile(PUBLIC, false)], &[rule(true), rule(false)]).unwrap(); + assert_eq!(title, "Windows Firewall is blocking EternalMonitor"); + assert!( + body.contains("Run the EternalMonitor installer again"), + "{body}" + ); + assert!(body.contains("A USB cable still works."), "{body}"); + + let (title, body) = notice( + &[profile(PRIVATE, false), profile(PUBLIC, true)], + &[rule(true)], + ) + .unwrap(); + assert_eq!(title, "Windows Firewall blocks every incoming connection"); + assert!(body.contains("choose the public network"), "{body}"); + + assert_eq!(notice(&[profile(PUBLIC, false)], &[rule(true)]), None); + // No rule it can read: a port rule may still let iPads in. + assert_eq!(notice(&[profile(PUBLIC, false)], &[]), None); + let off = Profile { + on: false, + ..profile(PUBLIC, true) + }; + assert_eq!(notice(&[off], &[rule(false)]), None); +} + #[test] fn audio_row_reports_rate_and_keeps_failure_separate_from_video() { let mut audio = crate::audio::AudioStats::started("USB Audio Device"); diff --git a/host/src/lib.rs b/host/src/lib.rs index 94cd3e26..1e91fcbf 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -12,6 +12,7 @@ pub mod discovery; pub mod display_guard; pub mod encoder; pub mod feedback; +pub mod firewall; pub mod gpu_preference; pub mod gui; pub mod handoff; diff --git a/host/src/main.rs b/host/src/main.rs index f6ecd1b9..5a888d98 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -160,6 +160,7 @@ fn main() -> Result<(), Box> { info!(" Listen: {}:{}", listen_ip, listen_port); info!("══════════════════════════════════"); eternal_host::network::log_addresses(); + eternal_host::firewall::watch(); let auth_token = persisted .ensure_auth_token() diff --git a/host/src/network.rs b/host/src/network.rs index 0238f108..bf01b624 100644 --- a/host/src/network.rs +++ b/host/src/network.rs @@ -379,8 +379,10 @@ pub fn default_route() -> Option { .map(|addr| addr.ip()) } -/// The startup line: address counts by reach, the default route's reach and -/// the kinds of adapters with an address. No address and no adapter name. +/// The startup line: address counts by reach, the default route's reach, the +/// kinds of adapters with an address and the kinds with a home network +/// address, which tell a PC on a cable from one on Wi-Fi. No address and no +/// adapter name. pub fn summary(adapters: Option<&[Adapter]>, route: Option) -> String { let Some(adapters) = adapters else { return "Network: addresses unavailable".to_string(); @@ -394,17 +396,25 @@ pub fn summary(adapters: Option<&[Adapter]>, route: Option) -> String { .find(|address| IpAddr::V4(address.ip) == ip) .map_or("unknown", |address| address.reach.label()), }; - let mut kinds: Vec<&str> = ranked.iter().map(|address| address.kind.label()).collect(); - kinds.sort_unstable(); - kinds.dedup(); + let kinds = |reach: Option| { + let mut kinds: Vec<&str> = ranked + .iter() + .filter(|address| reach.is_none_or(|reach| address.reach == reach)) + .map(|address| address.kind.label()) + .collect(); + kinds.sort_unstable(); + kinds.dedup(); + kinds.join(" ") + }; format!( - "Network: addresses lan={} other={} virtual={} vpn={} link_local={} route={route} adapters=[{}]", + "Network: addresses lan={} other={} virtual={} vpn={} link_local={} route={route} adapters=[{}] lan_adapters=[{}]", count(Reach::Lan), count(Reach::Other), count(Reach::Virtual), count(Reach::Vpn), count(Reach::LinkLocal), - kinds.join(" "), + kinds(None), + kinds(Some(Reach::Lan)), ) } @@ -897,7 +907,18 @@ mod tests { assert_eq!( line, "Network: addresses lan=1 other=0 virtual=1 vpn=1 link_local=0 route=vpn \ - adapters=[virtual vpn wifi]" + adapters=[virtual vpn wifi] lan_adapters=[wifi]" + ); + // A PC on a cable, its Wi-Fi without DHCP: which one iPads reach. + let cable = classify(6, "Realtek PCIe GbE Family Controller", "Ethernet"); + let both = [ + adapter(cable, true, [192, 168, 1, 21]), + adapter(wifi, false, [169, 254, 7, 9]), + ]; + assert_eq!( + summary(Some(&both), Some(IpAddr::V4(ip([192, 168, 1, 21])))), + "Network: addresses lan=1 other=0 virtual=0 vpn=0 link_local=1 route=lan \ + adapters=[ethernet wifi] lan_adapters=[ethernet]" ); for private in [ "100.", @@ -913,7 +934,7 @@ mod tests { assert!(!line.contains(private), "{private} in {line}"); } assert_eq!(summary(None, None), "Network: addresses unavailable"); - assert!(summary(Some(&[]), None).contains("route=none adapters=[]")); + assert!(summary(Some(&[]), None).contains("route=none adapters=[] lan_adapters=[]")); } /// Windows says the list grew every time: three bigger buffers, then no diff --git a/host/src/telemetry/redact.rs b/host/src/telemetry/redact.rs index f5988f98..06ff2e56 100644 --- a/host/src/telemetry/redact.rs +++ b/host/src/telemetry/redact.rs @@ -166,6 +166,7 @@ const DIAGNOSTIC_KEYS: &[&str] = &[ "choice", // The startup network line. "route", + "lan_adapters", // The virtual display snapshot. "driver", "tech", diff --git a/installer/EternalMonitor.iss b/installer/EternalMonitor.iss index b29b673b..d97ca563 100644 --- a/installer/EternalMonitor.iss +++ b/installer/EternalMonitor.iss @@ -206,6 +206,10 @@ Name: "{autodesktop}\EternalMonitor"; Filename: "{app}\EternalMonitor-host. ; listens only on UDP; the TCP rule older versions added is removed and not re-added. Filename: "{sys}\netsh.exe"; Parameters: "advfirewall firewall delete rule name=""EternalMonitor Host UDP"""; Flags: runhidden waituntilterminated Filename: "{sys}\netsh.exe"; Parameters: "advfirewall firewall delete rule name=""EternalMonitor Host TCP"""; Flags: runhidden waituntilterminated +; Every other inbound rule for the host goes too. Windows adds Block rules for it when its +; "allow access" prompt is dismissed, and a Block rule beats the Allow rule below: iPads then +; get no answer. Running Setup again is the fix the PC and the iPad offer for that. +Filename: "{sys}\netsh.exe"; Parameters: "advfirewall firewall delete rule name=all dir=in program=""{app}\EternalMonitor-host.exe"""; Flags: runhidden waituntilterminated Filename: "{sys}\netsh.exe"; Parameters: "advfirewall firewall add rule name=""EternalMonitor Host UDP"" dir=in action=allow program=""{app}\EternalMonitor-host.exe"" protocol=UDP profile=domain,private,public enable=yes"; StatusMsg: "{cm:FirewallStatus}"; Flags: runhidden waituntilterminated #ifdef IncludeDriver ; Save the screens before the driver steps: starting the virtual display can leave a PC with no From 1035382abfa1fb89ff2a1e307a435b9348a7f4d6 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Thu, 5 Nov 2026 21:14:14 -0500 Subject: [PATCH 005/226] Warn about the firewall only when the block is certain Review fixes for the Windows Firewall check: - A Block rule counts only when it reaches the stream's port. One scoped to remote or local addresses, remote ports, interfaces or a service is logged as scoped_block_rules and never shown. - Each profile in use keeps its own verdict. The Stream page warns only when every profile blocks and the PC has a home network address; a block on one of two networks is logged as partly_blocked. - The warning stays hidden while any iPad is connected, over Wi-Fi or the cable. - A failed read clears the last reading, so an old block can't stay on screen. - The line's keys are diagnostic keys: a PC or user named like a token no longer changes it in uploads. --- host/src/firewall.rs | 630 ++++++++++++++++++++++++++--------- host/src/gui/page_stream.rs | 59 ++-- host/src/gui/tests.rs | 79 +++-- host/src/main.rs | 2 +- host/src/telemetry/mod.rs | 34 ++ host/src/telemetry/redact.rs | 7 + 6 files changed, 604 insertions(+), 207 deletions(-) diff --git a/host/src/firewall.rs b/host/src/firewall.rs index 2e45f3c5..7f2b4fd0 100644 --- a/host/src/firewall.rs +++ b/host/src/firewall.rs @@ -5,8 +5,14 @@ //! on the PC: an inbound Block rule for this exe, which Windows adds when its //! "allow access" prompt is dismissed and which beats every Allow rule, or //! "Blocks all incoming connections" on the network's profile. A background -//! thread reads the firewall policy at startup and every [`RECHECK`], logs -//! the result when it changes, and the Stream page warns while it blocks. +//! thread reads the firewall policy at startup and every [`RECHECK`] and logs +//! the result when it changes. +//! +//! The Stream page warns only when the block is certain for the network an +//! iPad would use: every profile in use blocks, the PC has a home network +//! address, and no iPad is connected. A Block rule scoped to some addresses, +//! interfaces or a service, or a block on one of two networks, is only +//! logged. //! //! The log line names profiles and counts rules. It holds no path, rule name //! or address. @@ -15,7 +21,7 @@ use std::path::Path; use std::sync::Mutex; use std::time::Duration; -#[cfg(windows)] +#[cfg(any(windows, test))] use tracing::{info, warn}; /// How often the policy is read again: a network can turn Public, and a rule @@ -37,7 +43,8 @@ pub struct Profile { pub block_all_inbound: bool, } -/// An inbound rule, as far as the verdict needs it. +/// An inbound rule, as far as the verdict needs it. Scope fields hold what +/// Windows stores; `*` (or empty) means any. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Rule { /// The program path as stored, which may use `%ProgramFiles%` and the like. @@ -47,19 +54,100 @@ pub struct Rule { /// UDP or any protocol. pub udp: bool, pub profiles: u32, + pub local_ports: String, + pub remote_ports: String, + pub local_addresses: String, + pub remote_addresses: String, + /// `All`, or some of `Wireless`, `Lan`, `RemoteAccess`. + pub interface_types: String, + /// The rule names particular interfaces. + pub interfaces: bool, + /// The rule is for a service, not every process of the program. + pub service: String, +} + +impl Rule { + /// An inbound rule for `program` with no scope beyond its protocol. + pub fn unscoped(program: &str, allow: bool, profiles: u32) -> Self { + Self { + program: program.into(), + allow, + enabled: true, + udp: true, + profiles, + local_ports: "*".into(), + remote_ports: "*".into(), + local_addresses: "*".into(), + remote_addresses: "*".into(), + interface_types: "All".into(), + interfaces: false, + service: String::new(), + } + } +} + +/// How far a rule reaches the stream's traffic. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Reach { + /// Every datagram an iPad sends to the stream port. + All, + /// Only some: it names addresses, ports, interfaces or a service. + Some, + /// None of it: another port. + None, +} + +/// Whether `ports` (`*`, or numbers and ranges such as `80,9000-9999`) +/// covers `port`. Keywords such as `RPC` cover no stream port. +pub fn covers(ports: &str, port: u16) -> bool { + let ports = ports.trim(); + if ports.is_empty() || ports == "*" { + return true; + } + ports.split(',').map(str::trim).any(|part| { + let (low, high) = part.split_once('-').unwrap_or((part, part)); + match (low.trim().parse::(), high.trim().parse::()) { + (Ok(low), Ok(high)) => (low..=high).contains(&port), + _ => false, + } + }) } +fn any(value: &str) -> bool { + let value = value.trim(); + value.is_empty() || value == "*" +} + +/// How far `rule` reaches datagrams to `port`. +pub fn reach(rule: &Rule, port: u16) -> Reach { + if !covers(&rule.local_ports, port) { + return Reach::None; + } + let unscoped = any(&rule.remote_ports) + && any(&rule.local_addresses) + && any(&rule.remote_addresses) + && (any(&rule.interface_types) || rule.interface_types.trim().eq_ignore_ascii_case("all")) + && !rule.interfaces + && rule.service.trim().is_empty(); + if unscoped { + Reach::All + } else { + Reach::Some + } +} + +/// What the firewall does on one profile. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Verdict { - /// An Allow rule for this exe, and nothing that beats it. + /// An Allow rule for this exe, and nothing certain that beats it. Allowed, - /// An enabled Block rule for this exe in a profile in use. + /// An unscoped Block rule for this exe. Blocked, - /// A profile in use blocks all incoming connections. + /// The profile blocks all incoming connections. BlocksAll, /// Neither rule exists: Windows asks, or blocks quietly if it may not. NoRule, - /// Windows Firewall is off on every network: another firewall may be. + /// Windows Firewall is off on this profile: another firewall may be on. Off, } @@ -73,47 +161,87 @@ impl Verdict { Verdict::Off => "off", } } + + fn blocks(self) -> bool { + matches!(self, Verdict::Blocked | Verdict::BlocksAll) + } } -/// What the firewall does to this exe's inbound UDP on the networks in use. +/// What the firewall does to this exe's inbound UDP on each network in use. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Report { - pub profiles: Vec, + /// Each profile in use, with its verdict. + pub profiles: Vec<(u32, Verdict)>, pub allow_rules: usize, pub block_rules: usize, - pub verdict: Verdict, + /// Block rules that reach only some of the stream's traffic. + pub scoped_block_rules: usize, + /// The PC has a home network address (see [`crate::network::Reach::Lan`]). + pub lan: bool, +} + +/// What the Stream page may say: a block certain on every network in use. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CertainBlock { + /// Block rules for this exe, and on `all_blocked` networks "Blocks all + /// incoming connections" as well. + Rule { all_blocked: Vec<&'static str> }, + /// Only "Blocks all incoming connections", on these networks. + AllIncoming(Vec<&'static str>), } impl Report { - /// The profiles that block all incoming connections, by name. - pub fn blocking_profiles(&self) -> Vec<&'static str> { - self.profiles + /// The whole picture in one word for the log. + pub fn verdict(&self) -> &'static str { + let blocking = self.profiles.iter().filter(|(_, v)| v.blocks()).count(); + if !self.profiles.is_empty() && blocking == self.profiles.len() { + "blocked" + } else if blocking > 0 { + "partly_blocked" + } else if self.profiles.iter().all(|(_, v)| *v == Verdict::Off) { + "off" + } else if self.profiles.iter().any(|(_, v)| *v == Verdict::NoRule) { + "no_rule" + } else { + "allowed" + } + } + + /// A block certain for the network an iPad would use: every profile in + /// use blocks, and the PC has a home network address. Otherwise `None`, + /// and the log line says what is known. + pub fn certain_block(&self) -> Option { + if !self.lan || self.profiles.is_empty() || !self.profiles.iter().all(|(_, v)| v.blocks()) { + return None; + } + let all_blocked: Vec<_> = self + .profiles .iter() - .filter(|p| p.on && p.block_all_inbound) - .map(|p| profile_name(p.bit)) - .collect() + .filter(|(_, v)| *v == Verdict::BlocksAll) + .map(|(bit, _)| profile_name(*bit)) + .collect(); + Some(if all_blocked.len() == self.profiles.len() { + CertainBlock::AllIncoming(all_blocked) + } else { + CertainBlock::Rule { all_blocked } + }) } - /// The log line: profile names, switches and counts only. + /// The log line: profile names, verdicts and counts only. pub fn summary(&self) -> String { let profiles: Vec = self .profiles .iter() - .map(|p| { - let state = match (p.on, p.block_all_inbound) { - (false, _) => "off", - (true, true) => "blocks_all", - (true, false) => "on", - }; - format!("{}:{state}", profile_name(p.bit)) - }) + .map(|(bit, verdict)| format!("{}:{}", profile_name(*bit), verdict.label())) .collect(); format!( - "Windows Firewall: profiles=[{}] allow_rules={} block_rules={} verdict={}", + "Windows Firewall: profiles=[{}] allow_rules={} block_rules={} scoped_block_rules={} lan={} verdict={}", profiles.join(" "), self.allow_rules, self.block_rules, - self.verdict.label(), + self.scoped_block_rules, + self.lan, + self.verdict(), ) } } @@ -162,81 +290,98 @@ pub fn same_program(program: &str, exe: &Path, lookup: impl Fn(&str) -> Option Option, ) -> Report { let on: u32 = profiles.iter().filter(|p| p.on).map(|p| p.bit).sum(); - let counted = |allow: bool| { - rules - .iter() - .filter(|r| { - r.allow == allow - && r.enabled - && r.udp - && r.profiles & on != 0 - && same_program(&r.program, exe, &lookup) + let ours: Vec<(&Rule, Reach)> = rules + .iter() + .filter(|r| r.enabled && r.udp && same_program(&r.program, exe, &lookup)) + .map(|r| (r, reach(r, port))) + .filter(|(_, reach)| *reach != Reach::None) + .collect(); + let count = |allow: bool, wanted: Reach, mask: u32| { + ours.iter() + .filter(|(r, reach)| { + r.allow == allow && r.profiles & mask != 0 && (allow || *reach == wanted) }) .count() }; - let (allow_rules, block_rules) = (counted(true), counted(false)); - let verdict = if on == 0 { - Verdict::Off - } else if profiles.iter().any(|p| p.on && p.block_all_inbound) { - Verdict::BlocksAll - } else if block_rules > 0 { - Verdict::Blocked - } else if allow_rules > 0 { - Verdict::Allowed - } else { - Verdict::NoRule - }; + let verdicts = profiles + .iter() + .map(|p| { + let verdict = if !p.on { + Verdict::Off + } else if p.block_all_inbound { + Verdict::BlocksAll + } else if count(false, Reach::All, p.bit) > 0 { + Verdict::Blocked + } else if count(true, Reach::All, p.bit) > 0 { + Verdict::Allowed + } else { + Verdict::NoRule + }; + (p.bit, verdict) + }) + .collect(); Report { - profiles: profiles.to_vec(), - allow_rules, - block_rules, - verdict, + profiles: verdicts, + allow_rules: count(true, Reach::All, on), + block_rules: count(false, Reach::All, on), + scoped_block_rules: count(false, Reach::Some, on), + lan, } } static LATEST: Mutex> = Mutex::new(None); -/// The last reading, once there is one. `None` off Windows. +/// The last reading, if the last read worked. `None` off Windows. pub fn latest() -> Option { LATEST.lock().ok()?.clone() } -/// Stores `report` and says whether it differs from the one before. -#[cfg(windows)] -fn store(report: Report) -> bool { +/// Takes one read's result: stores and logs a report when it changes, and +/// clears it when the read failed, so an old block never stays on screen. +/// The first failure in a row is a warning. +#[cfg(any(windows, test))] +fn observe(read: Result, failed: &mut bool) { let Ok(mut latest) = LATEST.lock() else { - return false; + return; }; - let changed = latest.as_ref() != Some(&report); - *latest = Some(report); - changed -} - -/// Logs a new reading: a warning while it blocks. -#[cfg(windows)] -fn note(report: Report) { - let line = report.summary(); - let blocks = matches!(report.verdict, Verdict::Blocked | Verdict::BlocksAll); - if store(report) { - if blocks { - warn!("{line}"); - } else { - info!("{line}"); + match read { + Ok(report) => { + *failed = false; + if latest.as_ref() == Some(&report) { + return; + } + let line = report.summary(); + if report.verdict() == "allowed" || report.verdict() == "no_rule" { + info!("{line}"); + } else { + warn!("{line}"); + } + *latest = Some(report); + } + Err(error) => { + *latest = None; + if !std::mem::replace(failed, true) { + warn!(%error, "Could not read the Windows Firewall policy"); + } } } } -/// Reads the policy now and every [`RECHECK`] on its own thread. -pub fn watch() { +/// Reads the policy for the stream's `port` now and every [`RECHECK`] on +/// its own thread. +pub fn watch(port: u16) { #[cfg(windows)] { let Ok(exe) = std::env::current_exe() else { @@ -247,21 +392,19 @@ pub fn watch() { .spawn(move || { let mut failed = false; loop { - match windows_policy::read(&exe) { - Ok(report) => { - failed = false; - note(report); - } - Err(error) if !failed => { - failed = true; - warn!(%error, "Could not read the Windows Firewall policy"); - } - Err(_) => {} - } + let lan = crate::network::adapters().is_some_and(|adapters| { + crate::network::rank(&adapters) + .iter() + .any(|address| address.reach == crate::network::Reach::Lan) + }); + let read = windows_policy::read(&exe, port, lan).map_err(|e| e.to_string()); + observe(read, &mut failed); std::thread::sleep(RECHECK); } }); } + #[cfg(not(windows))] + let _ = port; } #[cfg(windows)] @@ -299,7 +442,7 @@ mod windows_policy { } } - pub fn read(exe: &Path) -> windows::core::Result { + pub fn read(exe: &Path, port: u16, lan: bool) -> windows::core::Result { let _apartment = Apartment::new()?; let policy: INetFwPolicy2 = unsafe { CoCreateInstance(&NetFwPolicy2, None, CLSCTX_INPROC_SERVER)? }; @@ -317,7 +460,7 @@ mod windows_policy { }); } let rules = inbound_rules(&policy)?; - Ok(judge(&profiles, &rules, exe, |name| { + Ok(judge(&profiles, &rules, exe, port, lan, |name| { std::env::var(name).ok() })) } @@ -354,9 +497,10 @@ mod windows_policy { if unsafe { rule.Direction()? } != NET_FW_RULE_DIR_IN { continue; } - let program = unsafe { rule.ApplicationName() } - .map(|name| name.to_string()) - .unwrap_or_default(); + let text = |value: windows::core::Result| { + value.map(|v| v.to_string()).unwrap_or_default() + }; + let program = text(unsafe { rule.ApplicationName() }); if program.is_empty() { continue; } @@ -367,6 +511,15 @@ mod windows_policy { enabled: unsafe { rule.Enabled()? }.as_bool(), udp: protocol == NET_FW_IP_PROTOCOL_UDP.0 || protocol == NET_FW_IP_PROTOCOL_ANY.0, profiles: unsafe { rule.Profiles()? } as u32, + local_ports: text(unsafe { rule.LocalPorts() }), + remote_ports: text(unsafe { rule.RemotePorts() }), + local_addresses: text(unsafe { rule.LocalAddresses() }), + remote_addresses: text(unsafe { rule.RemoteAddresses() }), + interface_types: text(unsafe { rule.InterfaceTypes() }), + // A rule read back without its list counts as scoped: never + // a certain block. + interfaces: unsafe { rule.Interfaces() }.map_or(true, |list| !list.is_empty()), + service: text(unsafe { rule.ServiceName() }), }); } Ok(rules) @@ -379,6 +532,7 @@ mod tests { use std::path::PathBuf; const EXE: &str = r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"; + const PORT: u16 = 9876; fn env(name: &str) -> Option { match name.to_lowercase().as_str() { @@ -397,48 +551,74 @@ mod tests { /// The installer's rule: all profiles, UDP. fn installer_rule() -> Rule { - Rule { - program: EXE.into(), - allow: true, - enabled: true, - udp: true, - profiles: 0x7fff_ffff, - } + Rule::unscoped(EXE, true, 0x7fff_ffff) } /// What Windows adds when its prompt is dismissed: lower case, the - /// profiles in use then. + /// profiles in use then, no other scope. fn prompt_block(profiles: u32) -> Rule { - Rule { - program: EXE.to_lowercase(), - allow: false, - enabled: true, - udp: true, - profiles, - } + Rule::unscoped(&EXE.to_lowercase(), false, profiles) } - fn verdict(profiles: &[Profile], rules: &[Rule]) -> Verdict { - judge(profiles, rules, &PathBuf::from(EXE), env).verdict + fn report(profiles: &[Profile], rules: &[Rule]) -> Report { + judge(profiles, rules, &PathBuf::from(EXE), PORT, true, env) + } + + fn verdicts(profiles: &[Profile], rules: &[Rule]) -> Vec { + report(profiles, rules) + .profiles + .iter() + .map(|(_, v)| *v) + .collect() } #[test] fn the_installer_rule_allows_on_every_network() { for bit in [DOMAIN, PRIVATE, PUBLIC] { - assert_eq!(verdict(&[on(bit)], &[installer_rule()]), Verdict::Allowed); + let report = report(&[on(bit)], &[installer_rule()]); + assert_eq!(report.profiles, [(bit, Verdict::Allowed)]); + assert_eq!(report.verdict(), "allowed"); + assert_eq!(report.certain_block(), None); } } #[test] - fn a_dismissed_prompt_beats_the_installer_rule_on_its_networks_only() { - let rules = [installer_rule(), prompt_block(PUBLIC)]; - assert_eq!(verdict(&[on(PUBLIC)], &rules), Verdict::Blocked); - assert_eq!(verdict(&[on(PRIVATE)], &rules), Verdict::Allowed); - // Ethernet on a private network, Wi-Fi on a public one. + fn a_dismissed_prompt_is_certain_on_the_only_network() { + let report = report(&[on(PUBLIC)], &[installer_rule(), prompt_block(PUBLIC)]); + assert_eq!(report.profiles, [(PUBLIC, Verdict::Blocked)]); + assert_eq!(report.verdict(), "blocked"); + assert_eq!( + report.certain_block(), + Some(CertainBlock::Rule { + all_blocked: vec![] + }) + ); + // Another network, the same rules: nothing blocks there. + let private = + super::tests::report(&[on(PRIVATE)], &[installer_rule(), prompt_block(PUBLIC)]); + assert_eq!(private.profiles, [(PRIVATE, Verdict::Allowed)]); + } + + /// Ethernet on a private network, Wi-Fi on a public one: which an iPad + /// uses isn't known, so the block is logged per network, not shown. + #[test] + fn a_block_on_one_of_two_networks_is_logged_not_certain() { + let report = report( + &[on(PRIVATE), on(PUBLIC)], + &[installer_rule(), prompt_block(PUBLIC)], + ); assert_eq!( - verdict(&[on(PRIVATE), on(PUBLIC)], &rules), - Verdict::Blocked + report.profiles, + [(PRIVATE, Verdict::Allowed), (PUBLIC, Verdict::Blocked)] + ); + assert_eq!(report.verdict(), "partly_blocked"); + assert_eq!(report.certain_block(), None); + // Blocked on both: certain again. + let both = super::tests::report( + &[on(PRIVATE), on(PUBLIC)], + &[installer_rule(), prompt_block(PRIVATE | PUBLIC)], ); + assert!(both.certain_block().is_some()); } #[test] @@ -447,14 +627,121 @@ mod tests { block_all_inbound: true, ..on(PUBLIC) }; - assert_eq!(verdict(&[shields], &[installer_rule()]), Verdict::BlocksAll); + let alone = report(&[shields], &[installer_rule()]); + assert_eq!(alone.profiles, [(PUBLIC, Verdict::BlocksAll)]); + assert_eq!( + alone.certain_block(), + Some(CertainBlock::AllIncoming(vec!["public"])) + ); + // A private network that lets iPads in: not certain. + let mixed = report(&[on(PRIVATE), shields], &[installer_rule()]); + assert_eq!(mixed.certain_block(), None); + // Both networks blocked, one by a rule and one by blocking all. + let both = report(&[on(PRIVATE), shields], &[prompt_block(PRIVATE)]); + assert_eq!( + both.certain_block(), + Some(CertainBlock::Rule { + all_blocked: vec!["public"] + }) + ); + } + + #[test] + fn a_block_for_another_port_does_not_count() { + for ports in ["443", "1-1024,9877", "RPC", "9000-9875"] { + let rule = Rule { + local_ports: ports.into(), + ..prompt_block(PUBLIC) + }; + let report = report(&[on(PUBLIC)], &[installer_rule(), rule]); + assert_eq!(report.profiles, [(PUBLIC, Verdict::Allowed)], "{ports}"); + assert_eq!((report.block_rules, report.scoped_block_rules), (0, 0)); + } + for ports in ["*", "", "9876", "53, 9000-9999"] { + let rule = Rule { + local_ports: ports.into(), + ..prompt_block(PUBLIC) + }; + assert_eq!( + verdicts(&[on(PUBLIC)], &[rule]), + [Verdict::Blocked], + "{ports}" + ); + } + // The port the stream really listens on decides. + let rule = Rule { + local_ports: "19876".into(), + ..prompt_block(PUBLIC) + }; + let moved = judge( + &[on(PUBLIC)], + &[rule], + &PathBuf::from(EXE), + 19876, + true, + env, + ); + assert_eq!(moved.profiles, [(PUBLIC, Verdict::Blocked)]); + } + + #[test] + fn a_scoped_block_is_logged_not_certain() { + let scoped = [ + Rule { + remote_addresses: "203.0.113.0/24".into(), + ..prompt_block(PUBLIC) + }, + Rule { + remote_addresses: "LocalSubnet".into(), + ..prompt_block(PUBLIC) + }, + Rule { + local_addresses: "10.0.0.5".into(), + ..prompt_block(PUBLIC) + }, + Rule { + remote_ports: "5000".into(), + ..prompt_block(PUBLIC) + }, + Rule { + interface_types: "RemoteAccess".into(), + ..prompt_block(PUBLIC) + }, + Rule { + interfaces: true, + ..prompt_block(PUBLIC) + }, + Rule { + service: "SomeService".into(), + ..prompt_block(PUBLIC) + }, + ]; + for rule in scoped { + let report = report(&[on(PUBLIC)], &[installer_rule(), rule.clone()]); + assert_eq!(report.profiles, [(PUBLIC, Verdict::Allowed)], "{rule:?}"); + assert_eq!((report.block_rules, report.scoped_block_rules), (0, 1)); + assert_eq!(report.certain_block(), None); + assert!(report.summary().contains("scoped_block_rules=1")); + } + let all_types = Rule { + interface_types: "All".into(), + ..prompt_block(PUBLIC) + }; + assert_eq!(verdicts(&[on(PUBLIC)], &[all_types]), [Verdict::Blocked]); + } + + #[test] + fn without_a_home_network_address_no_block_is_certain() { let report = judge( - &[on(PRIVATE), shields], - &[installer_rule()], + &[on(PUBLIC)], + &[prompt_block(PUBLIC)], &PathBuf::from(EXE), + PORT, + false, env, ); - assert_eq!(report.blocking_profiles(), ["public"]); + assert_eq!(report.verdict(), "blocked"); + assert_eq!(report.certain_block(), None); } #[test] @@ -467,11 +754,8 @@ mod tests { enabled: false, ..prompt_block(PUBLIC) }; - let other_exe = Rule { - program: r"C:\Tools\EternalMonitor-host.exe".into(), - ..prompt_block(PUBLIC) - }; - let report = judge( + let other_exe = Rule::unscoped(r"C:\Tools\EternalMonitor-host.exe", false, PUBLIC); + let report = report( &[on(PUBLIC)], &[ installer_rule(), @@ -480,35 +764,38 @@ mod tests { other_exe, prompt_block(PRIVATE), ], - &PathBuf::from(EXE), - env, ); assert_eq!((report.allow_rules, report.block_rules), (1, 0)); - assert_eq!(report.verdict, Verdict::Allowed); + assert_eq!(report.profiles, [(PUBLIC, Verdict::Allowed)]); } #[test] fn no_rule_and_firewall_off_are_told_apart() { - assert_eq!(verdict(&[on(PRIVATE)], &[]), Verdict::NoRule); + assert_eq!(report(&[on(PRIVATE)], &[]).verdict(), "no_rule"); let off = Profile { on: false, ..on(PRIVATE) }; - assert_eq!(verdict(&[off], &[prompt_block(PRIVATE)]), Verdict::Off); - // A rule for a profile whose firewall is off doesn't count. + let report_off = report(&[off], &[prompt_block(PRIVATE)]); + assert_eq!(report_off.verdict(), "off"); + assert_eq!(report_off.certain_block(), None); + // A network whose firewall is off lets iPads in: not certain. + let half = report(&[off, on(PUBLIC)], &[prompt_block(PUBLIC)]); assert_eq!( - verdict(&[off, on(PUBLIC)], &[prompt_block(PRIVATE)]), - Verdict::NoRule + half.profiles, + [(PRIVATE, Verdict::Off), (PUBLIC, Verdict::Blocked)] ); + assert_eq!(half.certain_block(), None); } #[test] fn a_rule_made_in_windows_security_matches_through_its_variables() { - let ui_rule = Rule { - program: r"%ProgramFiles%\EternalMonitor\EternalMonitor-host.exe".into(), - ..installer_rule() - }; - assert_eq!(verdict(&[on(PRIVATE)], &[ui_rule]), Verdict::Allowed); + let ui_rule = Rule::unscoped( + r"%ProgramFiles%\EternalMonitor\EternalMonitor-host.exe", + true, + 0x7fff_ffff, + ); + assert_eq!(verdicts(&[on(PRIVATE)], &[ui_rule]), [Verdict::Allowed]); assert!(same_program( "\"c:/program files/eternalmonitor/eternalmonitor-host.exe\"", Path::new(EXE), @@ -524,6 +811,24 @@ mod tests { assert_eq!(expand(r"%%\%ProgramFiles%", env), r"%%\C:\Program Files"); } + /// A read that fails drops the last reading, so a block that is gone + /// can't stay on the Stream page. + #[test] + fn a_failed_read_clears_the_last_reading() { + let blocked = report(&[on(PUBLIC)], &[prompt_block(PUBLIC)]); + let mut failed = false; + observe(Ok(blocked.clone()), &mut failed); + assert_eq!(latest(), Some(blocked.clone())); + observe(Err("RPC server unavailable".into()), &mut failed); + assert!(failed); + assert_eq!(latest(), None); + observe(Err("again".into()), &mut failed); + assert_eq!(latest(), None); + observe(Ok(blocked.clone()), &mut failed); + assert!(!failed); + assert_eq!(latest(), Some(blocked)); + } + /// `key=value` or `key="value with spaces"` from a netsh command line. fn netsh_arg(params: &str, key: &str) -> Option { let mut rest = params; @@ -577,15 +882,16 @@ mod tests { rules.push(( name, Rule { - program: arg("program").unwrap_or_default(), - allow: arg("action").as_deref() == Some("allow"), enabled: arg("enable").as_deref() != Some("no"), udp: arg("protocol").as_deref() == Some("UDP"), - profiles: bits - .iter() - .filter(|(word, _)| profiles.split(',').any(|p| p == *word)) - .map(|(_, bit)| bit) - .sum(), + ..Rule::unscoped( + &arg("program").unwrap_or_default(), + arg("action").as_deref() == Some("allow"), + bits.iter() + .filter(|(word, _)| profiles.split(',').any(|p| p == *word)) + .map(|(_, bit)| bit) + .sum(), + ) }, )); } else { @@ -598,10 +904,7 @@ mod tests { #[test] fn running_setup_again_removes_a_dismissed_prompts_block() { let app = r"C:\Program Files\EternalMonitor"; - let other = Rule { - program: r"C:\Tools\Other.exe".into(), - ..prompt_block(PUBLIC) - }; + let other = Rule::unscoped(r"C:\Tools\Other.exe", false, PUBLIC); let before = vec![ ("EternalMonitor Host UDP".to_string(), installer_rule()), ( @@ -611,37 +914,40 @@ mod tests { ("Other".to_string(), other.clone()), ]; let rules: Vec = before.iter().map(|(_, rule)| rule.clone()).collect(); - assert_eq!(verdict(&[on(PUBLIC)], &rules), Verdict::Blocked); + assert_eq!(verdicts(&[on(PUBLIC)], &rules), [Verdict::Blocked]); let after = after_setup(before, app); assert_eq!( - verdict(&[on(PUBLIC)], &after), - Verdict::Allowed, + verdicts(&[on(PUBLIC)], &after), + [Verdict::Allowed], "{after:?}" ); - assert_eq!(verdict(&[on(PRIVATE)], &after), Verdict::Allowed); + assert_eq!(verdicts(&[on(PRIVATE)], &after), [Verdict::Allowed]); assert!( after.contains(&other), "Setup removed another program's rule" ); // A first install: nothing to remove, the Allow rule still comes. assert_eq!( - verdict(&[on(DOMAIN)], &after_setup(Vec::new(), app)), - Verdict::Allowed + verdicts(&[on(DOMAIN)], &after_setup(Vec::new(), app)), + [Verdict::Allowed] ); } #[test] fn the_log_line_names_no_path_or_rule() { - let report = judge( + let scoped = Rule { + remote_addresses: "192.168.1.0/24".into(), + ..prompt_block(PRIVATE) + }; + let report = report( &[on(PRIVATE), on(PUBLIC)], - &[installer_rule(), prompt_block(PUBLIC)], - &PathBuf::from(EXE), - env, + &[installer_rule(), prompt_block(PUBLIC), scoped], ); assert_eq!( report.summary(), - "Windows Firewall: profiles=[private:on public:on] allow_rules=1 block_rules=1 verdict=blocked" + "Windows Firewall: profiles=[private:allowed public:blocked] allow_rules=1 \ + block_rules=1 scoped_block_rules=1 lan=true verdict=partly_blocked" ); let line = format!("2026-11-05T12:00:00.000000Z WARN {}", report.summary()); assert_eq!(crate::telemetry::uploaded(&line), line); diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index 2addf59b..2029ac57 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -56,10 +56,8 @@ impl AnalyzerApp { /// Update, restart and fault notices, most urgent first. fn notices(&mut self, ui: &mut Ui, snap: &StatsSnapshot, client: &ClientSnapshot) { self.port_notice(ui); - let on_wifi = client.peer.is_some() && !client.on_usb(); - if let Some((title, body)) = crate::firewall::latest() - .filter(|_| !on_wifi) - .and_then(|report| firewall_notice(&report)) + if let Some((title, body)) = + firewall_notice(crate::firewall::latest().as_ref(), client.peer.is_some()) { widgets::banner(ui, WARNING, Icon::Warning, &title, &body, 0.0, |_| {}); ui.add_space(GAP - 10.0); @@ -1360,26 +1358,43 @@ pub(super) fn usb_cable_notice( }) } -/// What the Stream page says while Windows Firewall keeps iPads out. Only -/// for blocks it is sure of: a missing rule may be made up for by a rule it -/// doesn't read, such as one for the port. -pub(super) fn firewall_notice(report: &crate::firewall::Report) -> Option<(String, String)> { - use crate::firewall::Verdict; +/// What the Stream page says while Windows Firewall keeps iPads out: only +/// for a block certain on the network an iPad would use (see +/// [`crate::firewall::Report::certain_block`]), and never while an iPad is +/// connected, over Wi-Fi or the cable. +pub(super) fn firewall_notice( + report: Option<&crate::firewall::Report>, + session_active: bool, +) -> Option<(String, String)> { + use crate::firewall::CertainBlock; + if session_active { + return None; + } + let lead = "iPads on Wi-Fi can't reach this PC, so they say it didn't answer."; let usb = "A USB cable still works."; - match report.verdict { - Verdict::Blocked => Some(( - "Windows Firewall is blocking EternalMonitor".into(), - format!("iPads on Wi-Fi can't reach this PC, so they say it didn't answer. Run the EternalMonitor installer again: it removes the block. {usb}"), - )), - Verdict::BlocksAll => { - let network = report.blocking_profiles().join(" and "); - Some(( - "Windows Firewall blocks every incoming connection".into(), - format!("iPads on Wi-Fi can't reach this PC, so they say it didn't answer. In Windows Security, open Firewall & network protection, choose the {network} network and turn off Blocks all incoming connections. {usb}"), - )) + let turn_off = |networks: &[&str]| { + format!( + "In Windows Security, open Firewall & network protection, choose the {} network and turn off Blocks all incoming connections.", + networks.join(" and ") + ) + }; + Some(match report?.certain_block()? { + CertainBlock::Rule { all_blocked } => { + let mut body = + format!("{lead} Run the EternalMonitor installer again: it removes the block."); + if !all_blocked.is_empty() { + body = format!("{body} {}", turn_off(&all_blocked)); + } + ( + "Windows Firewall is blocking EternalMonitor".into(), + format!("{body} {usb}"), + ) } - Verdict::Allowed | Verdict::NoRule | Verdict::Off => None, - } + CertainBlock::AllIncoming(networks) => ( + "Windows Firewall blocks every incoming connection".into(), + format!("{lead} {} {usb}", turn_off(&networks)), + ), + }) } /// What the USB row says. The Apple Devices hint appears only when Apple's diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index 8a410307..24d86977 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -13,41 +13,76 @@ fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { on: true, block_all_inbound, }; - let rule = |allow| Rule { - program: exe.to_string_lossy().to_lowercase(), - allow, - enabled: true, - udp: true, - profiles: PUBLIC, - }; - let notice = |profiles: &[Profile], rules: &[Rule]| { - firewall_notice(&judge(profiles, rules, exe, |_| None)) + let block = |profiles| Rule::unscoped(&exe.to_string_lossy().to_lowercase(), false, profiles); + let allow = Rule::unscoped(&exe.to_string_lossy(), true, 0x7fff_ffff); + let report = |profiles: &[Profile], rules: &[Rule], lan| { + judge(profiles, rules, exe, 9876, lan, |_| None) }; - let (title, body) = notice(&[profile(PUBLIC, false)], &[rule(true), rule(false)]).unwrap(); + let blocked = report( + &[profile(PUBLIC, false)], + &[allow.clone(), block(PUBLIC)], + true, + ); + let (title, body) = firewall_notice(Some(&blocked), false).unwrap(); assert_eq!(title, "Windows Firewall is blocking EternalMonitor"); assert!( body.contains("Run the EternalMonitor installer again"), "{body}" ); assert!(body.contains("A USB cable still works."), "{body}"); + // Any connected iPad, over Wi-Fi or the cable, hides it. + assert_eq!(firewall_notice(Some(&blocked), true), None); + // No reading, or the last read failed. + assert_eq!(firewall_notice(None, false), None); - let (title, body) = notice( - &[profile(PRIVATE, false), profile(PUBLIC, true)], - &[rule(true)], - ) - .unwrap(); + let shields = report(&[profile(PUBLIC, true)], std::slice::from_ref(&allow), true); + let (title, body) = firewall_notice(Some(&shields), false).unwrap(); assert_eq!(title, "Windows Firewall blocks every incoming connection"); assert!(body.contains("choose the public network"), "{body}"); + let both = report( + &[profile(PRIVATE, false), profile(PUBLIC, true)], + &[block(PRIVATE)], + true, + ); + let (title, body) = firewall_notice(Some(&both), false).unwrap(); + assert_eq!(title, "Windows Firewall is blocking EternalMonitor"); + assert!( + body.contains("installer again") && body.contains("choose the public network"), + "{body}" + ); - assert_eq!(notice(&[profile(PUBLIC, false)], &[rule(true)]), None); - // No rule it can read: a port rule may still let iPads in. - assert_eq!(notice(&[profile(PUBLIC, false)], &[]), None); - let off = Profile { - on: false, - ..profile(PUBLIC, true) + // Not certain for the network an iPad uses: logged, not shown. + let one_of_two = report( + &[profile(PRIVATE, false), profile(PUBLIC, false)], + &[allow.clone(), block(PUBLIC)], + true, + ); + assert_eq!(firewall_notice(Some(&one_of_two), false), None); + let scoped = Rule { + remote_addresses: "LocalSubnet".into(), + ..block(PUBLIC) }; - assert_eq!(notice(&[off], &[rule(false)]), None); + assert_eq!( + firewall_notice( + Some(&report(&[profile(PUBLIC, false)], &[scoped], true)), + false + ), + None + ); + let no_lan = report(&[profile(PUBLIC, false)], &[block(PUBLIC)], false); + assert_eq!(firewall_notice(Some(&no_lan), false), None); + assert_eq!( + firewall_notice( + Some(&report(&[profile(PUBLIC, false)], &[allow], true)), + false + ), + None + ); + assert_eq!( + firewall_notice(Some(&report(&[profile(PUBLIC, false)], &[], true)), false), + None + ); } #[test] diff --git a/host/src/main.rs b/host/src/main.rs index 5a888d98..0dbd39cb 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -160,7 +160,7 @@ fn main() -> Result<(), Box> { info!(" Listen: {}:{}", listen_ip, listen_port); info!("══════════════════════════════════"); eternal_host::network::log_addresses(); - eternal_host::firewall::watch(); + eternal_host::firewall::watch(listen_port); let auth_token = persisted .ensure_auth_token() diff --git a/host/src/telemetry/mod.rs b/host/src/telemetry/mod.rs index 7d4f3187..9a6ee450 100644 --- a/host/src/telemetry/mod.rs +++ b/host/src/telemetry/mod.rs @@ -1863,6 +1863,40 @@ mod tests { assert_eq!(event_with(&stamped, "info", &names).message, stamped); } + /// The Windows Firewall line uploads byte for byte, even when the PC or + /// user is named like one of its tokens; the same names elsewhere go. + #[test] + fn the_firewall_line_is_never_redacted() { + let names = redact::Names::new(Some(r"C:\Users\blocked"), Some("blocked"), Some("public")); + let profiles = |bit| crate::firewall::Profile { + bit, + on: true, + block_all_inbound: false, + }; + let exe = std::path::Path::new(r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"); + let report = crate::firewall::judge( + &[ + profiles(crate::firewall::PRIVATE), + profiles(crate::firewall::PUBLIC), + ], + &[crate::firewall::Rule::unscoped( + &exe.to_string_lossy().to_lowercase(), + false, + crate::firewall::PUBLIC, + )], + exe, + 9876, + true, + |_| None, + ); + let stamped = format!("2026-11-06T02:04:08.196847Z WARN {}", report.summary()); + assert!(stamped.contains("public:blocked"), "{stamped}"); + assert_eq!(event_with(&stamped, "warn", &names).message, stamped); + let named = "2026-11-06T02:04:08.196847Z INFO Signed in as blocked on public"; + let uploaded = event_with(named, "info", &names).message; + assert!(!uploaded.contains(" blocked on public"), "{uploaded}"); + } + /// Names that are also the summary's own tokens leave it byte for byte, /// while the same words elsewhere are names. /// A line is redacted as it is queued: a device name the registry lets diff --git a/host/src/telemetry/redact.rs b/host/src/telemetry/redact.rs index 06ff2e56..9334f2f4 100644 --- a/host/src/telemetry/redact.rs +++ b/host/src/telemetry/redact.rs @@ -167,6 +167,13 @@ const DIAGNOSTIC_KEYS: &[&str] = &[ // The startup network line. "route", "lan_adapters", + // The Windows Firewall line. + "profiles", + "allow_rules", + "block_rules", + "scoped_block_rules", + "lan", + "verdict", // The virtual display snapshot. "driver", "tech", From 03b5271286fa895a06763f852358ab775024aacb Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Thu, 5 Nov 2026 21:40:27 -0500 Subject: [PATCH 006/226] Judge all-programs firewall rules, and never trust a field it couldn't read Review fixes for the Windows Firewall check: - A Block rule with no program blocks every app, ours included. It is now judged like ours when it covers the stream's port, logged as all_programs_block_rules, and the Stream page points to Windows Security instead of Setup, which deletes only this exe's rules. - A rule field that could not be read is unknown, not "any". A rule with an unknown scope or program is logged as scoped and never makes a block certain. Rules are read through a small trait, so tests can fail each getter. --- host/src/firewall.rs | 542 ++++++++++++++++++++++++++++------- host/src/gui/page_stream.rs | 13 + host/src/gui/tests.rs | 25 +- host/src/telemetry/mod.rs | 19 +- host/src/telemetry/redact.rs | 1 + 5 files changed, 495 insertions(+), 105 deletions(-) diff --git a/host/src/firewall.rs b/host/src/firewall.rs index 7f2b4fd0..324ff898 100644 --- a/host/src/firewall.rs +++ b/host/src/firewall.rs @@ -43,55 +43,109 @@ pub struct Profile { pub block_all_inbound: bool, } -/// An inbound rule, as far as the verdict needs it. Scope fields hold what -/// Windows stores; `*` (or empty) means any. +/// An inbound rule, as far as the verdict needs it. Text fields hold what +/// Windows stores (`*` or empty means any); `None` is a field that could not +/// be read, which the verdict treats as unknown, never as any. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Rule { - /// The program path as stored, which may use `%ProgramFiles%` and the like. - pub program: String, + /// The program path as stored, which may use `%ProgramFiles%` and the + /// like. Empty: the rule is for every program. + pub program: Option, pub allow: bool, pub enabled: bool, /// UDP or any protocol. pub udp: bool, pub profiles: u32, - pub local_ports: String, - pub remote_ports: String, - pub local_addresses: String, - pub remote_addresses: String, + pub local_ports: Option, + pub remote_ports: Option, + pub local_addresses: Option, + pub remote_addresses: Option, /// `All`, or some of `Wireless`, `Lan`, `RemoteAccess`. - pub interface_types: String, + pub interface_types: Option, /// The rule names particular interfaces. - pub interfaces: bool, + pub interfaces: Option, /// The rule is for a service, not every process of the program. - pub service: String, + pub service: Option, } impl Rule { - /// An inbound rule for `program` with no scope beyond its protocol. + /// An inbound rule for `program` (empty: every program) with no scope + /// beyond its protocol. pub fn unscoped(program: &str, allow: bool, profiles: u32) -> Self { Self { - program: program.into(), + program: Some(program.into()), allow, enabled: true, udp: true, profiles, - local_ports: "*".into(), - remote_ports: "*".into(), - local_addresses: "*".into(), - remote_addresses: "*".into(), - interface_types: "All".into(), - interfaces: false, - service: String::new(), + local_ports: Some("*".into()), + remote_ports: Some("*".into()), + local_addresses: Some("*".into()), + remote_addresses: Some("*".into()), + interface_types: Some("All".into()), + interfaces: Some(false), + service: Some(String::new()), } } } +/// The text fields of a firewall rule, read one by one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Field { + Program, + LocalPorts, + RemotePorts, + LocalAddresses, + RemoteAddresses, + InterfaceTypes, + Service, +} + +/// A rule as Windows hands it out. Each getter can fail on its own. +pub trait RuleSource { + fn inbound(&self) -> Result; + fn allow(&self) -> Result; + fn enabled(&self) -> Result; + /// `NET_FW_IP_PROTOCOL`: 17 is UDP, 256 any. + fn protocol(&self) -> Result; + fn profiles(&self) -> Result; + fn text(&self, field: Field) -> Result; + /// Whether the rule names particular interfaces. + fn interfaces_listed(&self) -> Result; +} + +/// The inbound rule `source` describes, or `None` for an outbound one. A +/// failed direction, action, state, protocol or profile read fails the +/// whole reading; a text field that can't be read stays unknown. +pub fn rule_from(source: &impl RuleSource) -> Result, String> { + if !source.inbound()? { + return Ok(None); + } + let text = |field| source.text(field).ok(); + let protocol = source.protocol()?; + Ok(Some(Rule { + program: text(Field::Program), + allow: source.allow()?, + enabled: source.enabled()?, + udp: protocol == 17 || protocol == 256, + profiles: source.profiles()? as u32, + local_ports: text(Field::LocalPorts), + remote_ports: text(Field::RemotePorts), + local_addresses: text(Field::LocalAddresses), + remote_addresses: text(Field::RemoteAddresses), + interface_types: text(Field::InterfaceTypes), + interfaces: source.interfaces_listed().ok(), + service: text(Field::Service), + })) +} + /// How far a rule reaches the stream's traffic. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Reach { /// Every datagram an iPad sends to the stream port. All, - /// Only some: it names addresses, ports, interfaces or a service. + /// Only some, or unknown: it names addresses, ports, interfaces or a + /// service, or a field of it could not be read. Some, /// None of it: another port. None, @@ -113,22 +167,36 @@ pub fn covers(ports: &str, port: u16) -> bool { }) } -fn any(value: &str) -> bool { - let value = value.trim(); - value.is_empty() || value == "*" +/// Read, and any: an unknown field is not. +fn any(value: &Option) -> bool { + value + .as_deref() + .map(str::trim) + .is_some_and(|value| value.is_empty() || value == "*") } /// How far `rule` reaches datagrams to `port`. pub fn reach(rule: &Rule, port: u16) -> Reach { - if !covers(&rule.local_ports, port) { + if rule + .local_ports + .as_deref() + .is_some_and(|ports| !covers(ports, port)) + { return Reach::None; } - let unscoped = any(&rule.remote_ports) + let all_types = rule + .interface_types + .as_deref() + .map(str::trim) + .is_some_and(|types| types.is_empty() || types.eq_ignore_ascii_case("all")); + // Ports that cover the stream's are no limit; ports never read are. + let unscoped = rule.local_ports.is_some() + && any(&rule.remote_ports) && any(&rule.local_addresses) && any(&rule.remote_addresses) - && (any(&rule.interface_types) || rule.interface_types.trim().eq_ignore_ascii_case("all")) - && !rule.interfaces - && rule.service.trim().is_empty(); + && all_types + && rule.interfaces == Some(false) + && rule.service.as_deref().is_some_and(|s| s.trim().is_empty()); if unscoped { Reach::All } else { @@ -136,13 +204,37 @@ pub fn reach(rule: &Rule, port: u16) -> Reach { } } +/// Whose traffic a rule is for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Whose { + /// This exe. + Ours, + /// Every program: Setup's program-specific delete can't remove it. + AllPrograms, + /// The program could not be read. + Unknown, + Other, +} + +fn whose(rule: &Rule, exe: &Path, lookup: impl Fn(&str) -> Option) -> Whose { + match rule.program.as_deref() { + None => Whose::Unknown, + Some(program) if program.trim().is_empty() => Whose::AllPrograms, + Some(program) if same_program(program, exe, lookup) => Whose::Ours, + Some(_) => Whose::Other, + } +} + /// What the firewall does on one profile. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Verdict { - /// An Allow rule for this exe, and nothing certain that beats it. + /// An Allow rule for this exe or every program, and nothing certain + /// that beats it. Allowed, /// An unscoped Block rule for this exe. Blocked, + /// An unscoped Block rule for every program on the stream's port. + BlockedForAllApps, /// The profile blocks all incoming connections. BlocksAll, /// Neither rule exists: Windows asks, or blocks quietly if it may not. @@ -156,6 +248,7 @@ impl Verdict { match self { Verdict::Allowed => "allowed", Verdict::Blocked => "blocked", + Verdict::BlockedForAllApps => "blocked_all_programs", Verdict::BlocksAll => "blocks_all", Verdict::NoRule => "no_rule", Verdict::Off => "off", @@ -163,7 +256,10 @@ impl Verdict { } fn blocks(self) -> bool { - matches!(self, Verdict::Blocked | Verdict::BlocksAll) + matches!( + self, + Verdict::Blocked | Verdict::BlockedForAllApps | Verdict::BlocksAll + ) } } @@ -172,9 +268,15 @@ impl Verdict { pub struct Report { /// Each profile in use, with its verdict. pub profiles: Vec<(u32, Verdict)>, + /// The stream's port. + pub port: u16, pub allow_rules: usize, + /// Unscoped Block rules for this exe. pub block_rules: usize, - /// Block rules that reach only some of the stream's traffic. + /// Unscoped Block rules for every program that cover the stream's port. + pub all_programs_block_rules: usize, + /// Block rules that reach only some of the stream's traffic, or that + /// could not be read whole. pub scoped_block_rules: usize, /// The PC has a home network address (see [`crate::network::Reach::Lan`]). pub lan: bool, @@ -183,6 +285,12 @@ pub struct Report { /// What the Stream page may say: a block certain on every network in use. #[derive(Clone, Debug, PartialEq, Eq)] pub enum CertainBlock { + /// A rule for every program blocks the stream's port: Setup can't remove + /// it. On `all_blocked` networks "Blocks all incoming connections" too. + AllPrograms { + port: u16, + all_blocked: Vec<&'static str>, + }, /// Block rules for this exe, and on `all_blocked` networks "Blocks all /// incoming connections" as well. Rule { all_blocked: Vec<&'static str> }, @@ -220,11 +328,22 @@ impl Report { .filter(|(_, v)| *v == Verdict::BlocksAll) .map(|(bit, _)| profile_name(*bit)) .collect(); - Some(if all_blocked.len() == self.profiles.len() { - CertainBlock::AllIncoming(all_blocked) - } else { - CertainBlock::Rule { all_blocked } - }) + Some( + if self + .profiles + .iter() + .any(|(_, v)| *v == Verdict::BlockedForAllApps) + { + CertainBlock::AllPrograms { + port: self.port, + all_blocked, + } + } else if all_blocked.len() == self.profiles.len() { + CertainBlock::AllIncoming(all_blocked) + } else { + CertainBlock::Rule { all_blocked } + }, + ) } /// The log line: profile names, verdicts and counts only. @@ -235,10 +354,11 @@ impl Report { .map(|(bit, verdict)| format!("{}:{}", profile_name(*bit), verdict.label())) .collect(); format!( - "Windows Firewall: profiles=[{}] allow_rules={} block_rules={} scoped_block_rules={} lan={} verdict={}", + "Windows Firewall: profiles=[{}] allow_rules={} block_rules={} all_programs_block_rules={} scoped_block_rules={} lan={} verdict={}", profiles.join(" "), self.allow_rules, self.block_rules, + self.all_programs_block_rules, self.scoped_block_rules, self.lan, self.verdict(), @@ -291,8 +411,9 @@ pub fn same_program(program: &str, exe: &Path, lookup: impl Fn(&str) -> Option Option, ) -> Report { let on: u32 = profiles.iter().filter(|p| p.on).map(|p| p.bit).sum(); - let ours: Vec<(&Rule, Reach)> = rules + let relevant: Vec<(&Rule, Whose, Reach)> = rules .iter() - .filter(|r| r.enabled && r.udp && same_program(&r.program, exe, &lookup)) - .map(|r| (r, reach(r, port))) - .filter(|(_, reach)| *reach != Reach::None) + .filter(|r| r.enabled && r.udp) + .map(|r| (r, whose(r, exe, &lookup), reach(r, port))) + .filter(|(_, whose, reach)| *whose != Whose::Other && *reach != Reach::None) .collect(); - let count = |allow: bool, wanted: Reach, mask: u32| { - ours.iter() - .filter(|(r, reach)| { - r.allow == allow && r.profiles & mask != 0 && (allow || *reach == wanted) - }) + let count = |mask: u32, wanted: &dyn Fn(&Rule, Whose, Reach) -> bool| { + relevant + .iter() + .filter(|(r, whose, reach)| r.profiles & mask != 0 && wanted(r, *whose, *reach)) .count() }; + // Windows has many Allow rules for every program, each limited to a + // service or some addresses: only an unscoped one lets any app in. + let allows = |r: &Rule, whose, reach| { + r.allow && (whose == Whose::Ours || (whose == Whose::AllPrograms && reach == Reach::All)) + }; + let ours = |r: &Rule, whose, reach| !r.allow && whose == Whose::Ours && reach == Reach::All; + let everyone = + |r: &Rule, whose, reach| !r.allow && whose == Whose::AllPrograms && reach == Reach::All; + let unsure = + |r: &Rule, whose, reach| !r.allow && (whose == Whose::Unknown || reach == Reach::Some); let verdicts = profiles .iter() .map(|p| { @@ -322,9 +452,11 @@ pub fn judge( Verdict::Off } else if p.block_all_inbound { Verdict::BlocksAll - } else if count(false, Reach::All, p.bit) > 0 { + } else if count(p.bit, &everyone) > 0 { + Verdict::BlockedForAllApps + } else if count(p.bit, &ours) > 0 { Verdict::Blocked - } else if count(true, Reach::All, p.bit) > 0 { + } else if count(p.bit, &allows) > 0 { Verdict::Allowed } else { Verdict::NoRule @@ -334,9 +466,11 @@ pub fn judge( .collect(); Report { profiles: verdicts, - allow_rules: count(true, Reach::All, on), - block_rules: count(false, Reach::All, on), - scoped_block_rules: count(false, Reach::Some, on), + port, + allow_rules: count(on, &allows), + block_rules: count(on, &ours), + all_programs_block_rules: count(on, &everyone), + scoped_block_rules: count(on, &unsure), lan, } } @@ -412,9 +546,10 @@ mod windows_policy { use std::path::Path; use windows::core::{IUnknown, Interface, VARIANT}; + use windows::Win32::Foundation::E_FAIL; use windows::Win32::NetworkManagement::WindowsFirewall::{ - INetFwPolicy2, INetFwRule, NetFwPolicy2, NET_FW_ACTION_ALLOW, NET_FW_IP_PROTOCOL_ANY, - NET_FW_IP_PROTOCOL_UDP, NET_FW_PROFILE_TYPE2, NET_FW_RULE_DIR_IN, + INetFwPolicy2, INetFwRule, NetFwPolicy2, NET_FW_ACTION_ALLOW, NET_FW_PROFILE_TYPE2, + NET_FW_RULE_DIR_IN, }; use windows::Win32::System::Com::{ CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, @@ -422,7 +557,9 @@ mod windows_policy { }; use windows::Win32::System::Ole::IEnumVARIANT; - use super::{judge, Profile, Report, Rule, DOMAIN, PRIVATE, PUBLIC}; + use super::{ + judge, rule_from, Field, Profile, Report, Rule, RuleSource, DOMAIN, PRIVATE, PUBLIC, + }; /// VT_DISPATCH: what the rule enumerator hands out. const VT_DISPATCH: u16 = 9; @@ -465,7 +602,52 @@ mod windows_policy { })) } - /// Every inbound rule that names a program. + impl RuleSource for INetFwRule { + fn inbound(&self) -> Result { + Ok(unsafe { self.Direction() }.map_err(|e| e.to_string())? == NET_FW_RULE_DIR_IN) + } + + fn allow(&self) -> Result { + Ok(unsafe { self.Action() }.map_err(|e| e.to_string())? == NET_FW_ACTION_ALLOW) + } + + fn enabled(&self) -> Result { + Ok(unsafe { self.Enabled() } + .map_err(|e| e.to_string())? + .as_bool()) + } + + fn protocol(&self) -> Result { + unsafe { self.Protocol() }.map_err(|e| e.to_string()) + } + + fn profiles(&self) -> Result { + unsafe { self.Profiles() }.map_err(|e| e.to_string()) + } + + fn text(&self, field: Field) -> Result { + let value = unsafe { + match field { + Field::Program => self.ApplicationName(), + Field::LocalPorts => self.LocalPorts(), + Field::RemotePorts => self.RemotePorts(), + Field::LocalAddresses => self.LocalAddresses(), + Field::RemoteAddresses => self.RemoteAddresses(), + Field::InterfaceTypes => self.InterfaceTypes(), + Field::Service => self.ServiceName(), + } + }; + value.map(|v| v.to_string()).map_err(|e| e.to_string()) + } + + fn interfaces_listed(&self) -> Result { + Ok(!unsafe { self.Interfaces() } + .map_err(|e| e.to_string())? + .is_empty()) + } + } + + /// Every inbound rule, each with what could be read of it. fn inbound_rules(policy: &INetFwPolicy2) -> windows::core::Result> { let all = unsafe { policy.Rules()? }; let each: IEnumVARIANT = unsafe { all._NewEnum()? }.cast()?; @@ -494,33 +676,11 @@ mod windows_policy { let Ok(rule) = unknown.cast::() else { continue; }; - if unsafe { rule.Direction()? } != NET_FW_RULE_DIR_IN { - continue; - } - let text = |value: windows::core::Result| { - value.map(|v| v.to_string()).unwrap_or_default() - }; - let program = text(unsafe { rule.ApplicationName() }); - if program.is_empty() { - continue; + if let Some(rule) = + rule_from(&rule).map_err(|e| windows::core::Error::new(E_FAIL, e))? + { + rules.push(rule); } - let protocol = unsafe { rule.Protocol()? }; - rules.push(Rule { - program, - allow: unsafe { rule.Action()? } == NET_FW_ACTION_ALLOW, - enabled: unsafe { rule.Enabled()? }.as_bool(), - udp: protocol == NET_FW_IP_PROTOCOL_UDP.0 || protocol == NET_FW_IP_PROTOCOL_ANY.0, - profiles: unsafe { rule.Profiles()? } as u32, - local_ports: text(unsafe { rule.LocalPorts() }), - remote_ports: text(unsafe { rule.RemotePorts() }), - local_addresses: text(unsafe { rule.LocalAddresses() }), - remote_addresses: text(unsafe { rule.RemoteAddresses() }), - interface_types: text(unsafe { rule.InterfaceTypes() }), - // A rule read back without its list counts as scoped: never - // a certain block. - interfaces: unsafe { rule.Interfaces() }.map_or(true, |list| !list.is_empty()), - service: text(unsafe { rule.ServiceName() }), - }); } Ok(rules) } @@ -650,7 +810,7 @@ mod tests { fn a_block_for_another_port_does_not_count() { for ports in ["443", "1-1024,9877", "RPC", "9000-9875"] { let rule = Rule { - local_ports: ports.into(), + local_ports: Some(ports.into()), ..prompt_block(PUBLIC) }; let report = report(&[on(PUBLIC)], &[installer_rule(), rule]); @@ -659,7 +819,7 @@ mod tests { } for ports in ["*", "", "9876", "53, 9000-9999"] { let rule = Rule { - local_ports: ports.into(), + local_ports: Some(ports.into()), ..prompt_block(PUBLIC) }; assert_eq!( @@ -670,7 +830,7 @@ mod tests { } // The port the stream really listens on decides. let rule = Rule { - local_ports: "19876".into(), + local_ports: Some("19876".into()), ..prompt_block(PUBLIC) }; let moved = judge( @@ -688,31 +848,31 @@ mod tests { fn a_scoped_block_is_logged_not_certain() { let scoped = [ Rule { - remote_addresses: "203.0.113.0/24".into(), + remote_addresses: Some("203.0.113.0/24".into()), ..prompt_block(PUBLIC) }, Rule { - remote_addresses: "LocalSubnet".into(), + remote_addresses: Some("LocalSubnet".into()), ..prompt_block(PUBLIC) }, Rule { - local_addresses: "10.0.0.5".into(), + local_addresses: Some("10.0.0.5".into()), ..prompt_block(PUBLIC) }, Rule { - remote_ports: "5000".into(), + remote_ports: Some("5000".into()), ..prompt_block(PUBLIC) }, Rule { - interface_types: "RemoteAccess".into(), + interface_types: Some("RemoteAccess".into()), ..prompt_block(PUBLIC) }, Rule { - interfaces: true, + interfaces: Some(true), ..prompt_block(PUBLIC) }, Rule { - service: "SomeService".into(), + service: Some("SomeService".into()), ..prompt_block(PUBLIC) }, ]; @@ -724,12 +884,195 @@ mod tests { assert!(report.summary().contains("scoped_block_rules=1")); } let all_types = Rule { - interface_types: "All".into(), + interface_types: Some("All".into()), ..prompt_block(PUBLIC) }; assert_eq!(verdicts(&[on(PUBLIC)], &[all_types]), [Verdict::Blocked]); } + /// A rule for every program blocks the stream as surely as one for this + /// exe, but Setup's delete by program can't remove it. + #[test] + fn a_block_for_every_program_on_the_port_is_certain_and_told_apart() { + let everyone = report( + &[on(PUBLIC)], + &[installer_rule(), Rule::unscoped("", false, PUBLIC)], + ); + assert_eq!(everyone.profiles, [(PUBLIC, Verdict::BlockedForAllApps)]); + assert_eq!( + (everyone.block_rules, everyone.all_programs_block_rules), + (0, 1) + ); + assert_eq!(everyone.verdict(), "blocked"); + assert_eq!( + everyone.certain_block(), + Some(CertainBlock::AllPrograms { + port: PORT, + all_blocked: vec![] + }) + ); + // With this exe's own block as well, the all-programs one decides + // the advice: re-running Setup would leave it. + let both = report( + &[on(PUBLIC)], + &[prompt_block(PUBLIC), Rule::unscoped(" ", false, PUBLIC)], + ); + assert!(matches!( + both.certain_block(), + Some(CertainBlock::AllPrograms { .. }) + )); + // Another port, or scoped: not a block, or only logged. + let other_port = Rule { + local_ports: Some("443".into()), + ..Rule::unscoped("", false, PUBLIC) + }; + let scoped = Rule { + remote_addresses: Some("LocalSubnet".into()), + ..Rule::unscoped("", false, PUBLIC) + }; + let unsure = report(&[on(PUBLIC)], &[installer_rule(), other_port, scoped]); + assert_eq!(unsure.profiles, [(PUBLIC, Verdict::Allowed)]); + assert_eq!( + (unsure.all_programs_block_rules, unsure.scoped_block_rules), + (0, 1) + ); + assert_eq!(unsure.certain_block(), None); + // An Allow rule for every program lets the stream in too, unless it + // is one of Windows' own, limited to a service or some addresses. + let service_only = Rule { + service: Some("Dnscache".into()), + ..Rule::unscoped("", true, PRIVATE) + }; + let windows_own = report(&[on(PRIVATE)], &[service_only]); + assert_eq!(windows_own.profiles, [(PRIVATE, Verdict::NoRule)]); + assert_eq!(windows_own.allow_rules, 0); + let open = report(&[on(PRIVATE)], &[Rule::unscoped("", true, PRIVATE)]); + assert_eq!(open.profiles, [(PRIVATE, Verdict::Allowed)]); + } + + /// A rule as a fake COM object: every getter answers, except `broken`. + struct Fake { + rule: Rule, + protocol: i32, + broken: Option<&'static str>, + } + + impl Fake { + fn get(&self, name: &str, value: T) -> Result { + if self.broken == Some(name) { + Err(format!("{name} failed (0x80004005)")) + } else { + Ok(value) + } + } + } + + impl RuleSource for Fake { + fn inbound(&self) -> Result { + self.get("direction", true) + } + fn allow(&self) -> Result { + self.get("action", self.rule.allow) + } + fn enabled(&self) -> Result { + self.get("enabled", self.rule.enabled) + } + fn protocol(&self) -> Result { + self.get("protocol", self.protocol) + } + fn profiles(&self) -> Result { + self.get("profiles", self.rule.profiles as i32) + } + fn text(&self, field: Field) -> Result { + let (name, value) = match field { + Field::Program => ("program", &self.rule.program), + Field::LocalPorts => ("local_ports", &self.rule.local_ports), + Field::RemotePorts => ("remote_ports", &self.rule.remote_ports), + Field::LocalAddresses => ("local_addresses", &self.rule.local_addresses), + Field::RemoteAddresses => ("remote_addresses", &self.rule.remote_addresses), + Field::InterfaceTypes => ("interface_types", &self.rule.interface_types), + Field::Service => ("service", &self.rule.service), + }; + self.get(name, value.clone().unwrap_or_default()) + } + fn interfaces_listed(&self) -> Result { + self.get("interfaces", self.rule.interfaces.unwrap_or(false)) + } + } + + fn fake(rule: Rule, broken: Option<&'static str>) -> Fake { + Fake { + rule, + protocol: 17, + broken, + } + } + + /// A scope field that can't be read is unknown, never "any": the rule + /// is logged and can't make a block certain. + #[test] + fn a_scope_that_could_not_be_read_never_makes_a_block_certain() { + for block in [prompt_block(PUBLIC), Rule::unscoped("", false, PUBLIC)] { + let read = rule_from(&fake(block.clone(), None)).unwrap().unwrap(); + assert_eq!(read, block); + assert!(report(&[on(PUBLIC)], &[read]).certain_block().is_some()); + for broken in [ + "program", + "local_ports", + "remote_ports", + "local_addresses", + "remote_addresses", + "interface_types", + "interfaces", + "service", + ] { + let read = rule_from(&fake(block.clone(), Some(broken))) + .unwrap() + .unwrap(); + let judged = report(&[on(PUBLIC)], &[installer_rule(), read]); + assert_eq!(judged.profiles, [(PUBLIC, Verdict::Allowed)], "{broken}"); + assert_eq!( + ( + judged.block_rules, + judged.all_programs_block_rules, + judged.scoped_block_rules + ), + (0, 0, 1), + "{broken}" + ); + assert_eq!(judged.certain_block(), None, "{broken}"); + } + } + // Unknown ports on another program's rule: not ours, not counted. + let other = rule_from(&fake( + Rule::unscoped(r"C:\Tools\Other.exe", false, PUBLIC), + Some("local_ports"), + )) + .unwrap() + .unwrap(); + assert_eq!(report(&[on(PUBLIC)], &[other]).scoped_block_rules, 0); + } + + /// The fields the verdict can't do without fail the whole reading, + /// which clears what the Stream page shows. + #[test] + fn a_failed_action_or_profile_read_fails_the_reading() { + for broken in ["direction", "action", "enabled", "protocol", "profiles"] { + let failed = rule_from(&fake(prompt_block(PUBLIC), Some(broken))); + assert!(failed.is_err(), "{broken}"); + } + let any_protocol = Fake { + protocol: 256, + ..fake(prompt_block(PUBLIC), None) + }; + assert!(rule_from(&any_protocol).unwrap().unwrap().udp); + let tcp = Fake { + protocol: 6, + ..fake(prompt_block(PUBLIC), None) + }; + assert!(!rule_from(&tcp).unwrap().unwrap().udp); + } + #[test] fn without_a_home_network_address_no_block_is_certain() { let report = judge( @@ -870,7 +1213,10 @@ mod tests { Some(program) if name == "all" => { assert_ne!(arg("dir").as_deref(), Some("out")); rules.retain(|(_, rule)| { - !same_program(&rule.program, Path::new(&program), env) + !rule + .program + .as_deref() + .is_some_and(|ours| same_program(ours, Path::new(&program), env)) }); } _ => rules.retain(|(named, _)| *named != name), @@ -937,7 +1283,7 @@ mod tests { #[test] fn the_log_line_names_no_path_or_rule() { let scoped = Rule { - remote_addresses: "192.168.1.0/24".into(), + remote_addresses: Some("192.168.1.0/24".into()), ..prompt_block(PRIVATE) }; let report = report( @@ -947,7 +1293,7 @@ mod tests { assert_eq!( report.summary(), "Windows Firewall: profiles=[private:allowed public:blocked] allow_rules=1 \ - block_rules=1 scoped_block_rules=1 lan=true verdict=partly_blocked" + block_rules=1 all_programs_block_rules=0 scoped_block_rules=1 lan=true verdict=partly_blocked" ); let line = format!("2026-11-05T12:00:00.000000Z WARN {}", report.summary()); assert_eq!(crate::telemetry::uploaded(&line), line); diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index 2029ac57..5d4f9ac2 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -1379,6 +1379,19 @@ pub(super) fn firewall_notice( ) }; Some(match report?.certain_block()? { + // Setup removes only this exe's rules: it can't help here. + CertainBlock::AllPrograms { port, all_blocked } => { + let mut body = format!( + "{lead} A Windows Firewall rule blocks port {port} for all apps. In Windows Security, open Firewall & network protection, then Advanced settings, and turn off that rule under Inbound Rules." + ); + if !all_blocked.is_empty() { + body = format!("{body} {}", turn_off(&all_blocked)); + } + ( + "A Windows Firewall rule blocks EternalMonitor's port".into(), + format!("{body} {usb}"), + ) + } CertainBlock::Rule { all_blocked } => { let mut body = format!("{lead} Run the EternalMonitor installer again: it removes the block."); diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index 24d86977..f9313fe8 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -52,6 +52,29 @@ fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { "{body}" ); + // A rule for every program: Setup can't remove it, so no Setup advice. + let everyone = report( + &[profile(PUBLIC, false)], + &[allow.clone(), Rule::unscoped("", false, PUBLIC)], + true, + ); + let (title, body) = firewall_notice(Some(&everyone), false).unwrap(); + assert_eq!( + title, + "A Windows Firewall rule blocks EternalMonitor's port" + ); + assert!(body.contains("blocks port 9876 for all apps"), "{body}"); + assert!(body.contains("Windows Security"), "{body}"); + assert!(!body.contains("installer"), "{body}"); + let mine_too = report( + &[profile(PUBLIC, false)], + &[block(PUBLIC), Rule::unscoped("", false, PUBLIC)], + true, + ); + let (_, body) = firewall_notice(Some(&mine_too), false).unwrap(); + assert!(!body.contains("installer"), "{body}"); + assert_eq!(firewall_notice(Some(&everyone), true), None); + // Not certain for the network an iPad uses: logged, not shown. let one_of_two = report( &[profile(PRIVATE, false), profile(PUBLIC, false)], @@ -60,7 +83,7 @@ fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { ); assert_eq!(firewall_notice(Some(&one_of_two), false), None); let scoped = Rule { - remote_addresses: "LocalSubnet".into(), + remote_addresses: Some("LocalSubnet".into()), ..block(PUBLIC) }; assert_eq!( diff --git a/host/src/telemetry/mod.rs b/host/src/telemetry/mod.rs index 9a6ee450..b086a621 100644 --- a/host/src/telemetry/mod.rs +++ b/host/src/telemetry/mod.rs @@ -1879,18 +1879,25 @@ mod tests { profiles(crate::firewall::PRIVATE), profiles(crate::firewall::PUBLIC), ], - &[crate::firewall::Rule::unscoped( - &exe.to_string_lossy().to_lowercase(), - false, - crate::firewall::PUBLIC, - )], + &[ + crate::firewall::Rule::unscoped( + &exe.to_string_lossy().to_lowercase(), + false, + crate::firewall::PUBLIC, + ), + crate::firewall::Rule::unscoped("", false, crate::firewall::PRIVATE), + ], exe, 9876, true, |_| None, ); let stamped = format!("2026-11-06T02:04:08.196847Z WARN {}", report.summary()); - assert!(stamped.contains("public:blocked"), "{stamped}"); + assert!( + stamped.contains("private:blocked_all_programs public:blocked") + && stamped.contains("all_programs_block_rules=1"), + "{stamped}" + ); assert_eq!(event_with(&stamped, "warn", &names).message, stamped); let named = "2026-11-06T02:04:08.196847Z INFO Signed in as blocked on public"; let uploaded = event_with(named, "info", &names).message; diff --git a/host/src/telemetry/redact.rs b/host/src/telemetry/redact.rs index 9334f2f4..741d966e 100644 --- a/host/src/telemetry/redact.rs +++ b/host/src/telemetry/redact.rs @@ -171,6 +171,7 @@ const DIAGNOSTIC_KEYS: &[&str] = &[ "profiles", "allow_rules", "block_rules", + "all_programs_block_rules", "scoped_block_rules", "lan", "verdict", From 7ec08e4950b9d07ae86911198e71485a8ef82eb3 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Thu, 5 Nov 2026 22:00:54 -0500 Subject: [PATCH 007/226] Show the firewall warning only for what Setup or one switch fixes Review fixes for the Windows Firewall check: - Rules are also read through INetFwRule3: an app package, a user owning the process, authorized local or remote users or machines, or IPsec SecureFlags limit whose traffic a rule is for. Any of them, set or unread, makes a rule scoped, so it never makes a block certain. - The Stream page warns only for this exe's own unscoped Block rule, which Setup removes, or "Blocks all incoming connections" on every profile in use. A Block rule for every program is logged (blocked_all_programs) and never shown. - An enabled Allow rule for this exe or every program that asks for IPsec may override Block rules for authenticated traffic. Any such rule makes a block uncertain: verdict=uncertain, bypass_rules=N, no warning. --- host/src/firewall.rs | 289 ++++++++++++++++++++++++++++------- host/src/gui/page_stream.rs | 19 +-- host/src/gui/tests.rs | 23 ++- host/src/telemetry/redact.rs | 1 + 4 files changed, 249 insertions(+), 83 deletions(-) diff --git a/host/src/firewall.rs b/host/src/firewall.rs index 324ff898..60921c28 100644 --- a/host/src/firewall.rs +++ b/host/src/firewall.rs @@ -9,10 +9,13 @@ //! the result when it changes. //! //! The Stream page warns only when the block is certain for the network an -//! iPad would use: every profile in use blocks, the PC has a home network -//! address, and no iPad is connected. A Block rule scoped to some addresses, -//! interfaces or a service, or a block on one of two networks, is only -//! logged. +//! iPad would use, and only for what Setup or one switch removes: this +//! exe's own unscoped Block rule, or "Blocks all incoming connections", on +//! every profile in use, with a home network address and no iPad connected. +//! Everything else is only logged: a Block rule for every program or one +//! scoped to some addresses, interfaces, a service, a package, users or +//! machines, a block on one of two networks, and any block an Allow rule +//! asking for IPsec might let traffic past. //! //! The log line names profiles and counts rules. It holds no path, rule name //! or address. @@ -66,6 +69,16 @@ pub struct Rule { pub interfaces: Option, /// The rule is for a service, not every process of the program. pub service: Option, + /// INetFwRule3's restrictions: an app package, a user owning the + /// process, and users or machines allowed to connect. Any of them limits + /// whose traffic the rule is for. + pub package: Option, + pub local_user_owner: Option, + pub local_users: Option, + pub remote_users: Option, + pub remote_machines: Option, + /// `NET_FW_AUTHENTICATE_TYPE`: 0 is none; anything else asks for IPsec. + pub secure_flags: Option, } impl Rule { @@ -85,6 +98,12 @@ impl Rule { interface_types: Some("All".into()), interfaces: Some(false), service: Some(String::new()), + package: Some(String::new()), + local_user_owner: Some(String::new()), + local_users: Some(String::new()), + remote_users: Some(String::new()), + remote_machines: Some(String::new()), + secure_flags: Some(0), } } } @@ -99,6 +118,12 @@ pub enum Field { RemoteAddresses, InterfaceTypes, Service, + /// INetFwRule3 from here on. + Package, + LocalUserOwner, + LocalUsers, + RemoteUsers, + RemoteMachines, } /// A rule as Windows hands it out. Each getter can fail on its own. @@ -112,6 +137,8 @@ pub trait RuleSource { fn text(&self, field: Field) -> Result; /// Whether the rule names particular interfaces. fn interfaces_listed(&self) -> Result; + /// INetFwRule3's `SecureFlags`. + fn secure_flags(&self) -> Result; } /// The inbound rule `source` describes, or `None` for an outbound one. A @@ -136,6 +163,12 @@ pub fn rule_from(source: &impl RuleSource) -> Result, String> { interface_types: text(Field::InterfaceTypes), interfaces: source.interfaces_listed().ok(), service: text(Field::Service), + package: text(Field::Package), + local_user_owner: text(Field::LocalUserOwner), + local_users: text(Field::LocalUsers), + remote_users: text(Field::RemoteUsers), + remote_machines: text(Field::RemoteMachines), + secure_flags: source.secure_flags().ok(), })) } @@ -144,8 +177,9 @@ pub fn rule_from(source: &impl RuleSource) -> Result, String> { pub enum Reach { /// Every datagram an iPad sends to the stream port. All, - /// Only some, or unknown: it names addresses, ports, interfaces or a - /// service, or a field of it could not be read. + /// Only some, or unknown: it names addresses, ports, interfaces, a + /// service, a package, users or machines, asks for IPsec, or a field of + /// it could not be read. Some, /// None of it: another port. None, @@ -196,7 +230,17 @@ pub fn reach(rule: &Rule, port: u16) -> Reach { && any(&rule.remote_addresses) && all_types && rule.interfaces == Some(false) - && rule.service.as_deref().is_some_and(|s| s.trim().is_empty()); + && [ + &rule.service, + &rule.package, + &rule.local_user_owner, + &rule.local_users, + &rule.remote_users, + &rule.remote_machines, + ] + .iter() + .all(|value| value.as_deref().is_some_and(|v| v.trim().is_empty())) + && rule.secure_flags == Some(0); if unscoped { Reach::All } else { @@ -278,19 +322,18 @@ pub struct Report { /// Block rules that reach only some of the stream's traffic, or that /// could not be read whole. pub scoped_block_rules: usize, + /// Allow rules for this exe or every program that may let IPsec + /// traffic past Block rules (see [`judge`]). + pub bypass_rules: usize, /// The PC has a home network address (see [`crate::network::Reach::Lan`]). pub lan: bool, } -/// What the Stream page may say: a block certain on every network in use. +/// What the Stream page may say: a block certain on every network in use, +/// and one Setup or a single switch removes. A rule for every program is +/// only logged. #[derive(Clone, Debug, PartialEq, Eq)] pub enum CertainBlock { - /// A rule for every program blocks the stream's port: Setup can't remove - /// it. On `all_blocked` networks "Blocks all incoming connections" too. - AllPrograms { - port: u16, - all_blocked: Vec<&'static str>, - }, /// Block rules for this exe, and on `all_blocked` networks "Blocks all /// incoming connections" as well. Rule { all_blocked: Vec<&'static str> }, @@ -302,7 +345,9 @@ impl Report { /// The whole picture in one word for the log. pub fn verdict(&self) -> &'static str { let blocking = self.profiles.iter().filter(|(_, v)| v.blocks()).count(); - if !self.profiles.is_empty() && blocking == self.profiles.len() { + if blocking > 0 && self.bypass_rules > 0 { + "uncertain" + } else if !self.profiles.is_empty() && blocking == self.profiles.len() { "blocked" } else if blocking > 0 { "partly_blocked" @@ -315,11 +360,18 @@ impl Report { } } - /// A block certain for the network an iPad would use: every profile in - /// use blocks, and the PC has a home network address. Otherwise `None`, - /// and the log line says what is known. + /// A block certain for the network an iPad would use, and one Setup or + /// a single switch removes: every profile in use is blocked by this + /// exe's own rule or blocks all incoming connections, the PC has a home + /// network address, and no Allow rule may let traffic past. Otherwise + /// `None`, and the log line says what is known. pub fn certain_block(&self) -> Option { - if !self.lan || self.profiles.is_empty() || !self.profiles.iter().all(|(_, v)| v.blocks()) { + let fixable = |v: &Verdict| matches!(v, Verdict::Blocked | Verdict::BlocksAll); + if !self.lan + || self.bypass_rules > 0 + || self.profiles.is_empty() + || !self.profiles.iter().all(|(_, v)| fixable(v)) + { return None; } let all_blocked: Vec<_> = self @@ -328,22 +380,11 @@ impl Report { .filter(|(_, v)| *v == Verdict::BlocksAll) .map(|(bit, _)| profile_name(*bit)) .collect(); - Some( - if self - .profiles - .iter() - .any(|(_, v)| *v == Verdict::BlockedForAllApps) - { - CertainBlock::AllPrograms { - port: self.port, - all_blocked, - } - } else if all_blocked.len() == self.profiles.len() { - CertainBlock::AllIncoming(all_blocked) - } else { - CertainBlock::Rule { all_blocked } - }, - ) + Some(if all_blocked.len() == self.profiles.len() { + CertainBlock::AllIncoming(all_blocked) + } else { + CertainBlock::Rule { all_blocked } + }) } /// The log line: profile names, verdicts and counts only. @@ -354,12 +395,13 @@ impl Report { .map(|(bit, verdict)| format!("{}:{}", profile_name(*bit), verdict.label())) .collect(); format!( - "Windows Firewall: profiles=[{}] allow_rules={} block_rules={} all_programs_block_rules={} scoped_block_rules={} lan={} verdict={}", + "Windows Firewall: profiles=[{}] allow_rules={} block_rules={} all_programs_block_rules={} scoped_block_rules={} bypass_rules={} lan={} verdict={}", profiles.join(" "), self.allow_rules, self.block_rules, self.all_programs_block_rules, self.scoped_block_rules, + self.bypass_rules, self.lan, self.verdict(), ) @@ -445,6 +487,22 @@ pub fn judge( |r: &Rule, whose, reach| !r.allow && whose == Whose::AllPrograms && reach == Reach::All; let unsure = |r: &Rule, whose, reach| !r.allow && (whose == Whose::Unknown || reach == Reach::Some); + // An Allow rule that asks for IPsec (SecureFlags) can come with + // "Override block rules": IPsec-authenticated traffic then gets past + // Block rules for the exe. iPads never authenticate, so it changes + // nothing for them, but the COM API doesn't say whether the override is + // set: any such rule, or one whose flags couldn't be read, makes a block + // uncertain. Protocol and port aren't looked at: fewer warnings, never + // a wrong one. + let bypass_rules = rules + .iter() + .filter(|r| { + r.enabled + && r.allow + && r.secure_flags != Some(0) + && matches!(whose(r, exe, &lookup), Whose::Ours | Whose::AllPrograms) + }) + .count(); let verdicts = profiles .iter() .map(|p| { @@ -471,6 +529,7 @@ pub fn judge( block_rules: count(on, &ours), all_programs_block_rules: count(on, &everyone), scoped_block_rules: count(on, &unsure), + bypass_rules, lan, } } @@ -548,8 +607,8 @@ mod windows_policy { use windows::core::{IUnknown, Interface, VARIANT}; use windows::Win32::Foundation::E_FAIL; use windows::Win32::NetworkManagement::WindowsFirewall::{ - INetFwPolicy2, INetFwRule, NetFwPolicy2, NET_FW_ACTION_ALLOW, NET_FW_PROFILE_TYPE2, - NET_FW_RULE_DIR_IN, + INetFwPolicy2, INetFwRule, INetFwRule3, NetFwPolicy2, NET_FW_ACTION_ALLOW, + NET_FW_PROFILE_TYPE2, NET_FW_RULE_DIR_IN, }; use windows::Win32::System::Com::{ CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, @@ -635,6 +694,16 @@ mod windows_policy { Field::RemoteAddresses => self.RemoteAddresses(), Field::InterfaceTypes => self.InterfaceTypes(), Field::Service => self.ServiceName(), + restriction => { + let rule3: INetFwRule3 = self.cast().map_err(|e| e.to_string())?; + match restriction { + Field::Package => rule3.LocalAppPackageId(), + Field::LocalUserOwner => rule3.LocalUserOwner(), + Field::LocalUsers => rule3.LocalUserAuthorizedList(), + Field::RemoteUsers => rule3.RemoteUserAuthorizedList(), + _ => rule3.RemoteMachineAuthorizedList(), + } + } } }; value.map(|v| v.to_string()).map_err(|e| e.to_string()) @@ -645,6 +714,11 @@ mod windows_policy { .map_err(|e| e.to_string())? .is_empty()) } + + fn secure_flags(&self) -> Result { + let rule3: INetFwRule3 = self.cast().map_err(|e| e.to_string())?; + unsafe { rule3.SecureFlags() }.map_err(|e| e.to_string()) + } } /// Every inbound rule, each with what could be read of it. @@ -891,9 +965,10 @@ mod tests { } /// A rule for every program blocks the stream as surely as one for this - /// exe, but Setup's delete by program can't remove it. + /// exe, but Setup's delete by program can't remove it: it is logged and + /// never shown. #[test] - fn a_block_for_every_program_on_the_port_is_certain_and_told_apart() { + fn a_block_for_every_program_on_the_port_is_logged_not_shown() { let everyone = report( &[on(PUBLIC)], &[installer_rule(), Rule::unscoped("", false, PUBLIC)], @@ -904,23 +979,15 @@ mod tests { (0, 1) ); assert_eq!(everyone.verdict(), "blocked"); - assert_eq!( - everyone.certain_block(), - Some(CertainBlock::AllPrograms { - port: PORT, - all_blocked: vec![] - }) - ); - // With this exe's own block as well, the all-programs one decides - // the advice: re-running Setup would leave it. + assert_eq!(everyone.certain_block(), None); + assert!(everyone.summary().contains("public:blocked_all_programs")); + // With this exe's own block as well: Setup would leave the other. let both = report( &[on(PUBLIC)], &[prompt_block(PUBLIC), Rule::unscoped(" ", false, PUBLIC)], ); - assert!(matches!( - both.certain_block(), - Some(CertainBlock::AllPrograms { .. }) - )); + assert_eq!(both.profiles, [(PUBLIC, Verdict::BlockedForAllApps)]); + assert_eq!(both.certain_block(), None); // Another port, or scoped: not a block, or only logged. let other_port = Rule { local_ports: Some("443".into()), @@ -992,12 +1059,20 @@ mod tests { Field::RemoteAddresses => ("remote_addresses", &self.rule.remote_addresses), Field::InterfaceTypes => ("interface_types", &self.rule.interface_types), Field::Service => ("service", &self.rule.service), + Field::Package => ("package", &self.rule.package), + Field::LocalUserOwner => ("local_user_owner", &self.rule.local_user_owner), + Field::LocalUsers => ("local_users", &self.rule.local_users), + Field::RemoteUsers => ("remote_users", &self.rule.remote_users), + Field::RemoteMachines => ("remote_machines", &self.rule.remote_machines), }; self.get(name, value.clone().unwrap_or_default()) } fn interfaces_listed(&self) -> Result { self.get("interfaces", self.rule.interfaces.unwrap_or(false)) } + fn secure_flags(&self) -> Result { + self.get("secure_flags", self.rule.secure_flags.unwrap_or(0)) + } } fn fake(rule: Rule, broken: Option<&'static str>) -> Fake { @@ -1015,7 +1090,8 @@ mod tests { for block in [prompt_block(PUBLIC), Rule::unscoped("", false, PUBLIC)] { let read = rule_from(&fake(block.clone(), None)).unwrap().unwrap(); assert_eq!(read, block); - assert!(report(&[on(PUBLIC)], &[read]).certain_block().is_some()); + // Read whole, it blocks (this exe's rule or every program's). + assert!(report(&[on(PUBLIC)], &[read]).profiles[0].1.blocks()); for broken in [ "program", "local_ports", @@ -1025,6 +1101,12 @@ mod tests { "interface_types", "interfaces", "service", + "package", + "local_user_owner", + "local_users", + "remote_users", + "remote_machines", + "secure_flags", ] { let read = rule_from(&fake(block.clone(), Some(broken))) .unwrap() @@ -1053,6 +1135,103 @@ mod tests { assert_eq!(report(&[on(PUBLIC)], &[other]).scoped_block_rules, 0); } + /// INetFwRule3's restrictions limit whose traffic a rule is for: a + /// package-only rule for every program is not a block on this exe. Each + /// one, set or unread, makes a rule scoped. + #[test] + fn a_rule_limited_to_a_package_user_or_machine_is_never_certain() { + type Set = fn(&mut Rule); + let limits: [(&str, Set); 6] = [ + ("package", |r| r.package = Some("S-1-15-2-1".into())), + ("local_user_owner", |r| { + r.local_user_owner = Some("S-1-5-21-1".into()) + }), + ("local_users", |r| { + r.local_users = Some("O:LSD:(A;;CC;;;S-1-5-21-1)".into()) + }), + ("remote_users", |r| { + r.remote_users = Some("O:LSD:(A;;CC;;;S-1-5-21-2)".into()) + }), + ("remote_machines", |r| { + r.remote_machines = Some("O:LSD:(A;;CC;;;S-1-5-21-3)".into()) + }), + ("secure_flags", |r| r.secure_flags = Some(1)), + ]; + for program in [EXE.to_lowercase(), String::new()] { + for (name, set) in limits { + let mut limited = Rule::unscoped(&program, false, PUBLIC); + set(&mut limited); + let read = rule_from(&fake(limited, None)).unwrap().unwrap(); + let judged = report(&[on(PUBLIC)], &[installer_rule(), read]); + assert_eq!( + judged.profiles, + [(PUBLIC, Verdict::Allowed)], + "{name} {program:?}" + ); + assert_eq!(judged.scoped_block_rules, 1, "{name}"); + assert_eq!(judged.certain_block(), None, "{name}"); + } + } + // Empty restrictions and no IPsec: the prompt's rule as Windows + // stores it stays a certain block. + let plain = rule_from(&fake(prompt_block(PUBLIC), None)) + .unwrap() + .unwrap(); + assert!(report(&[on(PUBLIC)], &[plain]).certain_block().is_some()); + } + + /// An Allow rule asking for IPsec may override Block rules for + /// authenticated traffic. Whatever blocks, the verdict is uncertain and + /// nothing is shown; it is logged as bypass_rules. + #[test] + fn an_allow_rule_that_may_bypass_blocks_makes_any_block_uncertain() { + let secure = |program: &str, flags: Option| Rule { + secure_flags: flags, + ..Rule::unscoped(program, true, 0x7fff_ffff) + }; + let shields = Profile { + block_all_inbound: true, + ..on(PUBLIC) + }; + for bypass in [ + secure(EXE, Some(2)), + secure("", Some(4)), + secure(&EXE.to_lowercase(), None), + ] { + for (profiles, rules) in [ + (vec![on(PUBLIC)], vec![prompt_block(PUBLIC), bypass.clone()]), + (vec![shields], vec![bypass.clone()]), + ] { + let judged = report(&profiles, &rules); + assert_eq!(judged.bypass_rules, 1, "{bypass:?}"); + assert_eq!(judged.verdict(), "uncertain"); + assert_eq!(judged.certain_block(), None); + assert!(judged.summary().contains("bypass_rules=1 ")); + } + } + // Not ours, disabled, or a Block rule: no bypass. + let others = [ + secure(r"C:\Tools\Other.exe", Some(2)), + Rule { + enabled: false, + ..secure(EXE, Some(2)) + }, + Rule { + allow: false, + remote_addresses: Some("10.0.0.0/8".into()), + ..secure(EXE, Some(2)) + }, + ]; + for other in others { + let judged = report(&[on(PUBLIC)], &[prompt_block(PUBLIC), other.clone()]); + assert_eq!(judged.bypass_rules, 0, "{other:?}"); + assert!(judged.certain_block().is_some(), "{other:?}"); + } + // Nothing blocks: a bypass rule alone changes nothing. + let open = report(&[on(PUBLIC)], &[secure(EXE, Some(2))]); + assert_eq!(open.verdict(), "allowed"); + } + /// The fields the verdict can't do without fail the whole reading, /// which clears what the Stream page shows. #[test] @@ -1293,7 +1472,7 @@ mod tests { assert_eq!( report.summary(), "Windows Firewall: profiles=[private:allowed public:blocked] allow_rules=1 \ - block_rules=1 all_programs_block_rules=0 scoped_block_rules=1 lan=true verdict=partly_blocked" + block_rules=1 all_programs_block_rules=0 scoped_block_rules=1 bypass_rules=0 lan=true verdict=partly_blocked" ); let line = format!("2026-11-05T12:00:00.000000Z WARN {}", report.summary()); assert_eq!(crate::telemetry::uploaded(&line), line); diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index 5d4f9ac2..405e3d7f 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -1359,9 +1359,9 @@ pub(super) fn usb_cable_notice( } /// What the Stream page says while Windows Firewall keeps iPads out: only -/// for a block certain on the network an iPad would use (see -/// [`crate::firewall::Report::certain_block`]), and never while an iPad is -/// connected, over Wi-Fi or the cable. +/// for a block certain on the network an iPad would use that Setup or one +/// switch removes (see [`crate::firewall::Report::certain_block`]), and +/// never while an iPad is connected, over Wi-Fi or the cable. pub(super) fn firewall_notice( report: Option<&crate::firewall::Report>, session_active: bool, @@ -1379,19 +1379,6 @@ pub(super) fn firewall_notice( ) }; Some(match report?.certain_block()? { - // Setup removes only this exe's rules: it can't help here. - CertainBlock::AllPrograms { port, all_blocked } => { - let mut body = format!( - "{lead} A Windows Firewall rule blocks port {port} for all apps. In Windows Security, open Firewall & network protection, then Advanced settings, and turn off that rule under Inbound Rules." - ); - if !all_blocked.is_empty() { - body = format!("{body} {}", turn_off(&all_blocked)); - } - ( - "A Windows Firewall rule blocks EternalMonitor's port".into(), - format!("{body} {usb}"), - ) - } CertainBlock::Rule { all_blocked } => { let mut body = format!("{lead} Run the EternalMonitor installer again: it removes the block."); diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index f9313fe8..e74b54a8 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -52,28 +52,27 @@ fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { "{body}" ); - // A rule for every program: Setup can't remove it, so no Setup advice. + // A rule for every program: Setup can't remove it, so it is only + // logged, alone or beside this exe's own block. let everyone = report( &[profile(PUBLIC, false)], &[allow.clone(), Rule::unscoped("", false, PUBLIC)], true, ); - let (title, body) = firewall_notice(Some(&everyone), false).unwrap(); - assert_eq!( - title, - "A Windows Firewall rule blocks EternalMonitor's port" - ); - assert!(body.contains("blocks port 9876 for all apps"), "{body}"); - assert!(body.contains("Windows Security"), "{body}"); - assert!(!body.contains("installer"), "{body}"); + assert_eq!(firewall_notice(Some(&everyone), false), None); let mine_too = report( &[profile(PUBLIC, false)], &[block(PUBLIC), Rule::unscoped("", false, PUBLIC)], true, ); - let (_, body) = firewall_notice(Some(&mine_too), false).unwrap(); - assert!(!body.contains("installer"), "{body}"); - assert_eq!(firewall_notice(Some(&everyone), true), None); + assert_eq!(firewall_notice(Some(&mine_too), false), None); + // An Allow rule that asks for IPsec may let traffic past: not shown. + let bypass = Rule { + secure_flags: Some(2), + ..allow.clone() + }; + let maybe = report(&[profile(PUBLIC, false)], &[bypass, block(PUBLIC)], true); + assert_eq!(firewall_notice(Some(&maybe), false), None); // Not certain for the network an iPad uses: logged, not shown. let one_of_two = report( diff --git a/host/src/telemetry/redact.rs b/host/src/telemetry/redact.rs index 741d966e..69283075 100644 --- a/host/src/telemetry/redact.rs +++ b/host/src/telemetry/redact.rs @@ -173,6 +173,7 @@ const DIAGNOSTIC_KEYS: &[&str] = &[ "block_rules", "all_programs_block_rules", "scoped_block_rules", + "bypass_rules", "lan", "verdict", // The virtual display snapshot. From 443a143e40d3ec09bd7463d2ba4f48864817b06b Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 11:27:18 -0500 Subject: [PATCH 008/226] Split the BGRA to YUV conversion across threads FFmpeg 7.1 only uses swscale's threads through sws_scale_frame, so the encoder's scaler is now built with a thread count (half the logical processors, at most four) and converts capture's rows where they lie through sws_scale_frame, with a buffer that never frees them. YUV420 input, which AMF, Quick Sync, x264 and PCs that kept YUV420 from 0.3.4 use, took about 14 ms per 2732x2048 frame on one thread. The swscale line in the log now says how many threads it uses. --- host/src/encoder/input.rs | 380 ++++++++++++++++++++++++++++++-------- host/src/encoder/mod.rs | 10 +- 2 files changed, 313 insertions(+), 77 deletions(-) diff --git a/host/src/encoder/input.rs b/host/src/encoder/input.rs index ecc3d429..105e86d6 100644 --- a/host/src/encoder/input.rs +++ b/host/src/encoder/input.rs @@ -100,20 +100,126 @@ pub(super) fn conversion_context( format: Pixel, width: u32, height: u32, -) -> Result, ffmpeg_next::Error> { +) -> Result, ffmpeg_next::Error> { if !matches!(format, Pixel::YUV420P | Pixel::NV12) { return Ok(None); } - ffmpeg_next::software::scaling::Context::get( - Pixel::BGRA, - width, - height, - format, - width, - height, - ffmpeg_next::software::scaling::Flags::FAST_BILINEAR, - ) - .map(Some) + Scaler::new(format, width, height, scaler_threads()).map(Some) +} + +/// Threads one conversion is split across: half the logical processors, at +/// most four. A 2732x2048 frame takes about 14 ms on one thread of a desktop +/// CPU, most of a 60 fps frame's 16.7 ms. +fn scaler_threads() -> usize { + threads_for(std::thread::available_parallelism().map_or(1, |count| count.get())) +} + +fn threads_for(processors: usize) -> usize { + (processors / 2).clamp(1, 4) +} + +/// swscale converting BGRA to `output` at one size, FAST_BILINEAR, on +/// `threads` threads. FFmpeg 7.1 splits a conversion across threads only in +/// `sws_scale_frame`; `sws_scale`, which ffmpeg-next's `Context::run` calls, +/// always runs on one. +pub(super) struct Scaler { + context: *mut ffmpeg_next::ffi::SwsContext, + width: u32, + height: u32, + output: Pixel, + threads: usize, +} + +impl Scaler { + fn new( + output: Pixel, + width: u32, + height: u32, + threads: usize, + ) -> Result { + use ffmpeg_next::ffi::{ + av_opt_set_int, sws_alloc_context, sws_init_context, AVPixelFormat, SWS_FAST_BILINEAR, + }; + let context = unsafe { sws_alloc_context() }; + if context.is_null() { + return Err(ffmpeg_next::Error::Other { + errno: libc::ENOMEM, + }); + } + // Freed by Drop from here on, including on the errors below. + let scaler = Self { + context, + width, + height, + output, + threads, + }; + let options = [ + (c"srcw", i64::from(width)), + (c"srch", i64::from(height)), + (c"src_format", AVPixelFormat::from(Pixel::BGRA) as i64), + (c"dstw", i64::from(width)), + (c"dsth", i64::from(height)), + (c"dst_format", AVPixelFormat::from(output) as i64), + (c"sws_flags", i64::from(SWS_FAST_BILINEAR)), + (c"threads", threads as i64), + ]; + for (name, value) in options { + let result = unsafe { av_opt_set_int(context.cast(), name.as_ptr(), value, 0) }; + if result < 0 { + return Err(ffmpeg_next::Error::from(result)); + } + } + let result = + unsafe { sws_init_context(context, std::ptr::null_mut(), std::ptr::null_mut()) }; + if result < 0 { + return Err(ffmpeg_next::Error::from(result)); + } + Ok(scaler) + } + + pub(super) fn threads(&self) -> usize { + self.threads + } + + /// Converts `input`, a BGRA frame of the scaler's size, into `output`. + pub(super) fn run( + &mut self, + input: &ffmpeg_next::frame::Video, + output: &mut ffmpeg_next::frame::Video, + ) -> Result<(), ffmpeg_next::Error> { + if input.format() != Pixel::BGRA + || input.width() != self.width + || input.height() != self.height + { + return Err(ffmpeg_next::Error::InputChanged); + } + self.check_output(output)?; + // SAFETY: both frames own their buffers and have the sizes checked. + let result = unsafe { + ffmpeg_next::ffi::sws_scale_frame(self.context, output.as_mut_ptr(), input.as_ptr()) + }; + if result < 0 { + return Err(ffmpeg_next::Error::from(result)); + } + Ok(()) + } + + fn check_output(&self, output: &ffmpeg_next::frame::Video) -> Result<(), ffmpeg_next::Error> { + if output.format() != self.output + || output.width() != self.width + || output.height() != self.height + { + return Err(ffmpeg_next::Error::OutputChanged); + } + Ok(()) + } +} + +impl Drop for Scaler { + fn drop(&mut self) { + unsafe { ffmpeg_next::ffi::sws_freeContext(self.context) } + } } /// Converts capture's BGRA rows, `row_bytes` apart, into `output` where they @@ -128,49 +234,62 @@ pub(super) fn conversion_context( /// [`frame_buffer`](crate::capture::frame_buffer) has, are copied into a /// frame FFmpeg allocated first. pub(super) fn convert_packed_bgra( - scaler: &mut ffmpeg_next::software::scaling::Context, + scaler: &mut Scaler, rows: &[u8], row_bytes: usize, output: &mut ffmpeg_next::frame::Video, ) -> Result<(), ffmpeg_next::Error> { - let input = *scaler.input(); - let picture_bytes = input.width as usize * 4; + use ffmpeg_next::ffi::{ + av_buffer_create, av_frame_alloc, av_frame_free, sws_scale_frame, AVPixelFormat, + AV_BUFFER_FLAG_READONLY, + }; + let (width, height) = (scaler.width, scaler.height); + let picture_bytes = width as usize * 4; // The last row ends after its picture's pixels. - let frame_bytes = row_bytes * (input.height as usize).saturating_sub(1) + picture_bytes; - if input.format != Pixel::BGRA || row_bytes < picture_bytes || rows.len() < frame_bytes { + let frame_bytes = row_bytes * (height as usize).saturating_sub(1) + picture_bytes; + if row_bytes < picture_bytes || rows.len() < frame_bytes { return Err(ffmpeg_next::Error::InputChanged); } - let wanted = scaler.output(); - if output.format() != wanted.format - || output.width() != wanted.width - || output.height() != wanted.height - { - return Err(ffmpeg_next::Error::OutputChanged); - } + scaler.check_output(output)?; if rows.len() < frame_bytes + FRAME_PADDING { - let copied = copied_frame(&rows[..frame_bytes], row_bytes, input.width, input.height); + let copied = copied_frame(&rows[..frame_bytes], row_bytes, width, height); return scaler.run(&copied, output); } - let planes = [ - rows.as_ptr(), - std::ptr::null(), - std::ptr::null(), - std::ptr::null(), - ]; - let strides = [row_bytes as libc::c_int, 0, 0, 0]; - // swscale reads `input.height` rows of `picture_bytes`, `row_bytes` + // sws_scale_frame takes reference-counted frames, so `rows` is wrapped + // in a buffer that never frees it. sws_scale_frame drops its references + // before it returns, and the frame is freed below, so nothing refers to + // `rows` after this call. + unsafe extern "C" fn borrowed(_: *mut libc::c_void, _: *mut u8) {} + let mut source = unsafe { av_frame_alloc() }; + if source.is_null() { + return Err(ffmpeg_next::Error::Other { + errno: libc::ENOMEM, + }); + } + // SAFETY: swscale reads `height` rows of `picture_bytes`, `row_bytes` // apart, from `rows`, and at most FRAME_PADDING more, checked above, and // writes the planes `output` allocated at the output size. let result = unsafe { - ffmpeg_next::ffi::sws_scale( - scaler.as_mut_ptr(), - planes.as_ptr(), - strides.as_ptr(), - 0, - input.height as libc::c_int, - (*output.as_mut_ptr()).data.as_ptr(), - (*output.as_mut_ptr()).linesize.as_ptr(), - ) + let data = rows.as_ptr().cast_mut(); + (*source).format = AVPixelFormat::from(Pixel::BGRA) as libc::c_int; + (*source).width = width as libc::c_int; + (*source).height = height as libc::c_int; + (*source).data[0] = data; + (*source).linesize[0] = row_bytes as libc::c_int; + (*source).buf[0] = av_buffer_create( + data, + rows.len(), + Some(borrowed), + std::ptr::null_mut(), + AV_BUFFER_FLAG_READONLY, + ); + let result = if (*source).buf[0].is_null() { + -libc::ENOMEM + } else { + sws_scale_frame(scaler.context, output.as_mut_ptr(), source) + }; + av_frame_free(&mut source); + result }; if result < 0 { return Err(ffmpeg_next::Error::from(result)); @@ -395,9 +514,9 @@ mod tests { } } - /// What the copy into an aligned BGRA frame gives. + /// What the copy into an aligned BGRA frame gives, converted by + /// ffmpeg-next's single-threaded `sws_scale`. fn converted_from_copy( - scaler: &mut ffmpeg_next::software::scaling::Context, format: Pixel, rows: &[u8], width: u32, @@ -406,7 +525,18 @@ mod tests { let frame_bytes = width as usize * 4 * height as usize; let copied = copied_frame(&rows[..frame_bytes], width as usize * 4, width, height); let mut converted = ffmpeg_next::frame::Video::new(format, width, height); - scaler.run(&copied, &mut converted).unwrap(); + ffmpeg_next::software::scaling::Context::get( + Pixel::BGRA, + width, + height, + format, + width, + height, + ffmpeg_next::software::scaling::Flags::FAST_BILINEAR, + ) + .unwrap() + .run(&copied, &mut converted) + .unwrap(); converted } @@ -432,23 +562,36 @@ mod tests { } } - /// swscale reading a padded capture buffer gives the same picture as the - /// copy into an aligned BGRA frame it replaced, at widths whose rows are - /// and aren't 64-byte multiples. + /// swscale reading a padded capture buffer, on one thread or several, + /// gives the same picture as single-threaded swscale reading the copy into + /// an aligned BGRA frame it replaced, at widths whose rows are (1680) and + /// aren't (2732, 1366) 64-byte multiples, and whole iPad screens. #[test] fn converting_capture_rows_in_place_matches_the_copied_frame() { ffmpeg_next::init().unwrap(); - for (width, height) in [(2732u32, 64u32), (96, 64), (1366, 32), (102, 18)] { + for (width, height) in [ + (2732u32, 2048u32), + (2732, 64), + (1680, 1050), + (96, 64), + (1366, 32), + (102, 18), + ] { let row_bytes = width as usize * 4; let frame_bytes = row_bytes * height as usize; let mut rows = crate::capture::frame_buffer(frame_bytes); fill_pattern(&mut rows[..frame_bytes], row_bytes); for format in [Pixel::YUV420P, Pixel::NV12] { + let before = converted_from_copy(format, &rows, width, height); + for threads in [1, 4] { + let mut scaler = Scaler::new(format, width, height, threads).unwrap(); + let mut after = ffmpeg_next::frame::Video::new(format, width, height); + convert_packed_bgra(&mut scaler, &rows, row_bytes, &mut after).unwrap(); + let label = format!("{width}x{height} {format:?} threads={threads}"); + assert_same_picture(&before, &after, &label); + } let mut scaler = conversion_context(format, width, height).unwrap().unwrap(); - let before = converted_from_copy(&mut scaler, format, &rows, width, height); let mut after = ffmpeg_next::frame::Video::new(format, width, height); - convert_packed_bgra(&mut scaler, &rows, row_bytes, &mut after).unwrap(); - assert_same_picture(&before, &after, &format!("{width}x{height} {format:?}")); // A short buffer or another size is refused, never read past. assert_eq!( convert_packed_bgra( @@ -472,6 +615,29 @@ mod tests { } } + #[test] + fn conversions_use_half_the_processors_up_to_four_threads() { + ffmpeg_next::init().unwrap(); + for (processors, threads) in [(1, 1), (2, 1), (4, 2), (6, 3), (8, 4), (16, 4), (64, 4)] { + assert_eq!(threads_for(processors), threads, "{processors} processors"); + } + let scaler = conversion_context(Pixel::YUV420P, 2732, 2048) + .unwrap() + .unwrap(); + assert_eq!(scaler.threads(), scaler_threads()); + let mut set = 0; + let result = unsafe { + ffmpeg_next::ffi::av_opt_get_int( + scaler.context.cast(), + c"threads".as_ptr(), + 0, + &mut set, + ) + }; + assert!(result >= 0); + assert_eq!(set, scaler_threads() as i64); + } + #[test] fn capture_padding_is_ffmpegs() { assert_eq!( @@ -480,21 +646,22 @@ mod tests { ); } - /// Rows with no readable bytes after them, ending where an unreadable - /// page starts, convert to the same picture without a crash: swscale - /// can read past the last row, so they take the copy. + /// `len` bytes that end where a page that faults on any access starts. #[cfg(unix)] - #[test] - fn rows_without_padding_are_never_read_past() { - ffmpeg_next::init().unwrap(); - let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize; - for (width, height) in [(2733u32, 2048u32), (1365, 32), (17, 3)] { - let row_bytes = width as usize * 4; - let frame_bytes = row_bytes * height as usize; - let readable = frame_bytes.div_ceil(page) * page; + struct Guarded { + base: *mut libc::c_void, + mapped: usize, + len: usize, + } + + #[cfg(unix)] + impl Guarded { + fn new(len: usize) -> Self { + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize; + let readable = len.div_ceil(page) * page; // SAFETY: a fresh private mapping, `readable` bytes then one - // page that faults on any access; unmapped below. - let rows = unsafe { + // page that faults on any access; unmapped on drop. + unsafe { let base = libc::mmap( std::ptr::null_mut(), readable + page, @@ -506,8 +673,45 @@ mod tests { assert_ne!(base, libc::MAP_FAILED); let guard = base.cast::().add(readable); assert_eq!(libc::mprotect(guard.cast(), page, libc::PROT_NONE), 0); - std::slice::from_raw_parts_mut(guard.sub(frame_bytes), frame_bytes) - }; + Self { + base, + mapped: readable + page, + len, + } + } + } + + fn bytes(&mut self) -> &mut [u8] { + let start = self.mapped - unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize; + // SAFETY: the last `len` readable bytes of the mapping. + unsafe { + std::slice::from_raw_parts_mut( + self.base.cast::().add(start - self.len), + self.len, + ) + } + } + } + + #[cfg(unix)] + impl Drop for Guarded { + fn drop(&mut self) { + assert_eq!(unsafe { libc::munmap(self.base, self.mapped) }, 0); + } + } + + /// Rows with no readable bytes after them, ending where an unreadable + /// page starts, convert to the same picture without a crash: swscale + /// can read past the last row, so they take the copy. + #[cfg(unix)] + #[test] + fn rows_without_padding_are_never_read_past() { + ffmpeg_next::init().unwrap(); + for (width, height) in [(2733u32, 2048u32), (1365, 32), (17, 3)] { + let row_bytes = width as usize * 4; + let frame_bytes = row_bytes * height as usize; + let mut mapping = Guarded::new(frame_bytes); + let rows = mapping.bytes(); fill_pattern(rows, row_bytes); // The pixel swscale reads after each odd-width row is set. let copied = copied_frame(rows, row_bytes, width, height); @@ -517,15 +721,43 @@ mod tests { assert_eq!(line[row_bytes..], line[row_bytes - 4..row_bytes]); } for format in [Pixel::YUV420P, Pixel::NV12] { - let mut scaler = conversion_context(format, width, height).unwrap().unwrap(); - let mut after = ffmpeg_next::frame::Video::new(format, width, height); - convert_packed_bgra(&mut scaler, rows, row_bytes, &mut after).unwrap(); - let before = converted_from_copy(&mut scaler, format, rows, width, height); - assert_same_picture(&before, &after, &format!("{width}x{height} {format:?}")); + let before = converted_from_copy(format, rows, width, height); + for threads in [1, 4] { + let mut scaler = Scaler::new(format, width, height, threads).unwrap(); + let mut after = ffmpeg_next::frame::Video::new(format, width, height); + convert_packed_bgra(&mut scaler, rows, row_bytes, &mut after).unwrap(); + let label = format!("{width}x{height} {format:?} threads={threads}"); + assert_same_picture(&before, &after, &label); + } } - unsafe { - let base = rows.as_mut_ptr().add(frame_bytes).sub(readable); - assert_eq!(libc::munmap(base.cast(), readable + page), 0); + } + } + + /// Capture's padded rows are converted where they lie, on one thread or + /// several, without reading past the padding: an unreadable page starts + /// right after it. Even widths give the copy's picture; at odd widths the + /// last chroma column takes the next row's first pixel instead. + #[cfg(unix)] + #[test] + fn padded_rows_are_read_in_place_no_further_than_the_padding() { + ffmpeg_next::init().unwrap(); + for (width, height) in [(2732u32, 2048u32), (2733, 2048), (1680, 1050), (17, 3)] { + let row_bytes = width as usize * 4; + let frame_bytes = row_bytes * height as usize; + let mut mapping = Guarded::new(frame_bytes + FRAME_PADDING); + let rows = mapping.bytes(); + fill_pattern(&mut rows[..frame_bytes], row_bytes); + for format in [Pixel::YUV420P, Pixel::NV12] { + let before = converted_from_copy(format, rows, width, height); + for threads in [1, 4] { + let mut scaler = Scaler::new(format, width, height, threads).unwrap(); + let mut after = ffmpeg_next::frame::Video::new(format, width, height); + convert_packed_bgra(&mut scaler, rows, row_bytes, &mut after).unwrap(); + if width % 2 == 0 { + let label = format!("{width}x{height} {format:?} threads={threads}"); + assert_same_picture(&before, &after, &label); + } + } } } } diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index a865edf5..5a7e2940 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -636,7 +636,7 @@ struct EncoderState { native_converter: Option, encoder: ffmpeg_next::codec::encoder::video::Encoder, /// BGRA to the encoder's YUV input, reading capture's buffer directly. - scaler: Option, + scaler: Option, frame: ffmpeg_next::frame::Video, packet: ffmpeg_next::Packet, h264: H264BitstreamState, @@ -815,8 +815,12 @@ impl EncoderState { h264.refresh_parameter_sets_from_extradata(encoder_extradata(&encoder)); let scaler = input::conversion_context(pixel_format, width, height)?; - if scaler.is_some() { - info!(output = ?pixel_format, "swscale BGRA conversion context created"); + if let Some(scaler) = &scaler { + info!( + output = ?pixel_format, + threads = scaler.threads(), + "swscale BGRA conversion context created" + ); } Ok(Self { From 9b5a2ca7e35ee926e1b5d103687e05c488969f08 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 11:48:01 -0500 Subject: [PATCH 009/226] Hold the virtual display on when it takes a hybrid laptop's panel away On an AMD hybrid laptop, the virtual display's arrival takes the built-in panel away below the layout: Windows lists the panel's target as unavailable, refuses every layout that names it, and only a restart gives it back. The guard now tells that case apart on a PC with more than one graphics card (the virtual display on, the panel off, no built-in target offered) and makes no SetDisplayConfig call. Every turn off stays held, at start-up too, so the desktop stays on the display the iPad shows; only Windows ending the session lets the turn off run, so the restart starts without the display. Second screen falls back at once with second_screen_fallback="panel_lost" and leaves the display on. The failure is saved until Second screen is chosen again: no wait, update, or blocked mirror tries it by itself. It is saved as an attach_timeout with panel_lost set, which older versions read as a timeout. The Stream page says to save work on the iPad and restart. --- ARCHITECTURE.md | 5 +- host/src/capture/second_screen.rs | 115 +++++++-- host/src/capture/second_screen/memory.rs | 140 +++++++++- host/src/diagnostics.rs | 5 +- host/src/display_guard.rs | 311 +++++++++++++++++++++-- host/src/gui/page_stream.rs | 6 + host/src/gui/tests.rs | 29 +++ 7 files changed, 559 insertions(+), 52 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 26ad1d4b..28800203 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -199,7 +199,10 @@ re-apply the saved layout once, else switch to "PC screen only" if the built-in panel was on (`display_guard.rs`, `display-guard.ps1`). At runtime the host also acts while the virtual display is still on: a turn on that leaves it alone switches Windows to "Extend" (else the saved layout or "PC screen only"), and a -turn off never runs while it is the only screen. It also ships a UDP +turn off never runs while it is the only screen, until Windows ends the session. +On a hybrid laptop whose panel the virtual display takes away until a restart +(Windows withdraws its target), it tries nothing and keeps the display on, and +Second screen mirrors there until the user chooses it again. It also ships a UDP firewall rule. Runtime upgrade, task-ACL and uninstall checks remain hardware gates. diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index b92c697d..11ddffc4 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -40,6 +40,7 @@ use tracing::warn; use crate::control::{CaptureTarget, SharedControl, VddStatus}; use crate::gpu_preference::AdapterId; +pub(crate) use memory::PANEL_LOST; use memory::{Hold, Memory}; #[cfg(any(windows, test, debug_assertions))] @@ -225,8 +226,12 @@ impl State { } match *status { VddStatus::WaitingForClient if connected => { - let hold = if blocked { - if let Some(hold) = self.memory.hold(now) { + let held = self.memory.hold(now); + // A blocked mirror is no reason to take the panel again. + let hold = if held.is_some_and(|hold| hold.until_chosen) { + held + } else if blocked { + if let Some(hold) = held { tracing::info!( reason = hold.reason, failures = hold.failures, @@ -236,9 +241,7 @@ impl State { } None } else { - self.memory - .hold(now) - .or_else(|| self.memory.hold_for_rule(now)) + held.or_else(|| self.memory.hold_for_rule(now)) }; if let Some(hold) = hold { *status = VddStatus::Failed; @@ -368,6 +371,21 @@ impl State { } } + /// The display took the built-in panel away: save that until Second + /// screen is chosen again. Neither a blocked mirror nor a withheld + /// desktop excuses it, since trying again takes the panel again. + fn remember_panel_lost(&mut self, now: Instant) { + self.clear_blocked_retry(); + if let Some(hold) = self.memory.note_failure(PANEL_LOST, now) { + tracing::info!( + reason = PANEL_LOST, + failures = hold.failures, + "Remembering that the second screen took the built-in screen away on this PC — \ + Second screen sessions mirror until Second screen is chosen again" + ); + } + } + /// [`desktop_away`], or what the fake display an end-to-end test /// installed says. fn desktop_away(&self) -> bool { @@ -443,6 +461,16 @@ pub(crate) fn proving(shared: &SharedControl) -> bool { /// The one log line for a connection that mirrors because the second screen /// failed on this PC before. Diagnostics count `second_screen_fallback=remembered`. fn log_hold(hold: Hold) { + if hold.until_chosen { + tracing::info!( + second_screen_fallback = "remembered", + reason = hold.reason, + failures = hold.failures, + "The second screen took the built-in screen away on this PC before — mirroring the \ + primary display until Second screen is chosen again, or the user clicks Try again" + ); + return; + } tracing::info!( second_screen_fallback = "remembered", reason = hold.reason, @@ -489,7 +517,8 @@ pub(crate) fn note_connected(shared: &SharedControl, now: Instant) { let mut status = shared.vdd_status.lock(); let mut state = shared.second_screen.lock(); // The mirror would show nothing: the second screen is the only picture. - if *status == VddStatus::Failed && blocked { + // Not after a lost panel: the next try takes it again. + if *status == VddStatus::Failed && blocked && state.failure != Some(PANEL_LOST) { let reason = state.failure.unwrap_or("attach_timeout"); if let Some(at) = state.blocked_retry_at.filter(|at| now < *at) { tracing::info!( @@ -543,7 +572,9 @@ pub(crate) fn note_connected(shared: &SharedControl, now: Instant) { pub(crate) fn retries_later(shared: &SharedControl) -> bool { let status = *shared.vdd_status.lock(); let state = shared.second_screen.lock(); - status == VddStatus::Failed && (state.memory.saved() || state.retry_blocked) + status == VddStatus::Failed + && state.failure != Some(PANEL_LOST) + && (state.memory.saved() || state.retry_blocked) } /// Windows refused to duplicate the mirror's `screen` from `card`, the card @@ -1296,11 +1327,12 @@ pub(crate) trait Driver { fn settle(&self, chosen: OutputInfo) -> OutputInfo { chosen } - /// After an Enable that worked: Windows put the desktop on the display - /// alone, so it will not attach beside the PC's screens. The display - /// guard has already tried to bring them back. - fn alone(&self) -> bool { - false + /// After an Enable that worked: whether Windows put the desktop on the + /// display alone, so it will not attach beside the PC's screens. The + /// display guard has already tried to bring them back, unless the + /// display took the built-in panel away. + fn turned_on(&self) -> TurnedOn { + TurnedOn::Beside } /// Say why the display did not appear or showed no picture, while it is /// still on. @@ -1309,6 +1341,20 @@ pub(crate) trait Driver { fn sleep(&self, duration: Duration); } +/// How an Enable that worked left the PC's screens, for [`Driver::turned_on`]. +#[cfg(any(windows, test, debug_assertions))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TurnedOn { + /// As usual: the display can attach beside them. + Beside, + /// Windows put the desktop on the display alone. + Alone, + /// On a hybrid laptop, the display took the built-in panel away until + /// the PC restarts (`crate::display_guard`). The desktop is on the + /// display, and turning it off would leave the PC with no screen. + PanelLost, +} + /// Why a second screen failed, for [`Driver::report`]. #[cfg(any(windows, test, debug_assertions))] #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1414,8 +1460,8 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura } let waiting_since = driver.now(); // No wait for a display that can't attach. - let alone = driver.alone(); - let result = if alone { + let turned_on = driver.turned_on(); + let result = if turned_on != TurnedOn::Beside { AttachResult::TimedOut } else { wait_for_virtual_output( @@ -1465,8 +1511,23 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura disable(shared, driver); } } + AttachResult::TimedOut if turned_on == TurnedOn::PanelLost => { + warn!( + second_screen_fallback = "panel_lost", + "The laptop's built-in screen went dark when the virtual display turned on — \ + leaving the display on and mirroring it, so the iPad shows the desktop" + ); + driver.report(Failure::AttachTimeout); + // No turn off: it would leave Windows with no screen at all. + let now = driver.now(); + let mut status = shared.vdd_status.lock(); + *status = VddStatus::Failed; + let mut state = shared.second_screen.lock(); + state.failure = Some(PANEL_LOST); + state.remember_panel_lost(now); + } AttachResult::TimedOut => { - if alone { + if turned_on == TurnedOn::Alone { warn!( second_screen_fallback = "vdd_only", "Windows put the desktop on the virtual display alone — capturing the \ @@ -1617,8 +1678,14 @@ impl Driver for WindowsDriver<'_> { chosen } - fn alone(&self) -> bool { - crate::display_guard::last_turn_on_alone() + fn turned_on(&self) -> TurnedOn { + if crate::display_guard::last_turn_on_lost_panel() { + TurnedOn::PanelLost + } else if crate::display_guard::last_turn_on_alone() { + TurnedOn::Alone + } else { + TurnedOn::Beside + } } fn report(&self, failure: Failure) { @@ -1745,8 +1812,8 @@ mod tests { /// display attaches `attach_after` later, or never. `on_sleep` sees the /// clock each time the host waits, as the transport's ticks would. /// With `enable_fails`, the Enable task reports that instead. Setting - /// the mode after the attach takes `settle_takes`. With `alone`, the - /// Enable left the display as the only screen. + /// the mode after the attach takes `settle_takes`. `turned_on` says how the + /// Enable left the PC's screens. struct FakeDriver<'a> { start: Instant, elapsed: Cell, @@ -1757,7 +1824,7 @@ mod tests { toggles: RefCell>, on_sleep: &'a dyn Fn(Duration), enable_fails: Option, - alone: bool, + turned_on: TurnedOn, calls: RefCell>, } @@ -1777,7 +1844,7 @@ mod tests { toggles: RefCell::new(Vec::new()), on_sleep, enable_fails: None, - alone: false, + turned_on: TurnedOn::Beside, calls: RefCell::new(Vec::new()), } } @@ -1825,8 +1892,8 @@ mod tests { chosen } - fn alone(&self) -> bool { - self.alone + fn turned_on(&self) -> TurnedOn { + self.turned_on } fn report(&self, failure: Failure) { @@ -3035,7 +3102,7 @@ mod tests { let on_sleep = |_| {}; // It would attach later, if it were waited for. let driver = FakeDriver { - alone: true, + turned_on: TurnedOn::Alone, ..FakeDriver::new(secs(7.0), Some(secs(1.0)), &on_sleep) }; let out = Arc::new(parking_lot::Mutex::new(Vec::::new())); diff --git a/host/src/capture/second_screen/memory.rs b/host/src/capture/second_screen/memory.rs index 72773103..fd07daca 100644 --- a/host/src/capture/second_screen/memory.rs +++ b/host/src/capture/second_screen/memory.rs @@ -19,6 +19,12 @@ //! picture, or where Try again was clicked, is exempt from every rule for //! good: a rule never moves a PC that works into the mirror. //! +//! One failure has no wait that ends by itself: [`PANEL_LOST`], a hybrid +//! laptop whose built-in panel the virtual display took away until a +//! restart. Each try costs the user a black screen and a restart, so only +//! choosing Second screen again (Try again, or another mode and back) tries +//! it again, also after an update changes the fingerprint. +//! //! The files are written by a [`Saver`] thread, never by the caller: the //! caller holds the locks the transport takes at every pass, and a write on //! a slow disk took 3 s, long enough for the iPad to give up. The thread @@ -44,6 +50,12 @@ pub(super) const RULE_FILE_NAME: &str = "second-screen-rule.json"; /// Saved for a manifest rule: nothing was tried. const HARDWARE_RULE: &str = "hardware_rule"; +/// The virtual display took the built-in panel away until the PC restarts. +/// Saved as an `attach_timeout` with `panel_lost` set: an older version +/// reads that as a timeout and waits, where it would drop a reason it +/// doesn't know and try again at once. +pub(crate) const PANEL_LOST: &str = "panel_lost"; + /// The file's layout. Another value is ignored, like an unreadable file. const FORMAT: u32 = 1; @@ -72,11 +84,12 @@ const BACKOFF: [Duration; 5] = [ /// no usable display. A failed Enable task is not one: its notice already /// says how to repair the install, and it costs no wait. Nor is a manifest /// rule, but it is saved the same way. -const REMEMBERED: [&str; 4] = [ +const REMEMBERED: [&str; 5] = [ "attach_timeout", "no_frames", "duplication_failed", HARDWARE_RULE, + PANEL_LOST, ]; /// What decides whether the second screen can work on this PC. Every value @@ -189,6 +202,9 @@ struct Record { failures: u32, /// When the last one failed, in seconds since 1970. failed_at: u64, + /// The failure is [`PANEL_LOST`]. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + panel_lost: bool, } /// Connections mirror without trying: why, and when the host tries again. @@ -197,6 +213,9 @@ pub(crate) struct Hold { pub reason: &'static str, pub failures: u32, pub retry_in: Duration, + /// No wait ends it: only choosing Second screen again ([`PANEL_LOST`]). + /// `retry_in` is then zero. + pub until_chosen: bool, } #[derive(Debug, Clone, Copy)] @@ -277,7 +296,10 @@ impl Memory { let _ = remove_file(&path); None } - Ok(Some((record, _))) if record.fingerprint != fingerprint => { + // A lost panel stays saved through an update: see [`PANEL_LOST`]. + Ok(Some((record, reason))) + if record.fingerprint != fingerprint && reason != PANEL_LOST => + { tracing::info!( reason = %record.reason, changed = %record.fingerprint.changes(&fingerprint).join(","), @@ -311,10 +333,12 @@ impl Memory { rule: None, }; if let Some(failed) = memory.failed { + let hold = memory.hold(now); tracing::info!( reason = failed.reason, failures = failed.failures, - retry_in_s = memory.hold(now).map_or(0, |hold| hold.retry_in.as_secs()), + retry_in_s = hold.map_or(0, |hold| hold.retry_in.as_secs()), + until_chosen = hold.is_some_and(|hold| hold.until_chosen), "The second screen failed on this PC before" ); } @@ -403,6 +427,14 @@ impl Memory { /// waiting longer. pub(crate) fn hold(&self, now: Instant) -> Option { let failed = self.failed?; + if failed.reason == PANEL_LOST { + return Some(Hold { + reason: failed.reason, + failures: failed.failures, + retry_in: Duration::ZERO, + until_chosen: true, + }); + } let wall = self.wall(now)?; let retry_at = failed .failed_at @@ -411,6 +443,7 @@ impl Memory { reason: failed.reason, failures: failed.failures, retry_in: Duration::from_secs(retry_at - wall), + until_chosen: false, }) } @@ -433,11 +466,13 @@ impl Memory { let failed_at = self.wall(now)?; let store = self.store.as_ref()?; let failures = self.failed.map_or(0, |f| f.failures).saturating_add(1); + let panel_lost = reason == PANEL_LOST; match encode(Some(Record { fingerprint: store.fingerprint.clone(), - reason: reason.to_string(), + reason: if panel_lost { "attach_timeout" } else { reason }.to_string(), failures, failed_at, + panel_lost, })) { Ok(data) => store.save(store.path.clone(), Save::Failure(data), now), Err(error) => warn!(%error, "{}", FAILURE_NOT_SAVED), @@ -861,6 +896,11 @@ fn read(path: &Path) -> Result, String> { return Err("no failures".to_string()); } let reason = remembered(&record.reason).ok_or_else(|| format!("reason {}", record.reason))?; + let reason = if record.panel_lost { + PANEL_LOST + } else { + reason + }; Ok(Some((record, reason))) } @@ -892,8 +932,8 @@ fn overwrite_file(path: &Path, data: &[u8]) -> std::io::Result<()> { mod tests { use super::super::{ attach, next_step, note_attached, note_connected, note_image, remembered_failure, - retries_later, set_facts, start_proving, Driver, Failure, Step, BLOCKED_EXTEND_RETRY, - PROVE_TIMEOUT, + retries_later, set_facts, start_proving, Driver, Failure, Step, TurnedOn, + BLOCKED_EXTEND_RETRY, PROVE_TIMEOUT, }; use super::*; use crate::capture::tests::{connect_viewer, hybrid_laptop, refuse_moved_mirror, NVIDIA}; @@ -1011,11 +1051,14 @@ mod tests { store.saver.saving.lock().clear_failure_logged } - /// A display that never attaches, on a paused clock. + /// A display that never attaches, on a paused clock. `turned_on` says + /// how its Enable left the PC's screens. struct NeverAttaches { start: Instant, elapsed: Cell, enables: Cell, + disables: Cell, + turned_on: TurnedOn, } impl NeverAttaches { @@ -1024,6 +1067,8 @@ mod tests { start, elapsed: Cell::new(Duration::ZERO), enables: Cell::new(0), + disables: Cell::new(0), + turned_on: TurnedOn::Beside, } } } @@ -1046,12 +1091,19 @@ mod tests { fn prepare(&self, _outputs: &[OutputInfo]) {} fn set_enabled(&self, enabled: bool) -> bool { - if enabled { - self.enables.set(self.enables.get() + 1); - } + let count = if enabled { + &self.enables + } else { + &self.disables + }; + count.set(count.get() + 1); true } + fn turned_on(&self) -> TurnedOn { + self.turned_on + } + fn now(&self) -> Instant { self.start + self.elapsed.get() } @@ -1061,6 +1113,74 @@ mod tests { } } + /// The hybrid laptop: the Enable took the built-in panel away until a + /// restart. The display stays on, so the desktop stays on the iPad, and + /// no wait ends the hold: not the next app run a week later, not an + /// update, not a blocked mirror. Only choosing Second screen again tries + /// it. The record reads as a timeout to an older version, which then + /// waits instead of trying at once. + #[test] + fn a_lost_panel_is_held_until_second_screen_is_chosen_again() { + let path = temp_file("panel-lost"); + let (memory, start) = app_run(&path, Duration::ZERO); + let shared = second_screen_with(memory); + connect_viewer(&shared); + assert_eq!(next_step(&shared, start), Step::Attach); + let driver = NeverAttaches { + turned_on: TurnedOn::PanelLost, + ..NeverAttaches::new(start) + }; + attach(&shared, &driver, ATTACH_TIMEOUT); + assert_eq!((driver.enables.get(), driver.disables.get()), (1, 0)); + assert_eq!(driver.now(), start, "no attach wait, no toggle gap"); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + assert_eq!(remembered_failure(&shared), Some(PANEL_LOST)); + assert!(!retries_later(&shared), "the notice says how to try again"); + let record = &saved(&path)["failure"]; + assert_eq!( + ( + &record["reason"], + &record["panel_lost"], + &record["failures"] + ), + ( + &serde_json::json!("attach_timeout"), + &serde_json::json!(true), + &serde_json::json!(1) + ) + ); + + // A week later, after an update, with Windows refusing the mirror. + written(&path); + let later = Instant::now(); + let updated = Fingerprint { + app: "0.3.6".to_string(), + ..laptop() + }; + let memory = Memory::load(path.clone(), updated, T0 + 7 * DAY, later); + let shared = second_screen_with(memory); + refuse_moved_mirror(&shared); + connect_viewer(&shared); + note_connected(&shared, later); + for day in [0, 1, 30] { + let now = later + Duration::from_secs(day * DAY); + reconnect(&shared, now); + assert_eq!(next_step(&shared, now), Step::Nothing, "day {day}"); + } + assert_eq!(remembered_failure(&shared), Some(PANEL_LOST)); + let hold = shared.second_screen.lock().memory.hold(later).unwrap(); + assert!(hold.until_chosen); + + // Choosing it again (another mode, then Second screen) tries it. + *shared.capture_target.lock() = CaptureTarget::PrimaryAuto; + assert_eq!(next_step(&shared, later), Step::Nothing); + assert!(!exists(&path)); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + *shared.vdd_status.lock() = VddStatus::WaitingForClient; + assert_eq!(next_step(&shared, later), Step::Attach); + remove_folder(&path); + } + /// The field case: the display never attaches. The next app run mirrors /// at once, without the driver or the 30 s wait, says why once per /// connection, and tries again after the wait. diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 251ec553..c29b2c10 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -286,8 +286,8 @@ fn take_fallback(session_id: u32) -> Option<&'static str> { /// A `second_screen_fallback` reason as the summary names it. `remembered` /// is a failure saved by an earlier app run. The backend's check of the -/// summary lists each token, so a newer reason, `vdd_only`, goes as `other` -/// until it lists that one too. +/// summary lists each token, so the newer reasons, `vdd_only` and +/// `panel_lost`, go as `other` until it lists them too. fn fallback_token(reason: &str) -> &'static str { match reason { "attach_timeout" => "attach_timeout", @@ -886,6 +886,7 @@ pub(crate) mod tests { ("duplication_failed", "duplication_failed"), ("remembered", "remembered"), ("vdd_only", "other"), + ("panel_lost", "other"), ("C:\\Users\\someone", "other"), ] { assert_eq!(fallback_token(reason), token); diff --git a/host/src/display_guard.rs b/host/src/display_guard.rs index d3189e24..adc03793 100644 --- a/host/src/display_guard.rs +++ b/host/src/display_guard.rs @@ -28,6 +28,17 @@ //! One attempt per change. Between two calls the guard reads the screens //! again, and stops if one came back or they can't be read. //! +//! On a hybrid laptop (AMD integrated and discrete graphics), the virtual +//! display's arrival also takes the built-in panel away below the layout: +//! Windows lists the panel's target as unavailable, every layout that names +//! it is refused, and only a restart gives it back. The guard tells that +//! case apart, on a PC with more than one graphics card, by the panel's +//! target: the virtual display is on, the panel is not, and no built-in +//! target is offered any more. It then makes no `SetDisplayConfig` call and +//! never turns the display off: the desktop stays on it, which the iPad +//! still shows. Only when Windows ends the session does the turn off run, +//! so the restart that brings the panel back starts without the display. +//! //! The layout from before a turn on is also kept on disk, so the failure //! reporter, a new process, compares with it when the host that turned //! the display on ended. The record names that host by id and start time, @@ -142,6 +153,13 @@ pub trait Displays { fn internal_only(&mut self) -> Result<(), i32>; /// `SetDisplayConfig` with "Extend". fn extend(&mut self) -> Result<(), i32>; + /// What Windows offers of the built-in panel's targets now, active or + /// not. `None` when it can't say. + fn built_in(&mut self) -> Option; + /// The PC has more than one graphics card. + fn hybrid(&self) -> bool; + /// Windows is ending the session: shutting down, restarting or signing out. + fn session_ending(&self) -> bool; fn sleep(&mut self, duration: Duration); fn log(&mut self, warn: bool, text: &str); /// The host whose layout from before its turn on is kept on disk: this @@ -163,6 +181,18 @@ pub trait Saved: Sized { fn decode(text: &str) -> Option; } +/// The built-in panel's targets, as `QDC_ALL_PATHS` lists them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BuiltIn { + /// None listed: a desktop PC. + Missing, + /// At least one is available for a path. + Offered, + /// Listed, and none available: a closed lid, or the hybrid laptop's + /// panel after the virtual display took it. + Withdrawn, +} + /// How a change ended. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Outcome { @@ -179,6 +209,9 @@ pub enum Outcome { /// it changed nothing more. Recovered, Failed, + /// A turn on took the built-in panel away for good: see [`lost_panel`]. + /// Nothing was tried, and the display stays on. + PanelLost, } const RECORD_HEADER: &str = "eternalmonitor-screens 1"; @@ -283,6 +316,8 @@ pub struct Guard { /// The last turn on left the virtual display alone, and "Extend" did /// not put the PC's screens beside it: it will not attach. alone: bool, + /// The last turn on took the built-in panel away for good. + panel_lost: bool, } impl Default for Guard { @@ -296,6 +331,7 @@ impl Guard { Self { reference: None, alone: false, + panel_lost: false, } } @@ -304,6 +340,12 @@ impl Guard { pub fn alone(&self) -> bool { self.alone } + + /// Whether the last turn on took the built-in panel away until the PC + /// restarts. + pub fn panel_lost(&self) -> bool { + self.panel_lost + } } /// A `SetDisplayConfig` call that can bring a real screen back while the @@ -360,6 +402,7 @@ impl Guard { ); displays.log(false, &text); let alone = layout.as_ref().is_some_and(|layout| alone(&layout.screens)); + let screens = layout.as_ref().map(|layout| layout.screens.clone()); if enabled { // A turn on can run while the display is still on: after an // Enable that finished once the host had given up on it, or a @@ -394,16 +437,40 @@ impl Guard { } self.reference = kept.or(layout); } - if !enabled && alone { - return self.take_back_before_off(displays); + if let (false, true, Some(screens)) = (enabled, alone, screens) { + return self.take_back_before_off(displays, &screens); } true } /// The virtual display is the only screen on, and is about to turn off: /// put a real screen back first. False when none came back and the PC - /// had one before, so the turn off must not run. - fn take_back_before_off(&mut self, displays: &mut impl Displays) -> bool { + /// had one before, so the turn off must not run, nor while the panel is + /// lost. Windows ending the session lets it run: the desktop goes, and + /// the next start then begins without the display. `screens`: now. + fn take_back_before_off( + &mut self, + displays: &mut impl Displays, + screens: &[Screen], + ) -> bool { + if displays.session_ending() { + displays.log( + false, + "Windows is ending the session — turning off the virtual display, the only \ + screen on, so the next start begins without it outcome=session_end", + ); + return true; + } + if lost_panel(displays, screens) { + let text = format!( + "Leaving the virtual display on: the built-in screen is still unavailable, and \ + turning the display off would leave the PC with no screen outcome=kept_on \ + reason=panel_lost {}; restarting the PC brings the built-in screen back", + line(Some(screens)) + ); + displays.log(true, &text); + return false; + } let reference = self.reference.as_ref(); let had_real = reference.is_some_and(|layout| real(&layout.screens).next().is_some()); let internal = reference.is_some_and(|layout| real(&layout.screens).any(|s| s.internal)); @@ -465,7 +532,11 @@ impl Guard { ) -> Outcome { if enabled { let outcome = self.check_turn_on(displays); - self.alone = matches!(outcome, Outcome::Restored { .. } | Outcome::Failed); + self.alone = matches!( + outcome, + Outcome::Restored { .. } | Outcome::Failed | Outcome::PanelLost + ); + self.panel_lost = outcome == Outcome::PanelLost; return outcome; } let outcome = self.check(displays); @@ -491,6 +562,20 @@ impl Guard { let (Some(reference), Some(after)) = (reference, layout) else { return Outcome::Left; }; + // No layout can name a target Windows withdrew, so none is tried. + if real(&reference.screens).any(|screen| screen.internal) + && lost_panel(displays, &after.screens) + { + let text = format!( + "The built-in screen went dark when the virtual display turned on: Windows \ + reports it unavailable, and only a restart brings it back. Leaving the virtual \ + display on, so the desktop stays on it outcome=panel_lost {}. Before: {}", + line(Some(&after.screens)), + line(Some(&reference.screens)) + ); + displays.log(true, &text); + return Outcome::PanelLost; + } let Plan::Restore { internal } = plan(&reference.screens, &after.screens) else { return Outcome::Left; }; @@ -669,6 +754,19 @@ fn alone(screens: &[Screen]) -> bool { has_vdd(screens) && real(screens).next().is_none() } +/// The hybrid laptop's black panel: on a PC with more than one graphics +/// card, the virtual display is on in `screens`, the built-in panel is off, +/// and Windows no longer offers any built-in target. After a turn on this +/// is read once the screens settle ([`SETTLE`] after the Enable, while no +/// real screen came back). A closed lid reads the same; holding the display +/// on then costs nothing. +fn lost_panel(displays: &mut D, screens: &[Screen]) -> bool { + has_vdd(screens) + && !screens.iter().any(|screen| screen.internal) + && displays.hybrid() + && displays.built_in() == Some(BuiltIn::Withdrawn) +} + /// The screens once a real one is back, or as they are after [`SETTLE`]. A /// PC that has one returns at once; only one that lost them all waits. fn settle(displays: &mut D, before: &[Screen]) -> Option> { @@ -730,18 +828,27 @@ pub(crate) fn last_turn_on_alone() -> bool { GUARD.lock().alone() } +/// Whether the last turn on took the built-in panel away until the PC +/// restarts. +#[cfg(windows)] +pub(crate) fn last_turn_on_lost_panel() -> bool { + GUARD.lock().panel_lost() +} + #[cfg(windows)] mod windows_impl { - use super::{Layout, Screen}; + use super::{BuiltIn, Layout, Screen}; use std::time::Duration; use windows::Win32::Devices::Display::{ SetDisplayConfig, DISPLAYCONFIG_MODE_INFO, DISPLAYCONFIG_MODE_INFO_TYPE_SOURCE, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_DISPLAYPORT_EMBEDDED, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_INTERNAL, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_LVDS, DISPLAYCONFIG_OUTPUT_TECHNOLOGY_UDI_EMBEDDED, DISPLAYCONFIG_PATH_INFO, - QDC_ONLY_ACTIVE_PATHS, SDC_ALLOW_CHANGES, SDC_APPLY, SDC_TOPOLOGY_EXTEND, - SDC_TOPOLOGY_INTERNAL, SDC_USE_SUPPLIED_DISPLAY_CONFIG, SDC_VALIDATE, + DISPLAYCONFIG_VIDEO_OUTPUT_TECHNOLOGY, QDC_ALL_PATHS, QDC_ONLY_ACTIVE_PATHS, + SDC_ALLOW_CHANGES, SDC_APPLY, SDC_TOPOLOGY_EXTEND, SDC_TOPOLOGY_INTERNAL, + SDC_USE_SUPPLIED_DISPLAY_CONFIG, SDC_VALIDATE, }; + use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_SHUTTINGDOWN}; pub type Saved = (Vec, Vec); @@ -800,6 +907,17 @@ mod windows_impl { } } + /// A laptop's built-in panel. + fn is_built_in(technology: DISPLAYCONFIG_VIDEO_OUTPUT_TECHNOLOGY) -> bool { + [ + DISPLAYCONFIG_OUTPUT_TECHNOLOGY_INTERNAL, + DISPLAYCONFIG_OUTPUT_TECHNOLOGY_LVDS, + DISPLAYCONFIG_OUTPUT_TECHNOLOGY_DISPLAYPORT_EMBEDDED, + DISPLAYCONFIG_OUTPUT_TECHNOLOGY_UDI_EMBEDDED, + ] + .contains(&technology) + } + fn screen(path: &DISPLAYCONFIG_PATH_INFO, modes: &[DISPLAYCONFIG_MODE_INFO]) -> Screen { let technology = path.targetInfo.outputTechnology; // Without QDC_VIRTUAL_MODE_AWARE the index is the whole field. @@ -818,13 +936,7 @@ mod windows_impl { Screen { adapter: crate::vdd_snapshot::luid_value(path.targetInfo.adapterId), target: path.targetInfo.id, - internal: [ - DISPLAYCONFIG_OUTPUT_TECHNOLOGY_INTERNAL, - DISPLAYCONFIG_OUTPUT_TECHNOLOGY_LVDS, - DISPLAYCONFIG_OUTPUT_TECHNOLOGY_DISPLAYPORT_EMBEDDED, - DISPLAYCONFIG_OUTPUT_TECHNOLOGY_UDI_EMBEDDED, - ] - .contains(&technology), + internal: is_built_in(technology), virtual_display: crate::vdd_snapshot::is_vdd_monitor( &crate::vdd_snapshot::monitor_path(path), ), @@ -875,6 +987,29 @@ mod windows_impl { } } + fn built_in(&mut self) -> Option { + let (paths, _) = crate::vdd_snapshot::display_config(QDC_ALL_PATHS)?; + let mut panels = paths + .iter() + .filter(|path| is_built_in(path.targetInfo.outputTechnology)) + .peekable(); + Some(if panels.peek().is_none() { + BuiltIn::Missing + } else if panels.any(|path| path.targetInfo.targetAvailable.as_bool()) { + BuiltIn::Offered + } else { + BuiltIn::Withdrawn + }) + } + + fn hybrid(&self) -> bool { + crate::diagnostics::host_facts().hybrid == Some(true) + } + + fn session_ending(&self) -> bool { + unsafe { GetSystemMetrics(SM_SHUTTINGDOWN) != 0 } + } + fn sleep(&mut self, duration: Duration) { std::thread::sleep(duration); } @@ -985,6 +1120,10 @@ mod tests { reapply_calls: Vec, internal_calls: usize, extend_calls: usize, + /// What `QDC_ALL_PATHS` says of the panel; `None` (unread) by default. + built_in: Option, + hybrid: bool, + session_ending: bool, slept: Duration, logs: Vec<(bool, String)>, owner: Option, @@ -1074,6 +1213,18 @@ mod tests { Ok(()) } + fn built_in(&mut self) -> Option { + self.built_in + } + + fn hybrid(&self) -> bool { + self.hybrid + } + + fn session_ending(&self) -> bool { + self.session_ending + } + fn sleep(&mut self, duration: Duration) { self.slept += duration; } @@ -1344,6 +1495,127 @@ mod tests { ); } + /// The hybrid laptop as `QDC_ALL_PATHS` showed it on the second run: + /// after the turn on, the panel's target was listed for every source + /// but `available=False`, so "Extend" was refused (31), the saved layout + /// was refused (1610) and "PC screen only" brought nothing back. Each + /// later turn off was held back after the same three calls. + fn hybrid_laptop() -> Fake { + let mut fake = Fake::reading([vec![laptop_panel()], vec![lone_vdd()]]); + fake.owner = Some(HOST.into()); + (fake.hybrid, fake.built_in) = (true, Some(BuiltIn::Withdrawn)); + (fake.refuse_extend, fake.refuse_reapply) = (true, true); + fake + } + + /// Its signature, the panel's target withdrawn while the virtual + /// display is on, ends the turn on with no `SetDisplayConfig` call, and + /// keeps the display on: the desktop stays on it, which the iPad shows. + #[test] + fn a_turn_on_that_takes_the_hybrid_laptops_panel_away_changes_nothing() { + let mut fake = hybrid_laptop(); + let mut guard = Guard::new(); + assert!(guard.before(true, false, &mut fake)); + assert_eq!(guard.after(true, true, &mut fake), Outcome::PanelLost); + assert_eq!(fake.set_display_config_calls(), 0); + assert!(guard.alone() && guard.panel_lost()); + let (warn, text) = fake.logs.last().unwrap(); + assert!(*warn); + assert_eq!( + text, + "The built-in screen went dark when the virtual display turned on: Windows reports \ + it unavailable, and only a restart brings it back. Leaving the virtual display on, \ + so the desktop stays on it outcome=panel_lost paths=0 [] vdd=1. Before: paths=1 \ + [internal tgt=1298b/512 1920x1200@165Hz] vdd=0" + ); + // Every turn off, the quit's too, stays held, still without a call. + for _ in 0..2 { + assert!(!turn_off_allowed(&mut guard, &mut fake)); + } + assert_eq!(fake.set_display_config_calls(), 0); + assert_eq!( + fake.logs.last().unwrap().1, + "Leaving the virtual display on: the built-in screen is still unavailable, and \ + turning the display off would leave the PC with no screen outcome=kept_on \ + reason=panel_lost paths=0 [] vdd=1; restarting the PC brings the built-in screen \ + back" + ); + assert!(fake.kept.is_some(), "kept for the failure reporter"); + // Once Windows ends the session, the turn off runs, so the restart + // starts without the display. + fake.session_ending = true; + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.set_display_config_calls(), 0); + assert!(fake.logs.last().unwrap().1.ends_with("outcome=session_end")); + // A panel that comes back (the lid opened, say) lets it run as on any PC. + let mut fake = hybrid_laptop(); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + guard.after(true, true, &mut fake); + fake.built_in = Some(BuiltIn::Offered); + fake.refuse_reapply = false; + fake.reapplied = Some(vec![laptop_panel()]); + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.reapply_calls, [1]); + } + + /// The host that starts after a restart with the display still on + /// (Windows ended before the turn off ran) has no layout from before. + /// Its start-up turn off must not leave the PC with no screen either. + #[test] + fn a_start_up_turn_off_with_the_panel_withdrawn_leaves_the_display_on() { + let mut fake = hybrid_laptop(); + fake.reads = VecDeque::from([Some(vec![lone_vdd()])]); + let mut guard = Guard::new(); + assert!(!turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.set_display_config_calls(), 0); + // On a PC with one graphics card, it goes as before. + fake.hybrid = false; + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.extend_calls, 1); + } + + /// The other ways a turn on leaves the virtual display alone keep the + /// guard's calls: one graphics card, a panel Windows still offers, a + /// desktop PC, or a target list Windows can't give. + #[test] + fn only_a_withdrawn_panel_on_a_hybrid_pc_is_the_lost_panel() { + for (hybrid, built_in) in [ + (false, Some(BuiltIn::Withdrawn)), + (true, Some(BuiltIn::Offered)), + (true, Some(BuiltIn::Missing)), + (true, None), + ] { + let mut fake = hybrid_laptop(); + (fake.hybrid, fake.built_in) = (hybrid, built_in); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!( + guard.after(true, true, &mut fake), + Outcome::Failed, + "{hybrid} {built_in:?}" + ); + assert!(!guard.panel_lost()); + assert_eq!(fake.set_display_config_calls(), 3, "{hybrid} {built_in:?}"); + } + // The panel still on beside the display: nothing happened to it. + let mut fake = hybrid_laptop(); + fake.reads = VecDeque::from([ + Some(vec![laptop_panel()]), + Some(vec![laptop_panel(), lone_vdd()]), + ]); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); + assert!(!guard.panel_lost()); + // A closed laptop on an external monitor never had the panel on. + let mut fake = hybrid_laptop(); + fake.reads = VecDeque::from([Some(vec![monitor()]), Some(vec![lone_vdd()])]); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); + } + #[test] fn a_healthy_pc_sees_no_set_display_config_call() { // Panel and monitor stay through the session; the monitor as the @@ -1643,6 +1915,15 @@ mod tests { fn extend(&mut self) -> Result<(), i32> { self.0.extend() } + fn built_in(&mut self) -> Option { + self.0.built_in() + } + fn hybrid(&self) -> bool { + self.0.hybrid() + } + fn session_ending(&self) -> bool { + self.0.session_ending() + } fn sleep(&mut self, duration: Duration) { self.0.sleep(duration) } diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index 679e95e1..107b704d 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -917,6 +917,12 @@ pub(super) fn second_screen_notice( }) }; match (status, health, problem) { + // Windows left the device as not extended too, but Extend can't help. + (Failed, _, _) if failure == Some("panel_lost") => notice( + "Your laptop's built-in screen turned off when the second screen started, and Windows only turns it back on after a restart. Until then your desktop is on the iPad: save your work there, then restart the PC. On this PC the second screen can turn the built-in screen off again, so the iPad shows your main screen until you click Try again or choose Second screen again.", + true, + false, + ), (Failed | TaskFailed, VddTaskHealth::NotResponding, _) => notice( "Windows' display driver stopped responding. Restart the PC to use the second screen.", false, diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index d7f2b148..337c4ade 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -680,6 +680,35 @@ fn the_update_popup_waits_until_no_ipad_has_been_connected_for_a_while() { assert!(!ipad_nearby(false, &mut last, gone + quiet, quiet)); } +/// The hybrid laptop's lost panel: Windows also leaves the display "not +/// extended", but Extend can't bring the panel back, so the notice says to +/// restart, and that Try again can take the panel again. +#[test] +fn a_lost_built_in_screen_says_to_restart_not_to_extend() { + use crate::control::VddStatus; + use crate::vdd::VddTaskHealth; + use crate::vdd_device::VddProblem; + for problem in [None, Some(VddProblem::NotExtended)] { + let lost = second_screen_notice( + VddStatus::Failed, + VddTaskHealth::Ok, + problem, + Some("panel_lost"), + ) + .unwrap(); + assert!( + lost.body.starts_with( + "Your laptop's built-in screen turned off when the second screen started" + ), + "{}", + lost.body + ); + assert!(lost.body.contains("restart the PC"), "{}", lost.body); + assert!(!lost.body.contains("Extend"), "{}", lost.body); + assert!(lost.retry && !lost.installer); + } +} + /// A second screen failure remembered for this PC is tried again later by /// itself; the notice says so beside Try again. #[test] From 36b36c4d282219b9072e2e88d30118ea596065de Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 11:49:42 -0500 Subject: [PATCH 010/226] Find a saved PC again when its address changes When a saved PC's HELLO2s go unanswered, look it up by its Bonjour name. If exactly one PC claims that name at another address, connect there and send the saved pairing. The PC must give back the same name and token, otherwise the iPad leaves it and keeps its pairing. Addresses the user typed and host names are never looked up. A home network entry gives way to the new address; VPN and Tailscale entries stay. Recent entries now keep the PC's name and whether the address was typed. The abandoned attempt's summary reads outcome=no_reply end=superseded. If the new address does not answer either, the card says where the iPad found the PC. scripts/e2e_moved_pc.sh streams from 127.0.0.1, moves the host, and expects the app to stream again by itself. --- ios/EternalMonitor/App/ConnectView.swift | 30 ++- .../App/ConnectionManager.swift | 120 +++++++++- .../App/ConnectionProblem.swift | 8 + ios/EternalMonitor/App/MovedPC.swift | 61 +++++ ios/EternalMonitor/App/SessionSummary.swift | 12 +- ios/EternalMonitor/Core/PairingStore.swift | 8 + .../Network/NetworkScanner.swift | 5 +- .../ConnectionProblemTests.swift | 4 +- ios/EternalMonitorTests/FakeUDPHost.swift | 9 +- ios/EternalMonitorTests/MovedPCTests.swift | 224 ++++++++++++++++++ .../SessionSummaryTests.swift | 2 + scripts/e2e_moved_pc.sh | 167 +++++++++++++ 12 files changed, 628 insertions(+), 22 deletions(-) create mode 100644 ios/EternalMonitor/App/MovedPC.swift create mode 100644 ios/EternalMonitorTests/MovedPCTests.swift create mode 100755 scripts/e2e_moved_pc.sh diff --git a/ios/EternalMonitor/App/ConnectView.swift b/ios/EternalMonitor/App/ConnectView.swift index daaaa718..7b8aaea6 100644 --- a/ios/EternalMonitor/App/ConnectView.swift +++ b/ios/EternalMonitor/App/ConnectView.swift @@ -20,6 +20,9 @@ struct ConnectView: View { // track whether we've already pre-filled from lastHost so we // don't clobber user edits on every re-render. @State private var didPrefillFromLastHost = false + /// The address the app last put in the field: the last PC, or one + /// found, scanned or picked from Recent. + @State private var filledAddress: String? @State private var showQRScanner = false @State private var showDetails = false @State private var feedbackRequest: FeedbackRequest? @@ -62,6 +65,12 @@ struct ConnectView: View { && connectionManager.feedbackAsk.due(streaming: connectionManager.state == .connected) } + /// The address field holds something the app didn't put there: the + /// user typed it. + private var typedAddress: Bool { + normalizedHostIP != filledAddress + } + private var canConnect: Bool { !normalizedHostIP.isEmpty && parsedPort != nil } @@ -207,7 +216,7 @@ struct ConnectView: View { if UIPreview.opensSetup && !didSeeOnboarding { showSetup = true; return } #endif if !didPrefillFromLastHost && hostIP.isEmpty && !settings.lastHost.isEmpty { - hostIP = settings.lastHost + fill(settings.lastHost) if settings.lastPort != 0 { port = String(settings.lastPort) } @@ -273,7 +282,7 @@ struct ConnectView: View { setupOutcome = .finished switch outcome { case .connect(let host, let port): - hostIP = host + fill(host) self.port = String(port) connectionManager.connect(host: host, port: port) return @@ -447,7 +456,7 @@ struct ConnectView: View { if !settings.lastHost.isEmpty && settings.lastHost != normalizedHostIP { Button { - hostIP = settings.lastHost + fill(settings.lastHost) if settings.lastPort != 0 { port = String(settings.lastPort) } } label: { Label(L("Use last PC: \(settings.lastHost)"), systemImage: "clock.arrow.circlepath") @@ -467,7 +476,7 @@ struct ConnectView: View { focusedField = nil guard let p = parsedPort else { return } withAnimation(.easeInOut(duration: 0.2)) { - connectionManager.connect(host: normalizedHostIP, port: p) + connectionManager.connect(host: normalizedHostIP, port: p, typed: typedAddress) } } label: { Text(L("Connect")) @@ -540,7 +549,7 @@ struct ConnectView: View { focusedField = .port } else if canConnect, let p = parsedPort { focusedField = nil - connectionManager.connect(host: normalizedHostIP, port: p) + connectionManager.connect(host: normalizedHostIP, port: p, typed: typedAddress) } } .focused($focusedField, equals: field) @@ -549,6 +558,11 @@ struct ConnectView: View { } } + private func fill(_ address: String) { + hostIP = address + filledAddress = address.trimmingCharacters(in: .whitespacesAndNewlines) + } + // MARK: - Connecting private var connectingCard: some View { @@ -620,7 +634,7 @@ struct ConnectView: View { private func handleScannedQR(_ value: String) { switch PairingCode.parse(value) { case .success(let target): - hostIP = target.host + fill(target.host) port = String(target.port) connectionManager.connect(host: target.host, port: target.port, token: target.token) case .failure(.wrongScheme): @@ -697,7 +711,7 @@ struct ConnectView: View { Button { // Only an IPv4 address reaches the PC; see DiscoveredHost. guard let address = host.address else { return } - hostIP = address + fill(address) port = "\(host.port)" connectionManager.connect(host: address, port: host.port) } label: { @@ -743,7 +757,7 @@ struct ConnectView: View { if conn.isUSB { connectionManager.resumeUSBConnections() } else { - hostIP = conn.host + fill(conn.host) port = "\(conn.port)" connectionManager.connect(host: conn.host, port: conn.port) } diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 54023198..5d3c3b77 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -218,6 +218,23 @@ final class ConnectionManager: ObservableObject { /// The Wi-Fi link to a host's port. Tests pass a fake one, or replace it /// to fail a lookup without asking DNS. var makeUDPLink: (UInt16) -> any MediaLink + /// Looks for PCs with Bonjour, given the service name wanted. Tests + /// replace it. + var findPCs: (String) async -> [DiscoveredHost] = { await MovedPC.find($0) } + /// "host:port" of each address the user typed in this launch. Saved + /// entries keep the mark once they show a picture. + private var typedAddresses = Set() + /// A saved PC that didn't answer at its address and was found at another. + private struct Move { + let from: (host: String, port: UInt16) + let name: String + /// Its pairing token, or zeros: the PC at the new address must give + /// the same one back. + let token: Data + } + /// The move the next attempt makes, and the one the current attempt made. + private var pendingMove: Move? + private var attemptMove: Move? init(timing: Timing = Timing(), udpReceiver: @escaping (UInt16) -> any MediaLink = { UDPReceiver(port: $0) }) { self.timing = timing @@ -301,9 +318,11 @@ final class ConnectionManager: ObservableObject { // MARK: - Connect / Disconnect - func connect(host: String, port: UInt16, token: Data? = nil) { + /// `typed`: the user typed this address, so it is never looked up by name. + func connect(host: String, port: UInt16, token: Data? = nil, typed: Bool = false) { guard state == .disconnected else { return } explicitPairingToken = token + if typed { typedAddresses.insert(PairingStore.address(host: host, port: port)) } if let token { do { try pairingStore.save(token: token, hostName: "", address: PairingStore.address(host: host, port: port)) } catch { record(.warning, "pairing", "Could not save the scanned pairing on this iPad.") } @@ -373,6 +392,8 @@ final class ConnectionManager: ObservableObject { let pairingAddress = isUSB ? "USB" : PairingStore.address(host: normalizedHost, port: port) let pairingToken = explicitPairingToken ?? ((try? pairingStore.token(address: pairingAddress)) ?? Data(repeating: 0, count: 16)) explicitPairingToken = nil + attemptMove = isUSB ? nil : pendingMove + pendingMove = nil let connectionID = UUID() currentAttempt.withLock { $0 = connectionID } transportMode = isUSB ? "USB" : "WiFi" @@ -403,6 +424,7 @@ final class ConnectionManager: ObservableObject { audioStats = AudioStats() debugState = ConnectionDebugState(host: normalizedHost, port: port) record(.info, "connection", "Starting connection attempt=\(connectionID) transport=\(transportMode) to \(normalizedHost):\(port) drawing=\(drawing.enabled) input=\(wantsInput) audio=\(wantsAudio)") + if attemptMove != nil { record(.info, "connection", "Trying the saved PC at the new address Bonjour gave") } let decoder = VideoDecoder() let metrics = ReceiverMetrics() @@ -464,8 +486,19 @@ final class ConnectionManager: ObservableObject { UIApplication.shared.isIdleTimerDisabled = UserDefaults.standard.flag(forKey: "keepScreenAwake") E2E.firstFrame(width: frameWidth, height: frameHeight, link: isUSB ? "usb" : "udp") - RecentConnectionStore.shared.add(host: isUSB ? (self.hostInfo?.hostName ?? "USB host") : normalizedHost, - port: port, isUSB: isUSB) + if let move = self.attemptMove { + // The same PC by its name and pairing: its new address + // replaces the old one, and so does its pairing. + RecentConnectionStore.shared.moved(from: move.from, to: (normalizedHost, port), + name: self.hostInfo?.hostName ?? move.name) + if MovedPC.replacesSavedEntry(move.from.host) { + try? self.pairingStore.remove(address: PairingStore.address(host: move.from.host, port: move.from.port)) + } + } else { + RecentConnectionStore.shared.add(host: isUSB ? (self.hostInfo?.hostName ?? "USB host") : normalizedHost, + port: port, isUSB: isUSB, name: self.hostInfo?.hostName, + typed: self.typedAddresses.contains(pairingAddress)) + } if !isUSB { UserDefaults.standard.set(normalizedHost, forKey: "lastHost") UserDefaults.standard.set(Int(port), forKey: "lastPort") @@ -589,6 +622,12 @@ final class ConnectionManager: ObservableObject { let acceptedAt = ProcessInfo.processInfo.systemUptime Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } + if let move = self.attemptMove, !Self.isSamePC(move, info) { + self.record(.warning, "connection", "The PC at the new address isn't the saved one") + self.disconnect(.otherPC) + self.problem = .noReply(usb: false) + return + } self.sessionAcceptedAt = acceptedAt self.acceptedAddresses.insert("\(normalizedHost):\(port)") self.noPictureStreak.answered(by: info.hostName, pairingToken: info.authToken, @@ -617,6 +656,14 @@ final class ConnectionManager: ObservableObject { let mismatch = channel?.lastMismatch Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } + // A PC that wants a code didn't take the saved PC's pairing: + // it is another PC, and the saved pairing stays. + if self.attemptMove != nil, status == .unauthorized || status == .rateLimited { + self.record(.warning, "connection", "The PC at the new address isn't the saved one: it asked for pairing") + self.disconnect(.otherPC) + self.problem = .noReply(usb: false) + return + } // Busy passes: the PC is streaming elsewhere, often to this // iPad's own cable session that hasn't closed yet. An // automatic cycle keeps retrying and shows it if it runs out. @@ -664,6 +711,7 @@ final class ConnectionManager: ObservableObject { self.disconnect(.legacyPC) } else { self.record(.warning, "ctrl", "HELLO2 handshake got no reply yet") + if !isUSB { self.lookForMovedPC(connectionID, host: normalizedHost, port: port, token: pairingToken) } } } } @@ -1000,6 +1048,41 @@ final class ConnectionManager: ObservableObject { } } + /// The attempt's HELLO2 got no answer at `host:port`. A saved PC whose + /// address may have changed is looked for by name; where Bonjour has it + /// at another address, the attempt moves there with the PC's pairing. + private func lookForMovedPC(_ connectionID: UUID, host: String, port: UInt16, token: Data) { + let address = PairingStore.address(host: host, port: port) + let saved = RecentConnectionStore.shared.entry(host: host, port: port) + let name = saved?.name ?? pairingStore.pcName(address: address) + guard attemptMove == nil, let name, + MovedPC.canLookUp(host: host, pcName: name, typed: typedAddresses.contains(address) || saved?.typed == true) + else { return } + record(.info, "connection", "Looking for the saved PC by name: no answer at its address") + Task { @MainActor [weak self] in + guard let self else { return } + let found = await self.findPCs(MovedPC.serviceName(pcName: name)) + // The PC may have answered meanwhile, or the attempt ended. + guard self.currentAttempt.withLock({ $0 == connectionID }), self.state == .connecting, + self.hostInfo == nil, !self.awaitingPairing else { return } + guard let moved = MovedPC.newAddress(pcName: name, host: host, port: port, found: found) else { + self.record(.info, "connection", "The saved PC wasn't found at another address") + return + } + self.record(.info, "connection", "Found the saved PC at \(moved.host):\(moved.port)") + self.disconnect(.pcMoved) + self.pendingMove = Move(from: (host, port), name: name, token: token) + self.explicitPairingToken = PairingStore.valid(token) ? token : nil + self.startAttempt(.udp(host: moved.host, port: moved.port)) + } + } + + /// The PC that accepted a moved attempt is the saved one: it has the + /// saved name, and gave back the saved pairing token if there was one. + private static func isSamePC(_ move: Move, _ info: ControlChannel.SessionInfo) -> Bool { + info.hostName == move.name && (!PairingStore.valid(move.token) || info.authToken == move.token) + } + /// Retires the automatic reconnect cycle, if one is running: its task /// stops at its next check and can no longer write state. private func endReconnectCycle() { @@ -1036,10 +1119,12 @@ final class ConnectionManager: ObservableObject { self.record(level, "timeout", debug.timeoutSummary(handshakeDone: handshakeDone) + self.attemptTimings) // Only an attempt the PC answered counts. if handshakeDone { self.noPictureStreak.endedWithoutPicture() } - self.problem = self.noPictureStreak.shown(.timeout( + let problem = self.noPictureStreak.shown(.timeout( listenerReady: debug.listenerReadyPort != nil, helloAttempts: debug.helloAttempts, handshakeDone: handshakeDone, datagramsReceived: debug.datagramsReceived, isUSB: isUSB, host: debug.host), at: isUSB ? nil : "\(debug.host):\(debug.port)") + self.problem = problem == .noReply(usb: false) && self.attemptMove != nil + ? .noReplyAtNewAddress(debug.host) : problem // The app gave up, not the user: the PC's log tells them apart. self.disconnect(.connectTimeout) if handshakeDone && self.noPictureStreak.stopsReconnecting { self.stopAutomaticAttempts(isUSB: isUSB) } @@ -1545,6 +1630,10 @@ struct RecentConnection: Codable, Identifiable { let port: UInt16 let lastUsed: Date let isUSB: Bool + /// The name the PC gave. Entries saved before 0.3.5 have none. + var name: String? = nil + /// The user typed this address. It is never looked up by name again. + var typed: Bool? = nil } extension RecentConnection { @@ -1568,16 +1657,35 @@ final class RecentConnectionStore: ObservableObject { load() } - func add(host: String, port: UInt16, isUSB: Bool) { + /// Saves a connection that showed a picture. A Wi-Fi entry keeps its + /// name and typed mark when this connection gives none. + func add(host: String, port: UInt16, isUSB: Bool, name: String? = nil, typed: Bool = false) { + let previous = isUSB ? nil : entry(host: host, port: port) connections.removeAll { $0.host == host && $0.port == port && $0.isUSB == isUSB } connections.insert( - RecentConnection(host: host, port: port, lastUsed: Date(), isUSB: isUSB), + RecentConnection(host: host, port: port, lastUsed: Date(), isUSB: isUSB, + name: isUSB ? nil : name.flatMap { $0.isEmpty ? nil : $0 } ?? previous?.name, + typed: isUSB ? nil : (typed || previous?.typed == true)), at: 0 ) if connections.count > 10 { connections = Array(connections.prefix(10)) } save() } + /// The saved Wi-Fi entry for `host:port`. + func entry(host: String, port: UInt16) -> RecentConnection? { + connections.first { !$0.isUSB && $0.host == host && $0.port == port } + } + + /// The PC moved from `old` to `new`: the old entry gives way where + /// `MovedPC.replacesSavedEntry` says so. + func moved(from old: (host: String, port: UInt16), to new: (host: String, port: UInt16), name: String) { + if MovedPC.replacesSavedEntry(old.host) { + connections.removeAll { !$0.isUSB && $0.host == old.host && $0.port == old.port } + } + add(host: new.host, port: new.port, isUSB: false, name: name) + } + private func load() { guard let data = UserDefaults.standard.data(forKey: key), let decoded = try? JSONDecoder().decode([RecentConnection].self, from: data) else { return } diff --git a/ios/EternalMonitor/App/ConnectionProblem.swift b/ios/EternalMonitor/App/ConnectionProblem.swift index d476ad3c..5802bf8b 100644 --- a/ios/EternalMonitor/App/ConnectionProblem.swift +++ b/ios/EternalMonitor/App/ConnectionProblem.swift @@ -8,6 +8,9 @@ enum ConnectionProblem: Equatable { /// Nothing came back from the address (PC closed, asleep, other network, /// blocked by a firewall or the router). case noReply(usb: Bool) + /// The saved address didn't answer, and Bonjour found the PC at this + /// new one, which didn't answer either. + case noReplyAtNewAddress(String) /// The PC answered, but no picture could be shown. case noPicture /// This iPad could not open its side of the connection. @@ -61,6 +64,7 @@ enum ConnectionProblem: Equatable { switch self { case .noReply(usb: false): return "Your PC didn't answer" case .noReply(usb: true): return "Your PC didn't answer over the cable" + case .noReplyAtNewAddress: return "Your PC didn't answer at its new address" case .noPicture: return "Your PC answered, but no picture arrived" case .receiverNotReady: return "This iPad couldn't start the connection" case .badAddress: return "That address doesn't look right" @@ -88,6 +92,10 @@ enum ConnectionProblem: Equatable { return "Check that EternalMonitor is open on the PC and says Ready for your iPad, that the PC is awake, " + "and that both are on the same Wi-Fi, not a guest network. If all of that is right, Windows Firewall " + "or security software on the PC may be blocking EternalMonitor: run the PC installer again, or connect with a USB cable." + case .noReplyAtNewAddress(let address): + return "This iPad found your PC on the network at a new address, \(address), but EternalMonitor didn't answer there. " + + "Check that the PC says Ready for your iPad. If it does, Windows Firewall or security software on the PC " + + "may be blocking EternalMonitor: run the PC installer again, or connect with a USB cable." case .noReply(usb: true): return "Check that EternalMonitor and Apple Devices are both open on the PC, then unplug the cable and plug it in again." case .noPicture: diff --git a/ios/EternalMonitor/App/MovedPC.swift b/ios/EternalMonitor/App/MovedPC.swift new file mode 100644 index 00000000..368e0352 --- /dev/null +++ b/ios/EternalMonitor/App/MovedPC.swift @@ -0,0 +1,61 @@ +import Foundation +import Network + +/// Finding a saved PC again after its address changed: the router gave it a +/// new one, or it moved between a cable and Wi-Fi. The protocol has no PC id, +/// so a PC is the name it gives in its HELLO_ACK, which is also the name in +/// its Bonjour service. Its pairing token then proves it is the same PC. +enum MovedPC { + /// The Bonjour service a PC named `pcName` advertises. + static func serviceName(pcName: String) -> String { "EternalMonitor (\(pcName))" } + + /// Whether an attempt at `host` that got no answer may look for its PC + /// by name. Not an address the user typed, which may be a VPN address + /// they chose on purpose, and not a host name, which DNS keeps current. + static func canLookUp(host: String, pcName: String?, typed: Bool) -> Bool { + guard !typed, let pcName, !pcName.isEmpty else { return false } + return IPv4Address(host) != nil && host.split(separator: ".").count == 4 + } + + /// Where Bonjour found the PC named `pcName` instead of `host:port`, or + /// nil: not found, found at the same address, or not one PC. Another PC + /// with the same name makes Bonjour rename one of the two services, so + /// any service named for it besides the one found makes it ambiguous. + static func newAddress(pcName: String, host: String, port: UInt16, + found: [DiscoveredHost]) -> (host: String, port: UInt16)? { + let name = serviceName(pcName: pcName) + let claims = found.filter { $0.name == name || $0.name.hasPrefix(name + " (") } + guard claims.count == 1, let pc = claims.first, pc.name == name, let address = pc.address, + address != host || pc.port != port else { return nil } + return (address, pc.port) + } + + /// Whether the saved entry at `host` gives way to the PC's new address: + /// only a home network address, which the router hands to another + /// device in time. A VPN or Tailscale address stays for use away from home. + static func replacesSavedEntry(_ host: String) -> Bool { + guard host.split(separator: ".").count == 4, let bytes = IPv4Address(host)?.rawValue, bytes.count == 4 else { return false } + return bytes[0] == 10 || (bytes[0] == 172 && (16...31).contains(bytes[1])) || (bytes[0] == 192 && bytes[1] == 168) + || (bytes[0] == 169 && bytes[1] == 254) + } + + /// Looks for EternalMonitor PCs for up to `seconds`, stopping soon after + /// the service `name` has an address. The wait after it lets a second + /// service with the same name show up. + @MainActor + static func find(_ name: String, seconds: Double = 4) async -> [DiscoveredHost] { + let scanner = NetworkScanner() + scanner.retriesWhenEmpty = false + scanner.startScan() + let deadline = Date().addingTimeInterval(seconds) + var foundAt: Date? + while Date() < deadline { + try? await Task.sleep(nanoseconds: 250_000_000) + if foundAt == nil, scanner.hosts.contains(where: { $0.name == name && $0.address != nil }) { foundAt = Date() } + if let foundAt, Date().timeIntervalSince(foundAt) >= 0.75 { break } + } + let hosts = scanner.hosts + scanner.stopScan() + return hosts + } +} diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index 2103755b..dbd44769 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -27,6 +27,11 @@ enum AttemptEnd: Equatable { case closedBeforePicture /// This iPad's network side failed, for example Local Network denied. case linkFailed + /// No answer at the saved address, and Bonjour found the same PC at + /// another: a new attempt goes there. + case pcMoved + /// The PC found at a saved PC's new address turned out to be another PC. + case otherPC case receiverFailed /// The PC asked for the pairing code, and a new HELLO2 sends it: the /// refused one is over, the attempt goes on. @@ -93,6 +98,7 @@ struct SessionSummary: Equatable { case .userDisconnect, .usbTurnedOff: return "user_ended" case .appBackground, .superseded, .pcGoodbye: return "app_ended" case .connectTimeout: return sessionId == nil ? "no_reply" : "no_picture" + case .pcMoved: return "no_reply" case .closedBeforePicture: return "no_picture" default: return "error" } @@ -100,11 +106,11 @@ struct SessionSummary: Equatable { private var failure: String { switch end { - case .connectTimeout where sessionId == nil: return "no_reply" + case .connectTimeout where sessionId == nil, .pcMoved: return "no_reply" case .connectTimeout: return datagramsReceived == 0 ? "no_video" : "no_frame" case .pcStoppedResponding: return "signal_lost" case .cableUnplugged, .usbLinkFailed, .closedBeforePicture: return "usb_lost" - case .pcRefused, .legacyPC, .linkFailed, .receiverFailed, .pictureStalled: return "other" + case .pcRefused, .legacyPC, .linkFailed, .receiverFailed, .pictureStalled, .otherPC: return "other" default: return "none" } } @@ -113,7 +119,7 @@ struct SessionSummary: Equatable { switch end { case .userDisconnect, .usbTurnedOff: return "user" case .appBackground: return "background" - case .superseded, .retriedWithCode: return "superseded" + case .superseded, .retriedWithCode, .pcMoved: return "superseded" case .pcGoodbye: return "goodbye" case .connectTimeout: return "timeout" case .pcStoppedResponding: return "liveness" diff --git a/ios/EternalMonitor/Core/PairingStore.swift b/ios/EternalMonitor/Core/PairingStore.swift index 34bf6e72..5d43db38 100644 --- a/ios/EternalMonitor/Core/PairingStore.swift +++ b/ios/EternalMonitor/Core/PairingStore.swift @@ -75,6 +75,14 @@ final class PairingStore: ObservableObject { try storage.records().first { $0.account.hasSuffix("|\(address)") && Self.valid($0.token) }?.token } + /// The name of the PC paired at `address`, if it gave one. + func pcName(address: String) -> String? { + guard let account = (try? storage.records())?.first(where: { $0.account.hasSuffix("|\(address)") && Self.valid($0.token) })?.account + else { return nil } + let name = String(account.dropLast(address.count + 1)) + return name.isEmpty ? nil : name + } + func isPaired(host: String, port: UInt16) -> Bool { (try? token(address: Self.address(host: host, port: port))) != nil } diff --git a/ios/EternalMonitor/Network/NetworkScanner.swift b/ios/EternalMonitor/Network/NetworkScanner.swift index c63e6848..f0009668 100644 --- a/ios/EternalMonitor/Network/NetworkScanner.swift +++ b/ios/EternalMonitor/Network/NetworkScanner.swift @@ -86,6 +86,9 @@ final class NetworkScanner: NSObject, ObservableObject { private var scanTask: Task? // track auto-retry state so we don't bounce forever when discovery is broken. private var didAutoRetry = false + /// Search once more when a search finds nothing. Off for a one-off + /// lookup that stops the search itself. + var retriesWhenEmpty = true override init() { super.init() @@ -139,7 +142,7 @@ final class NetworkScanner: NSObject, ObservableObject { } if hosts.isEmpty { - if !didAutoRetry { + if !didAutoRetry && retriesWhenEmpty { // auto-retry once after 2s. Bonjour browse occasionally // misses the first multicast burst. didAutoRetry = true diff --git a/ios/EternalMonitorTests/ConnectionProblemTests.swift b/ios/EternalMonitorTests/ConnectionProblemTests.swift index 7450d989..cb8dadae 100644 --- a/ios/EternalMonitorTests/ConnectionProblemTests.swift +++ b/ios/EternalMonitorTests/ConnectionProblemTests.swift @@ -205,8 +205,8 @@ final class ConnectionProblemTests: XCTestCase { func testEveryProblemSaysWhatToDoWithoutTransportJargon() { let all: [ConnectionProblem] = [ - .noReply(usb: false), .noReply(usb: true), .noPicture, .receiverNotReady, .badAddress("pc"), - .noNetwork, .localNetworkBlocked, .couldNotConnect, .usbFailed, .lost(.pcStoppedResponding), .lost(.cableUnplugged), + .noReply(usb: false), .noReply(usb: true), .noReplyAtNewAddress("192.168.1.31"), .noPicture, .receiverNotReady, + .badAddress("pc"), .noNetwork, .localNetworkBlocked, .couldNotConnect, .usbFailed, .lost(.pcStoppedResponding), .lost(.cableUnplugged), .lost(.usbTurnedOff), .reconnecting(attempt: 2, of: 6), .reconnectFailed, .pcEndedSession, .pcEndedSessionOnError, .pcBusy, .pairingCodeAsAddress, .routerAddress, .pcUpdating(.reconnecting), .pcUpdating(.manual), .pcUpdating(.notBack), diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index 273ac591..cfc29ed9 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -14,6 +14,8 @@ final class FakeUDPHost: @unchecked Sendable { private var answered = Set() /// The nonce of the HELLO2 answered last. private(set) var lastNonce: UInt32 = 0 + /// The pairing token the HELLO2 answered last carried. + private(set) var lastAuthToken = Data() init() { var address = sockaddr_in() @@ -50,14 +52,17 @@ final class FakeUDPHost: @unchecked Sendable { /// Answers the next HELLO2 attempt, with session 41 unless `status` /// refuses it. False after thirty quiet seconds, a backstop that a /// loaded machine's slow start does not reach. - func acceptSession(status: HelloStatus = .ok, acceptedVersion: UInt8 = 2, hostCaps: UInt16 = 0) -> Bool { + func acceptSession(status: HelloStatus = .ok, acceptedVersion: UInt8 = 2, hostCaps: UInt16 = 0, + hostName: String = "Test PC", authToken: Data = Data(17...32)) -> Bool { for _ in 0..<100 { guard let (data, from, length) = receive() else { continue } guard case .hello2(let hello) = Wire.parseControl(data)?.message, answered.insert(hello.clientNonce).inserted else { continue } client = (from, length) lastNonce = hello.clientNonce - send(Self.ack(nonce: hello.clientNonce, status: status, acceptedVersion: acceptedVersion, hostCaps: hostCaps)) + lastAuthToken = hello.authToken + send(Self.ack(nonce: hello.clientNonce, status: status, acceptedVersion: acceptedVersion, + hostName: hostName, authToken: authToken, hostCaps: hostCaps)) return true } return false diff --git a/ios/EternalMonitorTests/MovedPCTests.swift b/ios/EternalMonitorTests/MovedPCTests.swift new file mode 100644 index 00000000..a1246b48 --- /dev/null +++ b/ios/EternalMonitorTests/MovedPCTests.swift @@ -0,0 +1,224 @@ +import Combine +import XCTest +@testable import EternalMonitor + +/// A saved PC whose address changed: a new lease from the router, or the +/// PC moved between its cable and Wi-Fi. +@MainActor +final class MovedPCTests: XCTestCase { + private static let token = Data(17...32) + private var hostsToForget: [String] = [] + + override func setUp() async throws { + let store = RecentConnectionStore.shared + let saved = store.connections + let defaults = UserDefaults.standard.object(forKey: "recentConnections") + addTeardownBlock { @MainActor in + store.connections = saved + UserDefaults.standard.set(defaults, forKey: "recentConnections") + for address in self.hostsToForget { try? PairingStore.shared.remove(address: address) } + } + } + + private func found(_ name: String, _ address: String?, port: UInt16 = 9876) -> DiscoveredHost { + DiscoveredHost(name: name, address: address, port: port) + } + + // MARK: - The choice + + func testAStaleSavedAddressMovesToTheFreshBonjourAddress() { + XCTAssertTrue(MovedPC.canLookUp(host: "192.168.1.20", pcName: "Studio PC", typed: false)) + let fresh = [found("EternalMonitor (Office PC)", "192.168.1.40"), found("EternalMonitor (Studio PC)", "192.168.1.31")] + let moved = MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: fresh) + XCTAssertEqual(moved?.host, "192.168.1.31") + XCTAssertEqual(moved?.port, 9876) + // A new port is a new address too. + let port = MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.31", port: 9876, + found: [found("EternalMonitor (Studio PC)", "192.168.1.31", port: 9900)]) + XCTAssertEqual(port?.port, 9900) + // Still at the saved address: it is just not answering. + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.31", port: 9876, found: fresh)) + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: [])) + } + + func testAManualAddressIsNeverLookedUpOrReplaced() { + XCTAssertFalse(MovedPC.canLookUp(host: "192.168.1.20", pcName: "Studio PC", typed: true)) + // No name, and host names that DNS keeps current, aren't looked up. + XCTAssertFalse(MovedPC.canLookUp(host: "192.168.1.20", pcName: nil, typed: false)) + XCTAssertFalse(MovedPC.canLookUp(host: "192.168.1.20", pcName: "", typed: false)) + XCTAssertFalse(MovedPC.canLookUp(host: "studio-pc.local", pcName: "Studio PC", typed: false)) + XCTAssertFalse(MovedPC.canLookUp(host: "fe80::1", pcName: "Studio PC", typed: false)) + + for home in ["192.168.1.20", "10.0.0.5", "172.20.1.2", "169.254.3.4"] { + XCTAssertTrue(MovedPC.replacesSavedEntry(home), home) + } + for kept in ["100.101.102.103", "127.0.0.1", "8.8.8.8", "studio-pc.local", "fe80::1", "192.168.1"] { + XCTAssertFalse(MovedPC.replacesSavedEntry(kept), kept) + } + + let store = RecentConnectionStore.shared + store.connections = [] + store.add(host: "192.168.1.20", port: 9876, isUSB: false, name: "Studio PC", typed: true) + store.add(host: "192.168.1.20", port: 9876, isUSB: false) + XCTAssertEqual(store.entry(host: "192.168.1.20", port: 9876)?.typed, true, "picked from Recent, still typed") + XCTAssertEqual(store.entry(host: "192.168.1.20", port: 9876)?.name, "Studio PC") + + // A Tailscale address stays for use away from home; a home address gives way. + store.add(host: "100.101.102.103", port: 9876, isUSB: false, name: "Studio PC") + store.moved(from: ("100.101.102.103", 9876), to: ("192.168.1.31", 9876), name: "Studio PC") + XCTAssertNotNil(store.entry(host: "100.101.102.103", port: 9876)) + store.add(host: "192.168.1.22", port: 9876, isUSB: false, name: "Studio PC") + store.moved(from: ("192.168.1.22", 9876), to: ("192.168.1.32", 9876), name: "Studio PC") + XCTAssertNil(store.entry(host: "192.168.1.22", port: 9876)) + XCTAssertEqual(store.connections.first?.host, "192.168.1.32") + XCTAssertEqual(store.connections.first?.name, "Studio PC") + } + + func testTwoPCsWithTheSameNameAreNeverConfused() { + let ours = found("EternalMonitor (Studio PC)", "192.168.1.31") + // Bonjour renamed the second service: either one may be the saved PC. + let renamed = found("EternalMonitor (Studio PC) (2)", "192.168.1.40") + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: [ours, renamed])) + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: [renamed])) + // A second claim still resolving counts as well. + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, + found: [ours, found("EternalMonitor (Studio PC) (2)", nil)])) + // Other PCs, however alike their names, are no claim. + let others = [found("EternalMonitor (Studio PC 2)", "192.168.1.41"), found("EternalMonitor (Studio)", "192.168.1.42")] + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: others)) + XCTAssertEqual(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: others + [ours])?.host, + "192.168.1.31") + } + + // MARK: - Connecting + + /// A saved PC named "Test PC" at `old`'s address, paired when `token` is set. + private func savePC(at old: FakeUDPHost, token: Data?) throws { + RecentConnectionStore.shared.connections = [ + RecentConnection(host: "127.0.0.1", port: old.port, lastUsed: .now, isUSB: false, name: "Test PC"), + ] + let address = PairingStore.address(host: "127.0.0.1", port: old.port) + hostsToForget.append(address) + if let token { try PairingStore.shared.save(token: token, hostName: "Test PC", address: address) } + } + + /// A manager whose Bonjour finds "Test PC" at `new`'s port, and the + /// service names it was asked for. + private func manager(finding new: FakeUDPHost, connectTimeout: Double) -> (ConnectionManager, () -> [String]) { + let manager = ConnectionManager(timing: .init(connectTimeoutSeconds: connectTimeout, reconnectGraceSeconds: 0.1, + reconnectBackoffSeconds: { _ in 0.05 })) + hostsToForget.append(PairingStore.address(host: "127.0.0.1", port: new.port)) + var asked: [String] = [] + manager.findPCs = { name in + asked.append(name) + return [DiscoveredHost(name: "EternalMonitor (Test PC)", address: "127.0.0.1", port: new.port)] + } + return (manager, { asked }) + } + + private func until(_ description: String, seconds: Double, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + private func logged(_ manager: ConnectionManager, _ prefix: String) -> String? { + manager.diagnostics.first { $0.message.hasPrefix(prefix) }?.message + } + + /// The saved address stays silent; Bonjour has the same PC at another + /// port, and the iPad streams from there with its pairing, no tap needed. + func testASilentSavedAddressReconnectsWhereBonjourFoundThePC() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + try savePC(at: old, token: Self.token) + let (manager, asked) = manager(finding: new, connectTimeout: 60) + defer { manager.cancel() } + // Every line logged: the new attempt clears Connection details. + var lines: [String] = [] + let log = manager.$diagnostics.sink { if let line = $0.last?.message { lines.append(line) } } + defer { log.cancel() } + manager.connect(host: "127.0.0.1", port: old.port) + let accepted = await Task.detached { new.acceptSession(authToken: Self.token) }.value + XCTAssertTrue(accepted, "the iPad never tried the new address") + XCTAssertEqual(asked(), ["EternalMonitor (Test PC)"]) + XCTAssertEqual(new.lastAuthToken, Self.token, "the saved pairing went along") + try await until("handshake", seconds: 60) { manager.hostInfo != nil } + new.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming", seconds: 60) { manager.state == .connected } + XCTAssertTrue(lines.contains("Disconnect state=connecting reason=pcMoved"), lines.joined(separator: "\n")) + // The saved address counts as not answering, in the closed set the backend accepts. + let summary = lines.first { $0.hasPrefix("session_summary ") } ?? "" + XCTAssertTrue(summary.contains(" stage=hello_sent outcome=no_reply failure=no_reply fallback=none end=superseded "), summary) + + let saved = RecentConnectionStore.shared.connections.first + XCTAssertEqual(saved?.port, new.port) + XCTAssertEqual(saved?.name, "Test PC") + XCTAssertNotNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: old.port), + "loopback is no home address, so its entry stays") + XCTAssertEqual(try PairingStore.shared.token(address: PairingStore.address(host: "127.0.0.1", port: new.port)), Self.token) + } + + func testATypedAddressIsNeverLookedUp() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + try savePC(at: old, token: nil) + let (manager, asked) = manager(finding: new, connectTimeout: 4) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: old.port, typed: true) + try await until("timed out", seconds: 15) { manager.state == .disconnected } + XCTAssertEqual(asked(), []) + XCTAssertEqual(manager.problem, .noReply(usb: false)) + } + + /// A PC with the saved name at the new address that doesn't give the + /// saved pairing back is another PC: the iPad leaves it and keeps its pairing. + func testAnotherPCWithTheSameNameIsLeftAndThePairingKept() async throws { + for answer in ["pairing off", "asks for a code"] { + let old = FakeUDPHost(), new = FakeUDPHost() + try savePC(at: old, token: Self.token) + let (manager, _) = manager(finding: new, connectTimeout: 60) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: old.port) + let accepted = await Task.detached { + answer == "pairing off" ? new.acceptSession(authToken: Data(repeating: 0, count: 16)) + : new.acceptSession(status: .unauthorized) + }.value + XCTAssertTrue(accepted, answer) + try await until("left it: \(answer)", seconds: 30) { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .noReply(usb: false), answer) + XCTAssertNil(manager.pairingPrompt, answer) + XCTAssertNotNil(logged(manager, "The PC at the new address isn't the saved one"), answer) + XCTAssertEqual(try PairingStore.shared.token(address: PairingStore.address(host: "127.0.0.1", port: old.port)), + Self.token, answer) + XCTAssertNil(try PairingStore.shared.token(address: PairingStore.address(host: "127.0.0.1", port: new.port)), answer) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: new.port), answer) + } + } + + /// Unpaired, the name is what tells PCs apart. + func testAnUnpairedPCWithAnotherNameIsLeft() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + try savePC(at: old, token: nil) + let (manager, _) = manager(finding: new, connectTimeout: 60) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: old.port) + let accepted = await Task.detached { new.acceptSession(hostName: "Other PC", authToken: Data(repeating: 0, count: 16)) }.value + XCTAssertTrue(accepted) + try await until("left it", seconds: 30) { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .noReply(usb: false)) + } + + /// Found at a new address that doesn't answer either: the card says so. + func testNoAnswerAtTheNewAddressSaysWhereItLooked() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + try savePC(at: old, token: nil) + let (manager, asked) = manager(finding: new, connectTimeout: 5) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: old.port) + let tried = await Task.detached { new.helloAttempts(within: 6) }.value + XCTAssertGreaterThan(tried, 0) + try await until("timed out", seconds: 15) { manager.state == .disconnected } + XCTAssertEqual(asked().count, 1, "looked once, not again from the new address") + XCTAssertEqual(manager.problem, .noReplyAtNewAddress("127.0.0.1")) + XCTAssertEqual(manager.problem?.actions, [.retry]) + } +} diff --git a/ios/EternalMonitorTests/SessionSummaryTests.swift b/ios/EternalMonitorTests/SessionSummaryTests.swift index 9e911677..9fcd4180 100644 --- a/ios/EternalMonitorTests/SessionSummaryTests.swift +++ b/ios/EternalMonitorTests/SessionSummaryTests.swift @@ -77,6 +77,8 @@ final class SessionSummaryTests: XCTestCase { (pairing, ["hello_sent", "pairing", "none", "user"]), (summary(.legacyPC, session: nil), ["hello_sent", "error", "other", "error"]), (summary(.connectTimeout, session: nil, duration: 10), ["hello_sent", "no_reply", "no_reply", "timeout"]), + (summary(.pcMoved, session: nil, duration: 3), ["hello_sent", "no_reply", "no_reply", "superseded"]), + (summary(.otherPC), ["accepted", "error", "other", "error"]), (summary(.connectTimeout, duration: 10), ["accepted", "no_picture", "no_video", "timeout"]), (noFrame, ["accepted", "no_picture", "no_frame", "timeout"]), (summary(.pcStoppedResponding, firstFrame: 0.3, duration: 90), ["steady", "streamed", "signal_lost", "liveness"]), diff --git a/scripts/e2e_moved_pc.sh b/scripts/e2e_moved_pc.sh new file mode 100755 index 00000000..368eac47 --- /dev/null +++ b/scripts/e2e_moved_pc.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# A saved PC changes address, as after a new lease from the router or a move +# between its cable and Wi-Fi. The real app streams from the real host at +# 127.0.0.1. The host is then killed and started again, and a silent socket +# takes 127.0.0.1:PORT: the saved address stops answering, while the PC +# still answers at its network addresses and advertises them with Bonjour +# under the same name. Success = the app streams again from another address +# by itself, after its HELLO2s to the old one went unanswered. +# +# On a shared Mac this is an iOS e2e run: hold the machine's e2e lock, and use +# your own simulator (EM_SIM_UDID) and a private port (EM_PORT). +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +export DEVELOPER_DIR="${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer}" +export PATH="$HOME/.cargo/bin:$PATH" +export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:-/opt/homebrew/opt/ffmpeg@7/lib/pkgconfig}" + +UDID="${EM_SIM_UDID:?set EM_SIM_UDID to your own simulator}" +PORT="${EM_PORT:-19893}" +TIMEOUT_SECS="${EM_TIMEOUT:-90}" +OUT="${EM_OUTPUT_DIR:-$ROOT/build/e2e/moved-pc}" +HOST_BIN="${EM_HOST_BIN:-$ROOT/target/release/eternal-host}" +# The PC's name: its Bonjour service and HELLO_ACK both carry it. +PC_NAME="EMMovedPC$$" +mkdir -p "$OUT" +rm -f "$OUT/result.json" "$OUT/old-address.count" +APP_LOG="$OUT/app.log" +HOST_LOG="$OUT/host.log" + +HOST_PID="" +SILENT_PID="" +APP_PID="" +cleanup() { + status=$? + trap - EXIT + [ -n "$SILENT_PID" ] && kill "$SILENT_PID" 2>/dev/null || true + [ -n "$SILENT_PID" ] && wait "$SILENT_PID" 2>/dev/null || true + [ -n "$HOST_PID" ] && kill -TERM "$HOST_PID" 2>/dev/null || true + [ -n "$HOST_PID" ] && wait "$HOST_PID" 2>/dev/null || true + xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true + if [ -f "$APP_LOG" ]; then + cp "$APP_LOG" "$APP_LOG.snapshot" && mv "$APP_LOG.snapshot" "$APP_LOG" || status=1 + fi + python3 - "$OUT/result.json" "$status" <<'PY' +import json,pathlib,sys +p=pathlib.Path(sys.argv[1]) +r=json.loads(p.read_text()) if p.exists() else {} +r.setdefault('status', 'PASS' if sys.argv[2]=='0' else 'FAIL') +p.write_text(json.dumps(r, indent=2)+'\n') +PY + [ "$status" -eq 0 ] || echo "FAIL: evidence at $OUT" >&2 + exit "$status" +} +trap cleanup EXIT + +if [ "${EM_SKIP_BUILD:-0}" != 1 ]; then + echo "==> Building app" + (cd "$ROOT/ios" && xcodegen generate >/dev/null) + xcodebuild build -project "$ROOT/ios/EternalMonitor.xcodeproj" -scheme EternalMonitor -configuration Release \ + -destination "platform=iOS Simulator,id=$UDID" -derivedDataPath "$ROOT/ios/build/e2e" \ + CODE_SIGNING_ALLOWED=YES CODE_SIGN_IDENTITY=- -quiet + if [ -z "${EM_HOST_BIN:-}" ]; then + echo "==> Building host" + (cd "$ROOT" && cargo build -q --release -p eternal-host --locked) + fi +fi +APP="$ROOT/ios/build/e2e/Build/Products/Release-iphonesimulator/EternalMonitor.app" +[ -d "$APP" ] || { echo "FAIL: app bundle not found at $APP"; exit 1; } +[ -x "$HOST_BIN" ] || { echo "FAIL: host binary not found at $HOST_BIN"; exit 1; } + +echo "==> Installing app with nothing saved" +xcrun simctl bootstatus "$UDID" -b >/dev/null +xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true +xcrun simctl uninstall "$UDID" com.eternal.monitor 2>/dev/null || true +STAGE="$(mktemp -d /tmp/em-moved-pc-install.XXXXXX)" +ditto "$APP" "$STAGE/EternalMonitor.app" +xcrun simctl install "$UDID" "$STAGE/EternalMonitor.app" +rm -rf "$STAGE" +APP_DATA=$(xcrun simctl get_app_container "$UDID" com.eternal.monitor data) +SIM_LOG="$APP_DATA/tmp/eternal-e2e.log" +mkdir -p "$(dirname "$SIM_LOG")" +rm -f "$SIM_LOG" "$APP_LOG" +touch "$SIM_LOG" +ln "$SIM_LOG" "$APP_LOG" + +mkdir -p "$OUT/state/EternalMonitor" +echo '{"bitrate_mbps": 8, "target_fps": 30, "start_on_boot": false, "stream_audio": false, "require_pairing": false}' \ + > "$OUT/state/EternalMonitor/settings.json" +start_host() { + APPDATA="$OUT/state" COMPUTERNAME="$PC_NAME" ETERNAL_HEADLESS=1 ETERNAL_E2E_LOG=1 ETERNAL_USB_DIRECT= \ + ETERNAL_CAPTURE=synthetic ETERNAL_AUDIO=synthetic ETERNAL_SYNTH_SIZE=640x360 ETERNAL_ENCODER=libx264 ETERNAL_FPS=30 \ + "$HOST_BIN" "$PORT" >>"$HOST_LOG" 2>&1 & + HOST_PID=$! +} +frames_at_least() { [ "$(grep -c 'E2E_FIRST_FRAME' "$APP_LOG" || true)" -ge "$1" ]; } +wait_for() { + local what=$1 limit=$2 elapsed=0 + shift 2 + until "$@"; do + sleep 1 + elapsed=$((elapsed + 1)) + if [ "$elapsed" -ge "$limit" ]; then + echo "FAIL: $what after ${limit}s" + echo "----- app milestones -----"; tail -40 "$APP_LOG" + exit 1 + fi + done +} + +: > "$HOST_LOG" +echo "==> Starting host $PC_NAME on 0.0.0.0:$PORT" +start_host +echo "==> Launching app, streaming from 127.0.0.1:$PORT" +LAUNCH=$(SIMCTL_CHILD_EM_AUTOCONNECT="127.0.0.1:$PORT" SIMCTL_CHILD_EM_E2E_LOG=1 SIMCTL_CHILD_EM_USB_PORT=0 \ + xcrun simctl launch "$UDID" com.eternal.monitor -didSeeOnboarding YES -allowUSB NO -autoReconnect YES -playPCaudio NO) +APP_PID="${LAUNCH##*: }" +wait_for "no first stream" "$TIMEOUT_SECS" frames_at_least 1 + +echo "==> Moving the PC: kill, restart, and silence 127.0.0.1:$PORT" +MOVED_AT=$(wc -l < "$APP_LOG") +kill -KILL "$HOST_PID" +wait "$HOST_PID" 2>/dev/null || true +start_host +wait_for "restarted host never bound UDP $PORT" 20 sh -c "lsof -nP -a -p $HOST_PID -iUDP:$PORT >/dev/null" +# More specific than the host's wildcard socket, so it takes every datagram +# sent to 127.0.0.1:PORT, and answers none. +python3 -u - "$PORT" "$OUT/old-address.count" <<'PY' & +import signal,socket,sys +s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) +s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('127.0.0.1',int(sys.argv[1]))) +count=0 +def done(*_): + open(sys.argv[2],'w').write(str(count)); sys.exit(0) +signal.signal(signal.SIGTERM,done) +while True: + s.recv(4096); count+=1 +PY +SILENT_PID=$! + +echo "==> Waiting for the app to stream again by itself (timeout ${TIMEOUT_SECS}s)" +wait_for "no stream after the move" "$TIMEOUT_SECS" frames_at_least 2 +sleep 2 +kill "$SILENT_PID"; wait "$SILENT_PID" 2>/dev/null || true; SILENT_PID="" + +python3 - "$APP_LOG" "$MOVED_AT" "$PORT" "$OUT/old-address.count" "$OUT/result.json" <<'PY' +import json,pathlib,re,sys +lines=pathlib.Path(sys.argv[1]).read_text(errors='replace').splitlines()[int(sys.argv[2]):] +port=sys.argv[3] +starts=[m.group(1) for l in lines if (m:=re.search(r'Starting connection attempt=\S+ transport=WiFi to (\S+):'+port+' ', l))] +frame=next(i for i,l in enumerate(lines) if 'E2E_FIRST_FRAME' in l) +last=[m.group(1) for l in lines[:frame] if (m:=re.search(r'Starting connection attempt=\S+ transport=WiFi to (\S+):'+port+' ', l))][-1] +old=int(pathlib.Path(sys.argv[4]).read_text()) +errors=[] +if '127.0.0.1' not in starts: errors.append('never retried the saved address') +if old==0: errors.append('the saved address got no HELLO2') +if last=='127.0.0.1': errors.append('streamed again from the saved address, not a new one') +if not any('Found the saved PC at ' in l for l in lines[:frame]): errors.append('no Bonjour lookup') +result=dict(scenario='moved-pc', status='FAIL' if errors else 'PASS', errors=errors, + attempts_after_move=len(starts), datagrams_at_old_address=old, streamed_from_loopback=last=='127.0.0.1') +pathlib.Path(sys.argv[5]).write_text(json.dumps(result, indent=2)+'\n') +print(json.dumps(result)) +sys.exit(1 if errors else 0) +PY +xcrun simctl io "$UDID" screenshot "$OUT/streaming-after-move.png" >/dev/null 2>&1 || true +echo "PASS: streamed again after the move. Evidence: $OUT" From d852931aebcd0f63982aaff1973c269d84ae2b95 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 11:52:02 -0500 Subject: [PATCH 011/226] Add a test hook that loses the window's graphics device --- host/Cargo.toml | 1 + host/src/gui/mod.rs | 3 ++- host/src/gui/renderer.rs | 41 ++++++++++++++++++++++++++++++++++++++++ host/src/startup/mod.rs | 17 +++++++++++++++-- 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/host/Cargo.toml b/host/Cargo.toml index d8ece94d..6220732d 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -100,6 +100,7 @@ features = [ "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", + "Win32_Graphics_Direct3D12", "Win32_Graphics_Gdi", "Win32_Globalization", "Win32_System_Registry", diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 53d7cf8e..708c5a3d 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -1074,8 +1074,9 @@ impl eframe::App for AnalyzerApp { CANVAS.to_normalized_gamma_f32() } - fn ui(&mut self, ui: &mut egui::Ui, _frame: &mut eframe::Frame) { + fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { renderer::frame_started(ui.ctx()); + renderer::simulate_device_loss(frame); crate::startup::simulate_window_failure(); let ctx = ui.ctx().clone(); if let Some(language) = self.pending_language.take() { diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 322c1d74..707042b9 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -453,6 +453,47 @@ pub(super) fn frame_started(ctx: &egui::Context) { } } +/// Lose the window's graphics device on purpose when +/// [`crate::startup::SIMULATE_ENV`] asks: DX12 removes the device, as a +/// driver reset does; other backends destroy it. +pub(super) fn simulate_device_loss(frame: &eframe::Frame) { + if !crate::startup::simulate_device_loss_now() { + return; + } + let Some(render_state) = frame.wgpu_render_state() else { + return; + }; + warn!("Simulated loss of the window's graphics device"); + #[cfg(windows)] + if remove_dx12_device(&render_state.device) { + return; + } + render_state.device.destroy(); +} + +/// `ID3D12Device5::RemoveDevice` on a DX12 device; false on other backends. +#[cfg(windows)] +fn remove_dx12_device(device: &wgpu::Device) -> bool { + use windows::core::Interface; + use windows::Win32::Graphics::Direct3D12::{ID3D12Device, ID3D12Device5}; + let Some(hal) = (unsafe { device.as_hal::() }) else { + return false; + }; + // wgpu-hal uses a newer windows crate, so the device crosses as its COM + // pointer, which is all either version's interface type holds. + let raw: *mut std::ffi::c_void = unsafe { std::mem::transmute_copy(hal.raw_device()) }; + let Some(device) = (unsafe { ID3D12Device::from_raw_borrowed(&raw) }) else { + return false; + }; + match device.cast::() { + Ok(device) => { + unsafe { device.RemoveDevice() }; + true + } + Err(_) => false, + } +} + /// The window closed normally. A launch that quits before its window is /// confirmed did not fail, so its marker goes without counting either way. pub(super) fn window_closed() { diff --git a/host/src/startup/mod.rs b/host/src/startup/mod.rs index 124cd263..13ef0209 100644 --- a/host/src/startup/mod.rs +++ b/host/src/startup/mod.rs @@ -34,8 +34,10 @@ static SHOWN_AT: OnceLock = OnceLock::new(); /// thread holds the heap lock) or `fail-fast` (past the exception filter) to /// fail while the window renderer starts; `early-error` to fail right after the failure reporter /// starts; `panic-later` or, on Windows, `fail-fast-later` to fail 15 s after -/// the window works; or `surface-error` to report a surface error on every -/// frame of the first two seconds. A `-quiet` suffix skips the message box. +/// the window works; `device-lost-later` to lose the window's graphics +/// device 15 s after the window works; or `surface-error` to report a surface +/// error on every frame of the first two seconds. A `-quiet` suffix skips the +/// message box. /// Only debug builds and builds with the `fault-injection` feature read it; /// a release build ignores it. pub const SIMULATE_ENV: &str = "ETERNAL_SIMULATE_STARTUP_FAILURE"; @@ -583,6 +585,17 @@ pub(crate) fn simulate_window_failure() { } } +/// True once, 15 s after the window works, when [`SIMULATE_ENV`] asks for +/// `device-lost-later`: the window's graphics device is to be lost then. +pub(crate) fn simulate_device_loss_now() -> bool { + static DONE: AtomicBool = AtomicBool::new(false); + simulated() == "device-lost-later" + && SHOWN_AT + .get() + .is_some_and(|shown| shown.elapsed() > Duration::from_secs(15)) + && !DONE.swap(true, Ordering::SeqCst) +} + /// Count surface errors for the first two seconds of frames, when /// [`SIMULATE_ENV`] asks. pub(crate) fn simulate_surface_error(failures: &std::sync::atomic::AtomicU64) { From 287860f3a9736b56048c580d2c73143337372119 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 11:54:42 -0500 Subject: [PATCH 012/226] Tell the iPad when the stream stops for good, and log a stuck transport loop Quitting, Stop and the storm brake ended the PC's stream without a word to the iPad. A quit or a self-restart showed up on the iPad as a lost signal or an unplugged cable. After Stop or Failed the session even stayed open with nothing ticking it: no heartbeats, no liveness expiry, and new HELLOs went unanswered until the next Start or quit. - Session::shut_down now ends the session with a goodbye that the transport sends on its way out, over UDP or the USB tunnel. Quit and Stop say user_disconnect, so the iPad shows "Your PC ended the session" and stays away. A restart or update handoff, and the storm brake, say error, so the iPad reconnects at once. - The supervisor ends the session when it stops for a manual Start (Stopped, Failed). A backoff or a held port keeps it, as before. - A goodbye left while no transport ran is dropped when the next one starts. - The transport loop marks a heartbeat with the step it is in. The supervisor's thread logs one line (stall="transport_loop" phase=...) when the mark is 2 s old, and one when the loop moves again. A gap in the supervisor's own checks (PC asleep) is not counted. --- host/src/control.rs | 4 + host/src/diagnostics.rs | 21 ++- host/src/supervisor.rs | 59 ++++++++ host/src/transport/beat.rs | 275 ++++++++++++++++++++++++++++++++++ host/src/transport/mod.rs | 35 ++++- host/src/transport/session.rs | 81 +++++++--- host/tests/goodbye.rs | 94 +++++++++--- host/tests/transport_stall.rs | 107 +++++++++++++ 8 files changed, 638 insertions(+), 38 deletions(-) create mode 100644 host/src/transport/beat.rs create mode 100644 host/tests/transport_stall.rs diff --git a/host/src/control.rs b/host/src/control.rs index 30ac372c..e7cd0ce8 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -76,6 +76,9 @@ pub struct SharedControl { pub hb_capture_loop_ms: Arc, pub hb_capture_frame_ms: Arc, pub hb_encode_frame_ms: Arc, + /// The transport loop's heartbeat and the step it was taking, which the + /// supervisor watches (`transport::beat`). + pub transport_beat: Arc, /// Desktop-space rectangle of the output currently being captured, set by /// the capture stage at (re)init. The input relay maps the client's /// normalized touch coordinates onto this rect. `None` until the first @@ -191,6 +194,7 @@ impl SharedControl { hb_capture_loop_ms: Arc::new(AtomicU64::new(0)), hb_capture_frame_ms: Arc::new(AtomicU64::new(0)), hb_encode_frame_ms: Arc::new(AtomicU64::new(0)), + transport_beat: Default::default(), capture_geometry: Arc::new(Mutex::new(None)), hevc_enabled: Arc::new(AtomicBool::new(false)), audio_enabled: Arc::new(AtomicBool::new(true)), diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 02ccf0d6..b7009110 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -6,6 +6,7 @@ use std::time::Duration; use crate::control::{CaptureTarget, SharedControl}; use crate::transport::link::LinkId; use crate::transport::session::{EndReason, SessionEnd}; +use eternal_wire::v2::control::ByeReason; use tracing::info; static SYSTEM_REPORT: OnceLock = OnceLock::new(); @@ -341,9 +342,25 @@ pub fn log_session_end(end: &SessionEnd, shared: &SharedControl) { ); } -/// The app is quitting: the summary of the stream it stops. +/// The app is quitting: the summary of the stream it stops. Its iPad is +/// told the PC ended the session, or, when a restart or an update hands over +/// to a new copy, to connect again. pub fn log_session_end_at_quit(shared: &SharedControl) { - let end = shared.session.lock().shut_down(std::time::Instant::now()); + let bye = if shared.handoff.holder().is_some() { + ByeReason::Error + } else { + ByeReason::UserDisconnect + }; + stop_session(shared, bye); +} + +/// The stream stops for good: ends the session, which the transport tells +/// `bye` on its way out, and logs its summary. +pub fn stop_session(shared: &SharedControl, bye: ByeReason) { + let end = shared + .session + .lock() + .shut_down(std::time::Instant::now(), bye); if let Some(end) = end { log_session_end(&end, shared); } diff --git a/host/src/supervisor.rs b/host/src/supervisor.rs index e12e4d0e..78a277ad 100644 --- a/host/src/supervisor.rs +++ b/host/src/supervisor.rs @@ -327,10 +327,14 @@ pub fn run( machine.wait_for_port(Instant::now() + crate::startup::port::RECHECK); } let mut healthy_since = Instant::now(); + let mut transport_watch = crate::transport::beat::Watch::default(); loop { let message = sup_rx.recv_timeout(Duration::from_millis(250)); let now = Instant::now(); + // A stuck transport stays in its generation's thread, which no join + // can end: this line says where it is stuck. + transport_watch.check_and_log(&shared.transport_beat, crate::clock::host_now_us() / 1000); // A minute of health clears the storm budget. if matches!(machine.state, SupervisorState::Running { .. }) @@ -363,6 +367,9 @@ pub fn run( match effect { Effect::None => {} Effect::StopPipeline => { + if let Some(bye) = goodbye_when_stopped(&machine.state) { + crate::diagnostics::stop_session(&shared, bye); + } shared.stop(); bounded_join(&mut pipeline, generation); } @@ -405,6 +412,24 @@ pub fn run( info!("Supervisor loop ended"); } +/// What the iPad of a session is told when the pipeline stops in `state`. +/// Stopped and Failed wait for a manual Start, so the session ends now: kept, +/// it went on unanswered while its iPad lost the signal and every HELLO it +/// sent went unanswered. Stop says the PC ended the session; giving up says +/// connect again. A backoff or a held port restarts the stream by itself, +/// and its session goes on. +fn goodbye_when_stopped(state: &SupervisorState) -> Option { + use eternal_wire::v2::control::ByeReason; + match state { + SupervisorState::Stopped => Some(ByeReason::UserDisconnect), + SupervisorState::Failed { .. } => Some(ByeReason::Error), + SupervisorState::Running { .. } + | SupervisorState::BackingOff { .. } + | SupervisorState::WaitingForPort { .. } + | SupervisorState::ShutDown => None, + } +} + /// Watchdog rules over the heartbeat atomics. Only meaningful while Running /// and after the pipeline has had a moment to warm up. fn wedge_reason(shared: &SharedControl, machine: &Machine) -> Option { @@ -629,6 +654,40 @@ mod tests { .unwrap(); } + /// Stop and giving up wait for a manual Start: their session ends with + /// a goodbye. A stream that restarts by itself keeps its session. + #[test] + fn only_a_stop_that_waits_for_start_ends_the_session() { + use eternal_wire::v2::control::ByeReason; + let mut machine = Machine::new(1); + assert_eq!( + machine.on_command(&SupervisorCommand::Stop), + Effect::StopPipeline + ); + assert_eq!( + goodbye_when_stopped(&machine.state), + Some(ByeReason::UserDisconnect) + ); + + let mut machine = Machine::new(1); + let t0 = Instant::now(); + let mut effect = Effect::None; + for _ in 0..=STORM_LIMIT { + effect = machine.on_stage_exit(1, Stage::Encoder, &failed("x"), t0); + if let SupervisorState::BackingOff { .. } = machine.state { + assert_eq!(goodbye_when_stopped(&machine.state), None); + machine.note_started(1); + } + } + assert_eq!(effect, Effect::StopPipeline); + assert!(matches!(machine.state, SupervisorState::Failed { .. })); + assert_eq!(goodbye_when_stopped(&machine.state), Some(ByeReason::Error)); + + let mut machine = Machine::new(1); + machine.wait_for_port(t0); + assert_eq!(goodbye_when_stopped(&machine.state), None); + } + fn failed(reason: &str) -> StageOutcome { StageOutcome::Failed(reason.to_string()) } diff --git a/host/src/transport/beat.rs b/host/src/transport/beat.rs new file mode 100644 index 00000000..f1911b20 --- /dev/null +++ b/host/src/transport/beat.rs @@ -0,0 +1,275 @@ +//! The transport loop's heartbeat, and the watch that notices when it stops. +//! +//! The loop wakes at least every 250 ms (its liveness tick). A loop stuck in +//! one call answers nobody: its iPad loses the heartbeats and gives up, the +//! session is never ended for silence, and new HELLOs wait unread. That +//! happened in the field for close to a minute while Windows switched the +//! virtual display, and once, perhaps, for 18 minutes, with nothing in the +//! log to say where. The loop now marks each step it takes with a [`Phase`], +//! and the supervisor's thread logs one line when the mark goes stale, and +//! one when it moves again. + +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering::Relaxed}; +use std::time::Duration; + +use tracing::{info, warn}; + +/// Where the transport loop was when it last marked its heartbeat. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum Phase { + /// Waiting for its next timer, datagram, frame or audio packet. + Waiting = 0, + /// A heartbeat or liveness tick. + Tick, + /// Handling a control datagram from an iPad, or a USB tunnel event. + Control, + /// Injecting the iPad's touch, pen, mouse or keyboard input into Windows. + Input, + /// Sending what the session decided: replies, goodbyes, summaries. + Actions, + /// The second screen's next step. + SecondScreen, + /// Sending a video frame, with its pacing waits and the repairs served + /// during them. + Video, + /// Sending PC audio. + Audio, +} + +impl Phase { + const ALL: [Phase; 8] = [ + Phase::Waiting, + Phase::Tick, + Phase::Control, + Phase::Input, + Phase::Actions, + Phase::SecondScreen, + Phase::Video, + Phase::Audio, + ]; + + /// Stable log value. + pub const fn name(self) -> &'static str { + match self { + Phase::Waiting => "waiting", + Phase::Tick => "tick", + Phase::Control => "control", + Phase::Input => "input", + Phase::Actions => "actions", + Phase::SecondScreen => "second_screen", + Phase::Video => "video", + Phase::Audio => "audio", + } + } +} + +/// The last mark: when (ms on the process clock, 0 while no transport runs) +/// and in which phase. +#[derive(Debug, Default)] +pub struct Beat { + at_ms: AtomicU64, + phase: AtomicU8, +} + +impl Beat { + pub fn mark(&self, phase: Phase) { + self.phase.store(phase as u8, Relaxed); + self.at_ms + .store((crate::clock::host_now_us() / 1000).max(1), Relaxed); + } + + /// The transport loop ended. + pub fn stop(&self) { + self.at_ms.store(0, Relaxed); + } + + /// The last mark, unless no transport runs. + pub fn read(&self) -> Option<(u64, Phase)> { + let at = self.at_ms.load(Relaxed); + let phase = Phase::ALL + .get(usize::from(self.phase.load(Relaxed))) + .copied() + .unwrap_or(Phase::Waiting); + (at != 0).then_some((at, phase)) + } +} + +/// A mark this old means the loop is stuck. An iPad gives up 3 s after the +/// last heartbeat it got, plus 5 s. +pub const STALL_AFTER: Duration = Duration::from_secs(2); + +/// A gap this long between two checks means the watch itself did not run: +/// the PC slept, or the whole process was paused. The loop did not run +/// either, through no fault of its own, so the gap is not counted. +const PAUSED_AFTER: Duration = Duration::from_secs(1); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// The loop has not marked for `silent_ms`, stuck in `phase`. + Stalled { phase: Phase, silent_ms: u64 }, + /// The loop marked again, or ended, after it was stuck for `stalled_ms`. + Moving { phase: Phase, stalled_ms: u64 }, +} + +/// Judges the heartbeat at each check. Pure: the caller passes the clock. +#[derive(Debug, Default)] +pub struct Watch { + checked_ms: u64, + /// Counting starts again here after a pause. + resumed_ms: u64, + /// The stale mark the last stall was reported for. + stalled: Option<(u64, Phase)>, +} + +impl Watch { + pub fn check(&mut self, beat: Option<(u64, Phase)>, now_ms: u64) -> Option { + if self.checked_ms != 0 + && now_ms.saturating_sub(self.checked_ms) >= PAUSED_AFTER.as_millis() as u64 + { + self.resumed_ms = now_ms; + } + self.checked_ms = now_ms; + if let Some((at, phase)) = self.stalled { + if beat.is_some_and(|(now_at, _)| now_at == at) { + return None; + } + self.stalled = None; + let moved = beat.map_or(now_ms, |(now_at, _)| now_at); + return Some(Verdict::Moving { + phase, + stalled_ms: moved.saturating_sub(at), + }); + } + let (at, phase) = beat?; + let silent_since = at.max(self.resumed_ms); + if now_ms.saturating_sub(silent_since) < STALL_AFTER.as_millis() as u64 { + return None; + } + self.stalled = Some((at, phase)); + Some(Verdict::Stalled { + phase, + silent_ms: now_ms.saturating_sub(at), + }) + } + + /// [`Self::check`], logged. + pub fn check_and_log(&mut self, beat: &Beat, now_ms: u64) { + match self.check(beat.read(), now_ms) { + Some(Verdict::Stalled { phase, silent_ms }) => warn!( + stall = "transport_loop", + phase = phase.name(), + silent_ms, + "The stream's transport loop stopped: iPads get no heartbeats or answers until it moves" + ), + Some(Verdict::Moving { phase, stalled_ms }) => info!( + phase = phase.name(), + stalled_ms, + "The stream's transport loop moved again" + ), + None => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_loop_that_keeps_marking_is_never_stalled() { + let mut watch = Watch::default(); + for step in 0..100u64 { + let now = 10_000 + step * 250; + assert_eq!(watch.check(Some((now - 200, Phase::Waiting)), now), None); + } + // Nothing runs, nothing to judge. + assert_eq!(watch.check(None, 40_000), None); + } + + #[test] + fn a_stuck_loop_is_reported_once_with_its_phase_then_when_it_moves() { + let mut watch = Watch::default(); + let stuck = Some((10_000, Phase::Input)); + let mut verdicts = Vec::new(); + for now in (10_000..=20_000).step_by(250) { + verdicts.extend(watch.check(stuck, now)); + } + assert_eq!( + verdicts, + [Verdict::Stalled { + phase: Phase::Input, + silent_ms: 2_000 + }] + ); + assert_eq!( + watch.check(Some((20_100, Phase::Actions)), 20_250), + Some(Verdict::Moving { + phase: Phase::Input, + stalled_ms: 10_100 + }) + ); + assert_eq!(watch.check(Some((20_300, Phase::Waiting)), 20_500), None); + } + + #[test] + fn a_stuck_loop_that_ends_is_reported_as_moving() { + let mut watch = Watch::default(); + for now in (5_000..=8_000).step_by(250) { + watch.check(Some((5_000, Phase::SecondScreen)), now); + } + assert_eq!( + watch.check(None, 8_250), + Some(Verdict::Moving { + phase: Phase::SecondScreen, + stalled_ms: 3_250 + }) + ); + } + + /// The PC slept: the loop's mark is minutes old when the watch runs + /// again, but the loop was never given a chance to run. + #[test] + fn a_sleeping_pc_is_not_a_stall() { + let mut watch = Watch::default(); + assert_eq!(watch.check(Some((9_900, Phase::Waiting)), 10_000), None); + let woke = 10_000 + 600_000; + assert_eq!(watch.check(Some((9_900, Phase::Waiting)), woke), None); + // The loop marks again within its tick after the wake. + assert_eq!( + watch.check(Some((woke + 100, Phase::Tick)), woke + 250), + None + ); + // A loop that stays stuck after the wake is still caught. + let mut stuck = Watch::default(); + stuck.check(Some((9_900, Phase::Video)), 10_000); + let mut verdicts = Vec::new(); + for now in (woke..woke + 3_000).step_by(250) { + verdicts.extend(stuck.check(Some((9_900, Phase::Video)), now)); + } + assert_eq!( + verdicts, + [Verdict::Stalled { + phase: Phase::Video, + silent_ms: woke + 2_000 - 9_900 + }] + ); + } + + #[test] + fn the_heartbeat_reads_back_what_was_marked() { + crate::clock::init(); + let beat = Beat::default(); + assert_eq!(beat.read(), None); + beat.mark(Phase::Audio); + let (at, phase) = beat.read().unwrap(); + assert!(at > 0); + assert_eq!(phase, Phase::Audio); + for phase in Phase::ALL { + beat.mark(phase); + assert_eq!(beat.read().unwrap().1, phase); + } + beat.stop(); + assert_eq!(beat.read(), None); + } +} diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 70cfb197..20a58bc3 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -1,6 +1,7 @@ pub mod abr; pub mod apple_usb; mod audio; +pub mod beat; pub mod fault; pub mod link; pub mod pacer; @@ -28,6 +29,7 @@ use crate::control::{CaptureTarget, SharedControl, SupervisorCommand, VddStatus} use crate::encoder::NALUnit; use crate::stats::PIPELINE_STATS; use abr::AbrController; +use beat::Phase; use fault::FaultInjector; use link::{LinkId, PeerId}; use pacer::{Deadline, FramePacer, PacingClock}; @@ -275,6 +277,13 @@ pub async fn start_sender( let mut links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); shared.session.lock().transport_resumed(now()); + // A goodbye left while no transport ran (a Stop while the port was held) + // is for a session that is over; its iPad may be in a new one by now. + if shared.session.lock().take_goodbye().is_some() { + debug!("Dropped a goodbye left while the stream was stopped"); + } + let beat = std::sync::Arc::clone(&shared.transport_beat); + let _beat_ends = BeatEnds(&beat); let mut deferred_control = VecDeque::new(); let mut dgram_scratch = [0u8; MAX_DGRAM_SIZE]; let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); @@ -298,11 +307,13 @@ pub async fn start_sender( .pen_spaced_until() .is_some_and(|at| at <= Instant::now()) { + beat.mark(Phase::Input); state .input_relay .keep_pen_alive(*shared.capture_geometry.lock()); } pen_spacing.arm(state.input_relay.pen_spaced_until()); + beat.mark(Phase::Waiting); tokio::select! { // Service timers and queued control before another paced frame. // Random branch selection can delay a NACK by multiple frame @@ -311,19 +322,23 @@ pub async fn start_sender( // Not while a backlog is held: its chunk went at the top of // this pass, and a second one would leave nothing between them. _ = pen_tick.tick(), if state.input_relay.pen_tick_needed() && state.input_relay.pen_spaced_until().is_none() => { + beat.mark(Phase::Input); state.input_relay.keep_pen_alive(*shared.capture_geometry.lock()); } _ = tokio::time::sleep_until(state.fault.next_deadline().unwrap_or_else(|| Instant::now() + Duration::from_secs(3600)).into()), if state.fault.enabled() => { + beat.mark(Phase::Video); let destination = *shared.target_addr.lock(); for datagram in state.fault.drain_due(Instant::now()) { let _ = links.send_to(&datagram, destination).await; } } _ = heartbeat.tick() => { + beat.mark(Phase::Tick); let actions = shared.session.lock().tick(&config, true, now()); act(actions, &mut state, &links, &shared, &config, &supervisor_tx).await; } _ = network_check.tick(), if listen_on != crate::network::ListenOn::All => { + beat.mark(Phase::Tick); if !shared.client_connected() && crate::network::listen_ip_now(&listen_on) != bind_ip { info!(listen_on = listen_on.label(), "The chosen network changed: restarting the stream to listen on it"); shared.stop(); @@ -333,6 +348,7 @@ pub async fn start_sender( } } _ = liveness_tick.tick() => { + beat.mark(Phase::Tick); links.update_stats(); if !shared.running.load(Ordering::SeqCst) { info!("Transport loop stopping on running=false"); @@ -343,6 +359,7 @@ pub async fn start_sender( } result = receive_control(&mut links, &mut deferred_control) => { + beat.mark(Phase::Control); match result { Ok(LinkEvent::Closed(id)) => { let actions = shared.session.lock().link_lost(id); @@ -385,9 +402,11 @@ pub async fn start_sender( } Some(packet) = audio_rx.recv() => { + beat.mark(Phase::Audio); audio_sender.send(packet, &links, &shared, stream_epoch).await; } nal_opt = nal_rx.recv() => { + beat.mark(Phase::Video); let Some(nal) = nal_opt else { break; }; let Some(session_id) = shared.session.lock().session_id() else { continue; }; let target_addr = *shared.target_addr.lock(); @@ -524,7 +543,8 @@ pub async fn start_sender( } // Every way out of the loop comes here, before the links close: an update - // that just stopped the stream must still tell the iPad. + // or a Stop that just stopped the stream must still tell the iPad. + beat.mark(Phase::Actions); let goodbye = shared.session.lock().take_goodbye(); if let Some(goodbye) = goodbye { say_goodbye(&links, goodbye).await; @@ -533,6 +553,16 @@ pub async fn start_sender( Ok(()) } +/// Marks the transport ended however `start_sender` stops, so the watch +/// never reads a finished loop's last mark as a stuck one. +struct BeatEnds<'a>(&'a beat::Beat); + +impl Drop for BeatEnds<'_> { + fn drop(&mut self) { + self.0.stop(); + } +} + #[derive(Debug, PartialEq, Eq)] pub(crate) enum FirstFrame { Send, @@ -718,6 +748,7 @@ async fn act( send_repairs(&links.udp.socket, &mut state.retransmit_ring, shared, &nack).await; } if let Some((session_id, event)) = actions.input.take() { + shared.transport_beat.mark(Phase::Input); if let Some(geometry) = *shared.capture_geometry.lock() { if actions.input_late { state.input_relay.relay_late(session_id, &event, geometry); @@ -780,6 +811,7 @@ async fn execute_actions( shared: &SharedControl, supervisor_tx: &std_mpsc::Sender, ) { + shared.transport_beat.mark(Phase::Actions); // An ended session's summary waits for this pass's second screen step, // which can give the second screen up for it, unless a new session // starts now: second screen news from here on is the new one's. @@ -855,6 +887,7 @@ async fn execute_actions( } } + shared.transport_beat.mark(Phase::SecondScreen); let restart = match second_screen::next_step(shared, now()) { second_screen::Step::Nothing => false, second_screen::Step::Attach => { diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index b6ef5504..f77c9962 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -298,7 +298,8 @@ pub struct Session { /// either closes the host or gives up. Changed only under the session lock, /// together with the check that nobody is streaming. updating: bool, - /// The goodbye for a stream an update stopped, sent by the next tick. + /// The goodbye for a stream an update or a stop ended, sent by the next + /// tick or by the transport on its way out. goodbye: Option<(PeerId, Vec)>, /// That stream's end, reported by the next tick or [`Self::shut_down`]. stopped: Option, @@ -364,21 +365,32 @@ impl Session { if !stop_stream { return false; } - let (peer, session_id) = (session.peer, session.session_id); - info!(%peer, "Stopping the stream for an update"); - let msg_seq = self.next_msg_seq(); - let bye = ControlMessage::Bye(ByeReason::HostShuttingDown); - self.goodbye = Some(( - peer, - eternal_wire::v2::control::encode_control(session_id, msg_seq, &bye), - )); - self.stopped = self.end(EndReason::Shutdown, Instant::now()); + info!(peer = %session.peer, "Stopping the stream for an update"); + self.stopped = self.end_saying(ByeReason::HostShuttingDown, Instant::now()); } self.updating = true; true } - /// The goodbye [`Self::begin_update`] left for a stopped stream, if any. + /// Ends the active session for [`EndReason::Shutdown`] and leaves `bye` + /// for the transport to send before its links close. + fn end_saying(&mut self, bye: ByeReason, now: Instant) -> Option { + let session = self.active.as_ref()?; + let (peer, session_id) = (session.peer, session.session_id); + let msg_seq = self.next_msg_seq(); + self.goodbye = Some(( + peer, + eternal_wire::v2::control::encode_control( + session_id, + msg_seq, + &ControlMessage::Bye(bye), + ), + )); + self.end(EndReason::Shutdown, now) + } + + /// The goodbye left for a stopped stream ([`Self::begin_update`], + /// [`Self::shut_down`]), if any. pub fn take_goodbye(&mut self) -> Option<(PeerId, Vec)> { self.goodbye.take() } @@ -388,10 +400,13 @@ impl Session { self.updating = false; } - /// The app is quitting: the end of the stream it stops, or of one an - /// update stopped that no tick reported yet. - pub fn shut_down(&mut self, now: Instant) -> Option { - let active = self.end(EndReason::Shutdown, now); + /// The stream stops for good (the app quits, or Stop): the end of the + /// session it stops, or of one an update stopped that no tick reported + /// yet. A session it stops is told `bye` by the transport on its way + /// out; the iPad used to hear nothing, and showed a lost signal or an + /// unplugged cable. + pub fn shut_down(&mut self, now: Instant, bye: ByeReason) -> Option { + let active = self.end_saying(bye, now); self.stopped.take().or(active) } @@ -2648,16 +2663,48 @@ mod tests { (ended(&stopped).client_nonce, ended(&stopped).reason), (4, EndReason::Shutdown) ); - assert_eq!(session.shut_down(now), None, "reported once"); + assert_eq!( + session.shut_down(now, ByeReason::UserDisconnect), + None, + "reported once" + ); + assert_eq!(session.take_goodbye(), None, "nobody left to tell"); session.end_update(); session.handle_control(usb, 0, identified_hello(5, 77), &TestConfig, now); - let quit = session.shut_down(now + Duration::from_secs(3)).unwrap(); + let quit = session + .shut_down(now + Duration::from_secs(3), ByeReason::UserDisconnect) + .unwrap(); assert_eq!((quit.client_nonce, quit.reason), (5, EndReason::Shutdown)); assert_eq!(quit.duration, Duration::from_secs(3)); assert!(!session.is_active()); } + /// Quitting or Stop ends the stream for good: the iPad hears why, from + /// the transport on its way out, instead of waiting out a lost signal. + /// A restart tells it to connect again. + #[test] + fn a_stopped_stream_tells_its_ipad() { + let now = Instant::now(); + for (peer, bye) in [ + (PeerId::usb(17), ByeReason::UserDisconnect), + (addr([10, 0, 0, 5], 50000), ByeReason::Error), + ] { + let mut session = Session::new(500); + session.handle_control(peer, 0, identified_hello(6, 77), &TestConfig, now); + let id = session.session_id().unwrap(); + let end = session.shut_down(now, bye).unwrap(); + assert_eq!(end.reason, EndReason::Shutdown); + let (to, bytes) = session.take_goodbye().expect("a goodbye"); + let (header, message) = eternal_wire::v2::control::parse_control(&bytes).unwrap(); + assert_eq!( + (to, header.session_id, message), + (peer, id, ControlMessage::Bye(bye)) + ); + assert_eq!(session.take_goodbye(), None, "sent once"); + } + } + #[test] fn a_refused_hello_is_one_attempt_without_a_session() { let now = Instant::now(); diff --git a/host/tests/goodbye.rs b/host/tests/goodbye.rs index b669a9de..32eb3c21 100644 --- a/host/tests/goodbye.rs +++ b/host/tests/goodbye.rs @@ -1,6 +1,6 @@ -//! When Stop and update ends a stream, the iPad must hear the host's goodbye -//! however the transport stops, over UDP and over the USB tunnel. Drives the -//! real transport loop with a fake iPad on loopback. +//! When Stop and update ends a stream, or the app quits, the iPad must hear +//! the host's goodbye however the transport stops, over UDP and over the USB +//! tunnel. Drives the real transport loop with a fake iPad on loopback. use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; @@ -58,11 +58,8 @@ fn hello() -> ControlMessage { }) } -fn is_goodbye(bytes: &[u8]) -> bool { - matches!( - parse_control(bytes), - Ok((_, ControlMessage::Bye(ByeReason::HostShuttingDown))) - ) +fn is_bye(bytes: &[u8], reason: ByeReason) -> bool { + matches!(parse_control(bytes), Ok((_, ControlMessage::Bye(said))) if said == reason) } fn free_udp_port() -> u16 { @@ -72,7 +69,18 @@ fn free_udp_port() -> u16 { .unwrap() } -/// How the transport stops after Stop and update ended the stream. +/// How the stream ends. +#[derive(Clone, Copy)] +enum End { + /// Stop and update. + Update, + /// The app quits. + Quit, + /// The app quits for "Restart EternalMonitor". + Restart, +} + +/// How the transport stops after the stream ended. enum Stop { /// The host is closing: `running` goes false. Shutdown, @@ -80,9 +88,10 @@ enum Stop { EncoderEof, } -/// Runs the transport with a UDP iPad, stops the stream for an update, stops -/// the transport the given way and counts the goodbyes the iPad received. -fn udp_goodbyes(stop: Stop) -> usize { +/// Runs the transport with a UDP iPad, ends the stream the given way, stops +/// the transport the given way and counts the goodbyes the iPad received +/// with `reason`. +fn udp_goodbyes(end: End, stop: Stop, reason: ByeReason) -> usize { let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); eternal_host::clock::init(); // Never reach a real iPad: this unreachable endpoint stands in for USB. @@ -115,7 +124,17 @@ fn udp_goodbyes(stop: Stop) -> usize { }); // Let the loop take its first ticks. tokio::time::sleep(Duration::from_millis(40)).await; - assert!(shared.session.lock().begin_update(true)); + match end { + End::Update => assert!(shared.session.lock().begin_update(true)), + End::Quit => eternal_host::diagnostics::log_session_end_at_quit(&shared), + End::Restart => { + shared + .handoff + .claim(eternal_host::handoff::Handoff::Restart) + .unwrap(); + eternal_host::diagnostics::log_session_end_at_quit(&shared); + } + } match stop { Stop::Shutdown => shared.stop(), Stop::EncoderEof => drop(nal_tx), @@ -136,7 +155,7 @@ fn udp_goodbyes(stop: Stop) -> usize { while let Ok(Ok((len, _))) = tokio::time::timeout(Duration::from_millis(100), ipad.recv_from(&mut bytes)).await { - goodbyes += usize::from(is_goodbye(&bytes[..len])); + goodbyes += usize::from(is_bye(&bytes[..len], reason)); } goodbyes }) @@ -145,14 +164,39 @@ fn udp_goodbyes(stop: Stop) -> usize { /// One lost datagram must not turn a planned update into a lost connection. #[test] fn udp_goodbye_is_a_short_burst() { - assert_eq!(udp_goodbyes(Stop::Shutdown), 3); + assert_eq!( + udp_goodbyes(End::Update, Stop::Shutdown, ByeReason::HostShuttingDown), + 3 + ); } /// The encoder can close its channel before the next liveness tick while the /// host shuts down; the goodbye must still go out. #[test] fn encoder_eof_still_says_goodbye() { - assert!(udp_goodbyes(Stop::EncoderEof) >= 1); + assert!(udp_goodbyes(End::Update, Stop::EncoderEof, ByeReason::HostShuttingDown) >= 1); +} + +/// Quitting used to end the session without a word: the iPad waited out a +/// lost signal and then tried to reconnect to a PC that had closed. It now +/// hears that the PC ended the session, the goodbye that keeps it from +/// reconnecting by itself. +#[test] +fn quitting_tells_the_ipad_the_pc_ended_the_session() { + assert_eq!( + udp_goodbyes(End::Quit, Stop::Shutdown, ByeReason::UserDisconnect), + 3 + ); +} + +/// A restart hands over to a new copy that is back in seconds: the iPad is +/// told to connect again, at once. +#[test] +fn a_restart_tells_the_ipad_to_connect_again() { + assert_eq!( + udp_goodbyes(End::Restart, Stop::Shutdown, ByeReason::Error), + 3 + ); } /// Over USB the goodbye is queued for a writer task; the transport must wait @@ -161,6 +205,17 @@ fn encoder_eof_still_says_goodbye() { /// one goodbye arrives on a link that stays open. #[test] fn usb_goodbye_arrives_before_the_link_closes() { + usb_goodbye(End::Update, ByeReason::HostShuttingDown); +} + +/// Quitting over the cable: the iPad used to see the tunnel close and showed +/// an unplugged cable. +#[test] +fn quitting_over_usb_says_goodbye_before_the_link_closes() { + usb_goodbye(End::Quit, ByeReason::UserDisconnect); +} + +fn usb_goodbye(end: End, reason: ByeReason) { let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); eternal_host::clock::init(); let runtime = tokio::runtime::Builder::new_current_thread() @@ -204,7 +259,10 @@ fn usb_goodbye_arrives_before_the_link_closes() { } } tokio::time::sleep(Duration::from_millis(40)).await; - assert!(shared.session.lock().begin_update(true)); + match end { + End::Update => assert!(shared.session.lock().begin_update(true)), + End::Quit | End::Restart => eternal_host::diagnostics::log_session_end_at_quit(&shared), + } shared.stop(); let stopping = Instant::now(); tokio::time::timeout(Duration::from_secs(2), transport) @@ -217,7 +275,7 @@ fn usb_goodbye_arrives_before_the_link_closes() { let mut goodbyes = 0; loop { match tokio::time::timeout(Duration::from_millis(500), read_datagram(&mut ipad)).await { - Ok(Ok(bytes)) => goodbyes += usize::from(is_goodbye(&bytes)), + Ok(Ok(bytes)) => goodbyes += usize::from(is_bye(&bytes, reason)), Ok(Err(error)) => panic!("the USB hub closed the tunnel: {error}"), Err(_) => break, } diff --git a/host/tests/transport_stall.rs b/host/tests/transport_stall.rs new file mode 100644 index 00000000..fe20e731 --- /dev/null +++ b/host/tests/transport_stall.rs @@ -0,0 +1,107 @@ +//! A transport loop stuck in one call is named by the supervisor's watch, with +//! the step it is stuck in, and again when it moves. Drives the real loop and +//! blocks it the way a stuck call would: the session lock it needs at its +//! next tick is held elsewhere. + +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use eternal_host::control::SharedControl; +use eternal_host::transport::beat::{Phase, Verdict, Watch}; +use eternal_host::transport::{self}; +use tokio::sync::mpsc; + +fn free_udp_port() -> u16 { + std::net::UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap() +} + +fn now_ms() -> u64 { + eternal_host::clock::host_now_us() / 1000 +} + +/// Checks the beat as the supervisor does, every 100 ms, until `until`. +fn watch_until(watch: &mut Watch, shared: &SharedControl, until: Instant) -> Vec { + let mut verdicts = Vec::new(); + while Instant::now() < until { + verdicts.extend(watch.check(shared.transport_beat.read(), now_ms())); + std::thread::sleep(Duration::from_millis(100)); + } + verdicts +} + +#[test] +fn a_stuck_transport_loop_is_named_with_its_step() { + eternal_host::clock::init(); + // Never reach a real iPad over USB. + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + let port = free_udp_port(); + let shared = SharedControl::new(port, 10_000_000); + shared.running.store(true, Ordering::SeqCst); + let (nal_tx, nal_rx) = mpsc::channel(4); + let (_audio_tx, audio_rx) = mpsc::channel(4); + let control = shared.clone(); + let transport = std::thread::spawn(move || { + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(transport::start_sender( + nal_rx, + audio_rx, + port, + control, + supervisor_tx, + )) + .unwrap(); + }); + let mut watch = Watch::default(); + let started = Instant::now(); + while shared.transport_beat.read().is_none() { + assert!( + started.elapsed() < Duration::from_secs(30), + "the loop starts" + ); + std::thread::sleep(Duration::from_millis(10)); + } + // Running freely, it ticks every 250 ms. + assert_eq!( + watch_until( + &mut watch, + &shared, + Instant::now() + Duration::from_millis(1500) + ), + [] + ); + + let held = shared.session.lock(); + let stuck = watch_until(&mut watch, &shared, Instant::now() + Duration::from_secs(4)); + drop(held); + assert!( + matches!( + stuck.as_slice(), + [Verdict::Stalled { phase: Phase::Tick, silent_ms }] if *silent_ms >= 2_000 + ), + "{stuck:?}" + ); + let moved = watch_until(&mut watch, &shared, Instant::now() + Duration::from_secs(1)); + assert!( + matches!( + moved.as_slice(), + [Verdict::Moving { phase: Phase::Tick, stalled_ms }] if *stalled_ms >= 2_000 + ), + "{moved:?}" + ); + + shared.stop(); + drop(nal_tx); + transport.join().unwrap(); + assert_eq!( + shared.transport_beat.read(), + None, + "an ended loop is not stuck" + ); +} From 75c0dff656e2fdb056ff3de25e088da2f71888c5 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 11:55:45 -0500 Subject: [PATCH 013/226] Vendor egui-wgpu 0.36.2 unchanged --- Cargo.lock | 2 - Cargo.toml | 5 + vendor/README.md | 10 + vendor/egui-wgpu/Cargo.toml | 340 +++++++ vendor/egui-wgpu/LICENSE-APACHE | 201 ++++ vendor/egui-wgpu/LICENSE-MIT | 25 + vendor/egui-wgpu/README.md | 10 + vendor/egui-wgpu/src/capture.rs | 267 ++++++ vendor/egui-wgpu/src/egui.wgsl | 162 ++++ vendor/egui-wgpu/src/lib.rs | 555 +++++++++++ vendor/egui-wgpu/src/renderer.rs | 1178 ++++++++++++++++++++++++ vendor/egui-wgpu/src/setup.rs | 331 +++++++ vendor/egui-wgpu/src/texture_copy.wgsl | 43 + vendor/egui-wgpu/src/winit.rs | 800 ++++++++++++++++ 14 files changed, 3927 insertions(+), 2 deletions(-) create mode 100644 vendor/README.md create mode 100644 vendor/egui-wgpu/Cargo.toml create mode 100644 vendor/egui-wgpu/LICENSE-APACHE create mode 100644 vendor/egui-wgpu/LICENSE-MIT create mode 100644 vendor/egui-wgpu/README.md create mode 100644 vendor/egui-wgpu/src/capture.rs create mode 100644 vendor/egui-wgpu/src/egui.wgsl create mode 100644 vendor/egui-wgpu/src/lib.rs create mode 100644 vendor/egui-wgpu/src/renderer.rs create mode 100644 vendor/egui-wgpu/src/setup.rs create mode 100644 vendor/egui-wgpu/src/texture_copy.wgsl create mode 100644 vendor/egui-wgpu/src/winit.rs diff --git a/Cargo.lock b/Cargo.lock index 03c5906c..5427b912 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -998,8 +998,6 @@ dependencies = [ [[package]] name = "egui-wgpu" version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cef1f6d1bba40b09a4d0d62cb822b81030922cfbf65c5373398c3a3caafc5bd" dependencies = [ "ahash", "bytemuck", diff --git a/Cargo.toml b/Cargo.toml index c25514d0..2b301e23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,13 @@ # docs/ contains the static website (GitHub Pages). Never write build output here. [workspace] members = ["host", "proto", "updater"] +exclude = ["vendor"] resolver = "2" [profile.release] strip = "symbols" opt-level = 3 + +# Patched copies of upstream crates; vendor/README.md says why. +[patch.crates-io] +egui-wgpu = { path = "vendor/egui-wgpu" } diff --git a/vendor/README.md b/vendor/README.md new file mode 100644 index 00000000..781290d2 --- /dev/null +++ b/vendor/README.md @@ -0,0 +1,10 @@ +# Vendored crates + +Cargo builds these in place of the crates.io release (`[patch.crates-io]` in +the workspace `Cargo.toml`). Each keeps its upstream license files. + +## egui-wgpu 0.36.2 + +Copied from the crates.io release of egui-wgpu 0.36.2 +(https://github.com/emilk/egui, MIT OR Apache-2.0), without its `Cargo.lock` +and `Cargo.toml.orig`. diff --git a/vendor/egui-wgpu/Cargo.toml b/vendor/egui-wgpu/Cargo.toml new file mode 100644 index 00000000..cd9495a2 --- /dev/null +++ b/vendor/egui-wgpu/Cargo.toml @@ -0,0 +1,340 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.95" +name = "egui-wgpu" +version = "0.36.2" +authors = [ + "Nils Hasenbanck ", + "embotech ", + "Emil Ernerfeldt ", +] +build = false +include = [ + "../../LICENSE-APACHE", + "../../LICENSE-MIT", + "**/*.rs", + "**/*.wgsl", + "Cargo.toml", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Bindings for using egui natively using the wgpu library" +homepage = "https://github.com/emilk/egui/tree/main/crates/egui-wgpu" +readme = "README.md" +keywords = [ + "wgpu", + "egui", + "gui", + "gamedev", +] +categories = [ + "gui", + "game-development", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/emilk/egui/tree/main/crates/egui-wgpu" +resolver = "2" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--generate-link-to-definition"] + +[features] +capture = ["dep:egui"] +default = [ + "fragile-send-sync-non-atomic-wasm", + "macos-window-resize-jitter-fix", + "wgpu/default", + "wgpu/webgl", +] +fragile-send-sync-non-atomic-wasm = ["wgpu/fragile-send-sync-non-atomic-wasm"] +macos-window-resize-jitter-fix = ["wgpu/metal"] +wayland = ["winit?/wayland"] +winit = [ + "dep:winit", + "winit/rwh_06", + "dep:egui", + "capture", +] +x11 = ["winit?/x11"] + +[lib] +name = "egui_wgpu" +path = "src/lib.rs" + +[dependencies.ahash] +version = "0.8.12" +features = [ + "no-rng", + "std", +] +default-features = false + +[dependencies.bytemuck] +version = "1.25" + +[dependencies.document-features] +version = "0.2.12" + +[dependencies.egui] +version = "0.36.2" +optional = true +default-features = false + +[dependencies.epaint] +version = "0.36.2" +features = ["bytemuck"] +default-features = false + +[dependencies.log] +version = "0.4.33" +features = ["std"] + +[dependencies.profiling] +version = "1.0" +default-features = false + +[dependencies.thiserror] +version = "2.0" + +[dependencies.type-map] +version = "0.5.1" + +[dependencies.web-time] +version = "1.1" + +[dependencies.wgpu] +version = "30.0" +features = [ + "std", + "wgsl", +] +default-features = false + +[dependencies.winit] +version = "0.30.13" +optional = true +default-features = false + +[lints.clippy] +allow_attributes = "warn" +as_ptr_cast_mut = "warn" +assigning_clones = "allow" +bool_to_int_with_if = "warn" +branches_sharing_code = "warn" +cast_possible_truncation = "allow" +cast_possible_wrap = "allow" +checked_conversions = "warn" +clear_with_drain = "warn" +clone_on_ref_ptr = "warn" +cloned_instead_of_copied = "warn" +coerce_container_to_any = "warn" +comparison_chain = "allow" +dbg_macro = "warn" +debug_assert_with_mut_call = "warn" +decimal_bitwise_operands = "warn" +default_union_representation = "warn" +derive_partial_eq_without_eq = "warn" +disallowed_script_idents = "warn" +doc_broken_link = "warn" +doc_comment_double_space_linebreaks = "warn" +doc_include_without_cfg = "warn" +doc_link_with_quotes = "warn" +doc_markdown = "warn" +duration_suboptimal_units = "warn" +elidable_lifetime_names = "warn" +empty_enum_variants_with_brackets = "warn" +empty_enums = "warn" +enum_glob_use = "warn" +equatable_if_let = "warn" +exit = "warn" +expl_impl_clone_on_copy = "warn" +explicit_deref_methods = "warn" +explicit_into_iter_loop = "warn" +explicit_iter_loop = "warn" +fallible_impl_from = "warn" +filter_map_next = "warn" +flat_map_option = "warn" +float_cmp_const = "warn" +fn_params_excessive_bools = "warn" +fn_to_numeric_cast_any = "warn" +format_push_string = "warn" +from_iter_instead_of_collect = "warn" +get_unwrap = "warn" +ignore_without_reason = "warn" +ignored_unit_patterns = "warn" +implicit_clone = "warn" +imprecise_flops = "warn" +inconsistent_struct_constructor = "warn" +index_refutable_slice = "warn" +inefficient_to_string = "warn" +infinite_loop = "warn" +into_iter_without_iter = "warn" +invalid_upcast_comparisons = "warn" +ip_constant = "warn" +iter_filter_is_ok = "warn" +iter_filter_is_some = "warn" +iter_not_returning_iterator = "warn" +iter_on_empty_collections = "warn" +iter_on_single_items = "warn" +iter_over_hash_type = "warn" +iter_without_into_iter = "warn" +large_digit_groups = "warn" +large_futures = "warn" +large_include_file = "warn" +large_stack_arrays = "warn" +large_stack_frames = "warn" +large_types_passed_by_value = "warn" +let_underscore_must_use = "allow" +let_underscore_untyped = "allow" +linkedlist = "warn" +literal_string_with_formatting_args = "warn" +lossy_float_literal = "warn" +macro_use_imports = "warn" +manual_assert = "warn" +manual_ilog2 = "warn" +manual_instant_elapsed = "warn" +manual_is_power_of_two = "warn" +manual_is_variant_and = "warn" +manual_let_else = "warn" +manual_midpoint = "warn" +manual_range_contains = "allow" +manual_string_new = "warn" +map_err_ignore = "warn" +map_unwrap_or = "allow" +match_bool = "warn" +match_same_arms = "warn" +match_wild_err_arm = "warn" +match_wildcard_for_single_variants = "warn" +mem_forget = "warn" +mismatching_type_param_order = "warn" +missing_assert_message = "warn" +missing_errors_doc = "warn" +missing_fields_in_debug = "warn" +mut_mut = "warn" +mutex_integer = "warn" +needless_continue = "warn" +needless_for_each = "warn" +needless_pass_by_ref_mut = "warn" +needless_pass_by_value = "warn" +needless_raw_string_hashes = "warn" +needless_type_cast = "warn" +negative_feature_names = "warn" +non_std_lazy_statics = "warn" +non_zero_suggestions = "warn" +nonstandard_macro_braces = "warn" +option_as_ref_cloned = "warn" +option_option = "warn" +or_fun_call = "warn" +path_buf_push_overwrite = "warn" +pathbuf_init_then_push = "warn" +precedence_bits = "warn" +print_stderr = "warn" +print_stdout = "warn" +ptr_as_ptr = "warn" +ptr_cast_constness = "warn" +pub_underscore_fields = "warn" +pub_without_shorthand = "warn" +rc_mutex = "warn" +redundant_type_annotations = "warn" +ref_as_ptr = "warn" +ref_option = "warn" +ref_option_ref = "warn" +ref_patterns = "warn" +rest_pat_in_fully_bound_structs = "warn" +return_and_then = "warn" +same_functions_in_if_condition = "warn" +same_length_and_capacity = "warn" +self_named_module_files = "allow" +self_only_used_in_recursion = "warn" +semicolon_if_nothing_returned = "warn" +set_contains_or_insert = "warn" +should_panic_without_expect = "allow" +significant_drop_tightening = "allow" +single_char_pattern = "warn" +single_match_else = "warn" +single_option_map = "warn" +std_instead_of_core = "warn" +str_split_at_newline = "warn" +str_to_string = "warn" +string_add = "warn" +string_add_assign = "warn" +string_lit_as_bytes = "warn" +string_lit_chars_any = "warn" +suspicious_xor_used_as_pow = "warn" +todo = "warn" +too_long_first_doc_paragraph = "warn" +too_many_lines = "allow" +trailing_empty_array = "warn" +trait_duplication_in_bounds = "warn" +transmute_ptr_to_ptr = "warn" +tuple_array_conversions = "warn" +unchecked_time_subtraction = "warn" +undocumented_unsafe_blocks = "warn" +unimplemented = "warn" +uninhabited_references = "warn" +uninlined_format_args = "warn" +unnecessary_box_returns = "warn" +unnecessary_debug_formatting = "warn" +unnecessary_literal_bound = "warn" +unnecessary_safety_comment = "warn" +unnecessary_safety_doc = "warn" +unnecessary_self_imports = "warn" +unnecessary_semicolon = "warn" +unnecessary_struct_initialization = "warn" +unnecessary_trailing_comma = "warn" +unnecessary_wraps = "warn" +unnested_or_patterns = "warn" +unused_async = "warn" +unused_peekable = "warn" +unused_rounding = "warn" +unused_self = "warn" +unused_trait_names = "warn" +unwrap_used = "warn" +use_self = "warn" +useless_let_if_seq = "warn" +verbose_file_reads = "warn" +wildcard_dependencies = "warn" +wildcard_imports = "allow" +zero_sized_map_values = "warn" + +[lints.clippy.all] +level = "warn" +priority = -1 + +[lints.rust] +trivial_casts = "allow" +trivial_numeric_casts = "warn" +unsafe_code = "deny" +unused_import_braces = "warn" +unused_lifetimes = "warn" +unused_qualifications = "allow" + +[lints.rust.future_incompatible] +level = "warn" +priority = -1 + +[lints.rust.nonstandard_style] +level = "warn" +priority = -1 + +[lints.rust.rust_2018_idioms] +level = "warn" +priority = -1 + +[lints.rustdoc] +all = "warn" diff --git a/vendor/egui-wgpu/LICENSE-APACHE b/vendor/egui-wgpu/LICENSE-APACHE new file mode 100644 index 00000000..11069edd --- /dev/null +++ b/vendor/egui-wgpu/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/vendor/egui-wgpu/LICENSE-MIT b/vendor/egui-wgpu/LICENSE-MIT new file mode 100644 index 00000000..673ea5f0 --- /dev/null +++ b/vendor/egui-wgpu/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2018-2021 Emil Ernerfeldt + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/vendor/egui-wgpu/README.md b/vendor/egui-wgpu/README.md new file mode 100644 index 00000000..9aab3159 --- /dev/null +++ b/vendor/egui-wgpu/README.md @@ -0,0 +1,10 @@ +# egui-wgpu + +[![Latest version](https://img.shields.io/crates/v/egui-wgpu.svg)](https://crates.io/crates/egui-wgpu) +[![Documentation](https://docs.rs/egui-wgpu/badge.svg)](https://docs.rs/egui-wgpu) +![MIT](https://img.shields.io/badge/license-MIT-blue.svg) +![Apache](https://img.shields.io/badge/license-Apache-blue.svg) + +This crates provides bindings between [`egui`](https://github.com/emilk/egui) and [wgpu](https://crates.io/crates/wgpu). + +This was originally hosted at https://github.com/hasenbanck/egui_wgpu_backend diff --git a/vendor/egui-wgpu/src/capture.rs b/vendor/egui-wgpu/src/capture.rs new file mode 100644 index 00000000..54ddeb88 --- /dev/null +++ b/vendor/egui-wgpu/src/capture.rs @@ -0,0 +1,267 @@ +use egui::{UserData, ViewportId}; +use epaint::ColorImage; +use std::sync::{Arc, mpsc}; +use wgpu::{BindGroupLayout, MultisampleState, StoreOp}; + +/// A texture and a buffer for reading the rendered frame back to the cpu. +/// +/// The texture is required since [`wgpu::TextureUsages::COPY_SRC`] is not an allowed +/// flag for the surface texture on all platforms. This means that anytime we want to +/// capture the frame, we first render it to this texture, and then we can copy it to +/// both the surface texture (via a render pass) and the buffer (via a texture to buffer copy), +/// from where we can pull it back +/// to the cpu. +pub struct CaptureState { + padding: BufferPadding, + pub texture: wgpu::Texture, + pipeline: wgpu::RenderPipeline, + bind_group: wgpu::BindGroup, +} + +pub type CaptureReceiver = mpsc::Receiver<(ViewportId, Vec, ColorImage)>; +pub type CaptureSender = mpsc::Sender<(ViewportId, Vec, ColorImage)>; +pub use mpsc::channel as capture_channel; + +impl CaptureState { + pub fn new(device: &wgpu::Device, surface_texture: &wgpu::Texture) -> Self { + let shader = device.create_shader_module(wgpu::include_wgsl!("texture_copy.wgsl")); + + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("texture_copy"), + layout: None, + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(surface_texture.format().into())], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + ..Default::default() + }, + depth_stencil: None, + multisample: MultisampleState::default(), + multiview_mask: None, + cache: None, + }); + + let bind_group_layout = pipeline.get_bind_group_layout(0); + + let (texture, padding, bind_group) = + Self::create_texture(device, surface_texture, &bind_group_layout); + + Self { + padding, + texture, + pipeline, + bind_group, + } + } + + fn create_texture( + device: &wgpu::Device, + surface_texture: &wgpu::Texture, + layout: &BindGroupLayout, + ) -> (wgpu::Texture, BufferPadding, wgpu::BindGroup) { + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("egui_screen_capture_texture"), + size: surface_texture.size(), + mip_level_count: surface_texture.mip_level_count(), + sample_count: surface_texture.sample_count(), + dimension: surface_texture.dimension(), + format: surface_texture.format(), + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + + let padding = BufferPadding::new(surface_texture.width()); + + let view = texture.create_view(&Default::default()); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&view), + }], + label: None, + }); + + (texture, padding, bind_group) + } + + /// Updates the [`CaptureState`] if the size of the surface texture has changed + pub fn update(&mut self, device: &wgpu::Device, texture: &wgpu::Texture) { + if self.texture.size() != texture.size() { + let (new_texture, padding, bind_group) = + Self::create_texture(device, texture, &self.pipeline.get_bind_group_layout(0)); + self.texture = new_texture; + self.padding = padding; + self.bind_group = bind_group; + } + } + + /// Handles copying from the [`CaptureState`] texture to the surface texture and the buffer. + /// Pass the returned buffer to [`CaptureState::read_screen_rgba`] to read the data back to the cpu. + pub fn copy_textures( + &mut self, + device: &wgpu::Device, + output_frame: &wgpu::SurfaceTexture, + encoder: &mut wgpu::CommandEncoder, + ) -> wgpu::Buffer { + debug_assert_eq!( + self.texture.size(), + output_frame.texture.size(), + "Texture sizes must match, `CaptureState::update` was probably not called" + ); + + // It would be more efficient to reuse the Buffer, e.g. via some kind of ring buffer, but + // for most screenshot use cases this should be fine. When taking many screenshots (e.g. for a video) + // it might make sense to revisit this and implement a more efficient solution. + #[allow(clippy::allow_attributes, clippy::arc_with_non_send_sync)] // For wasm + let buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("egui_screen_capture_buffer"), + size: (self.padding.padded_bytes_per_row * self.texture.height()) as u64, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let padding = self.padding; + let tex = &mut self.texture; + + let tex_extent = tex.size(); + + encoder.copy_texture_to_buffer( + tex.as_image_copy(), + wgpu::TexelCopyBufferInfo { + buffer: &buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(padding.padded_bytes_per_row), + rows_per_image: None, + }, + }, + tex_extent, + ); + + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("texture_copy"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &output_frame.texture.create_view(&Default::default()), + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: StoreOp::Store, + }, + depth_slice: None, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + multiview_mask: None, + }); + + pass.set_pipeline(&self.pipeline); + pass.set_bind_group(0, &self.bind_group, &[]); + pass.draw(0..3, 0..1); + + buffer + } + + /// Handles copying from the [`CaptureState`] texture to the surface texture and the cpu + /// This function is non-blocking and will send the data to the given sender when it's ready. + /// Pass in the buffer returned from [`CaptureState::copy_textures`]. + /// Make sure to call this after the encoder has been submitted. + pub fn read_screen_rgba( + &self, + ctx: egui::Context, + buffer: wgpu::Buffer, + data: Vec, + tx: CaptureSender, + viewport_id: ViewportId, + ) { + #[allow(clippy::allow_attributes, clippy::arc_with_non_send_sync)] // For wasm + let buffer = Arc::new(buffer); + let buffer_clone = Arc::clone(&buffer); + let buffer_slice = buffer_clone.slice(..); + let format = self.texture.format(); + let tex_extent = self.texture.size(); + let padding = self.padding; + let to_rgba = match format { + wgpu::TextureFormat::Rgba8Unorm => [0, 1, 2, 3], + wgpu::TextureFormat::Bgra8Unorm => [2, 1, 0, 3], + _ => { + log::error!( + "Screen can't be captured unless the surface format is Rgba8Unorm or Bgra8Unorm. Current surface format is {format:?}" + ); + return; + } + }; + buffer_slice.map_async(wgpu::MapMode::Read, move |result| { + if let Err(err) = result { + log::error!("Failed to map buffer for reading: {err}"); + return; + } + let buffer_slice = buffer.slice(..); + + let mut pixels = Vec::with_capacity((tex_extent.width * tex_extent.height) as usize); + let mapped_range = match buffer_slice.get_mapped_range() { + Ok(range) => range, + Err(err) => { + log::error!("Failed to get mapped range for reading: {err}"); + return; + } + }; + for padded_row in mapped_range.chunks(padding.padded_bytes_per_row as usize) { + let row = &padded_row[..padding.unpadded_bytes_per_row as usize]; + for color in row.chunks(4) { + pixels.push(epaint::Color32::from_rgba_premultiplied( + color[to_rgba[0]], + color[to_rgba[1]], + color[to_rgba[2]], + color[to_rgba[3]], + )); + } + } + drop(mapped_range); + buffer.unmap(); + + tx.send(( + viewport_id, + data, + ColorImage::new( + [tex_extent.width as usize, tex_extent.height as usize], + pixels, + ), + )) + .ok(); + ctx.request_repaint(); + }); + } +} + +#[derive(Copy, Clone)] +struct BufferPadding { + unpadded_bytes_per_row: u32, + padded_bytes_per_row: u32, +} + +impl BufferPadding { + fn new(width: u32) -> Self { + let bytes_per_pixel = core::mem::size_of::() as u32; + let unpadded_bytes_per_row = width * bytes_per_pixel; + let padded_bytes_per_row = + wgpu::util::align_to(unpadded_bytes_per_row, wgpu::COPY_BYTES_PER_ROW_ALIGNMENT); + Self { + unpadded_bytes_per_row, + padded_bytes_per_row, + } + } +} diff --git a/vendor/egui-wgpu/src/egui.wgsl b/vendor/egui-wgpu/src/egui.wgsl new file mode 100644 index 00000000..39210841 --- /dev/null +++ b/vendor/egui-wgpu/src/egui.wgsl @@ -0,0 +1,162 @@ +// Vertex shader bindings + +struct VertexOutput { + @location(0) tex_coord: vec2, + @location(1) color: vec4, // gamma 0-1 + @builtin(position) position: vec4, +}; + +struct Locals { + screen_size: vec2, + + /// 1 if dithering is enabled, 0 otherwise + dithering: u32, + + /// 1 to do manual filtering for more predictable kittest snapshot images. + /// See also https://github.com/emilk/egui/issues/5295 + predictable_texture_filtering: u32, +}; +@group(0) @binding(0) var r_locals: Locals; + + +// ----------------------------------------------- +// Adapted from +// https://www.shadertoy.com/view/llVGzG +// Originally presented in: +// Jimenez 2014, "Next Generation Post-Processing in Call of Duty" +// +// A good overview can be found in +// https://blog.demofox.org/2022/01/01/interleaved-gradient-noise-a-different-kind-of-low-discrepancy-sequence/ +// via https://github.com/rerun-io/rerun/ +fn interleaved_gradient_noise(n: vec2) -> f32 { + let f = 0.06711056 * n.x + 0.00583715 * n.y; + return fract(52.9829189 * fract(f)); +} + +fn dither_interleaved(rgb: vec3, levels: f32, frag_coord: vec4) -> vec3 { + var noise = interleaved_gradient_noise(frag_coord.xy); + // scale down the noise slightly to ensure flat colors aren't getting dithered + noise = (noise - 0.5) * 0.95; + return rgb + noise / (levels - 1.0); +} + +// 0-1 linear from 0-1 sRGB gamma +fn linear_from_gamma_rgb(srgb: vec3) -> vec3 { + let cutoff = srgb < vec3(0.04045); + let lower = srgb / vec3(12.92); + let higher = pow((srgb + vec3(0.055)) / vec3(1.055), vec3(2.4)); + return select(higher, lower, cutoff); +} + +// 0-1 sRGB gamma from 0-1 linear +fn gamma_from_linear_rgb(rgb: vec3) -> vec3 { + let cutoff = rgb < vec3(0.0031308); + let lower = rgb * vec3(12.92); + let higher = vec3(1.055) * pow(rgb, vec3(1.0 / 2.4)) - vec3(0.055); + return select(higher, lower, cutoff); +} + +// 0-1 sRGBA gamma from 0-1 linear +fn gamma_from_linear_rgba(linear_rgba: vec4) -> vec4 { + return vec4(gamma_from_linear_rgb(linear_rgba.rgb), linear_rgba.a); +} + +// [u8; 4] SRGB as u32 -> [r, g, b, a] in 0.-1 +fn unpack_color(color: u32) -> vec4 { + return vec4( + f32(color & 255u), + f32((color >> 8u) & 255u), + f32((color >> 16u) & 255u), + f32((color >> 24u) & 255u), + ) / 255.0; +} + +fn position_from_screen(screen_pos: vec2) -> vec4 { + return vec4( + 2.0 * screen_pos.x / r_locals.screen_size.x - 1.0, + 1.0 - 2.0 * screen_pos.y / r_locals.screen_size.y, + 0.0, + 1.0, + ); +} + +@vertex +fn vs_main( + @location(0) a_pos: vec2, + @location(1) a_tex_coord: vec2, + @location(2) a_color: u32, +) -> VertexOutput { + var out: VertexOutput; + out.tex_coord = a_tex_coord; + out.color = unpack_color(a_color); + out.position = position_from_screen(a_pos); + return out; +} + +// Fragment shader bindings + +@group(1) @binding(0) var r_tex_color: texture_2d; +@group(1) @binding(1) var r_tex_sampler: sampler; + +fn sample_texture(in: VertexOutput) -> vec4 { + if r_locals.predictable_texture_filtering == 0 { + // Hardware filtering: fast, but varies across GPUs and drivers. + return textureSample(r_tex_color, r_tex_sampler, in.tex_coord); + } else { + // Manual bilinear filtering with four taps at pixel centers using textureLoad + let texture_size = vec2(textureDimensions(r_tex_color, 0)); + let texture_size_f = vec2(texture_size); + let pixel_coord = in.tex_coord * texture_size_f - 0.5; + let pixel_fract = fract(pixel_coord); + let pixel_floor = vec2(floor(pixel_coord)); + + // Manual texture clamping + let max_coord = texture_size - vec2(1, 1); + let p00 = clamp(pixel_floor + vec2(0, 0), vec2(0, 0), max_coord); + let p10 = clamp(pixel_floor + vec2(1, 0), vec2(0, 0), max_coord); + let p01 = clamp(pixel_floor + vec2(0, 1), vec2(0, 0), max_coord); + let p11 = clamp(pixel_floor + vec2(1, 1), vec2(0, 0), max_coord); + + // Load at pixel centers + let tl = textureLoad(r_tex_color, p00, 0); + let tr = textureLoad(r_tex_color, p10, 0); + let bl = textureLoad(r_tex_color, p01, 0); + let br = textureLoad(r_tex_color, p11, 0); + + // Manual bilinear interpolation + let top = mix(tl, tr, pixel_fract.x); + let bottom = mix(bl, br, pixel_fract.x); + return mix(top, bottom, pixel_fract.y); + } +} + +@fragment +fn fs_main_linear_framebuffer(in: VertexOutput) -> @location(0) vec4 { + // We expect "normal" textures that are NOT sRGB-aware. + let tex_gamma = sample_texture(in); + var out_color_gamma = in.color * tex_gamma; + // Dither the float color down to eight bits to reduce banding. + // This step is optional for egui backends. + // Note that dithering is performed on the gamma encoded values, + // because this function is used together with a srgb converting target. + if r_locals.dithering == 1 { + let out_color_gamma_rgb = dither_interleaved(out_color_gamma.rgb, 256.0, in.position); + out_color_gamma = vec4(out_color_gamma_rgb, out_color_gamma.a); + } + let out_color_linear = linear_from_gamma_rgb(out_color_gamma.rgb); + return vec4(out_color_linear, out_color_gamma.a); +} + +@fragment +fn fs_main_gamma_framebuffer(in: VertexOutput) -> @location(0) vec4 { + // We expect "normal" textures that are NOT sRGB-aware. + let tex_gamma = sample_texture(in); + var out_color_gamma = in.color * tex_gamma; + // Dither the float color down to eight bits to reduce banding. + // This step is optional for egui backends. + if r_locals.dithering == 1 { + let out_color_gamma_rgb = dither_interleaved(out_color_gamma.rgb, 256.0, in.position); + out_color_gamma = vec4(out_color_gamma_rgb, out_color_gamma.a); + } + return out_color_gamma; +} diff --git a/vendor/egui-wgpu/src/lib.rs b/vendor/egui-wgpu/src/lib.rs new file mode 100644 index 00000000..1eb975ae --- /dev/null +++ b/vendor/egui-wgpu/src/lib.rs @@ -0,0 +1,555 @@ +//! This crates provides bindings between [`egui`](https://github.com/emilk/egui) and [wgpu](https://crates.io/crates/wgpu). +//! +//! If you're targeting WebGL you also need to turn on the +//! `webgl` feature of the `wgpu` crate: +//! +//! ```toml +//! # Enable both WebGL and WebGPU backends on web. +//! wgpu = { version = "*", features = ["webgpu", "webgl"] } +//! ``` +//! +//! You can control whether WebGL or WebGPU will be picked at runtime by configuring +//! [`WgpuConfiguration::wgpu_setup`]. +//! The default is to prefer WebGPU and fall back on WebGL. +//! +//! ## Feature flags +#![doc = document_features::document_features!()] +//! + +pub use wgpu; + +/// Low-level painting of [`egui`](https://github.com/emilk/egui) on [`wgpu`]. +mod renderer; + +mod setup; + +pub use renderer::*; +pub use setup::{ + EguiDisplayHandle, NativeAdapterSelectorMethod, WgpuSetup, WgpuSetupCreateNew, + WgpuSetupExisting, +}; + +/// Helpers for capturing screenshots of the UI. +#[cfg(feature = "capture")] +pub mod capture; + +/// Module for painting [`egui`](https://github.com/emilk/egui) with [`wgpu`] on [`winit`]. +#[cfg(feature = "winit")] +pub mod winit; + +use std::sync::Arc; + +use epaint::mutex::RwLock; + +/// An error produced by egui-wgpu. +#[derive(thiserror::Error, Debug)] +pub enum WgpuError { + #[error(transparent)] + RequestAdapterError(#[from] wgpu::RequestAdapterError), + + #[error("Adapter selection failed: {0}")] + CustomNativeAdapterSelectionError(String), + + #[error("There was no valid format for the surface at all.")] + NoSurfaceFormatsAvailable, + + #[error(transparent)] + RequestDeviceError(#[from] wgpu::RequestDeviceError), + + #[error(transparent)] + CreateSurfaceError(#[from] wgpu::CreateSurfaceError), + + #[cfg(feature = "winit")] + #[error(transparent)] + HandleError(#[from] ::winit::raw_window_handle::HandleError), +} + +/// Runtime-mutable subset of [`WgpuConfiguration`]. +/// +/// Edit any field to have the surface reconfigured on the next paint. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct SurfaceConfig { + /// Present mode used for the primary surface. + pub present_mode: wgpu::PresentMode, + + /// Desired maximum number of frames that the presentation engine should queue in advance. + /// + /// Use `1` for low-latency, and `2` for high-throughput. + /// + /// See [`wgpu::SurfaceConfiguration::desired_maximum_frame_latency`] for details. + /// + /// `None` => Let `wgpu` pick a default (currently `2`). + pub desired_maximum_frame_latency: Option, +} + +impl SurfaceConfig { + /// Good default for GUIs with very little (or no) extra GPU work. + pub const LOW_LATENCY: Self = Self { + present_mode: wgpu::PresentMode::AutoVsync, + + desired_maximum_frame_latency: if cfg!(target_os = "ios") { + None // The default is good on iOS, while `Some(1)` cuts FPS in half + } else { + Some(1) + }, + }; + + /// Good default for GUIs with a lot of extra GPU work, + /// or that want to prioritize smoothness over latency. + pub const HIGH_THROUGHPUT: Self = Self { + present_mode: wgpu::PresentMode::AutoVsync, + desired_maximum_frame_latency: Some(2), // High-throughput. + }; +} + +/// Access to the render state for egui. +#[derive(Clone)] +pub struct RenderState { + /// Wgpu adapter used for rendering. + pub adapter: wgpu::Adapter, + + /// All the available adapters. + /// + /// This is not available on web. + /// On web, we always select WebGPU is available, then fall back to WebGL if not. + #[cfg(not(target_arch = "wasm32"))] + pub available_adapters: Vec, + + /// Wgpu instance used for creating surfaces and adapters. + pub instance: wgpu::Instance, + + /// Wgpu device used for rendering, created from the adapter. + pub device: wgpu::Device, + + /// Wgpu queue used for rendering, created from the adapter. + pub queue: wgpu::Queue, + + /// The target texture format used for presenting to the window. + pub target_format: wgpu::TextureFormat, + + /// Egui renderer responsible for drawing the UI. + pub renderer: Arc>, + + /// Runtime-mutable subset of the wgpu configuration. + /// + /// Update this to have the surface reconfigured on the next paint. + pub surface_config: SurfaceConfig, +} + +async fn request_adapter( + instance: &wgpu::Instance, + power_preference: wgpu::PowerPreference, + compatible_surface: Option<&wgpu::Surface<'_>>, + available_adapters: &[wgpu::Adapter], +) -> Result { + profiling::function_scope!(); + + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions { + power_preference, + compatible_surface, + // We don't expose this as an option right now since it's fairly rarely useful: + // * only has an effect on native + // * fails if there's no software rasterizer available + // * can achieve the same with `native_adapter_selector` + force_fallback_adapter: false, + apply_limit_buckets: false, + }) + .await + .inspect_err(|_err| { + if cfg!(target_arch = "wasm32") { + // Nothing to add here + } else if available_adapters.is_empty() { + if std::env::var("DYLD_LIBRARY_PATH").is_ok() { + // DYLD_LIBRARY_PATH can sometimes lead to loading dylibs that cause + // us to find zero adapters. Very strange. + // I don't want to debug this again. + // See https://github.com/rerun-io/rerun/issues/11351 for more + log::warn!( + "No wgpu adapter found. This could be because DYLD_LIBRARY_PATH causes dylibs to be loaded that interfere with Metal device creation. Try restarting with DYLD_LIBRARY_PATH=''" + ); + } else { + log::info!("No wgpu adapter found"); + } + } else if available_adapters.len() == 1 { + log::info!( + "The only available wgpu adapter was not suitable: {}", + adapter_info_summary(&available_adapters[0].get_info()) + ); + } else { + log::info!( + "No suitable wgpu adapter found out of the {} available ones: {}", + available_adapters.len(), + describe_adapters(available_adapters) + ); + } + })?; + + if 1 < available_adapters.len() { + log::info!( + "There are {} available wgpu adapters: {}", + available_adapters.len(), + describe_adapters(available_adapters) + ); + } + + Ok(adapter) +} + +impl RenderState { + /// Creates a new [`RenderState`], containing everything needed for drawing egui with wgpu. + /// + /// # Errors + /// Wgpu initialization may fail due to incompatible hardware or driver for a given config. + pub async fn create( + config: &WgpuConfiguration, + instance: &wgpu::Instance, + compatible_surface: Option<&wgpu::Surface<'static>>, + options: RendererOptions, + ) -> Result { + profiling::scope!("RenderState::create"); // async yield give bad names using `profile_function` + + // This is always an empty list on web. + #[cfg(not(target_arch = "wasm32"))] + let available_adapters = { + let backends = if let WgpuSetup::CreateNew(create_new) = &config.wgpu_setup { + create_new.instance_descriptor.backends + } else { + wgpu::Backends::all() + }; + + instance.enumerate_adapters(backends).await + }; + + let (instance, adapter, device, queue) = match config.wgpu_setup.clone() { + WgpuSetup::CreateNew(WgpuSetupCreateNew { + instance_descriptor: _, + display_handle: _, + power_preference, + native_adapter_selector: _native_adapter_selector, + device_descriptor, + }) => { + let adapter = { + #[cfg(target_arch = "wasm32")] + { + request_adapter(instance, power_preference, compatible_surface, &[]).await + } + #[cfg(not(target_arch = "wasm32"))] + if let Some(native_adapter_selector) = _native_adapter_selector { + native_adapter_selector(&available_adapters, compatible_surface) + .map_err(WgpuError::CustomNativeAdapterSelectionError) + } else { + request_adapter( + instance, + power_preference, + compatible_surface, + &available_adapters, + ) + .await + } + }?; + + let (device, queue) = { + profiling::scope!("request_device"); + adapter + .request_device(&(*device_descriptor)(&adapter)) + .await? + }; + + (instance.clone(), adapter, device, queue) + } + WgpuSetup::Existing(WgpuSetupExisting { + instance, + adapter, + device, + queue, + }) => (instance, adapter, device, queue), + }; + + log_adapter_info(&adapter.get_info()); + + let surface_formats = { + profiling::scope!("get_capabilities"); + compatible_surface.map_or_else( + || vec![wgpu::TextureFormat::Rgba8Unorm], + |s| s.get_capabilities(&adapter).formats, + ) + }; + let target_format = crate::preferred_framebuffer_format(&surface_formats)?; + + let renderer = Renderer::new(&device, target_format, options); + + // On wasm, depending on feature flags, wgpu objects may or may not implement sync. + // It doesn't make sense to switch to Rc for that special usecase, so simply disable the lint. + #[allow(clippy::allow_attributes, clippy::arc_with_non_send_sync)] // For wasm + Ok(Self { + instance, + adapter, + #[cfg(not(target_arch = "wasm32"))] + available_adapters, + device, + queue, + target_format, + renderer: Arc::new(RwLock::new(renderer)), + surface_config: config.surface, + }) + } +} + +fn describe_adapters(adapters: &[wgpu::Adapter]) -> String { + if adapters.is_empty() { + "(none)".to_owned() + } else if adapters.len() == 1 { + adapter_info_summary(&adapters[0].get_info()) + } else { + adapters + .iter() + .map(|a| format!("{{{}}}", adapter_info_summary(&a.get_info()))) + .collect::>() + .join(", ") + } +} + +/// Specifies which action should be taken as consequence of a surface error. +pub enum SurfaceErrorAction { + /// Do nothing and skip the current frame. + SkipFrame, + + /// Reconfigure the existing surface, then skip the current frame. + /// + /// Calls [`wgpu::Surface::configure`] on the current surface object. + /// Use for [`wgpu::CurrentSurfaceTexture::Outdated`]. + Reconfigure, + + /// Drop the surface, create a new one via [`wgpu::Instance::create_surface`], configure it, + /// then skip the current frame. + /// + /// Use for [`wgpu::CurrentSurfaceTexture::Lost`], where reconfiguring the same surface + /// object cannot recover. + RecreateSurface, +} + +/// Configuration for using wgpu with eframe or the egui-wgpu winit feature. +#[derive(Clone)] +pub struct WgpuConfiguration { + /// Runtime-mutable configuration for the surface (present mode, frame latency). + /// + /// These are the fields exposed via [`RenderState::surface_config`] for live + /// reconfiguration at runtime. + pub surface: SurfaceConfig, + + /// How to create the wgpu adapter & device + pub wgpu_setup: WgpuSetup, + + /// Callback for surface status changes. + /// + /// Called with the [`wgpu::CurrentSurfaceTexture`] result whenever acquiring a frame + /// does not return [`wgpu::CurrentSurfaceTexture::Success`]. For + /// [`wgpu::CurrentSurfaceTexture::Suboptimal`], egui uses the frame as-is and + /// defers surface reconfiguration to the next frame — the callback is not invoked + /// in that case either. + pub on_surface_status: + Arc SurfaceErrorAction + Send + Sync>, +} + +#[test] +fn wgpu_config_impl_send_sync() { + fn assert_send_sync() {} + assert_send_sync::(); +} + +impl core::fmt::Debug for WgpuConfiguration { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let Self { + surface, + wgpu_setup, + on_surface_status: _, + } = self; + f.debug_struct("WgpuConfiguration") + .field("surface", &surface) + .field("wgpu_setup", &wgpu_setup) + .finish_non_exhaustive() + } +} + +impl WgpuConfiguration { + #[inline] + pub fn with_surface_config(mut self, surface_config: SurfaceConfig) -> Self { + self.surface = surface_config; + self + } +} + +impl Default for WgpuConfiguration { + fn default() -> Self { + Self { + surface: SurfaceConfig::HIGH_THROUGHPUT, + + // No display handle available at this point — callers should replace this with + // `WgpuSetup::from_display_handle(...)` before creating the instance if one is available. + wgpu_setup: WgpuSetup::without_display_handle(), + on_surface_status: Arc::new(|status| match status { + wgpu::CurrentSurfaceTexture::Outdated => { + // The compositor changed the surface (resize, scale, output, …). wgpu + // requires us to reconfigure before the next acquire. Skipping would mean + // we are stuck in `Outdated` forever. + log::trace!("Dropped frame with error: {status:?}"); + SurfaceErrorAction::Reconfigure + } + wgpu::CurrentSurfaceTexture::Lost => { + // The underlying surface is gone and we need a fresh one from the `wgpu::Instance`. + log::debug!("Dropped frame with error: {status:?}"); + SurfaceErrorAction::RecreateSurface + } + wgpu::CurrentSurfaceTexture::Occluded => { + // App is hidden (minimized / behind another window). Skip silently. + log::trace!("Skipping frame due to occlusion."); + SurfaceErrorAction::SkipFrame + } + _ => { + log::warn!("Dropped frame with error: {status:?}"); + SurfaceErrorAction::SkipFrame + } + }), + } + } +} + +/// Find the framebuffer format that egui prefers +/// +/// # Errors +/// Returns [`WgpuError::NoSurfaceFormatsAvailable`] if the given list of formats is empty. +pub fn preferred_framebuffer_format( + formats: &[wgpu::TextureFormat], +) -> Result { + for &format in formats { + if matches!( + format, + wgpu::TextureFormat::Rgba8Unorm | wgpu::TextureFormat::Bgra8Unorm + ) { + return Ok(format); + } + } + + formats + .first() + .copied() + .ok_or(WgpuError::NoSurfaceFormatsAvailable) +} + +/// Take's epi's depth/stencil bits and returns the corresponding wgpu format. +pub fn depth_format_from_bits(depth_buffer: u8, stencil_buffer: u8) -> Option { + match (depth_buffer, stencil_buffer) { + (0, 8) => Some(wgpu::TextureFormat::Stencil8), + (16, 0) => Some(wgpu::TextureFormat::Depth16Unorm), + (24, 0) => Some(wgpu::TextureFormat::Depth24Plus), + (24, 8) => Some(wgpu::TextureFormat::Depth24PlusStencil8), + (32, 0) => Some(wgpu::TextureFormat::Depth32Float), + (32, 8) => Some(wgpu::TextureFormat::Depth32FloatStencil8), + _ => None, + } +} + +// --------------------------------------------------------------------------- + +fn log_adapter_info(info: &wgpu::AdapterInfo) { + let summary = adapter_info_summary(info); + + let is_test = cfg!(test); // Software rasterizers are expected (and preferred) during testing! + + if info.device_type == wgpu::DeviceType::Cpu && !is_test { + log::warn!("Software rasterizer detected - loss of performance expected. {summary}"); + } else { + log::debug!("wgpu adapter: {summary}"); + } +} + +/// A human-readable summary about an adapter +pub fn adapter_info_summary(info: &wgpu::AdapterInfo) -> String { + let wgpu::AdapterInfo { + name, + vendor, + device, + device_type, + driver, + driver_info, + backend, + device_pci_bus_id, + subgroup_min_size, + subgroup_max_size, + transient_saves_memory, + limit_bucket, + } = &info; + + // Example values: + // > name: "llvmpipe (LLVM 16.0.6, 256 bits)", device_type: Cpu, backend: Vulkan, driver: "llvmpipe", driver_info: "Mesa 23.1.6-arch1.4 (LLVM 16.0.6)" + // > name: "Apple M1 Pro", device_type: IntegratedGpu, backend: Metal, driver: "", driver_info: "" + // > name: "ANGLE (Apple, Apple M1 Pro, OpenGL 4.1)", device_type: IntegratedGpu, backend: Gl, driver: "", driver_info: "" + + use core::fmt::Write as _; + + let mut summary = format!("backend: {backend:?}, device_type: {device_type:?}"); + + if !name.is_empty() { + write!(summary, ", name: {name:?}").ok(); + } + if !driver.is_empty() { + write!(summary, ", driver: {driver:?}").ok(); + } + if !driver_info.is_empty() { + write!(summary, ", driver_info: {driver_info:?}").ok(); + } + if *vendor != 0 { + #[cfg(not(target_arch = "wasm32"))] + { + write!( + summary, + ", vendor: {} (0x{vendor:04X})", + parse_vendor_id(*vendor) + ) + .ok(); + } + #[cfg(target_arch = "wasm32")] + { + write!(summary, ", vendor: 0x{vendor:04X}").ok(); + } + } + if *device != 0 { + write!(summary, ", device: 0x{device:02X}").ok(); + } + if !device_pci_bus_id.is_empty() { + write!(summary, ", pci_bus_id: {device_pci_bus_id:?}").ok(); + } + if *subgroup_min_size != 0 || *subgroup_max_size != 0 { + write!( + summary, + ", subgroup_size: {subgroup_min_size}..={subgroup_max_size}" + ) + .ok(); + } + write!( + summary, + ", transient_saves_memory: {transient_saves_memory:?}" + ) + .ok(); + write!(summary, ", limit_bucket: {limit_bucket:?}").ok(); + + summary +} + +/// Tries to parse the adapter's vendor ID to a human-readable string. +#[cfg(not(target_arch = "wasm32"))] +pub fn parse_vendor_id(vendor_id: u32) -> &'static str { + match vendor_id { + wgpu::hal::auxil::db::amd::VENDOR => "AMD", + wgpu::hal::auxil::db::apple::VENDOR => "Apple", + wgpu::hal::auxil::db::arm::VENDOR => "ARM", + wgpu::hal::auxil::db::broadcom::VENDOR => "Broadcom", + wgpu::hal::auxil::db::imgtec::VENDOR => "Imagination Technologies", + wgpu::hal::auxil::db::intel::VENDOR => "Intel", + wgpu::hal::auxil::db::mesa::VENDOR => "Mesa", + wgpu::hal::auxil::db::nvidia::VENDOR => "NVIDIA", + wgpu::hal::auxil::db::qualcomm::VENDOR => "Qualcomm", + _ => "Unknown", + } +} diff --git a/vendor/egui-wgpu/src/renderer.rs b/vendor/egui-wgpu/src/renderer.rs new file mode 100644 index 00000000..de8808de --- /dev/null +++ b/vendor/egui-wgpu/src/renderer.rs @@ -0,0 +1,1178 @@ +use core::{num::NonZeroU64, ops::Range}; +use std::borrow::Cow; + +use ahash::HashMap; +use bytemuck::Zeroable as _; +use epaint::{PaintCallbackInfo, Primitive, Vertex, emath::NumExt as _}; + +use wgpu::util::DeviceExt as _; + +// Only implements Send + Sync on wasm32 in order to allow storing wgpu resources on the type map. +#[cfg(not(all( + target_arch = "wasm32", + not(feature = "fragile-send-sync-non-atomic-wasm"), +)))] +/// You can use this for storage when implementing [`CallbackTrait`]. +pub type CallbackResources = type_map::concurrent::TypeMap; +#[cfg(all( + target_arch = "wasm32", + not(feature = "fragile-send-sync-non-atomic-wasm"), +))] +/// You can use this for storage when implementing [`CallbackTrait`]. +pub type CallbackResources = type_map::TypeMap; + +/// You can use this to do custom [`wgpu`] rendering in an egui app. +/// +/// Implement [`CallbackTrait`] and call [`Callback::new_paint_callback`]. +/// +/// This can be turned into a [`epaint::PaintCallback`] and [`epaint::Shape`]. +pub struct Callback(Box); + +impl Callback { + /// Creates a new [`epaint::PaintCallback`] from a callback trait instance. + pub fn new_paint_callback( + rect: epaint::emath::Rect, + callback: impl CallbackTrait + 'static, + ) -> epaint::PaintCallback { + epaint::PaintCallback { + rect, + callback: std::sync::Arc::new(Self(Box::new(callback))), + } + } +} + +/// A callback trait that can be used to compose an [`epaint::PaintCallback`] via [`Callback`] +/// for custom WGPU rendering. +/// +/// Callbacks in [`Renderer`] are done in three steps: +/// * [`CallbackTrait::prepare`]: called for all registered callbacks before the main egui render pass. +/// * [`CallbackTrait::finish_prepare`]: called for all registered callbacks after all callbacks finished calling prepare. +/// * [`CallbackTrait::paint`]: called for all registered callbacks during the main egui render pass. +/// +/// Each callback has access to an instance of [`CallbackResources`] that is stored in the [`Renderer`]. +/// This can be used to store wgpu resources that need to be accessed during the [`CallbackTrait::paint`] step. +/// +/// The callbacks implementing [`CallbackTrait`] itself must always be Send + Sync, but resources stored in +/// [`Renderer::callback_resources`] are not required to implement Send + Sync when building for wasm. +/// (this is because wgpu stores references to the JS heap in most of its resources which can not be shared with other threads). +/// +/// +/// # Command submission +/// +/// ## Command Encoder +/// +/// The passed-in [`wgpu::CommandEncoder`] is egui's and can be used directly to register +/// wgpu commands for simple use cases. +/// This allows reusing the same [`wgpu::CommandEncoder`] for all callbacks and egui +/// rendering itself. +/// +/// ## Command Buffers +/// +/// For more complicated use cases, one can also return a list of arbitrary +/// [`wgpu::CommandBuffer`]s and have complete control over how they get created and fed. +/// In particular, this gives an opportunity to parallelize command registration and +/// prevents a faulty callback from poisoning the main wgpu pipeline. +/// +/// When using eframe, the main egui command buffer, as well as all user-defined +/// command buffers returned by this function, are guaranteed to all be submitted +/// at once in a single call. +/// +/// Command Buffers returned by [`CallbackTrait::finish_prepare`] will always be issued *after* +/// those returned by [`CallbackTrait::prepare`]. +/// Order within command buffers returned by [`CallbackTrait::prepare`] is dependent +/// on the order the respective [`epaint::Shape::Callback`]s were submitted in. +/// +/// # Example +/// +/// See the [`custom3d_wgpu`](https://github.com/emilk/egui/blob/main/crates/egui_demo_app/src/apps/custom3d_wgpu.rs) demo source for a detailed usage example. +pub trait CallbackTrait: Send + Sync { + fn prepare( + &self, + _device: &wgpu::Device, + _queue: &wgpu::Queue, + _screen_descriptor: &ScreenDescriptor, + _egui_encoder: &mut wgpu::CommandEncoder, + _callback_resources: &mut CallbackResources, + ) -> Vec { + Vec::new() + } + + /// Called after all [`CallbackTrait::prepare`] calls are done. + fn finish_prepare( + &self, + _device: &wgpu::Device, + _queue: &wgpu::Queue, + _egui_encoder: &mut wgpu::CommandEncoder, + _callback_resources: &mut CallbackResources, + ) -> Vec { + Vec::new() + } + + /// Called after all [`CallbackTrait::finish_prepare`] calls are done. + /// + /// It is given access to the [`wgpu::RenderPass`] so that it can issue draw commands + /// into the same [`wgpu::RenderPass`] that is used for all other egui elements. + fn paint( + &self, + info: PaintCallbackInfo, + render_pass: &mut wgpu::RenderPass<'static>, + callback_resources: &CallbackResources, + ); +} + +/// Information about the screen used for rendering. +pub struct ScreenDescriptor { + /// Size of the window in physical pixels. + pub size_in_pixels: [u32; 2], + + /// High-DPI scale factor (pixels per point). + pub pixels_per_point: f32, +} + +impl ScreenDescriptor { + /// size in "logical" points + fn screen_size_in_points(&self) -> [f32; 2] { + [ + self.size_in_pixels[0] as f32 / self.pixels_per_point, + self.size_in_pixels[1] as f32 / self.pixels_per_point, + ] + } +} + +/// Uniform buffer used when rendering. +#[derive(Clone, Copy, Debug, PartialEq, bytemuck::Pod, bytemuck::Zeroable)] +#[repr(C)] +struct UniformBuffer { + screen_size_in_points: [f32; 2], + dithering: u32, + + /// 1 to do manual filtering for more predictable kittest snapshot images. + /// + /// See also . + predictable_texture_filtering: u32, +} + +struct SlicedBuffer { + buffer: wgpu::Buffer, + slices: Vec>, + capacity: wgpu::BufferAddress, +} + +pub struct Texture { + /// The texture may be None if the `TextureId` is just a handle to a user-provided bind-group. + pub texture: Option, + + /// Bindgroup for the texture + sampler. + pub bind_group: wgpu::BindGroup, + + /// Options describing the sampler used in the bind group. This may be None if the `TextureId` + /// is just a handle to a user-provided bind-group. + pub options: Option, +} + +/// Ways to configure [`Renderer`] during creation. +#[derive(Clone, Copy, Debug)] +pub struct RendererOptions { + /// Set the level of the multisampling anti-aliasing (MSAA). + /// + /// Must be a power-of-two. Higher = more smooth 3D. + /// + /// A value of `0` or `1` turns it off (default). + /// + /// `egui` already performs anti-aliasing via "feathering" + /// (controlled by [`egui::epaint::TessellationOptions`]), + /// but if you are embedding 3D in egui you may want to turn on multisampling. + pub msaa_samples: u32, + + /// What format to use for the depth and stencil buffers, + /// e.g. [`wgpu::TextureFormat::Depth32FloatStencil8`]. + /// + /// egui doesn't need depth/stencil, so the default value is `None` (no depth or stancil buffers). + pub depth_stencil_format: Option, + + /// Controls whether to apply dithering to minimize banding artifacts. + /// + /// Dithering assumes an sRGB output and thus will apply noise to any input value that lies between + /// two 8bit values after applying the sRGB OETF function, i.e. if it's not a whole 8bit value in "gamma space". + /// This means that only inputs from texture interpolation and vertex colors should be affected in practice. + /// + /// Defaults to true. + pub dithering: bool, + + /// Perform texture filtering in software? + /// + /// This is useful when you want predictable rendering across + /// different hardware, e.g. for kittest snapshots. + /// + /// Default is `false`. + /// + /// See also . + pub predictable_texture_filtering: bool, +} + +impl RendererOptions { + /// Set options that produce the most predicatable output. + /// + /// Useful for image snapshot tests. + pub const PREDICTABLE: Self = Self { + msaa_samples: 1, + depth_stencil_format: None, + dithering: false, + predictable_texture_filtering: true, + }; +} + +impl Default for RendererOptions { + fn default() -> Self { + Self { + msaa_samples: 0, + depth_stencil_format: None, + dithering: true, + predictable_texture_filtering: false, + } + } +} + +/// Renderer for a egui based GUI. +pub struct Renderer { + pipeline: wgpu::RenderPipeline, + + index_buffer: SlicedBuffer, + vertex_buffer: SlicedBuffer, + + uniform_buffer: wgpu::Buffer, + previous_uniform_buffer_content: UniformBuffer, + uniform_bind_group: wgpu::BindGroup, + texture_bind_group_layout: wgpu::BindGroupLayout, + + /// Map of egui texture IDs to textures and their associated bindgroups (texture view + + /// sampler). The texture may be None if the `TextureId` is just a handle to a user-provided + /// sampler. + textures: HashMap, + next_user_texture_id: u64, + samplers: HashMap, + + options: RendererOptions, + + /// Storage for resources shared with all invocations of [`CallbackTrait`]'s methods. + /// + /// See also [`CallbackTrait`]. + pub callback_resources: CallbackResources, +} + +impl Renderer { + /// Creates a renderer for a egui UI. + /// + /// `output_color_format` should preferably be [`wgpu::TextureFormat::Rgba8Unorm`] or + /// [`wgpu::TextureFormat::Bgra8Unorm`], i.e. in gamma-space. + pub fn new( + device: &wgpu::Device, + output_color_format: wgpu::TextureFormat, + options: RendererOptions, + ) -> Self { + profiling::function_scope!(); + + let shader = wgpu::ShaderModuleDescriptor { + label: Some("egui"), + source: wgpu::ShaderSource::Wgsl(Cow::Borrowed(include_str!("egui.wgsl"))), + }; + let module = { + profiling::scope!("create_shader_module"); + device.create_shader_module(shader) + }; + + let uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("egui_uniform_buffer"), + contents: bytemuck::cast_slice(&[UniformBuffer { + screen_size_in_points: [0.0, 0.0], + dithering: u32::from(options.dithering), + predictable_texture_filtering: u32::from(options.predictable_texture_filtering), + }]), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + + let uniform_bind_group_layout = { + profiling::scope!("create_bind_group_layout"); + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("egui_uniform_bind_group_layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: NonZeroU64::new( + core::mem::size_of::() as _ + ), + ty: wgpu::BufferBindingType::Uniform, + }, + count: None, + }], + }) + }; + + let uniform_bind_group = { + profiling::scope!("create_bind_group"); + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("egui_uniform_bind_group"), + layout: &uniform_bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding { + buffer: &uniform_buffer, + offset: 0, + size: None, + }), + }], + }) + }; + + let texture_bind_group_layout = { + profiling::scope!("create_bind_group_layout"); + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("egui_texture_bind_group_layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + multisampled: false, + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }) + }; + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("egui_pipeline_layout"), + bind_group_layouts: &[ + Some(&uniform_bind_group_layout), + Some(&texture_bind_group_layout), + ], + immediate_size: 0, + }); + + let depth_stencil = options + .depth_stencil_format + .map(|format| wgpu::DepthStencilState { + format, + depth_write_enabled: Some(false), + depth_compare: Some(wgpu::CompareFunction::Always), + stencil: wgpu::StencilState::default(), + bias: wgpu::DepthBiasState::default(), + }); + + let pipeline = { + profiling::scope!("create_render_pipeline"); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("egui_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + entry_point: Some("vs_main"), + module: &module, + buffers: &[Some(wgpu::VertexBufferLayout { + array_stride: 5 * 4, + step_mode: wgpu::VertexStepMode::Vertex, + // 0: vec2 position + // 1: vec2 texture coordinates + // 2: uint color + attributes: &wgpu::vertex_attr_array![0 => Float32x2, 1 => Float32x2, 2 => Uint32], + })], + compilation_options: wgpu::PipelineCompilationOptions::default() + }, + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + unclipped_depth: false, + conservative: false, + cull_mode: None, + front_face: wgpu::FrontFace::default(), + polygon_mode: wgpu::PolygonMode::default(), + strip_index_format: None, + }, + depth_stencil, + multisample: wgpu::MultisampleState { + alpha_to_coverage_enabled: false, + count: options.msaa_samples.max(1), + mask: !0, + }, + + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some(if output_color_format.is_srgb() { + log::warn!("Detected a linear (sRGBA aware) framebuffer {output_color_format:?}. egui prefers Rgba8Unorm or Bgra8Unorm"); + "fs_main_linear_framebuffer" + } else { + "fs_main_gamma_framebuffer" // this is what we prefer + }), + targets: &[Some(wgpu::ColorTargetState { + format: output_color_format, + blend: Some(wgpu::BlendState { + color: wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::One, + dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha, + operation: wgpu::BlendOperation::Add, + }, + alpha: wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::OneMinusDstAlpha, + dst_factor: wgpu::BlendFactor::One, + operation: wgpu::BlendOperation::Add, + }, + }), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default() + }), + multiview_mask: None, + cache: None, + } + ) + }; + + const VERTEX_BUFFER_START_CAPACITY: wgpu::BufferAddress = + (core::mem::size_of::() * 1024) as _; + const INDEX_BUFFER_START_CAPACITY: wgpu::BufferAddress = + (core::mem::size_of::() * 1024 * 3) as _; + + Self { + pipeline, + vertex_buffer: SlicedBuffer { + buffer: create_vertex_buffer(device, VERTEX_BUFFER_START_CAPACITY), + slices: Vec::with_capacity(64), + capacity: VERTEX_BUFFER_START_CAPACITY, + }, + index_buffer: SlicedBuffer { + buffer: create_index_buffer(device, INDEX_BUFFER_START_CAPACITY), + slices: Vec::with_capacity(64), + capacity: INDEX_BUFFER_START_CAPACITY, + }, + uniform_buffer, + // Buffers on wgpu are zero initialized, so this is indeed its current state! + previous_uniform_buffer_content: UniformBuffer::zeroed(), + uniform_bind_group, + texture_bind_group_layout, + textures: HashMap::default(), + next_user_texture_id: 0, + samplers: HashMap::default(), + options, + callback_resources: CallbackResources::default(), + } + } + + /// Executes the egui renderer onto an existing wgpu renderpass. + /// + /// Note that the lifetime of `render_pass` is `'static` which requires a call to [`wgpu::RenderPass::forget_lifetime`]. + /// This allows users to pass resources that live outside of the callback resources to the render pass. + /// The render pass internally keeps all referenced resources alive as long as necessary. + /// The only consequence of `forget_lifetime` is that any operation on the parent encoder will cause a runtime error + /// instead of a compile time error. + /// + /// # Panic + /// Always ensure that [`Renderer::update_buffers`] has been called otherwise calling [`Renderer::render`] will panic! + pub fn render( + &self, + render_pass: &mut wgpu::RenderPass<'static>, + paint_jobs: &[epaint::ClippedPrimitive], + screen_descriptor: &ScreenDescriptor, + ) { + profiling::function_scope!(); + + let pixels_per_point = screen_descriptor.pixels_per_point; + let size_in_pixels = screen_descriptor.size_in_pixels; + + // Whether or not we need to reset the render pass because a paint callback has just + // run. + let mut needs_reset = true; + + let mut index_buffer_slices = self.index_buffer.slices.iter(); + let mut vertex_buffer_slices = self.vertex_buffer.slices.iter(); + + for epaint::ClippedPrimitive { + clip_rect, + primitive, + } in paint_jobs + { + if needs_reset { + render_pass.set_viewport( + 0.0, + 0.0, + size_in_pixels[0] as f32, + size_in_pixels[1] as f32, + 0.0, + 1.0, + ); + render_pass.set_pipeline(&self.pipeline); + render_pass.set_bind_group(0, &self.uniform_bind_group, &[]); + needs_reset = false; + } + + { + let rect = ScissorRect::new(clip_rect, pixels_per_point, size_in_pixels); + + if rect.width == 0 || rect.height == 0 { + // Skip rendering zero-sized clip areas. + if let Primitive::Mesh(_) = primitive { + // If this is a mesh, we need to advance the index and vertex buffer iterators: + index_buffer_slices + .next() + .expect("You must call .update_buffers() before .render()"); + vertex_buffer_slices + .next() + .expect("You must call .update_buffers() before .render()"); + } + continue; + } + + render_pass.set_scissor_rect(rect.x, rect.y, rect.width, rect.height); + } + + match primitive { + Primitive::Mesh(mesh) => { + let index_buffer_slice = index_buffer_slices + .next() + .expect("You must call .update_buffers() before .render()"); + let vertex_buffer_slice = vertex_buffer_slices + .next() + .expect("You must call .update_buffers() before .render()"); + + if let Some(Texture { bind_group, .. }) = self.textures.get(&mesh.texture_id) { + render_pass.set_bind_group(1, bind_group, &[]); + render_pass.set_index_buffer( + self.index_buffer.buffer.slice( + index_buffer_slice.start as u64..index_buffer_slice.end as u64, + ), + wgpu::IndexFormat::Uint32, + ); + render_pass.set_vertex_buffer( + 0, + self.vertex_buffer.buffer.slice( + vertex_buffer_slice.start as u64..vertex_buffer_slice.end as u64, + ), + ); + render_pass.draw_indexed(0..mesh.indices.len() as u32, 0, 0..1); + } else { + log::warn!("Missing texture: {:?}", mesh.texture_id); + } + } + Primitive::Callback(callback) => { + let Some(cbfn) = callback.callback.downcast_ref::() else { + // We already warned in the `prepare` callback + continue; + }; + + let info = PaintCallbackInfo { + viewport: callback.rect, + clip_rect: *clip_rect, + pixels_per_point, + screen_size_px: size_in_pixels, + }; + + let viewport_px = info.viewport_in_pixels(); + if viewport_px.width_px > 0 && viewport_px.height_px > 0 { + profiling::scope!("callback"); + + needs_reset = true; + + // We're setting a default viewport for the render pass as a + // courtesy for the user, so that they don't have to think about + // it in the simple case where they just want to fill the whole + // paint area. + // + // The user still has the possibility of setting their own custom + // viewport during the paint callback, effectively overriding this + // one. + render_pass.set_viewport( + viewport_px.left_px as f32, + viewport_px.top_px as f32, + viewport_px.width_px as f32, + viewport_px.height_px as f32, + 0.0, + 1.0, + ); + + cbfn.0.paint(info, render_pass, &self.callback_resources); + } + } + } + } + + render_pass.set_scissor_rect(0, 0, size_in_pixels[0], size_in_pixels[1]); + } + + /// Should be called before [`Self::render`]. + pub fn update_texture( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + id: epaint::TextureId, + image_delta: &epaint::ImageDelta, + ) { + profiling::function_scope!(); + + let width = image_delta.image.width() as u32; + let height = image_delta.image.height() as u32; + + let size = wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }; + + let data_color32 = match &image_delta.image { + epaint::ImageData::Color(image) => { + assert_eq!( + width as usize * height as usize, + image.pixels.len(), + "Mismatch between texture size and texel count" + ); + Cow::Borrowed(&image.pixels) + } + }; + let data_bytes: &[u8] = bytemuck::cast_slice(data_color32.as_slice()); + + let queue_write_data_to_texture = |texture, origin| { + profiling::scope!("write_texture"); + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture, + mip_level: 0, + origin, + aspect: wgpu::TextureAspect::All, + }, + data_bytes, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(4 * width), + rows_per_image: Some(height), + }, + size, + ); + }; + + // Use same label for all resources associated with this texture id (no point in retyping the type) + let label_str = format!("egui_texid_{id:?}"); + let label = Some(label_str.as_str()); + + let (texture, origin, bind_group) = if let Some(pos) = image_delta.pos { + // update the existing texture + let Texture { + texture, + bind_group, + options, + } = self + .textures + .remove(&id) + .expect("Tried to update a texture that has not been allocated yet."); + let texture = texture.expect("Tried to update user texture."); + let options = options.expect("Tried to update user texture."); + let origin = wgpu::Origin3d { + x: pos[0] as u32, + y: pos[1] as u32, + z: 0, + }; + + ( + texture, + origin, + // If the TextureOptions are the same as the previous ones, we can reuse the bind group. Otherwise we + // have to recreate it. + if image_delta.options == options { + Some(bind_group) + } else { + None + }, + ) + } else { + // allocate a new texture + let texture = { + profiling::scope!("create_texture"); + device.create_texture(&wgpu::TextureDescriptor { + label, + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[wgpu::TextureFormat::Rgba8Unorm], + }) + }; + let origin = wgpu::Origin3d::ZERO; + (texture, origin, None) + }; + + let bind_group = bind_group.unwrap_or_else(|| { + let sampler = self + .samplers + .entry(image_delta.options) + .or_insert_with(|| create_sampler(image_delta.options, device)); + device.create_bind_group(&wgpu::BindGroupDescriptor { + label, + layout: &self.texture_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView( + &texture.create_view(&wgpu::TextureViewDescriptor::default()), + ), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(sampler), + }, + ], + }) + }); + + queue_write_data_to_texture(&texture, origin); + + // A full update must (re)create the texture at exactly the delta's size, + // or glyph UVs (normalized by the CPU atlas size) will sample the wrong rows. + debug_assert!( + image_delta.pos.is_some() || [texture.width(), texture.height()] == [width, height], + "egui texture {id:?}: GPU texture is {}x{} but full delta is {width}x{height}", + texture.width(), + texture.height(), + ); + + self.textures.insert( + id, + Texture { + texture: Some(texture), + bind_group, + options: Some(image_delta.options), + }, + ); + } + + pub fn free_texture(&mut self, id: &epaint::TextureId) { + if let Some(texture) = self.textures.remove(id).and_then(|t| t.texture) { + texture.destroy(); + } + } + + /// Get the WGPU texture and bind group associated to a texture that has been allocated by egui. + /// + /// This could be used by custom paint hooks to render images that have been added through + /// [`epaint::Context::load_texture`](https://docs.rs/egui/latest/egui/struct.Context.html#method.load_texture). + pub fn texture(&self, id: &epaint::TextureId) -> Option<&Texture> { + self.textures.get(id) + } + + /// Registers a [`wgpu::Texture`] with a [`epaint::TextureId`]. + /// + /// This enables the application to reference the texture inside an image ui element. + /// This effectively enables off-screen rendering inside the egui UI. Texture must have + /// the texture format [`wgpu::TextureFormat::Rgba8Unorm`]. + pub fn register_native_texture( + &mut self, + device: &wgpu::Device, + texture: &wgpu::TextureView, + texture_filter: wgpu::FilterMode, + ) -> epaint::TextureId { + self.register_native_texture_with_sampler_options( + device, + texture, + wgpu::SamplerDescriptor { + label: Some(format!("egui_user_image_{}", self.next_user_texture_id).as_str()), + mag_filter: texture_filter, + min_filter: texture_filter, + ..Default::default() + }, + ) + } + + /// Registers a [`wgpu::Texture`] with an existing [`epaint::TextureId`]. + /// + /// This enables applications to reuse [`epaint::TextureId`]s. + pub fn update_egui_texture_from_wgpu_texture( + &mut self, + device: &wgpu::Device, + texture: &wgpu::TextureView, + texture_filter: wgpu::FilterMode, + id: epaint::TextureId, + ) { + self.update_egui_texture_from_wgpu_texture_with_sampler_options( + device, + texture, + wgpu::SamplerDescriptor { + label: Some(format!("egui_user_image_{}", self.next_user_texture_id).as_str()), + mag_filter: texture_filter, + min_filter: texture_filter, + ..Default::default() + }, + id, + ); + } + + /// Registers a [`wgpu::Texture`] with a [`epaint::TextureId`] while also accepting custom + /// [`wgpu::SamplerDescriptor`] options. + /// + /// This allows applications to specify individual minification/magnification filters as well as + /// custom mipmap and tiling options. + /// + /// The texture must have the format [`wgpu::TextureFormat::Rgba8Unorm`]. + /// Any compare function supplied in the [`wgpu::SamplerDescriptor`] will be ignored. + #[expect(clippy::needless_pass_by_value)] // false positive + pub fn register_native_texture_with_sampler_options( + &mut self, + device: &wgpu::Device, + texture: &wgpu::TextureView, + sampler_descriptor: wgpu::SamplerDescriptor<'_>, + ) -> epaint::TextureId { + profiling::function_scope!(); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + compare: None, + ..sampler_descriptor + }); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some(format!("egui_user_image_{}", self.next_user_texture_id).as_str()), + layout: &self.texture_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(texture), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&sampler), + }, + ], + }); + + let id = epaint::TextureId::User(self.next_user_texture_id); + self.textures.insert( + id, + Texture { + texture: None, + bind_group, + options: None, + }, + ); + self.next_user_texture_id += 1; + + id + } + + /// Registers a [`wgpu::Texture`] with an existing [`epaint::TextureId`] while also accepting custom + /// [`wgpu::SamplerDescriptor`] options. + /// + /// This allows applications to reuse [`epaint::TextureId`]s created with custom sampler options. + #[expect(clippy::needless_pass_by_value)] // false positive + pub fn update_egui_texture_from_wgpu_texture_with_sampler_options( + &mut self, + device: &wgpu::Device, + texture: &wgpu::TextureView, + sampler_descriptor: wgpu::SamplerDescriptor<'_>, + id: epaint::TextureId, + ) { + profiling::function_scope!(); + + let Texture { + bind_group: user_texture_binding, + .. + } = self + .textures + .get_mut(&id) + .expect("Tried to update a texture that has not been allocated yet."); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + compare: None, + ..sampler_descriptor + }); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some(format!("egui_user_image_{}", self.next_user_texture_id).as_str()), + layout: &self.texture_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(texture), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&sampler), + }, + ], + }); + + *user_texture_binding = bind_group; + } + + /// Uploads the uniform, vertex and index data used by the renderer. + /// Should be called before [`Self::render`]. + /// + /// Returns all user-defined command buffers gathered from [`CallbackTrait::prepare`] & [`CallbackTrait::finish_prepare`] callbacks. + pub fn update_buffers( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + encoder: &mut wgpu::CommandEncoder, + paint_jobs: &[epaint::ClippedPrimitive], + screen_descriptor: &ScreenDescriptor, + ) -> Vec { + profiling::function_scope!(); + + let screen_size_in_points = screen_descriptor.screen_size_in_points(); + + let uniform_buffer_content = UniformBuffer { + screen_size_in_points, + dithering: u32::from(self.options.dithering), + predictable_texture_filtering: u32::from(self.options.predictable_texture_filtering), + }; + if uniform_buffer_content != self.previous_uniform_buffer_content { + profiling::scope!("update uniforms"); + queue.write_buffer( + &self.uniform_buffer, + 0, + bytemuck::cast_slice(&[uniform_buffer_content]), + ); + self.previous_uniform_buffer_content = uniform_buffer_content; + } + + // Determine how many vertices & indices need to be rendered, and gather prepare callbacks + let mut callbacks = Vec::new(); + let (vertex_count, index_count) = { + profiling::scope!("count_vertices_indices"); + paint_jobs.iter().fold((0, 0), |acc, clipped_primitive| { + match &clipped_primitive.primitive { + Primitive::Mesh(mesh) => { + (acc.0 + mesh.vertices.len(), acc.1 + mesh.indices.len()) + } + Primitive::Callback(callback) => { + if let Some(c) = callback.callback.downcast_ref::() { + callbacks.push(c.0.as_ref()); + } else { + log::warn!("Unknown paint callback: expected `egui_wgpu::Callback`"); + } + acc + } + } + }) + }; + + if index_count > 0 { + profiling::scope!("indices", index_count.to_string().as_str()); + + self.index_buffer.slices.clear(); + + let required_index_buffer_size = (core::mem::size_of::() * index_count) as u64; + if self.index_buffer.capacity < required_index_buffer_size { + // Resize index buffer if needed. + self.index_buffer.capacity = + (self.index_buffer.capacity * 2).at_least(required_index_buffer_size); + self.index_buffer.buffer = create_index_buffer(device, self.index_buffer.capacity); + } + + let index_buffer_staging = queue.write_buffer_with( + &self.index_buffer.buffer, + 0, + #[expect(clippy::unwrap_used)] // Checked above + NonZeroU64::new(required_index_buffer_size).unwrap(), + ); + + let Some(mut index_buffer_staging) = index_buffer_staging else { + panic!( + "Failed to create staging buffer for index data. Index count: {index_count}. Required index buffer size: {required_index_buffer_size}. Actual size {} and capacity: {} (bytes)", + self.index_buffer.buffer.size(), + self.index_buffer.capacity + ); + }; + + let mut index_offset = 0; + for epaint::ClippedPrimitive { primitive, .. } in paint_jobs { + match primitive { + Primitive::Mesh(mesh) => { + let size = mesh.indices.len() * core::mem::size_of::(); + let slice = index_offset..(size + index_offset); + index_buffer_staging + .slice(slice.clone()) + .copy_from_slice(bytemuck::cast_slice(&mesh.indices)); + self.index_buffer.slices.push(slice); + index_offset += size; + } + Primitive::Callback(_) => {} + } + } + } + if vertex_count > 0 { + profiling::scope!("vertices", vertex_count.to_string().as_str()); + + self.vertex_buffer.slices.clear(); + + let required_vertex_buffer_size = + (core::mem::size_of::() * vertex_count) as u64; + if self.vertex_buffer.capacity < required_vertex_buffer_size { + // Resize vertex buffer if needed. + self.vertex_buffer.capacity = + (self.vertex_buffer.capacity * 2).at_least(required_vertex_buffer_size); + self.vertex_buffer.buffer = + create_vertex_buffer(device, self.vertex_buffer.capacity); + } + + let vertex_buffer_staging = queue.write_buffer_with( + &self.vertex_buffer.buffer, + 0, + #[expect(clippy::unwrap_used)] // Checked above + NonZeroU64::new(required_vertex_buffer_size).unwrap(), + ); + + let Some(mut vertex_buffer_staging) = vertex_buffer_staging else { + panic!( + "Failed to create staging buffer for vertex data. Vertex count: {vertex_count}. Required vertex buffer size: {required_vertex_buffer_size}. Actual size {} and capacity: {} (bytes)", + self.vertex_buffer.buffer.size(), + self.vertex_buffer.capacity + ); + }; + + let mut vertex_offset = 0; + for epaint::ClippedPrimitive { primitive, .. } in paint_jobs { + match primitive { + Primitive::Mesh(mesh) => { + let size = mesh.vertices.len() * core::mem::size_of::(); + let slice = vertex_offset..(size + vertex_offset); + vertex_buffer_staging + .slice(slice.clone()) + .copy_from_slice(bytemuck::cast_slice(&mesh.vertices)); + self.vertex_buffer.slices.push(slice); + vertex_offset += size; + } + Primitive::Callback(_) => {} + } + } + } + + let mut user_cmd_bufs = Vec::new(); + { + profiling::scope!("prepare callbacks"); + for callback in &callbacks { + user_cmd_bufs.extend(callback.prepare( + device, + queue, + screen_descriptor, + encoder, + &mut self.callback_resources, + )); + } + } + { + profiling::scope!("finish prepare callbacks"); + for callback in &callbacks { + user_cmd_bufs.extend(callback.finish_prepare( + device, + queue, + encoder, + &mut self.callback_resources, + )); + } + } + + user_cmd_bufs + } +} + +fn create_sampler( + options: epaint::textures::TextureOptions, + device: &wgpu::Device, +) -> wgpu::Sampler { + let mag_filter = match options.magnification { + epaint::textures::TextureFilter::Nearest => wgpu::FilterMode::Nearest, + epaint::textures::TextureFilter::Linear => wgpu::FilterMode::Linear, + }; + let min_filter = match options.minification { + epaint::textures::TextureFilter::Nearest => wgpu::FilterMode::Nearest, + epaint::textures::TextureFilter::Linear => wgpu::FilterMode::Linear, + }; + let address_mode = match options.wrap_mode { + epaint::textures::TextureWrapMode::ClampToEdge => wgpu::AddressMode::ClampToEdge, + epaint::textures::TextureWrapMode::Repeat => wgpu::AddressMode::Repeat, + epaint::textures::TextureWrapMode::MirroredRepeat => wgpu::AddressMode::MirrorRepeat, + }; + device.create_sampler(&wgpu::SamplerDescriptor { + label: Some(&format!( + "egui sampler (mag: {mag_filter:?}, min {min_filter:?})" + )), + mag_filter, + min_filter, + address_mode_u: address_mode, + address_mode_v: address_mode, + ..Default::default() + }) +} + +fn create_vertex_buffer(device: &wgpu::Device, size: u64) -> wgpu::Buffer { + profiling::function_scope!(); + device.create_buffer(&wgpu::BufferDescriptor { + label: Some("egui_vertex_buffer"), + usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, + size, + mapped_at_creation: false, + }) +} + +fn create_index_buffer(device: &wgpu::Device, size: u64) -> wgpu::Buffer { + profiling::function_scope!(); + device.create_buffer(&wgpu::BufferDescriptor { + label: Some("egui_index_buffer"), + usage: wgpu::BufferUsages::INDEX | wgpu::BufferUsages::COPY_DST, + size, + mapped_at_creation: false, + }) +} + +/// A Rect in physical pixel space, used for setting clipping rectangles. +struct ScissorRect { + x: u32, + y: u32, + width: u32, + height: u32, +} + +impl ScissorRect { + fn new(clip_rect: &epaint::Rect, pixels_per_point: f32, target_size: [u32; 2]) -> Self { + // Transform clip rect to physical pixels: + let clip_min_x = pixels_per_point * clip_rect.min.x; + let clip_min_y = pixels_per_point * clip_rect.min.y; + let clip_max_x = pixels_per_point * clip_rect.max.x; + let clip_max_y = pixels_per_point * clip_rect.max.y; + + // Round to integer: + let clip_min_x = clip_min_x.round() as u32; + let clip_min_y = clip_min_y.round() as u32; + let clip_max_x = clip_max_x.round() as u32; + let clip_max_y = clip_max_y.round() as u32; + + // Clamp: + let clip_min_x = clip_min_x.clamp(0, target_size[0]); + let clip_min_y = clip_min_y.clamp(0, target_size[1]); + let clip_max_x = clip_max_x.clamp(clip_min_x, target_size[0]); + let clip_max_y = clip_max_y.clamp(clip_min_y, target_size[1]); + + Self { + x: clip_min_x, + y: clip_min_y, + width: clip_max_x - clip_min_x, + height: clip_max_y - clip_min_y, + } + } +} + +// Look at the feature flag for an explanation. +#[cfg(not(all( + target_arch = "wasm32", + not(feature = "fragile-send-sync-non-atomic-wasm"), +)))] +#[test] +fn renderer_impl_send_sync() { + fn assert_send_sync() {} + assert_send_sync::(); +} diff --git a/vendor/egui-wgpu/src/setup.rs b/vendor/egui-wgpu/src/setup.rs new file mode 100644 index 00000000..3c9ceae5 --- /dev/null +++ b/vendor/egui-wgpu/src/setup.rs @@ -0,0 +1,331 @@ +use std::sync::Arc; + +/// A cloneable display handle for use with [`wgpu::InstanceDescriptor`]. +/// +/// [`wgpu::InstanceDescriptor`] stores its display handle as a non-cloneable +/// `Box`. This trait wraps it so it can be cloned +/// alongside the rest of the egui wgpu configuration. +/// +/// Automatically implemented for all types that satisfy the bounds +/// (including [`winit::event_loop::OwnedDisplayHandle`]). +pub trait EguiDisplayHandle: + wgpu::rwh::HasDisplayHandle + core::fmt::Debug + Send + Sync + 'static +{ + /// Clone into a `Box` for [`wgpu::InstanceDescriptor::display`]. + fn clone_for_wgpu(&self) -> Box; + + /// Clone into a new `Box`. + fn clone_display_handle(&self) -> Box; +} + +impl Clone for Box { + fn clone(&self) -> Self { + // We need to deref here, otherwise this causes infinite recursion stack overflow. + (**self).clone_display_handle() + } +} + +impl EguiDisplayHandle for T +where + T: wgpu::rwh::HasDisplayHandle + Clone + core::fmt::Debug + Send + Sync + 'static, +{ + fn clone_for_wgpu(&self) -> Box { + Box::new(self.clone()) + } + + fn clone_display_handle(&self) -> Box { + Box::new(self.clone()) + } +} + +#[derive(Clone)] +#[expect(clippy::large_enum_variant)] +pub enum WgpuSetup { + /// Construct a wgpu setup using some predefined settings & heuristics. + /// This is the default option. You can customize most behaviours overriding the + /// supported backends, power preferences, and device description. + /// + /// By default can also be configured with various environment variables: + /// * `WGPU_BACKEND`: `vulkan`, `dx12`, `metal`, `opengl`, `webgpu` + /// * `WGPU_POWER_PREF`: `low`, `high` or `none` + /// * `WGPU_TRACE`: Path to a file to output a wgpu trace file. + /// + /// Each instance flag also comes with an environment variable (for details see [`wgpu::InstanceFlags`]): + /// * `WGPU_VALIDATION`: Enables validation (enabled by default in debug builds). + /// * `WGPU_DEBUG`: Generate debug information in shaders and objects (enabled by default in debug builds). + /// * `WGPU_ALLOW_UNDERLYING_NONCOMPLIANT_ADAPTER`: Whether wgpu should expose adapters that run on top of non-compliant adapters. + /// * `WGPU_GPU_BASED_VALIDATION`: Enable GPU-based validation. + CreateNew(WgpuSetupCreateNew), + + /// Run on an existing wgpu setup. + Existing(WgpuSetupExisting), +} + +impl WgpuSetup { + /// Creates a new [`WgpuSetup::CreateNew`] with the given display handle. + /// + /// See [`WgpuSetupCreateNew::from_display_handle`] for details. + pub fn from_display_handle(display_handle: impl EguiDisplayHandle) -> Self { + Self::CreateNew(WgpuSetupCreateNew::from_display_handle(display_handle)) + } + + /// Creates a new [`WgpuSetup::CreateNew`] without a display handle. + /// + /// See [`WgpuSetupCreateNew::without_display_handle`] for details. + pub fn without_display_handle() -> Self { + Self::CreateNew(WgpuSetupCreateNew::without_display_handle()) + } +} + +impl core::fmt::Debug for WgpuSetup { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::CreateNew(create_new) => f + .debug_tuple("WgpuSetup::CreateNew") + .field(create_new) + .finish(), + Self::Existing { .. } => f.debug_tuple("WgpuSetup::Existing").finish(), + } + } +} + +impl WgpuSetup { + /// Creates a new [`wgpu::Instance`] or clones the existing one. + /// + /// Does *not* store the wgpu instance, so calling this repeatedly may + /// create a new instance every time! + pub async fn new_instance(&self) -> wgpu::Instance { + match self { + Self::CreateNew(create_new) => { + #[allow(clippy::allow_attributes, unused_mut)] + let mut backends = create_new.instance_descriptor.backends; + + // Don't try WebGPU if we're not in a secure context. + // Emscripten is excluded: wgpu gates both its `webgpu` backend and + // its `web_sys` re-export on `not(Emscripten)` (see the cfg aliases + // in `wgpu/build.rs`). + #[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))] + if backends.contains(wgpu::Backends::BROWSER_WEBGPU) { + let is_secure_context = + wgpu::web_sys::window().is_some_and(|w| w.is_secure_context()); + if !is_secure_context { + log::info!( + "WebGPU is only available in secure contexts, i.e. on HTTPS and on localhost." + ); + backends.remove(wgpu::Backends::BROWSER_WEBGPU); + } + } + + log::debug!("Creating wgpu instance with backends {backends:?}"); + let desc = &create_new.instance_descriptor; + let descriptor = wgpu::InstanceDescriptor { + backends: desc.backends, + flags: desc.flags, + backend_options: desc.backend_options.clone(), + memory_budget_thresholds: desc.memory_budget_thresholds, + display: create_new + .display_handle + .as_ref() + .map(|handle| handle.clone_for_wgpu()), + }; + wgpu::util::new_instance_with_webgpu_detection(descriptor).await + } + Self::Existing(existing) => existing.instance.clone(), + } + } +} + +impl From for WgpuSetup { + fn from(create_new: WgpuSetupCreateNew) -> Self { + Self::CreateNew(create_new) + } +} + +impl From for WgpuSetup { + fn from(existing: WgpuSetupExisting) -> Self { + Self::Existing(existing) + } +} + +/// Method for selecting an adapter on native. +/// +/// This can be used for fully custom adapter selection. +/// If available, `wgpu::Surface` is passed to allow checking for surface compatibility. +pub type NativeAdapterSelectorMethod = Arc< + dyn Fn(&[wgpu::Adapter], Option<&wgpu::Surface<'_>>) -> Result + + Send + + Sync, +>; + +/// Configuration for creating a new wgpu setup. +/// +/// Used for [`WgpuSetup::CreateNew`]. +/// +/// Prefer [`Self::from_display_handle`] when you have a display handle available. +/// Most platforms work without one, but some (e.g. Wayland with GLES, or WebGL) +/// require it, so providing one ensures maximum compatibility. +/// With winit, pass [`EventLoop::owned_display_handle`](winit::event_loop::EventLoop::owned_display_handle). +/// +/// Note: The display handle is stored in [`Self::display_handle`] rather than in +/// [`Self::instance_descriptor`] so the config can be cloned +/// ([`wgpu::InstanceDescriptor`] is not `Clone`). It is injected at instance creation time. +pub struct WgpuSetupCreateNew { + /// Descriptor for the wgpu instance. + /// + /// Leave [`wgpu::InstanceDescriptor::display`] as `None` — use [`Self::display_handle`] + /// instead (injected at instance creation time). + /// + /// The most important field is [`wgpu::InstanceDescriptor::backends`], which controls + /// which backends are supported (wgpu will pick one of these). For example, set it to + /// [`wgpu::Backends::GL`] to use only WebGL. By default on web, WebGPU is preferred + /// with WebGL as a fallback (requires the `webgl` feature of crate `wgpu`). + pub instance_descriptor: wgpu::InstanceDescriptor, + + /// Display handle passed to wgpu at instance creation time. + /// + /// Required on some platforms (e.g. Wayland with GLES, WebGL); optional elsewhere. + /// With winit, use [`winit::event_loop::OwnedDisplayHandle`]. + /// + /// `eframe` 's winit & web integrations will attempt to fill the display handle automatically if it is left empty. + pub display_handle: Option>, + + /// Power preference for the adapter if [`Self::native_adapter_selector`] is not set or targeting web. + pub power_preference: wgpu::PowerPreference, + + /// Optional selector for native adapters. + /// + /// This field has no effect when targeting web! + /// Otherwise, if set [`Self::power_preference`] is ignored and the adapter is instead selected by this method. + /// Note that [`Self::instance_descriptor`]'s [`wgpu::InstanceDescriptor::backends`] + /// are still used to filter the adapter enumeration in the first place. + /// + /// Defaults to `None`. + pub native_adapter_selector: Option, + + /// Configuration passed on device request, given an adapter + pub device_descriptor: + Arc wgpu::DeviceDescriptor<'static> + Send + Sync>, +} + +impl WgpuSetupCreateNew { + /// Creates a new configuration with the given display handle. + /// + /// This is the recommended constructor. Most platforms (Windows, macOS/iOS, Android, web) + /// work fine without a display handle, but some (e.g. Wayland on Linux with GLES) require + /// one. Providing it unconditionally ensures your app works everywhere. + /// + /// If you don't have a display handle available, use [`Self::without_display_handle`] + /// instead — it will still work on the majority of platforms. + /// + /// With winit, pass [`EventLoop::owned_display_handle`](winit::event_loop::EventLoop::owned_display_handle). + pub fn from_display_handle(display_handle: impl EguiDisplayHandle) -> Self { + Self { + display_handle: Some(Box::new(display_handle)), + ..Self::without_display_handle() + } + } + + /// Creates a new configuration without a display handle. + /// + /// A display handle is not required for headless operation (offscreen rendering, tests, + /// compute-only workloads). It also isn't needed on most platforms even when presenting + /// to a window — only some configurations (e.g. Wayland on Linux with GLES) require one. + /// + /// If you do have a display handle available, prefer [`Self::from_display_handle`] for + /// maximum compatibility. + /// + /// With winit you can obtain one via [`EventLoop::owned_display_handle`](winit::event_loop::EventLoop::owned_display_handle). + /// + /// `eframe` 's winit & web integrations will attempt to fill the display handle automatically if it is left empty. + pub fn without_display_handle() -> Self { + Self { + instance_descriptor: wgpu::InstanceDescriptor { + // Add GL backend, primarily because WebGPU is not stable enough yet. + // (note however, that the GL backend needs to be opted-in via the wgpu feature flag "webgl") + backends: wgpu::Backends::from_env() + .unwrap_or(wgpu::Backends::PRIMARY | wgpu::Backends::GL), + flags: wgpu::InstanceFlags::from_build_config().with_env(), + backend_options: wgpu::BackendOptions::from_env_or_default(), + memory_budget_thresholds: wgpu::MemoryBudgetThresholds::default(), + display: None, + }, + + display_handle: None, + + power_preference: wgpu::PowerPreference::from_env() + .unwrap_or(wgpu::PowerPreference::HighPerformance), + + native_adapter_selector: None, + + device_descriptor: Arc::new(|adapter| { + let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl { + wgpu::Limits::downlevel_webgl2_defaults() + } else { + wgpu::Limits::default() + }; + + wgpu::DeviceDescriptor { + label: Some("egui wgpu device"), + required_limits: wgpu::Limits { + // When using a depth buffer, we have to be able to create a texture + // large enough for the entire surface, and we want to support 4k+ displays. + max_texture_dimension_2d: 8192, + ..base_limits + }, + ..Default::default() + } + }), + } + } +} + +impl Clone for WgpuSetupCreateNew { + fn clone(&self) -> Self { + let desc = &self.instance_descriptor; + Self { + instance_descriptor: wgpu::InstanceDescriptor { + backends: desc.backends, + flags: desc.flags, + backend_options: desc.backend_options.clone(), + memory_budget_thresholds: desc.memory_budget_thresholds, + display: None, + }, + display_handle: self.display_handle.clone(), + power_preference: self.power_preference, + native_adapter_selector: self.native_adapter_selector.clone(), + device_descriptor: Arc::clone(&self.device_descriptor), + } + } +} + +impl core::fmt::Debug for WgpuSetupCreateNew { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let Self { + instance_descriptor, + display_handle, + power_preference, + native_adapter_selector, + device_descriptor: _, + } = self; + f.debug_struct("WgpuSetupCreateNew") + .field("instance_descriptor", instance_descriptor) + .field("display_handle", display_handle) + .field("power_preference", power_preference) + .field( + "native_adapter_selector", + &native_adapter_selector.is_some(), + ) + .finish_non_exhaustive() + } +} + +/// Configuration for using an existing wgpu setup. +/// +/// Used for [`WgpuSetup::Existing`]. +#[derive(Clone)] +pub struct WgpuSetupExisting { + pub instance: wgpu::Instance, + pub adapter: wgpu::Adapter, + pub device: wgpu::Device, + pub queue: wgpu::Queue, +} diff --git a/vendor/egui-wgpu/src/texture_copy.wgsl b/vendor/egui-wgpu/src/texture_copy.wgsl new file mode 100644 index 00000000..4096d164 --- /dev/null +++ b/vendor/egui-wgpu/src/texture_copy.wgsl @@ -0,0 +1,43 @@ +struct VertexOutput { + @builtin(position) position: vec4, +}; + +var positions: array = array( + vec2f(-1.0, -3.0), + vec2f(-1.0, 1.0), + vec2f(3.0, 1.0) +); + +// meant to be called with 3 vertex indices: 0, 1, 2 +// draws one large triangle over the clip space like this: +// (the asterisks represent the clip space bounds) +//-1,1 1,1 +// --------------------------------- +// | * . +// | * . +// | * . +// | * . +// | * . +// | * . +// |*************** +// | . 1,-1 +// | . +// | . +// | . +// | . +// |. +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var result: VertexOutput; + result.position = vec4f(positions[vertex_index], 0.0, 1.0); + return result; +} + +@group(0) +@binding(0) +var r_color: texture_2d; + +@fragment +fn fs_main(vertex: VertexOutput) -> @location(0) vec4 { + return textureLoad(r_color, vec2i(vertex.position.xy), 0); +} diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs new file mode 100644 index 00000000..84b32927 --- /dev/null +++ b/vendor/egui-wgpu/src/winit.rs @@ -0,0 +1,800 @@ +#![expect(clippy::missing_errors_doc)] +#![expect(clippy::undocumented_unsafe_blocks)] +#![expect(clippy::unwrap_used)] // TODO(emilk): avoid unwraps +#![expect(unsafe_code)] + +use crate::{RenderState, SurfaceConfig, SurfaceErrorAction, WgpuConfiguration, renderer}; +use crate::{ + RendererOptions, + capture::{CaptureReceiver, CaptureSender, CaptureState, capture_channel}, +}; +use core::num::NonZeroU32; +use egui::{Context, Event, UserData, ViewportId, ViewportIdMap, ViewportIdSet}; +use std::sync::Arc; + +struct SurfaceState { + surface: wgpu::Surface<'static>, + alpha_mode: wgpu::CompositeAlphaMode, + width: u32, + height: u32, + resizing: bool, + needs_reconfigure: bool, + needs_recreate: bool, +} + +/// Everything you need to paint egui with [`wgpu`] on [`winit`]. +/// +/// Alternatively you can use [`crate::Renderer`] directly. +/// +/// NOTE: all egui viewports share the same painter. +pub struct Painter { + context: Context, + config: WgpuConfiguration, + options: RendererOptions, + support_transparent_backbuffer: bool, + screen_capture_state: Option, + + instance: wgpu::Instance, + render_state: Option, + + // Per viewport/window: + depth_texture_view: ViewportIdMap, + msaa_texture_view: ViewportIdMap, + surfaces: ViewportIdMap, + capture_tx: CaptureSender, + capture_rx: CaptureReceiver, +} + +impl Painter { + /// Manages [`wgpu`] state, including surface state, required to render egui. + /// + /// Only the [`wgpu::Instance`] is initialized here. Device selection and the initialization + /// of render + surface state is deferred until the painter is given its first window target + /// via [`set_window()`](Self::set_window). (Ensuring that a device that's compatible with the + /// native window is chosen) + /// + /// Before calling [`paint_and_update_textures()`](Self::paint_and_update_textures) a + /// [`wgpu::Surface`] must be initialized (and corresponding render state) by calling + /// [`set_window()`](Self::set_window) once you have + /// a [`winit::window::Window`] with a valid `.raw_window_handle()` + /// associated. + pub async fn new( + context: Context, + config: WgpuConfiguration, + support_transparent_backbuffer: bool, + options: RendererOptions, + ) -> Self { + let (capture_tx, capture_rx) = capture_channel(); + let instance = config.wgpu_setup.new_instance().await; + + Self { + context, + config, + options, + support_transparent_backbuffer, + screen_capture_state: None, + + instance, + render_state: None, + + depth_texture_view: Default::default(), + surfaces: Default::default(), + msaa_texture_view: Default::default(), + + capture_tx, + capture_rx, + } + } + + /// Get the [`RenderState`]. + /// + /// Will return [`None`] if the render state has not been initialized yet. + pub fn render_state(&self) -> Option { + self.render_state.clone() + } + + fn configure_surface( + surface_state: &SurfaceState, + render_state: &RenderState, + config: &SurfaceConfig, + ) { + profiling::function_scope!(); + + let SurfaceConfig { + present_mode, + desired_maximum_frame_latency, + } = *config; + + // Transaction presentation can hold a drawable during AppKit live resize. Keep the + // configured low-latency path normally, but use three Metal drawables while resizing. + #[cfg(all(target_os = "macos", feature = "macos-window-resize-jitter-fix"))] + let desired_maximum_frame_latency = if surface_state.resizing { + Some(desired_maximum_frame_latency.unwrap_or(2).max(2)) + } else { + desired_maximum_frame_latency + }; + + let width = surface_state.width; + let height = surface_state.height; + + let mut surf_config = wgpu::SurfaceConfiguration { + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + format: render_state.target_format, + present_mode, + alpha_mode: surface_state.alpha_mode, + view_formats: vec![render_state.target_format], + ..surface_state + .surface + .get_default_config(&render_state.adapter, width, height) + .expect("The surface isn't supported by this adapter") + }; + + if let Some(desired_maximum_frame_latency) = desired_maximum_frame_latency { + surf_config.desired_maximum_frame_latency = desired_maximum_frame_latency; + } + + surface_state + .surface + .configure(&render_state.device, &surf_config); + } + + /// Drop the existing [`wgpu::Surface`] for `viewport_id` and create a fresh one for the + /// given window via [`wgpu::Instance::create_surface`], then configure it. + /// + /// Used to recover from [`wgpu::CurrentSurfaceTexture::Lost`], where reconfiguring the + /// existing surface object cannot recover. + fn recreate_surface( + &mut self, + viewport_id: ViewportId, + window: &Arc, + ) -> Result<(), crate::WgpuError> { + profiling::function_scope!(); + + let Some(old_state) = self.surfaces.remove(&viewport_id) else { + return Ok(()); + }; + + let surface = self.instance.create_surface(Arc::clone(window))?; + self.install_surface( + surface, + viewport_id, + old_state.width, + old_state.height, + old_state.resizing, + ); + Ok(()) + } + + /// Updates (or clears) the [`winit::window::Window`] associated with the [`Painter`] + /// + /// This creates a [`wgpu::Surface`] for the given Window (as well as initializing render + /// state if needed) that is used for egui rendering. + /// + /// This must be called before trying to render via + /// [`paint_and_update_textures`](Self::paint_and_update_textures) + /// + /// # Portability + /// + /// _In particular it's important to note that on Android a it's only possible to create + /// a window surface between `Resumed` and `Paused` lifecycle events, and Winit will panic on + /// attempts to query the raw window handle while paused._ + /// + /// On Android [`set_window`](Self::set_window) should be called with `Some(window)` for each + /// `Resumed` event and `None` for each `Paused` event. Currently, on all other platforms + /// [`set_window`](Self::set_window) may be called with `Some(window)` as soon as you have a + /// valid [`winit::window::Window`]. + /// + /// # Errors + /// If the provided wgpu configuration does not match an available device. + pub async fn set_window( + &mut self, + viewport_id: ViewportId, + window: Option>, + ) -> Result<(), crate::WgpuError> { + profiling::scope!("Painter::set_window"); // profile_function gives bad names for async functions + + if let Some(window) = window { + let size = window.inner_size(); + if !self.surfaces.contains_key(&viewport_id) { + let surface = self.instance.create_surface(window)?; + self.add_surface(surface, viewport_id, size).await?; + } + } else { + log::warn!("No window - clearing all surfaces"); + self.surfaces.clear(); + } + Ok(()) + } + + /// Updates (or clears) the [`winit::window::Window`] associated with the [`Painter`] without taking ownership of the window. + /// + /// Like [`set_window`](Self::set_window) except: + /// + /// # Safety + /// The user is responsible for ensuring that the window is alive for as long as it is set. + pub async unsafe fn set_window_unsafe( + &mut self, + viewport_id: ViewportId, + window: Option<&winit::window::Window>, + ) -> Result<(), crate::WgpuError> { + profiling::scope!("Painter::set_window_unsafe"); // profile_function gives bad names for async functions + + if let Some(window) = window { + let size = window.inner_size(); + if !self.surfaces.contains_key(&viewport_id) { + let surface = unsafe { + self.instance + .create_surface_unsafe(wgpu::SurfaceTargetUnsafe::from_window(&window)?)? + }; + self.add_surface(surface, viewport_id, size).await?; + } + } else { + log::warn!("No window - clearing all surfaces"); + self.surfaces.clear(); + } + Ok(()) + } + + async fn add_surface( + &mut self, + surface: wgpu::Surface<'static>, + viewport_id: ViewportId, + size: winit::dpi::PhysicalSize, + ) -> Result<(), crate::WgpuError> { + if self.render_state.is_none() { + let render_state = + RenderState::create(&self.config, &self.instance, Some(&surface), self.options) + .await?; + self.render_state = Some(render_state); + } + self.install_surface(surface, viewport_id, size.width, size.height, false); + Ok(()) + } + + /// Inserts a freshly created surface into [`Self::surfaces`] and configures it. + /// + /// Render state must already be initialised before calling this. + // NOTE: The same assumption is already required by `resize_and_generate_depth_texture_view_and_msaa_view`. + fn install_surface( + &mut self, + surface: wgpu::Surface<'static>, + viewport_id: ViewportId, + width: u32, + height: u32, + resizing: bool, + ) { + let alpha_mode = { + // Panic: We use the same failure mode as `resize_and_generate_depth_texture_view_and_msaa_view` + let render_state = self + .render_state + .as_ref() + .expect("install_surface called before render_state initialization"); + if self.support_transparent_backbuffer { + let supported_alpha_modes = + surface.get_capabilities(&render_state.adapter).alpha_modes; + // Prefer pre multiplied over post multiplied! + if supported_alpha_modes.contains(&wgpu::CompositeAlphaMode::PreMultiplied) { + wgpu::CompositeAlphaMode::PreMultiplied + } else if supported_alpha_modes.contains(&wgpu::CompositeAlphaMode::PostMultiplied) + { + wgpu::CompositeAlphaMode::PostMultiplied + } else { + log::warn!( + "Transparent window was requested, but the active wgpu surface does not support a `CompositeAlphaMode` with transparency." + ); + wgpu::CompositeAlphaMode::Auto + } + } else { + wgpu::CompositeAlphaMode::Auto + } + }; + self.surfaces.insert( + viewport_id, + SurfaceState { + surface, + width, + height, + alpha_mode, + resizing, + needs_reconfigure: false, + needs_recreate: false, + }, + ); + let Some(width) = NonZeroU32::new(width) else { + log::debug!("The window width was zero; skipping generate textures"); + return; + }; + let Some(height) = NonZeroU32::new(height) else { + log::debug!("The window height was zero; skipping generate textures"); + return; + }; + self.resize_and_generate_depth_texture_view_and_msaa_view(viewport_id, width, height); + } + + /// Returns the maximum texture dimension supported if known + /// + /// This API will only return a known dimension after `set_window()` has been called + /// at least once, since the underlying device and render state are initialized lazily + /// once we have a window (that may determine the choice of adapter/device). + pub fn max_texture_side(&self) -> Option { + self.render_state + .as_ref() + .map(|rs| rs.device.limits().max_texture_dimension_2d as usize) + } + + fn resize_and_generate_depth_texture_view_and_msaa_view( + &mut self, + viewport_id: ViewportId, + width_in_pixels: NonZeroU32, + height_in_pixels: NonZeroU32, + ) { + profiling::function_scope!(); + + let width = width_in_pixels.get(); + let height = height_in_pixels.get(); + + let render_state = self.render_state.as_ref().unwrap(); + let surface_state = self.surfaces.get_mut(&viewport_id).unwrap(); + + surface_state.width = width; + surface_state.height = height; + + Self::configure_surface(surface_state, render_state, &self.config.surface); + + if let Some(depth_format) = self.options.depth_stencil_format { + self.depth_texture_view.insert( + viewport_id, + render_state + .device + .create_texture(&wgpu::TextureDescriptor { + label: Some("egui_depth_texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: self.options.msaa_samples.max(1), + dimension: wgpu::TextureDimension::D2, + format: depth_format, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[depth_format], + }) + .create_view(&wgpu::TextureViewDescriptor::default()), + ); + } + + if let Some(render_state) = (self.options.msaa_samples > 1) + .then_some(self.render_state.as_ref()) + .flatten() + { + let texture_format = render_state.target_format; + self.msaa_texture_view.insert( + viewport_id, + render_state + .device + .create_texture(&wgpu::TextureDescriptor { + label: Some("egui_msaa_texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: self.options.msaa_samples.max(1), + dimension: wgpu::TextureDimension::D2, + format: texture_format, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + view_formats: &[texture_format], + }) + .create_view(&wgpu::TextureViewDescriptor::default()), + ); + } + } + + /// Handles changes of the resizing state. + /// + /// Should be called prior to the first [`Painter::on_window_resized`] call and after the last in + /// the chain. Used to apply platform-specific logic, e.g. OSX Metal window resize jitter fix. + pub fn on_window_resize_state_change(&mut self, viewport_id: ViewportId, resizing: bool) { + profiling::function_scope!(); + + let Some(state) = self.surfaces.get_mut(&viewport_id) else { + return; + }; + if state.resizing == resizing { + if resizing { + log::debug!( + "Painter::on_window_resize_state_change() redundant call while resizing" + ); + } else { + log::debug!( + "Painter::on_window_resize_state_change() redundant call after resizing" + ); + } + return; + } + + // Set before reconfiguring so macOS live resize uses the temporary latency bump above. + state.resizing = resizing; + + // Resizing is a bit tricky on macOS. + // It requires enabling ["present_with_transaction"](https://developer.apple.com/documentation/quartzcore/cametallayer/presentswithtransaction) + // flag to avoid jittering during the resize. Even though resize jittering on macOS + // is common across rendering backends, the solution for wgpu/metal is known. + // + // See https://github.com/emilk/egui/issues/903 + #[cfg(all(target_os = "macos", feature = "macos-window-resize-jitter-fix"))] + { + // SAFETY: `as_hal::()` returns `None` unless this surface is backed by wgpu's + // Metal backend. + unsafe { + if let (Some(render_state), Some(hal_surface)) = ( + self.render_state.as_ref(), + state.surface.as_hal::(), + ) { + hal_surface + .render_layer() + .lock() + .setPresentsWithTransaction(resizing); + + Self::configure_surface(state, render_state, &self.config.surface); + } + } + } + } + + pub fn on_window_resized( + &mut self, + viewport_id: ViewportId, + width_in_pixels: NonZeroU32, + height_in_pixels: NonZeroU32, + ) { + profiling::function_scope!(); + + if self.surfaces.contains_key(&viewport_id) { + self.resize_and_generate_depth_texture_view_and_msaa_view( + viewport_id, + width_in_pixels, + height_in_pixels, + ); + } else { + log::warn!( + "Ignoring window resize notification with no surface created via Painter::set_window()" + ); + } + } + + /// Returns two things: + /// + /// The approximate number of seconds spent on vsync-waiting (if any), + /// and the captures captured screenshot if it was requested. + /// + /// If `capture_data` isn't empty, a screenshot will be captured. + #[expect(clippy::too_many_arguments)] + pub fn paint_and_update_textures( + &mut self, + viewport_id: ViewportId, + pixels_per_point: f32, + clear_color: [f32; 4], + clipped_primitives: &[epaint::ClippedPrimitive], + textures_delta: &mut epaint::textures::TexturesDelta, + capture_data: Vec, + window: &Arc, + ) -> f32 { + profiling::function_scope!(); + + /// Guard to ensure that commands are always submitted to the renderer queue + /// so that calls to [`write_buffer()`](https://docs.rs/wgpu/latest/wgpu/struct.Queue.html#method.write_buffer) + /// are completed even if we take a codepath which doesn't submit commands and avoids + /// internal buffers growing indefinitely. + /// + /// This may happen, for example, if no output frame is resolved. + /// See for full context. + struct RendererQueueGuard<'q> { + queue: &'q wgpu::Queue, + commands_submitted: bool, + } + + impl Drop for RendererQueueGuard<'_> { + fn drop(&mut self) { + // Only submit an empty command buffer array if no commands were + // explicitly submitted. + if !self.commands_submitted { + self.queue.submit([]); + } + } + } + + let capture = !capture_data.is_empty(); + let mut vsync_sec = 0.0; + + // If the previous frame produced `CurrentSurfaceTexture::Lost`, the action match + // below set `needs_recreate`. Recreate the surface now, before re-borrowing + // `self.render_state` / `self.surfaces` for the rest of the paint. + if self + .surfaces + .get(&viewport_id) + .is_some_and(|s| s.needs_recreate) + && let Err(err) = self.recreate_surface(viewport_id, window) + { + log::error!("Failed to recreate surface for {viewport_id:?}: {err}"); + return vsync_sec; + } + + // Apply any runtime changes requested via `RenderState::surface_config`. + // We diff against the already-applied values in `self.config.surface` + // and, if anything differs, mark every surface as needing reconfiguration so + // the existing `needs_reconfigure` pathway below picks them up. + if let Some(render_state) = self.render_state.as_ref() + && render_state.surface_config != self.config.surface + { + self.config.surface = render_state.surface_config; + #[expect(clippy::iter_over_hash_type)] + for surface in self.surfaces.values_mut() { + surface.needs_reconfigure = true; + } + } + + let Some(render_state) = self.render_state.as_mut() else { + return vsync_sec; + }; + + let mut render_queue_guard = RendererQueueGuard { + queue: &render_state.queue, + commands_submitted: false, + }; + + let Some(surface_state) = self.surfaces.get_mut(&viewport_id) else { + return vsync_sec; + }; + + let mut encoder = + render_state + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("encoder"), + }); + + // Upload all resources for the GPU. + let screen_descriptor = renderer::ScreenDescriptor { + size_in_pixels: [surface_state.width, surface_state.height], + pixels_per_point, + }; + + let user_cmd_bufs = { + let mut renderer = render_state.renderer.write(); + #[expect(clippy::iter_over_hash_type)] // Order doesn't matter here + for (id, image_deltas) in textures_delta.set.drain() { + for image_delta in image_deltas { + renderer.update_texture( + &render_state.device, + &render_state.queue, + id, + &image_delta, + ); + } + } + + renderer.update_buffers( + &render_state.device, + &render_state.queue, + &mut encoder, + clipped_primitives, + &screen_descriptor, + ) + }; + + if surface_state.needs_reconfigure { + Self::configure_surface(surface_state, render_state, &self.config.surface); + surface_state.needs_reconfigure = false; + } + + let output_frame = { + profiling::scope!("get_current_texture"); + // This is what vsync-waiting happens on my Mac. + let start = web_time::Instant::now(); + let output_frame = surface_state.surface.get_current_texture(); + vsync_sec += start.elapsed().as_secs_f32(); + output_frame + }; + + let output_frame = match output_frame { + wgpu::CurrentSurfaceTexture::Success(frame) => frame, + wgpu::CurrentSurfaceTexture::Suboptimal(frame) => { + surface_state.needs_reconfigure = true; + frame + } + other => { + match (*self.config.on_surface_status)(&other) { + SurfaceErrorAction::Reconfigure => { + Self::configure_surface(surface_state, render_state, &self.config.surface); + self.context.request_repaint_of(viewport_id); + } + SurfaceErrorAction::RecreateSurface => { + // Because of ownership, I could not find an easy way to do a full recovery here, + // as that would involve dropping the old surface and creating a new one. + // For now, we defer the recreation to the beginning of the next frame (which + // we ensure to arrive via `request_repaint_of`). A cleaner solution would be + // to untangle the ownership of `RenderState`. + surface_state.needs_recreate = true; + self.context.request_repaint_of(viewport_id); + } + SurfaceErrorAction::SkipFrame => {} + } + return vsync_sec; + } + }; + + let mut capture_buffer = None; + { + let renderer = render_state.renderer.read(); + + let target_texture = if capture { + let capture_state = self.screen_capture_state.get_or_insert_with(|| { + CaptureState::new(&render_state.device, &output_frame.texture) + }); + capture_state.update(&render_state.device, &output_frame.texture); + + &capture_state.texture + } else { + &output_frame.texture + }; + let target_view = target_texture.create_view(&wgpu::TextureViewDescriptor::default()); + + let (view, resolve_target) = (self.options.msaa_samples > 1) + .then_some(self.msaa_texture_view.get(&viewport_id)) + .flatten() + .map_or((&target_view, None), |texture_view| { + (texture_view, Some(&target_view)) + }); + + let render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("egui_render"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view, + resolve_target, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: clear_color[0] as f64, + g: clear_color[1] as f64, + b: clear_color[2] as f64, + a: clear_color[3] as f64, + }), + store: wgpu::StoreOp::Store, + }, + depth_slice: None, + })], + depth_stencil_attachment: self.depth_texture_view.get(&viewport_id).map(|view| { + wgpu::RenderPassDepthStencilAttachment { + view, + depth_ops: self + .options + .depth_stencil_format + .is_some_and(|depth_stencil_format| { + depth_stencil_format.has_depth_aspect() + }) + .then_some(wgpu::Operations { + load: wgpu::LoadOp::Clear(1.0), + // It is very unlikely that the depth buffer is needed after egui finished rendering + // so no need to store it. (this can improve performance on tiling GPUs like mobile chips or Apple Silicon) + store: wgpu::StoreOp::Discard, + }), + stencil_ops: self + .options + .depth_stencil_format + .is_some_and(|depth_stencil_format| { + depth_stencil_format.has_stencil_aspect() + }) + .then_some(wgpu::Operations { + load: wgpu::LoadOp::Clear(0), + store: wgpu::StoreOp::Discard, + }), + } + }), + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + + // Forgetting the pass' lifetime means that we are no longer compile-time protected from + // runtime errors caused by accessing the parent encoder before the render pass is dropped. + // Since we don't pass it on to the renderer, we should be perfectly safe against this mistake here! + renderer.render( + &mut render_pass.forget_lifetime(), + clipped_primitives, + &screen_descriptor, + ); + + if capture && let Some(capture_state) = &mut self.screen_capture_state { + capture_buffer = Some(capture_state.copy_textures( + &render_state.device, + &output_frame, + &mut encoder, + )); + } + } + + let encoded = { + profiling::scope!("CommandEncoder::finish"); + encoder.finish() + }; + + // Submit the commands: both the main buffer and user-defined ones. + { + profiling::scope!("Queue::submit"); + // wgpu doesn't document where vsync can happen. Maybe here? + let start = web_time::Instant::now(); + render_state + .queue + .submit(core::iter::chain(user_cmd_bufs, [encoded])); + vsync_sec += start.elapsed().as_secs_f32(); + }; + + // Ensure that the queue guard does not do unnecessary work when dropped + render_queue_guard.commands_submitted = true; + + // Free textures marked for destruction **after** queue submit since they might still be used in the current frame. + // Calling `wgpu::Texture::destroy` on a texture that is still in use would invalidate the command buffer(s) it is used in. + // However, once we called `wgpu::Queue::submit`, it is up for wgpu to determine how long the underlying gpu resource has to live. + { + let mut renderer = render_state.renderer.write(); + #[expect(clippy::iter_over_hash_type)] // Order doesn't matter here + for id in textures_delta.free.drain() { + renderer.free_texture(&id); + } + } + + if let Some(capture_buffer) = capture_buffer + && let Some(screen_capture_state) = &mut self.screen_capture_state + { + screen_capture_state.read_screen_rgba( + self.context.clone(), + capture_buffer, + capture_data, + self.capture_tx.clone(), + viewport_id, + ); + } + + window.pre_present_notify(); + + { + profiling::scope!("present"); + // wgpu doesn't document where vsync can happen. Maybe here? + let start = web_time::Instant::now(); + render_state.queue.present(output_frame); + vsync_sec += start.elapsed().as_secs_f32(); + } + + vsync_sec + } + + /// Call this at the beginning of each frame to receive the requested screenshots. + pub fn handle_screenshots(&self, events: &mut Vec) { + for (viewport_id, user_data, screenshot) in self.capture_rx.try_iter() { + let screenshot = Arc::new(screenshot); + for data in user_data { + events.push(Event::Screenshot { + viewport_id, + user_data: data, + image: Arc::clone(&screenshot), + }); + } + } + } + + pub fn gc_viewports(&mut self, active_viewports: &ViewportIdSet) { + self.surfaces.retain(|id, _| active_viewports.contains(id)); + self.depth_texture_view + .retain(|id, _| active_viewports.contains(id)); + self.msaa_texture_view + .retain(|id, _| active_viewports.contains(id)); + } + + #[expect(clippy::needless_pass_by_ref_mut, clippy::unused_self)] + pub fn destroy(&mut self) { + // TODO(emilk): something here? + } +} From 08ffddcc9bf33d484f8b729775dfbc02438c6f97 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:02:44 -0500 Subject: [PATCH 014/226] Recover a first frame that never completes, and say when Wi-Fi is off, Apple Devices is missing or the PC is old - No complete frame 2 s after video began: the reassembler names the missing fragments of the latest incomplete frames and the largest fragment received, the app logs it and asks the PC for a fresh keyframe with reason Startup, and again every 2 s while video still arrives. Startup passes the 500 ms request spacing, which the dropped frames' own requests keep busy. The PC's packet size has no control message, so the iPad can't ask for smaller datagrams. - A Wi-Fi link failure with no network ends the attempt as noNetwork, summarized as failure=no_network instead of other. The card says "Wi-Fi is off". - A PC whose HELLO_ACK lacks the USB capability can't reach Apple's USB service: the USB card says Apple Devices isn't running there. - The update check keeps the manifest's Windows version. A PC found by Bonjour with an older version gets "Update EternalMonitor on your PC" under its row. --- ios/EternalMonitor/App/ConnectView.swift | 21 +++++- .../App/ConnectionManager.swift | 16 +++- .../App/ConnectionProblem.swift | 11 ++- ios/EternalMonitor/App/SessionSummary.swift | 3 + ios/EternalMonitor/App/UIPreview.swift | 4 +- ios/EternalMonitor/Core/UpdateChecker.swift | 27 +++++++ .../Network/ControlChannel.swift | 5 +- .../Network/FrameAssembler.swift | 62 ++++++++++++++++ .../ConnectionProblemTests.swift | 15 ++++ ios/EternalMonitorTests/FakeUDPHost.swift | 12 +++ .../FrameAssemblerTests.swift | 74 +++++++++++++++++++ .../KeyframeRequestTests.swift | 28 +++++-- ios/EternalMonitorTests/PCNoticeTests.swift | 30 ++++++++ .../SessionSummaryTests.swift | 25 ++++++- .../StreamRecoveryTests.swift | 30 ++++++++ .../UpdateCheckerTests.swift | 41 ++++++++++ 16 files changed, 388 insertions(+), 16 deletions(-) create mode 100644 ios/EternalMonitorTests/PCNoticeTests.swift diff --git a/ios/EternalMonitor/App/ConnectView.swift b/ios/EternalMonitor/App/ConnectView.swift index daaaa718..c1506e73 100644 --- a/ios/EternalMonitor/App/ConnectView.swift +++ b/ios/EternalMonitor/App/ConnectView.swift @@ -596,7 +596,8 @@ struct ConnectView: View { Text(L("USB cable")) .font(.app(15, .medium, relativeTo: .subheadline)) .foregroundStyle(Theme.text) - Text(USBStatusText.describe(connectionManager.usbStatus)) + Text(USBStatusText.describe(connectionManager.usbStatus, + pcLacksAppleDevices: connectionManager.pcLacksAppleDevices)) .font(.app(13.5, relativeTo: .footnote)) .foregroundStyle(Theme.textMuted) .fixedSize(horizontal: false, vertical: true) @@ -706,7 +707,9 @@ struct ConnectView: View { title: host.name, subtitle: Self.discoveredSubtitle(host), badge: nil, - paired: false + paired: false, + note: updates.pcNeedsUpdate(PCVersions.version(address: host.address, name: host.name)) + ? Self.pcUpdateNote : nil ) } .buttonStyle(RowButtonStyle()) @@ -720,6 +723,10 @@ struct ConnectView: View { .transition(.opacity.combined(with: .move(edge: .bottom))) } + /// Under a found PC whose EternalMonitor is older than the newest for + /// Windows. Update text stays English, like the other update text. + static let pcUpdateNote = "Update EternalMonitor on your PC" + static func discoveredSubtitle(_ host: DiscoveredHost) -> String { if let address = host.address { return L("Tap to connect · \(address)") } return host.finding @@ -772,7 +779,8 @@ struct ConnectView: View { .padding(.leading, 70) } - private func hostRow(icon: String, title: String, subtitle: String, badge: String?, paired: Bool) -> some View { + private func hostRow(icon: String, title: String, subtitle: String, badge: String?, paired: Bool, + note: String? = nil) -> some View { HStack(spacing: 14) { Image(systemName: icon) .font(.system(size: 16, weight: .medium)) @@ -791,6 +799,13 @@ struct ConnectView: View { .font(.app(13, relativeTo: .footnote)) .foregroundStyle(Theme.textMuted) .lineLimit(1) + if let note { + Label(note, systemImage: "arrow.up.circle") + .font(.app(12.5, relativeTo: .caption)) + .foregroundStyle(Theme.textMuted) + .lineLimit(2) + .accessibilityIdentifier("discovered.pcUpdate") + } } Spacer(minLength: 8) if paired { diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 54023198..755f027b 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -140,6 +140,9 @@ final class ConnectionManager: ObservableObject { @Published private(set) var hostStreamConfig: StreamConfig? @Published private(set) var usbStatus = "USB: waiting for a cable" + /// The PC of the last session could not reach Apple's USB service; see + /// `USBStatusText.describe`. + @Published private(set) var pcLacksAppleDevices = false private var mediaLink: (any MediaLink)? private var usbListener: USBListener? private var isForeground = true @@ -558,6 +561,16 @@ final class ConnectionManager: ObservableObject { assembler.onNeedsKeyframe = { [weak channel] epoch, seq in channel?.sendKeyframeRequest(streamEpoch: epoch, lastCompleteSeq: seq, reason: .gapLoss) } + // Video arrives but the first frame never completes: ask for a fresh + // keyframe. The PC's packet size is its own setting; no message + // asks it for smaller datagrams. + assembler.onNoFrameYet = { [weak self, weak channel] epoch, missing in + channel?.sendKeyframeRequest(streamEpoch: epoch, lastCompleteSeq: 0, reason: .startup) + Task { @MainActor in + guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } + self.record(.warning, "assembly", "Video arrives but no frame completed in 2 s; asked the PC for a fresh keyframe: \(missing)") + } + } channel.reportProvider = { [weak assembler, weak channel, weak decoder, weak audio] in var report = metrics.makeReport(assembler: assembler?.counters.withLock { $0 } ?? .init(), decodeDepth: decoder?.decodeDepth ?? 0, @@ -595,6 +608,7 @@ final class ConnectionManager: ObservableObject { at: isUSB ? nil : "\(normalizedHost):\(port)") self.livenessTimeoutUs = UInt64(info.livenessTimeoutMs) * 1000 self.hostInfo = info + self.pcLacksAppleDevices = info.hostCaps & HelloAck.hostCapUSB == 0 if self.awaitingPairing { self.awaitingPairing = false self.pairingPrompt = nil @@ -778,7 +792,7 @@ final class ConnectionManager: ObservableObject { return } guard isUSB else { - self.disconnect(.linkFailed) + self.disconnect(failure == .noNetwork ? .noNetwork : .linkFailed) self.problem = failure return } diff --git a/ios/EternalMonitor/App/ConnectionProblem.swift b/ios/EternalMonitor/App/ConnectionProblem.swift index d476ad3c..2cd8e27d 100644 --- a/ios/EternalMonitor/App/ConnectionProblem.swift +++ b/ios/EternalMonitor/App/ConnectionProblem.swift @@ -66,7 +66,7 @@ enum ConnectionProblem: Equatable { case .badAddress: return "That address doesn't look right" case .pairingCodeAsAddress: return "That looks like the pairing code, not the PC's address" case .routerAddress: return "That looks like your router's address, not your PC's" - case .noNetwork: return "This iPad isn't connected to a network" + case .noNetwork: return "Wi-Fi is off" case .localNetworkBlocked: return "Local Network is turned off for EternalMonitor" case .couldNotConnect: return "Couldn't connect to your PC" case .usbFailed: return "The connection over the cable stopped" @@ -346,9 +346,16 @@ struct NoPictureStreak: Equatable { /// What the USB cable card says for each listener status. enum USBStatusText { - static func describe(_ status: String) -> String { + /// `pcLacksAppleDevices`: the PC said in its last session that it can't + /// reach Apple's USB service, so Apple Devices isn't running there or + /// isn't installed. Only a cable session would show the PC more. + static func describe(_ status: String, pcLacksAppleDevices: Bool = false) -> String { switch status { case "USB: off": return L("Off. Turn on to connect with a cable.") + case "USB: waiting for a cable" where pcLacksAppleDevices: + // A problem: stays English, like the other problem text. + return "Apple Devices isn't running on your PC, so the cable can't connect. " + + "Open it, or install it from the Microsoft Store. Wi-Fi still works." case "USB: waiting for a cable": return L("Plug this iPad into your PC. Keep Apple Devices open on the PC.") case "USB: disconnected": return L("Paused. Tap Connect USB to use the cable again.") case "USB: connecting": return L("Connecting over the cable…") diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index 2103755b..587ae9fd 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -27,6 +27,8 @@ enum AttemptEnd: Equatable { case closedBeforePicture /// This iPad's network side failed, for example Local Network denied. case linkFailed + /// This iPad has no network: Wi-Fi is off or joined to none. + case noNetwork case receiverFailed /// The PC asked for the pairing code, and a new HELLO2 sends it: the /// refused one is over, the attempt goes on. @@ -104,6 +106,7 @@ struct SessionSummary: Equatable { case .connectTimeout: return datagramsReceived == 0 ? "no_video" : "no_frame" case .pcStoppedResponding: return "signal_lost" case .cableUnplugged, .usbLinkFailed, .closedBeforePicture: return "usb_lost" + case .noNetwork: return "no_network" case .pcRefused, .legacyPC, .linkFailed, .receiverFailed, .pictureStalled: return "other" default: return "none" } diff --git a/ios/EternalMonitor/App/UIPreview.swift b/ios/EternalMonitor/App/UIPreview.swift index ca0e247a..e6f7242e 100644 --- a/ios/EternalMonitor/App/UIPreview.swift +++ b/ios/EternalMonitor/App/UIPreview.swift @@ -4,7 +4,7 @@ import SwiftUI /// Debug builds only: put the UI in a state that normally needs a live PC, /// for screenshots and design review. Never opens a connection. /// -/// `EM_UI_PREVIEW=display|quality|restoring|connecting|error|error-no-picture|error-pairing-code|error-router|error-updating|error-local-network|mismatch|pairing|settings|settings-about|feedback|recent|searchFailed|searchBlocked|setup|setup-blocked`. +/// `EM_UI_PREVIEW=display|quality|restoring|connecting|error|error-no-picture|error-pairing-code|error-router|error-updating|error-local-network|error-wifi-off|mismatch|pairing|settings|settings-about|feedback|recent|searchFailed|searchBlocked|setup|setup-blocked`. /// `setup` opens first-run setup whatever the saved state (launch with /// `-didSeeOnboarding NO`); `setup-blocked` also fails its search as if /// Local Network were off. @@ -60,6 +60,8 @@ enum UIPreview { manager.problem = .pcUpdating(.reconnecting) case "error-local-network": manager.problem = .localNetworkBlocked + case "error-wifi-off": + manager.problem = .noNetwork case "mismatch": manager.versionMismatch = .iPadOlder manager.connectionError = VersionMismatch.iPadOlder.message diff --git a/ios/EternalMonitor/Core/UpdateChecker.swift b/ios/EternalMonitor/Core/UpdateChecker.swift index 88380b8c..91fb2078 100644 --- a/ios/EternalMonitor/Core/UpdateChecker.swift +++ b/ios/EternalMonitor/Core/UpdateChecker.swift @@ -53,6 +53,20 @@ struct UpdateManifest: Decodable, Equatable { else { return nil } return note } + + /// The Windows app's version, read on its own so that a malformed + /// Windows entry never hides the iPad's update, or the other way round. + static func windowsVersion(_ data: Data) -> String? { + struct Manifest: Decodable { + struct Windows: Decodable { let version: String } + let schema: Int + let windows: Windows? + } + guard let manifest = try? JSONDecoder().decode(Manifest.self, from: data), manifest.schema == 1, + let version = manifest.windows?.version, AppVersion(version) != nil + else { return nil } + return version + } } extension UpdateManifest.IPad { @@ -212,6 +226,7 @@ final class UpdateChecker: ObservableObject { static let rolloutDays = "updateAvailableRolloutDays" /// The staged build whose prompt this iPad has reached. static let rolloutReached = "updateRolloutReachedBuild" + static let windowsVersion = "updateWindowsVersion" static let saved = [version, build, testflight, recommended, note, rolloutStart, rolloutDays] } @@ -224,6 +239,8 @@ final class UpdateChecker: ObservableObject { @Published private(set) var available: UpdateManifest.IPad? @Published private(set) var dismissed: (build: Int, at: Date)? @Published private(set) var popupClosed: (build: Int, at: Date)? + /// The newest EternalMonitor for Windows, from the same manifest. + @Published private(set) var windowsVersion: String? @Published var automatic: Bool { didSet { defaults.set(automatic, forKey: Key.automatic) @@ -245,6 +262,7 @@ final class UpdateChecker: ObservableObject { self.installed = installed self.installationID = installationID automatic = defaults.object(forKey: Key.automatic) as? Bool ?? true + windowsVersion = defaults.string(forKey: Key.windowsVersion) if let version = defaults.string(forKey: Key.version) { let found = UpdateManifest.IPad( version: version, @@ -322,6 +340,13 @@ final class UpdateChecker: ObservableObject { defaults.set(now, forKey: Key.popupAt) } + /// A PC that announced `pcVersion` should update EternalMonitor. Only + /// plain release versions compare; a beta or an unknown version never does. + func pcNeedsUpdate(_ pcVersion: String?) -> Bool { + guard let pc = pcVersion.flatMap(AppVersion.init), let newest = windowsVersion.flatMap(AppVersion.init) else { return false } + return pc < newest + } + /// The TestFlight invite named by the manifest, or the public link. var joinURL: URL { if let link = available?.testflight, let url = URL(string: link), @@ -366,6 +391,8 @@ final class UpdateChecker: ObservableObject { defaults.set(now, forKey: Key.checkedAt) // The first-picture wait applies from the next launch. RemoteConfig.save(RemoteConfig.parse(data), to: defaults, now: now) + windowsVersion = UpdateManifest.windowsVersion(data) + defaults.set(windowsVersion, forKey: Key.windowsVersion) guard let found = UpdateManifest.parse(data), installed.isOlder(than: found) else { available = nil for key in Key.saved { defaults.removeObject(forKey: key) } diff --git a/ios/EternalMonitor/Network/ControlChannel.swift b/ios/EternalMonitor/Network/ControlChannel.swift index a65961e1..2c2f2d58 100644 --- a/ios/EternalMonitor/Network/ControlChannel.swift +++ b/ios/EternalMonitor/Network/ControlChannel.swift @@ -296,7 +296,10 @@ final class ControlChannel { queue.async { [self] in guard sessionId != 0 else { return } let now = nowUs() - if let last = lastKeyframeRequestUs, now &- last < Self.keyframeRequestSpacingUs { return } + // Startup comes at most every 2 s, while no frame completes. The + // dropped frames' own requests then keep the spacing busy, and + // would always hide it from the PC's log. + if reason != .startup, let last = lastKeyframeRequestUs, now &- last < Self.keyframeRequestSpacingUs { return } lastKeyframeRequestUs = now onDiagnostic?("Requesting keyframe (\(reason))") sendMessage(.keyframeRequest(KeyframeRequest( diff --git a/ios/EternalMonitor/Network/FrameAssembler.swift b/ios/EternalMonitor/Network/FrameAssembler.swift index e75f7e74..83644991 100644 --- a/ios/EternalMonitor/Network/FrameAssembler.swift +++ b/ios/EternalMonitor/Network/FrameAssembler.swift @@ -17,6 +17,10 @@ final class FrameAssembler { var onInfo: ((String) -> Void)? var onNack: ((Nack) -> Void)? var onNeedsKeyframe: ((_ epoch: UInt32, _ lastCompleteSeq: UInt32) -> Void)? + /// Video has arrived for `noFrameYetUs` and no frame has completed, and + /// again after each further `noFrameYetUs` while video still arrives. + /// `missing` names the fragments the latest incomplete frames lack. + var onNoFrameYet: ((_ epoch: UInt32, _ missing: String) -> Void)? var repairEnabled = false var framePeriodUs: UInt64 = 16_667 var rttUs: () -> UInt64 = { 10_000 } @@ -62,6 +66,10 @@ final class FrameAssembler { /// Without repair, a partial frame is dropped when no fragment of it has /// arrived for this long. static let staleFrameUs: UInt64 = 100_000 + /// The first keyframe is the largest frame of a session. On lossy Wi-Fi, + /// or a path that drops full-size datagrams, it can lose fragments every + /// time while the frames after it wait for it. + static let noFrameYetUs: UInt64 = 2_000_000 /// Cumulative per-connection loss accounting, safe to read from any /// thread (feeds receiver reports and the HUD). @@ -118,6 +126,16 @@ final class FrameAssembler { private var staleEpochStreak: UInt32 = 0 private static let epochResyncThreshold: UInt32 = 512 + /// Until a frame completes: when the first fragment arrived, then when + /// `onNoFrameYet` last ran. The incomplete frames dropped so far, as + /// `onNoFrameYet` words them, and the largest fragment payload, which + /// is smaller than the PC sends when full-size datagrams are lost. + /// `reset()` keeps them: they span the attempt, not one stream epoch. + private var noFrameSince: UInt64? + private var deliveredAny = false + private var droppedBeforeFirstFrame: [String] = [] + private var largestPayload = 0 + struct PendingFrame { let fragmentCount: UInt16 let isKeyframe: Bool @@ -199,6 +217,7 @@ final class FrameAssembler { if repairEnabled { resumeAfterStall(at: now) } lastArrivalUs = now if isRetransmit { lastRepairUs = now } + if !deliveredAny && noFrameSince == nil { noFrameSince = now } let latestRetired = max(latestCompletedSeq, retiredSeq) if latestRetired > 0 { @@ -275,6 +294,7 @@ final class FrameAssembler { frame.lastFragmentAt = now frame.byteCount += payload.count frame.highestIndex = max(frame.highestIndex, index) + largestPayload = max(largestPayload, payload.count) while frame.contiguous < count && frame.fragments[frame.contiguous] != nil { frame.contiguous += 1 } @@ -393,6 +413,43 @@ final class FrameAssembler { } if repairEnabled { drainReady(at: now) } updateDepth() + reportNoFrameYet(at: now) + } + + /// See `onNoFrameYet`. Repeats only while fragments still arrive: a PC + /// that went silent is the liveness and connect timeouts' business. + private func reportNoFrameYet(at now: UInt64) { + guard !deliveredAny, let since = noFrameSince, now >= since + Self.noFrameYetUs, + let arrived = lastArrivalUs, arrived >= since else { return } + noFrameSince = now + let waiting = pending.keys.sorted().compactMap { seq in + pending[seq].flatMap { $0.isComplete ? nil : Self.describe(seq, $0, dropped: false) } + } + let frames = (droppedBeforeFirstFrame + waiting).suffix(3) + onNoFrameYet?(currentEpoch ?? 0, (frames.isEmpty ? "no incomplete frame" : frames.joined(separator: "; ")) + + "; largest fragment \(largestPayload) bytes") + } + + /// One incomplete frame for the support log, such as + /// "seq=1 keyframe dropped with 95/97 fragments, missing 3,90". + static func describe(_ seq: UInt32, _ frame: PendingFrame, dropped: Bool) -> String { + let missing = (0.. String { + var runs: [ClosedRange] = [] + for index in indices { + if let last = runs.last, last.upperBound &+ 1 == index { + runs[runs.count - 1] = last.lowerBound...index + } else { + runs.append(index...index) + } + } + let shown = runs.prefix(8).map { $0.count == 1 ? "\($0.lowerBound)" : "\($0.lowerBound)-\($0.upperBound)" } + return shown.joined(separator: ",") + (runs.count > 8 ? " and \(runs.count - 8) more ranges" : "") } /// How long this waiting frame has been stalled, not yet counted, within @@ -516,6 +573,8 @@ final class FrameAssembler { } latestCompletedSeq = seq retiredSeq = max(retiredSeq, seq) + deliveredAny = true + droppedBeforeFirstFrame = [] counters.withLock { $0.framesComplete += 1 $0.fragsRepaired += frame.repaired @@ -526,6 +585,9 @@ final class FrameAssembler { private func dropPending(_ seq: UInt32) { requestKeyframe(seq) if let frame = pending.removeValue(forKey: seq) { + if !deliveredAny && !frame.isComplete { + droppedBeforeFirstFrame = Array(droppedBeforeFirstFrame.suffix(2)) + [Self.describe(seq, frame, dropped: true)] + } pendingBytes -= frame.byteCount counters.withLock { $0.framesDropped += 1 diff --git a/ios/EternalMonitorTests/ConnectionProblemTests.swift b/ios/EternalMonitorTests/ConnectionProblemTests.swift index 7450d989..48bc68f6 100644 --- a/ios/EternalMonitorTests/ConnectionProblemTests.swift +++ b/ios/EternalMonitorTests/ConnectionProblemTests.swift @@ -162,6 +162,8 @@ final class ConnectionProblemTests: XCTestCase { XCTAssertEqual(wifi(LinkError(.connectionFailed, "UDP connection failed: PolicyDenied", underlying: NWError.dns(-65570))), .localNetworkBlocked) // Offline. XCTAssertEqual(wifi(LinkError(.pathUnavailable, "UDP path not ready: Network is down", underlying: NWError.posix(.ENETDOWN))), .noNetwork) + XCTAssertEqual(ConnectionProblem.noNetwork.title, "Wi-Fi is off") + XCTAssertTrue(ConnectionProblem.noNetwork.detail.contains("Turn on Wi-Fi"), ConnectionProblem.noNetwork.detail) // Unknown failures get general advice, never the raw error. let unknown = LinkError(.connectionFailed, "UDP socket failed: Address already in use", underlying: POSIXError(.EADDRINUSE)) XCTAssertEqual(wifi(unknown), .couldNotConnect) @@ -301,6 +303,19 @@ final class ConnectionProblemTests: XCTestCase { } } + /// The PC's last session said it can't reach Apple's USB service: the + /// waiting card says why the cable won't connect. Other states don't + /// change, and a cable session (which reaches it) clears the flag. + func testUSBCardSaysWhenThePCLacksAppleDevices() { + let waiting = USBStatusText.describe("USB: waiting for a cable", pcLacksAppleDevices: true) + XCTAssertEqual(waiting, "Apple Devices isn't running on your PC, so the cable can't connect. " + + "Open it, or install it from the Microsoft Store. Wi-Fi still works.") + XCTAssertNotEqual(waiting, USBStatusText.describe("USB: waiting for a cable")) + for status in ["USB: off", "USB: disconnected", "USB: connecting", "USB: unavailable", "USB: connected to PC"] { + XCTAssertEqual(USBStatusText.describe(status, pcLacksAppleDevices: true), USBStatusText.describe(status), status) + } + } + /// A HELLO no one answered is a warning (a PC off, asleep or elsewhere) /// unless that PC answered at this address earlier in this launch. func testAnUnansweredHelloIsAnErrorOnlyWhereAPCAnsweredBefore() { diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index 273ac591..c05946be 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -132,6 +132,18 @@ final class FakeUDPHost: @unchecked Sendable { timedKeyframeRequests(within: seconds, count: count).map(\.reason) } + /// The reasons of the keyframe requests received within `seconds`, up to + /// and including the first for which `last` is true. + func keyframeRequests(within seconds: Double, until last: (KeyframeReason) -> Bool) -> [KeyframeReason] { + var reasons: [KeyframeReason] = [] + let deadline = Date().addingTimeInterval(seconds) + while Date() < deadline, !(reasons.last.map(last) ?? false) { + guard let (data, _, _) = receive() else { continue } + if case .keyframeRequest(let request) = Wire.parseControl(data)?.message { reasons.append(request.reason) } + } + return reasons + } + /// The keyframe requests received within `seconds`, with when each came, /// or the first `count` of them as soon as they have come. func timedKeyframeRequests(within seconds: Double, count: Int = .max) -> [(reason: KeyframeReason, at: Date)] { diff --git a/ios/EternalMonitorTests/FrameAssemblerTests.swift b/ios/EternalMonitorTests/FrameAssemblerTests.swift index 1e11195f..9780e58a 100644 --- a/ios/EternalMonitorTests/FrameAssemblerTests.swift +++ b/ios/EternalMonitorTests/FrameAssemblerTests.swift @@ -649,4 +649,78 @@ final class FrameAssemblerTests: XCTestCase { XCTAssertEqual(assembler.mismatchedCountFragments.withLock { $0 }, 6) XCTAssertEqual(FrameAssembler().mismatchedCountFragments.withLock { $0 }, 0, "a new session starts at zero") } + + /// The session's first keyframe loses fragments for good, and the frames + /// after it never complete either. 2 s after the first fragment, and + /// again every 2 s while fragments still arrive, the app hears which + /// fragments are missing, so it can ask for a fresh keyframe. A + /// completed frame ends it. + func testAFirstFrameThatNeverCompletesIsReportedWithItsMissingFragments() { + now = 0 + assembler = FrameAssembler(nowUs: { [weak self] in self?.now ?? 0 }) + assembler.onFrameAssembled = { [weak self] data, _, _, _ in self?.completed.append(data) } + var reports: [(epoch: UInt32, missing: String)] = [] + assembler.onNoFrameYet = { reports.append(($0, $1)) } + + for index: UInt16 in [0, 1, 3] { add(seq: 1, index: index, count: 5, epoch: 7, byte: 1, keyframe: true) } + now = 200_000 + assembler.tick() + now = 1_000_000 + add(seq: 2, index: 0, count: 2, epoch: 7, byte: 2) + now = 1_999_999 + assembler.tick() + XCTAssertTrue(reports.isEmpty, "\(reports)") + + now = 2_000_000 + assembler.tick() + XCTAssertEqual(reports.map(\.epoch), [7]) + XCTAssertEqual(reports.first?.missing, "seq=1 keyframe dropped with 3/5 fragments, missing 2,4;" + + " seq=2 dropped with 1/2 fragments, missing 1; largest fragment 1 bytes") + + // Nothing arrived since: no second report, however long it waits. + now = 9_000_000 + assembler.tick() + XCTAssertEqual(reports.count, 1) + // Video again: reported at once, as 2 s have passed. + add(seq: 3, index: 1, count: 2, epoch: 7, byte: 3) + XCTAssertEqual(reports.count, 2) + + add(seq: 4, index: 0, count: 1, epoch: 7, byte: 4) + XCTAssertEqual(completed, [Data([4])]) + for step in UInt64(1)...5 { + now = 9_000_000 + step * 2_000_000 + add(seq: 4 + UInt32(step), index: 0, count: 2, epoch: 7, byte: 5) + assembler.tick() + } + XCTAssertEqual(reports.count, 2, "a frame completed") + } + + /// With repair on, the missing fragment is asked for as before; the + /// report names it once the frame has waited 2 s without completing. + func testARepairedSessionAlsoReportsAFirstKeyframeThatNeverCompletes() { + enableRepair() + var reports: [String] = [] + assembler.onNoFrameYet = { _, missing in reports.append(missing) } + add(seq: 1, index: 0, count: 3, byte: 1, keyframe: true) + add(seq: 1, index: 2, count: 3, byte: 3, keyframe: true) + while now < 1_990_000 { + now += 10_000 + assembler.tick() + } + XCTAssertTrue(reports.isEmpty, "\(reports)") + XCTAssertEqual(nacks.first?.missing, [1]) + now = 2_000_000 + add(seq: 2, index: 0, count: 2, byte: 4) + XCTAssertEqual(reports.count, 1, "\(reports)") + XCTAssertTrue(reports.first?.hasPrefix("seq=1 keyframe dropped with 2/3 fragments, missing 1;") == true, "\(reports)") + XCTAssertTrue(completed.isEmpty) + } + + func testMissingFragmentsAreListedAsRanges() { + XCTAssertEqual(FrameAssembler.ranges([0, 1, 2, 5, 7, 8]), "0-2,5,7-8") + XCTAssertEqual(FrameAssembler.ranges(stride(from: 0, to: 40, by: 4).map { UInt16($0) }), + "0,4,8,12,16,20,24,28 and 2 more ranges") + XCTAssertEqual(FrameAssembler.ranges([UInt16.max - 1, UInt16.max]), "65534-65535") + XCTAssertEqual(FrameAssembler.ranges([]), "") + } } diff --git a/ios/EternalMonitorTests/KeyframeRequestTests.swift b/ios/EternalMonitorTests/KeyframeRequestTests.swift index 14741746..d572bc21 100644 --- a/ios/EternalMonitorTests/KeyframeRequestTests.swift +++ b/ios/EternalMonitorTests/KeyframeRequestTests.swift @@ -6,6 +6,22 @@ final class KeyframeRequestTests: XCTestCase { /// A stall that damages many frames at once must ask the host once, as the /// host grants only one keyframe per 500 ms. func testRequestsWithinTheHostWindowAreCoalesced() async throws { + let requests = try await sent([(1_000_000, KeyframeReason.gapLoss), (1_010_000, .gapLoss), + (1_400_000, .decodeError), (1_500_000, .gapLoss)]) + XCTAssertEqual(requests, [.gapLoss, .gapLoss], "one request per 500 ms window") + } + + /// No first frame yet: the Startup request goes out although dropped + /// frames asked moments ago, so the PC's log shows it. + func testAStartupRequestIsNeverCoalesced() async throws { + let requests = try await sent([(1_000_000, KeyframeReason.gapLoss), (1_100_000, .startup), + (1_200_000, .gapLoss), (1_700_000, .gapLoss)]) + XCTAssertEqual(requests, [.gapLoss, .startup, .gapLoss]) + } + + /// The keyframe requests a session's channel sends for `requests`, each + /// made at its time on the channel's clock. + private func sent(_ requests: [(UInt64, KeyframeReason)]) async throws -> [KeyframeReason] { let queue = DispatchQueue(label: "keyframe-request-test") let clock = OSAllocatedUnfairLock(initialState: 1_000_000) let (stream, continuation) = AsyncStream.makeStream() @@ -16,7 +32,8 @@ final class KeyframeRequestTests: XCTestCase { var messages = stream.makeAsyncIterator() let first = await messages.next() guard case .hello2(let hello) = try XCTUnwrap(Wire.parseControl(XCTUnwrap(first))).message else { - return XCTFail("missing handshake") + XCTFail("missing handshake") + return [] } queue.sync { channel.handleControl(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .helloAck(HelloAck( @@ -24,18 +41,17 @@ final class KeyframeRequestTests: XCTestCase { heartbeatIntervalMs: 1000, reportIntervalMs: 500, livenessTimeoutMs: 3000, streamConfig: StreamConfig(), hostName: "Test PC", authToken: Data(17...32), hostCaps: 0)))) } - for (at, reason) in [(1_000_000, KeyframeReason.gapLoss), (1_010_000, .gapLoss), - (1_400_000, .decodeError), (1_500_000, .gapLoss)] as [(UInt64, KeyframeReason)] { + for (at, reason) in requests { clock.withLock { $0 = at } channel.sendKeyframeRequest(streamEpoch: 1, lastCompleteSeq: 7, reason: reason) queue.sync {} } channel.stop() continuation.finish() - var requests: [KeyframeReason] = [] + var reasons: [KeyframeReason] = [] while let data = await messages.next() { - if case .keyframeRequest(let request) = Wire.parseControl(data)?.message { requests.append(request.reason) } + if case .keyframeRequest(let request) = Wire.parseControl(data)?.message { reasons.append(request.reason) } } - XCTAssertEqual(requests, [.gapLoss, .gapLoss], "one request per 500 ms window") + return reasons } } diff --git a/ios/EternalMonitorTests/PCNoticeTests.swift b/ios/EternalMonitorTests/PCNoticeTests.swift new file mode 100644 index 00000000..3c2fecda --- /dev/null +++ b/ios/EternalMonitorTests/PCNoticeTests.swift @@ -0,0 +1,30 @@ +import XCTest +@testable import EternalMonitor + +/// What a session's PC tells this iPad about itself beyond the stream. +@MainActor +final class PCNoticeTests: XCTestCase { + private func until(_ description: String, seconds: Double = 10, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + /// A PC that can't reach Apple's USB service (Apple Devices not running + /// or not installed) leaves the bit out of its HELLO_ACK. The USB card + /// then says so; a PC that reaches it clears that. + func testThePCsAppleDevicesStateReachesTheUSBCard() async throws { + let manager = ConnectionManager() + addTeardownBlock { await MainActor.run { manager.cancel() } } + XCTAssertFalse(manager.pcLacksAppleDevices, "nothing known before a session") + for (caps, lacks) in [(UInt16(0), true), (HelloAck.hostCapUSB, false)] { + let host = FakeUDPHost() + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession(hostCaps: caps) }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + try await until("accepted") { manager.hostInfo?.hostCaps == caps && manager.pcLacksAppleDevices == lacks } + manager.cancel() + XCTAssertEqual(manager.pcLacksAppleDevices, lacks, "kept after the session for the Connect screen") + } + } +} diff --git a/ios/EternalMonitorTests/SessionSummaryTests.swift b/ios/EternalMonitorTests/SessionSummaryTests.swift index 9e911677..0a4a5ef8 100644 --- a/ios/EternalMonitorTests/SessionSummaryTests.swift +++ b/ios/EternalMonitorTests/SessionSummaryTests.swift @@ -1,3 +1,4 @@ +import Network import XCTest @testable import EternalMonitor @@ -13,7 +14,7 @@ final class SessionSummaryTests: XCTestCase { + #" transport=(usb|wifi) mode=(mirror|second_screen|unknown)"# + #" stage=(started|hello_sent|accepted|video_sent|first_frame|steady)"# + #" outcome=(streamed|no_reply|no_picture|pairing|user_ended|app_ended|error)"# - + #" failure=(none|no_reply|no_video|no_frame|vdd_enable_failed|vdd_attach_timeout|vdd_not_attached|capture_failed|encoder_failed|usb_lost|signal_lost|other)"# + + #" failure=(none|no_reply|no_video|no_frame|vdd_enable_failed|vdd_attach_timeout|vdd_not_attached|capture_failed|encoder_failed|usb_lost|signal_lost|no_network|other)"# + #" fallback=(none|attach_timeout|enable_failed|no_driver|no_frames|duplication_failed|remembered|other)"# + #" end=(goodbye|liveness|superseded|shutdown|timeout|user|background|error)"# + #" ttff_ms=(0|[1-9][0-9]{0,5}|-) duration_ms=(0|[1-9][0-9]{0,9})"# @@ -85,6 +86,7 @@ final class SessionSummaryTests: XCTestCase { (summary(.usbLinkFailed, usb: true), ["accepted", "error", "usb_lost", "error"]), (summary(.closedBeforePicture, usb: true), ["accepted", "no_picture", "usb_lost", "error"]), (summary(.linkFailed, session: nil, nonce: 0, duration: 0.2), ["started", "error", "other", "error"]), + (summary(.noNetwork, session: nil, nonce: 0, duration: 0.2), ["started", "error", "no_network", "error"]), (summary(.receiverFailed, session: nil, nonce: 0, duration: 0), ["started", "error", "other", "error"]), (retried, ["hello_sent", "pairing", "none", "superseded"]), ] @@ -104,7 +106,7 @@ final class SessionSummaryTests: XCTestCase { XCTAssertEqual(AttemptEnd.superseded.bye, .superseded) for end in [AttemptEnd.pcGoodbye, .pcRefused, .legacyPC, .connectTimeout, .pcStoppedResponding, .pictureStalled, .cableUnplugged, .usbLinkFailed, .closedBeforePicture, .linkFailed, - .receiverFailed, .retriedWithCode] { + .noNetwork, .receiverFailed, .retriedWithCode] { XCTAssertEqual(end.bye, .error, "\(end)") } XCTAssertEqual(AttemptEnd.linkFailed.name, "linkFailed") @@ -165,6 +167,25 @@ final class SessionSummaryTests: XCTestCase { Self.assertContract(line) } + /// Wi-Fi is off: Network.framework waits with "Network is down". The + /// card says so, and the summary gives it its own failure instead of + /// `other`. + func testWiFiOffHasItsOwnCardAndFailure() async throws { + let link = DeniedLink() + let manager = ConnectionManager(udpReceiver: { _ in link }) + manager.connect(host: "192.168.1.20", port: 9876) + link.onError?(LinkError(.pathUnavailable, "UDP path not ready: The operation couldn’t be completed. (Network.NWError error 50 - Network is down)", + underlying: NWError.posix(.ENETDOWN))) + try await until("ended") { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .noNetwork) + XCTAssertEqual(manager.problem?.title, "Wi-Fi is off") + XCTAssertEqual(logged(manager, "connection").filter { $0.hasPrefix("Disconnect ") }, + ["Disconnect state=connecting reason=noNetwork"]) + let line = logged(manager, "summary").first ?? "" + XCTAssertTrue(line.contains(" stage=started outcome=error failure=no_network fallback=none end=error "), line) + Self.assertContract(line) + } + /// Try again opens a new connection, which asks iPadOS again: once /// Local Network is on, the old denial isn't shown or remembered. func testTryAgainAfterLocalNetworkDeniedAsksIPadOSAgain() async throws { diff --git a/ios/EternalMonitorTests/StreamRecoveryTests.swift b/ios/EternalMonitorTests/StreamRecoveryTests.swift index 3610be20..82fcebbb 100644 --- a/ios/EternalMonitorTests/StreamRecoveryTests.swift +++ b/ios/EternalMonitorTests/StreamRecoveryTests.swift @@ -217,4 +217,34 @@ final class StreamRecoveryTests: XCTestCase { let after = await Task.detached { host.keyframeRequests(within: 1) }.value XCTAssertEqual(after, []) } + + /// Video arrives but no frame ever completes, as when every full-size + /// datagram is lost: 2 s on, the iPad logs the missing fragments and + /// asks for a fresh keyframe with reason Startup, which the PC logs, + /// besides the requests the dropped frames make. + func testAFirstFrameThatNeverCompletesAsksForAFreshKeyframe() async throws { + let host = FakeUDPHost() + let manager = ConnectionManager() + addTeardownBlock { await MainActor.run { manager.cancel() } } + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + let fragments = Task.detached { + for seq in UInt32(1)...80 where !Task.isCancelled { + host.sendFirstFragment(Data([0, 0, 0, 1, 0x65]), seq: seq) + try? await Task.sleep(nanoseconds: 100_000_000) + } + } + defer { fragments.cancel() } + let reasons = await Task.detached { host.keyframeRequests(within: 8) { $0 == .startup } }.value + XCTAssertEqual(reasons.last, .startup, "\(reasons)") + try await until("logged") { + manager.diagnostics.contains { $0.category == "assembly" && $0.message.hasPrefix( + "Video arrives but no frame completed in 2 s; asked the PC for a fresh keyframe: seq=") } + } + let logged = manager.diagnostics.first { $0.message.hasPrefix("Video arrives") } + XCTAssertEqual(logged?.level, .warning) + XCTAssertTrue(logged?.message.contains(" dropped with 1/2 fragments, missing 1") == true, logged?.message ?? "") + XCTAssertEqual(manager.state, .connecting) + } } diff --git a/ios/EternalMonitorTests/UpdateCheckerTests.swift b/ios/EternalMonitorTests/UpdateCheckerTests.swift index d838a807..568dd396 100644 --- a/ios/EternalMonitorTests/UpdateCheckerTests.swift +++ b/ios/EternalMonitorTests/UpdateCheckerTests.swift @@ -223,6 +223,47 @@ final class UpdateCheckerTests: XCTestCase { XCTAssertFalse(UpdateChecker(defaults: defaults, source: .disabled, installed: installed).automatic) } + func testManifestGivesTheWindowsVersionOnItsOwn() { + XCTAssertEqual(UpdateManifest.windowsVersion(manifest("0.3.4", 698)), "0.3.4") + XCTAssertEqual(UpdateManifest.windowsVersion(Data(#"{"schema":1,"windows":{"version":"0.3.5"}}"#.utf8)), "0.3.5") + XCTAssertNil(UpdateManifest.windowsVersion(manifest("0.3.4", 698, schema: 2))) + XCTAssertNil(UpdateManifest.windowsVersion(Data(#"{"schema":1,"windows":{"version":"0.3.5-beta.1"}}"#.utf8))) + XCTAssertNil(UpdateManifest.windowsVersion(Data(#"{"schema":1,"windows":{}}"#.utf8))) + XCTAssertNil(UpdateManifest.windowsVersion(Data(#"{"schema":1,"ipad":{"version":"0.3.4","build":698}}"#.utf8))) + // A broken Windows entry doesn't hide the iPad's update. + XCTAssertNotNil(UpdateManifest.parse(Data(#"{"schema":1,"windows":{"version":3},"ipad":{"version":"0.3.4","build":698}}"#.utf8))) + } + + /// A PC older than the manifest's Windows version gets the note, also + /// when this iPad is up to date and after a relaunch offline. Betas, + /// unknown versions and newer PCs don't. + @MainActor + func testAnOlderPCIsToldToUpdate() throws { + let suite = "UpdateCheckerTests-\(UUID().uuidString)" + let defaults = try XCTUnwrap(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + let current = InstalledApp(version: "0.3.4", build: 698) + + let checker = UpdateChecker(defaults: defaults, source: .injected(manifest("0.3.4", 698)), installed: current) + XCTAssertFalse(checker.pcNeedsUpdate("0.3.2"), "before any check") + checker.checkIfDue() + XCTAssertNil(checker.available) + XCTAssertEqual(checker.windowsVersion, "0.3.4") + for old in ["0.3.3", "0.3.2", "0.2.9"] { XCTAssertTrue(checker.pcNeedsUpdate(old), old) } + for fine in ["0.3.4", "0.3.5", "0.3.3-beta.1", "", "unknown", nil] as [String?] { + XCTAssertFalse(checker.pcNeedsUpdate(fine), fine ?? "nil") + } + + let relaunched = UpdateChecker(defaults: defaults, source: .disabled, installed: current) + XCTAssertTrue(relaunched.pcNeedsUpdate("0.3.3")) + + // A manifest without a Windows entry forgets it. + let without = UpdateChecker(defaults: defaults, source: .injected(Data(#"{"schema":1}"#.utf8)), installed: current) + without.checkIfDue() + XCTAssertNil(without.windowsVersion) + XCTAssertFalse(UpdateChecker(defaults: defaults, source: .disabled, installed: current).pcNeedsUpdate("0.3.3")) + } + @MainActor func testUntrustedTestFlightLinkFallsBackToTheInvite() throws { let suite = "UpdateCheckerTests-\(UUID().uuidString)" From b12abfa713c6aa5ca54a3159d6d1ed17ffcf81e2 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:03:06 -0500 Subject: [PATCH 015/226] Log pairing-off at info and collapse unchanged mDNS re-advertisements Turning pairing off is the user's setting, so the startup line is info, not a warning. The host re-advertises mDNS every minute and logged each one; it now logs a re-advertisement when its addresses change and otherwise once an hour, with how many went unlogged. --- host/src/discovery.rs | 57 +++++++++++++++++++++++++++++++++++++++++-- host/src/gui/mod.rs | 2 +- host/src/main.rs | 2 +- host/src/pairing.rs | 42 ++++++++++++++++++++++++++++--- 4 files changed, 96 insertions(+), 7 deletions(-) diff --git a/host/src/discovery.rs b/host/src/discovery.rs index ed39b782..f9dcac4c 100644 --- a/host/src/discovery.rs +++ b/host/src/discovery.rs @@ -1,5 +1,6 @@ +use std::net::IpAddr; use std::thread; -use std::time::Duration; +use std::time::{Duration, Instant}; use mdns_sd::{ServiceDaemon, ServiceInfo}; use tracing::{error, info, warn}; @@ -9,6 +10,33 @@ use crate::stats::PIPELINE_STATS; const SERVICE_TYPE: &str = "_eternaldisplay._udp.local."; const INSTANCE_NAME: &str = "EternalMonitor"; const RE_ADVERT_INTERVAL: Duration = Duration::from_secs(60); +/// A re-advertisement with the addresses of the last logged one is logged +/// once in this long, with how many there were. Every minute, it was one of +/// the most common lines in the uploads. +const RE_ADVERT_LOG_INTERVAL: Duration = Duration::from_secs(3600); + +/// Which re-advertisements get a log line. +#[derive(Default)] +struct ReAdvertLog { + last: Option<(Vec, Instant)>, + unchanged: u32, +} + +impl ReAdvertLog { + /// For a re-advertisement of `addrs` at `now`: `Some(n)` to log it, where + /// `n` is how many unchanged ones went unlogged before it. + fn note(&mut self, mut addrs: Vec, now: Instant) -> Option { + addrs.sort(); + if let Some((last, logged)) = &self.last { + if *last == addrs && now.duration_since(*logged) < RE_ADVERT_LOG_INTERVAL { + self.unchanged += 1; + return None; + } + } + self.last = Some((addrs, now)); + Some(std::mem::take(&mut self.unchanged)) + } +} /// Registers the EternalMonitor service on the local network via mDNS/DNS-SD /// so the iOS app's NetworkScanner (NWBrowser) can discover it. @@ -55,6 +83,7 @@ pub fn advertise_service(port: u16) -> Option { let host_fqdn_for_thread = host_fqdn.clone(); let instance_for_thread = instance_name.clone(); let _ = full_name; // fullname only mattered for the old unregister cycle + let mut log = ReAdvertLog::default(); thread::spawn(move || loop { thread::sleep(RE_ADVERT_INTERVAL); let Some(info) = build_service_info(port, &host_fqdn_for_thread, &instance_for_thread) @@ -66,7 +95,11 @@ pub fn advertise_service(port: u16) -> Option { // Plain upsert: the service stays continuously advertised (addresses // may have changed, e.g. DHCP renew or interface flap). match mdns_clone.register(info) { - Ok(_) => info!(addrs = ?addrs, "mDNS re-advertisement"), + Ok(_) => { + if let Some(unchanged) = log.note(addrs.clone(), Instant::now()) { + info!(addrs = ?addrs, unchanged, "mDNS re-advertisement"); + } + } Err(error) => warn!(error = %error, "mDNS re-advertisement register failed"), } }); @@ -128,4 +161,24 @@ mod tests { assert!(info.is_addr_auto()); assert!(info.get_addresses().is_empty()); } + + /// The minute's re-advertisement is logged when its addresses change and + /// otherwise once an hour, with the count since the last line. + #[test] + fn unchanged_re_advertisements_are_logged_hourly() { + let start = Instant::now(); + let minute = |n: u32| start + RE_ADVERT_INTERVAL * n; + let wifi: IpAddr = "192.0.2.10".parse().unwrap(); + let usb: IpAddr = "198.51.100.1".parse().unwrap(); + let mut log = ReAdvertLog::default(); + assert_eq!(log.note(vec![], minute(1)), Some(0)); + let logged: Vec<_> = (2..=121) + .filter_map(|n| log.note(vec![], minute(n)).map(|c| (n, c))) + .collect(); + assert_eq!(logged, [(61, 59), (121, 59)]); + // A new address list is logged at once; the order doesn't matter. + assert_eq!(log.note(vec![wifi, usb], minute(122)), Some(0)); + assert_eq!(log.note(vec![usb, wifi], minute(123)), None); + assert_eq!(log.note(vec![wifi], minute(124)), Some(1)); + } } diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 53d7cf8e..e49d475b 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -791,7 +791,7 @@ impl AnalyzerApp { { let mut pairing = self.control.shared.pairing.lock(); pairing.required = required; - pairing.warn_if_off(); + pairing.log_if_off(); } self.persist_settings(); } diff --git a/host/src/main.rs b/host/src/main.rs index 0dbd39cb..9fae06d2 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -176,7 +176,7 @@ fn main() -> Result<(), Box> { let mut pairing = shared.pairing.lock(); pairing.configure(persisted.require_pairing, auth_token); pairing.log_code(); - pairing.warn_if_off(); + pairing.log_if_off(); } // Each Troubleshooting knob: an environment variable (developers, CI) // wins, then the saved setting, then Automatic. diff --git a/host/src/pairing.rs b/host/src/pairing.rs index 61c6696b..1978aefd 100644 --- a/host/src/pairing.rs +++ b/host/src/pairing.rs @@ -105,10 +105,11 @@ impl Pairing { } } - /// Said at startup and whenever pairing is turned off. - pub fn warn_if_off(&self) { + /// Said at startup and whenever pairing is turned off. The user chose + /// it, so it is not a warning. + pub fn log_if_off(&self) { if !self.required { - warn!("Pairing is off: any iPad on the network can connect without a code"); + info!("Pairing is off: any iPad on the network can connect without a code"); } } @@ -355,6 +356,41 @@ impl RateLimiter { mod tests { use super::*; + /// Log lines `run` writes on this thread. + fn logged(run: impl FnOnce()) -> String { + let buffer = std::sync::Arc::new(parking_lot::Mutex::new(Vec::new())); + let writer = buffer.clone(); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || Captured(writer.clone())) + .finish(); + tracing::subscriber::with_default(subscriber, run); + let text = String::from_utf8_lossy(&buffer.lock()).into_owned(); + text + } + + struct Captured(std::sync::Arc>>); + impl std::io::Write for Captured { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + /// The user turned pairing off: that is a setting, not a warning. + #[test] + fn pairing_off_is_logged_at_info() { + let off = Pairing::new(false, [7; 16]).unwrap(); + let text = logged(|| off.log_if_off()); + assert!(text.contains(" INFO "), "{text}"); + assert!(text.contains("Pairing is off"), "{text}"); + let on = Pairing::new(true, [7; 16]).unwrap(); + assert_eq!(logged(|| on.log_if_off()), ""); + } + fn guess(nonce: u32) -> FailedRequest { FailedRequest { nonce, From e2cd001a1dda48c34b837689e1b0f5a534a67ea2 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:03:37 -0500 Subject: [PATCH 016/226] Draw the PC's pointer on the iPad, sent apart from the video An iPad that says it draws the pointer (HELLO2 feature bit 5) gets the pointer image and its position as control messages from a host that says it sends them (host capability bit 6). The host then leaves the pointer out of the video, and a pointer-only update needs no frame, so pointer motion no longer waits for the encoder. - POINTER_SHAPE (0x1A) carries the image in chunks: premultiplied BGRA and, when a pixel inverts the picture under it, an invert mask. Monochrome, color and masked-color Windows pointers convert to it. - POINTER_STATE (0x1B) carries the position and visibility, at most one per frame budget, the newest, and once more 100 ms after the last. - Receiver reports append the image id the iPad holds, so a lost chunk is sent again; the image also goes with every session start and IDR. - The iPad draws the image over the video in the Metal pass, through the same content rect the touch relay uses, with a second pass that inverts the picture where the mask says. - iPads and hosts without the bits behave as before. Fixes #183 --- host/src/capture/dxgi.rs | 158 +++++-- host/src/capture/mod.rs | 2 +- host/src/capture/pointer.rs | 411 +++++++++++++++++ host/src/capture/synthetic.rs | 194 +++++++- host/src/control.rs | 55 +++ host/src/diagnostics.rs | 1 + host/src/stats.rs | 7 + host/src/transport/mod.rs | 78 ++++ host/src/transport/pointer.rs | 421 ++++++++++++++++++ host/src/transport/session.rs | 67 +++ host/tests/e2e_synthetic.rs | 173 +++++++ host/tests/gui_snapshots.rs | 1 + .../App/ConnectionManager.swift | 10 + ios/EternalMonitor/Core/E2ESignpost.swift | 12 + .../Network/ControlChannel.swift | 11 + ios/EternalMonitor/Network/WireProtocol.swift | 115 ++++- ios/EternalMonitor/Render/MetalView.swift | 91 +++- .../Render/PointerOverlay.swift | 179 ++++++++ ios/EternalMonitor/Render/Shaders.metal | 40 ++ .../PointerOverlayTests.swift | 192 ++++++++ .../WireProtocolGoldenTests.swift | 90 +++- proto/src/v2/control.rs | 317 +++++++++++++ proto/src/v2/mod.rs | 7 + proto/testdata/v2_vectors.txt | 6 + proto/tests/golden.rs | 130 +++++- scripts/e2e_pointer.sh | 147 ++++++ 26 files changed, 2851 insertions(+), 64 deletions(-) create mode 100644 host/src/capture/pointer.rs create mode 100644 host/src/transport/pointer.rs create mode 100644 ios/EternalMonitor/Render/PointerOverlay.swift create mode 100644 ios/EternalMonitorTests/PointerOverlayTests.swift create mode 100755 scripts/e2e_pointer.sh diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 1139b91f..0bad0f4d 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -316,6 +316,9 @@ pub fn run_capture_loop( let mut idle = IdleGate::new(); // Set once this duplication has delivered a real desktop image. let mut seen_image = false; + // Whether `last_frame` has the pointer drawn in: it has unless the + // client draws the pointer itself (`SharedControl::client_draws_pointer`). + let mut last_frame_has_pointer = true; loop { if !shared.running.load(Ordering::SeqCst) @@ -346,6 +349,46 @@ pub fn run_capture_loop( continue; } + // An iPad that draws the pointer gets a picture without it; any + // other client, or none, one with it. The staging texture still + // holds the last desktop image, so the cached frame is redrawn from + // it when the connected client's answer differs from the cache's. + let draws_pointer = shared.client_draws_pointer(); + if seen_image && last_frame_has_pointer == draws_pointer { + let mut pixel_buf = frame_buffer(frame_bytes); + copy_staging( + &device_context, + &staging_texture, + tex_height, + row_bytes, + &mut pixel_buf, + )?; + if !draws_pointer && cursor_visible && cursor.has_shape { + draw_cursor( + &mut pixel_buf, + tex_width, + tex_height, + row_bytes, + &cursor, + cursor_x, + cursor_y, + ); + } + spare = None; + last_frame = Arc::new(pixel_buf); + last_frame_has_pointer = !draws_pointer; + if client_connected(&shared) { + resend_last_frame( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + (tex_width, tex_height), + ); + } + } + let frame_start = Instant::now(); let frame_budget = frame_budget_for(shared.effective_fps()); // Half the frame budget, clamped: at 60 fps this is the classic 8 ms; @@ -397,9 +440,26 @@ pub fn run_capture_loop( cursor_visible = frame_info.PointerPosition.Visible.as_bool(); cursor_x = frame_info.PointerPosition.Position.x; cursor_y = frame_info.PointerPosition.Position.y; + shared.pointer.set_position(super::pointer::Position { + visible: cursor_visible, + x: cursor_x, + y: cursor_y, + host_time_us: crate::clock::host_now_us(), + }); } - if frame_info.PointerShapeBufferSize > 0 { - cursor.update_shape(dupl, frame_info.PointerShapeBufferSize); + if frame_info.PointerShapeBufferSize > 0 + && cursor.update_shape(dupl, frame_info.PointerShapeBufferSize) + { + if let Some(image) = super::pointer::convert_shape( + cursor.shape_type, + cursor.width, + cursor.height, + cursor.pitch, + cursor.hot, + &cursor.shape, + ) { + shared.pointer.set_shape(image); + } } // --- Get dirty rect count --- @@ -438,7 +498,11 @@ pub fn run_capture_loop( // --- Copy desktop texture to staging and read pixels into a // recycled buffer (the readback is the ONLY full-frame copy) --- - if !frame_has_image(frame_info.LastPresentTime, seen_image) { + // A pointer-only update needs no frame when the iPad draws + // the pointer: the transport sends the position. + if !frame_has_image(frame_info.LastPresentTime, seen_image) + || (draws_pointer && frame_info.LastPresentTime == 0) + { unsafe { dupl.ReleaseFrame()?; } @@ -465,39 +529,16 @@ pub fn run_capture_loop( _ => frame_buffer(frame_bytes), }; - // Map staging texture for CPU read - let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); - unsafe { - device_context.Map( - &staging_texture, - 0, - D3D11_MAP_READ, - 0, - Some(&mut mapped), - )?; - } - - // Copy row-by-row (pitch may differ from width*4) - let src_pitch = mapped.RowPitch as usize; - let src_ptr = mapped.pData as *const u8; - for row in 0..tex_height as usize { - let src_offset = row * src_pitch; - let dst_offset = row * row_bytes; - unsafe { - std::ptr::copy_nonoverlapping( - src_ptr.add(src_offset), - pixel_buf.as_mut_ptr().add(dst_offset), - row_bytes, - ); - } - } - - unsafe { - device_context.Unmap(&staging_texture, 0); - } + copy_staging( + &device_context, + &staging_texture, + tex_height, + row_bytes, + &mut pixel_buf, + )?; // Composite the mouse cursor into the BGRA frame. - if cursor_visible && cursor.has_shape { + if !draws_pointer && cursor_visible && cursor.has_shape { draw_cursor( &mut pixel_buf, tex_width, @@ -547,6 +588,7 @@ pub fn run_capture_loop( // two buffers alive; steady state recycles them. spare = Some(Arc::clone(&last_frame)); last_frame = data; + last_frame_has_pointer = !draws_pointer; } } Err(e) if e.code() == DXGI_ERROR_WAIT_TIMEOUT => { @@ -903,6 +945,37 @@ fn publish_capture_geometry(output: &IDXGIOutput1, shared: &SharedControl) { } } +/// Reads the staging texture's image into `buf`, row by row (its pitch may +/// differ from `row_bytes`). +fn copy_staging( + device_context: &ID3D11DeviceContext, + staging_texture: &ID3D11Texture2D, + height: u32, + row_bytes: usize, + buf: &mut [u8], +) -> windows::core::Result<()> { + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + unsafe { + device_context.Map(staging_texture, 0, D3D11_MAP_READ, 0, Some(&mut mapped))?; + } + let src_pitch = mapped.RowPitch as usize; + let src_ptr = mapped.pData as *const u8; + debug_assert!(buf.len() >= height as usize * row_bytes); + for row in 0..height as usize { + unsafe { + std::ptr::copy_nonoverlapping( + src_ptr.add(row * src_pitch), + buf.as_mut_ptr().add(row * row_bytes), + row_bytes, + ); + } + } + unsafe { + device_context.Unmap(staging_texture, 0); + } + Ok(()) +} + fn make_staging_desc(width: u32, height: u32) -> D3D11_TEXTURE2D_DESC { D3D11_TEXTURE2D_DESC { Width: width, @@ -948,10 +1021,10 @@ fn get_dirty_rect_count( } } -// DXGI pointer shape types (DXGI_OUTDUPL_POINTER_SHAPE_TYPE_*). -const DXGI_POINTER_SHAPE_MONOCHROME: u32 = 1; -const DXGI_POINTER_SHAPE_COLOR: u32 = 2; -const DXGI_POINTER_SHAPE_MASKED_COLOR: u32 = 4; +use super::pointer::{ + SHAPE_COLOR as DXGI_POINTER_SHAPE_COLOR, SHAPE_MASKED_COLOR as DXGI_POINTER_SHAPE_MASKED_COLOR, + SHAPE_MONOCHROME as DXGI_POINTER_SHAPE_MONOCHROME, +}; /// Caches the latest mouse-cursor shape from DXGI Desktop Duplication. The shape only /// arrives when it changes, so it's kept across frames and composited at the per-frame @@ -963,13 +1036,15 @@ struct CursorState { width: u32, height: u32, pitch: u32, + hot: (u32, u32), has_shape: bool, } impl CursorState { /// Fetch the current pointer shape. Must be called while a frame is acquired - /// (before `ReleaseFrame`). On failure the previous shape is kept. - fn update_shape(&mut self, duplication: &IDXGIOutputDuplication, buffer_size: u32) { + /// (before `ReleaseFrame`). On failure the previous shape is kept and + /// this returns false. + fn update_shape(&mut self, duplication: &IDXGIOutputDuplication, buffer_size: u32) -> bool { self.shape.resize(buffer_size as usize, 0); let mut required = 0u32; let mut info = DXGI_OUTDUPL_POINTER_SHAPE_INFO::default(); @@ -986,10 +1061,13 @@ impl CursorState { self.width = info.Width; self.height = info.Height; self.pitch = info.Pitch; + self.hot = (info.HotSpot.x.max(0) as u32, info.HotSpot.y.max(0) as u32); self.has_shape = true; + true } Err(e) => { warn!(error = %e, "GetFramePointerShape failed — cursor shape not updated"); + false } } } diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 1473f9a9..e83117b1 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -22,6 +22,7 @@ use crate::vdd_device::DeviceState; #[cfg(windows)] pub mod dxgi; +pub mod pointer; pub mod recovery; pub mod second_screen; pub mod synthetic; @@ -36,7 +37,6 @@ pub(crate) const FPS_WINDOW: usize = 60; /// DXGI delivers no new frames. If a client is connected and we haven't sent anything for this /// long, resend the last frame so the encoder emits the pending IDR and the iPad gets a sync /// sample instead of timing out on a black screen. -#[cfg(any(windows, test))] pub(crate) const IDLE_KEEPALIVE: Duration = Duration::from_millis(750); /// With nobody connected the cached picture is refreshed this often, so a new diff --git a/host/src/capture/pointer.rs b/host/src/capture/pointer.rs new file mode 100644 index 00000000..469545aa --- /dev/null +++ b/host/src/capture/pointer.rs @@ -0,0 +1,411 @@ +//! The pointer, sent apart from the video to an iPad that draws it itself +//! (`FEATURE_DRAWS_POINTER`). Capture converts each new Windows pointer +//! image once ([`convert_shape`]) and records every position Windows +//! reports in the [`PointerFeed`]; the transport sends them at most once a +//! frame budget (`transport::pointer`). For any other client capture keeps +//! drawing the pointer into the video, as before. +//! +//! Positions stay in the captured output's pixels, where the video has +//! always drawn the pointer: Desktop Duplication reports them relative to +//! that output, so another monitor's offset never enters, and the iPad +//! scales them with the picture. + +use std::sync::Arc; + +use eternal_wire::v2::control::{pointer_shape_len, POINTER_MAX_SIDE, POINTER_SHAPE_INVERT}; +use parking_lot::Mutex; + +// DXGI_OUTDUPL_POINTER_SHAPE_TYPE_*. +pub const SHAPE_MONOCHROME: u32 = 1; +pub const SHAPE_COLOR: u32 = 2; +pub const SHAPE_MASKED_COLOR: u32 = 4; + +/// A pointer image in the wire's form: premultiplied BGRA, then an invert +/// mask when any pixel inverts the picture under it (see +/// `eternal_wire::v2::control::pointer_shape_len`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PointerImage { + pub source_type: u8, + pub flags: u8, + pub width: u16, + pub height: u16, + pub hot: (u16, u16), + pub bytes: Vec, +} + +/// Converts a shape as `GetFramePointerShape` returns it. `height` is the +/// buffer's: a monochrome shape stacks its AND mask over its XOR mask. +/// +/// What the iPad can draw of each kind: +/// - Color: alpha-blended, as the video drew it. +/// - Monochrome: exact. AND 0 is opaque black (XOR 0) or white (XOR 1); +/// AND 1 is clear (XOR 0) or inverts the picture (XOR 1), as the I-beam +/// over text does. +/// - Masked color: mask 0 pixels are opaque. Mask 1 pixels XOR the picture +/// with their color; black changes nothing, so it is clear, and every +/// other color inverts the picture. That is exact for white, the color +/// Windows uses, and a close match otherwise. +/// +/// A side past `POINTER_MAX_SIDE` keeps its top-left part. `None` for an +/// unknown kind or an empty image. +pub fn convert_shape( + source_type: u32, + width: u32, + height: u32, + pitch: u32, + hot: (u32, u32), + raw: &[u8], +) -> Option { + let rows = match source_type { + SHAPE_MONOCHROME => height / 2, + SHAPE_COLOR | SHAPE_MASKED_COLOR => height, + _ => return None, + }; + let max = u32::from(POINTER_MAX_SIDE); + let (w, h) = (width.min(max), rows.min(max)); + if w == 0 || h == 0 { + return None; + } + let (wu, hu, pitch) = (w as usize, h as usize, pitch as usize); + let mut image = vec![0u8; wu * hu * 4]; + let mut mask = vec![0u8; wu.div_ceil(8) * hu]; + let mut inverts = false; + let byte = |index: usize| raw.get(index).copied(); + for y in 0..hu { + for x in 0..wu { + let invert = match source_type { + SHAPE_MONOCHROME => { + let (at, bit) = (x / 8, 7 - (x % 8) as u8); + let (Some(and), Some(xor)) = + (byte(y * pitch + at), byte((y + rows as usize) * pitch + at)) + else { + continue; + }; + match ((and >> bit) & 1, (xor >> bit) & 1) { + (0, xor) => { + let v = if xor == 1 { 255 } else { 0 }; + image[(y * wu + x) * 4..][..4].copy_from_slice(&[v, v, v, 255]); + false + } + (_, xor) => xor == 1, + } + } + _ => { + let at = y * pitch + x * 4; + let Some(px) = raw.get(at..at + 4) else { + continue; + }; + let (b, g, r, a) = (px[0], px[1], px[2], px[3]); + let out = &mut image[(y * wu + x) * 4..][..4]; + if source_type == SHAPE_COLOR { + let premultiply = |c: u8| ((u32::from(c) * u32::from(a) + 127) / 255) as u8; + out.copy_from_slice(&[premultiply(b), premultiply(g), premultiply(r), a]); + false + } else if a == 0 { + out.copy_from_slice(&[b, g, r, 255]); + false + } else { + (b, g, r) != (0, 0, 0) + } + } + }; + if invert { + mask[y * wu.div_ceil(8) + x / 8] |= 0x80 >> (x % 8); + inverts = true; + } + } + } + let flags = if inverts { POINTER_SHAPE_INVERT } else { 0 }; + if inverts { + image.extend_from_slice(&mask); + } + let (width, height) = (w as u16, h as u16); + debug_assert_eq!(image.len(), pointer_shape_len(width, height, flags)); + Some(PointerImage { + source_type: source_type as u8, + flags, + width, + height, + hot: ( + hot.0.min(u32::from(u16::MAX)) as u16, + hot.1.min(u32::from(u16::MAX)) as u16, + ), + bytes: image, + }) +} + +/// The current pointer image with the id the wire names it by. +#[derive(Debug, PartialEq, Eq)] +pub struct Shape { + pub id: u32, + pub image: PointerImage, +} + +/// Where Windows last put the pointer, in the captured output's pixels. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Position { + pub visible: bool, + /// The image's top-left corner. + pub x: i32, + pub y: i32, + pub host_time_us: u64, +} + +/// The latest pointer image and position, written by capture and read by +/// the transport, which [`PointerFeed::changed`] wakes. +#[derive(Default)] +pub struct PointerFeed { + state: Mutex, + changed: tokio::sync::Notify, +} + +#[derive(Default)] +struct FeedState { + shape: Option>, + position: Option, + last_id: u32, +} + +impl PointerFeed { + /// A new image gets a new id; the same image again (as a re-created + /// duplication reports it) keeps its id, so it is not sent again. + pub fn set_shape(&self, image: PointerImage) { + let mut state = self.state.lock(); + if state + .shape + .as_ref() + .is_some_and(|shape| shape.image == image) + { + return; + } + state.last_id = state.last_id.wrapping_add(1).max(1); + state.shape = Some(Arc::new(Shape { + id: state.last_id, + image, + })); + drop(state); + self.changed.notify_one(); + } + + pub fn set_position(&self, position: Position) { + self.state.lock().position = Some(position); + self.changed.notify_one(); + } + + pub fn latest(&self) -> (Option>, Option) { + let state = self.state.lock(); + (state.shape.clone(), state.position) + } + + /// Resolves after the next [`Self::set_shape`] or [`Self::set_position`], + /// or at once when one came since the last wait. + pub async fn changed(&self) { + self.changed.notified().await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// One monochrome row per string, AND mask over XOR mask, 1 byte pitch. + fn monochrome(and: &[u8], xor: &[u8]) -> Vec { + [and, xor].concat() + } + + fn pixel(image: &PointerImage, x: usize, y: usize) -> [u8; 4] { + let at = (y * usize::from(image.width) + x) * 4; + image.bytes[at..at + 4].try_into().unwrap() + } + + fn inverts(image: &PointerImage, x: usize, y: usize) -> bool { + if image.flags & POINTER_SHAPE_INVERT == 0 { + return false; + } + let (w, h) = (usize::from(image.width), usize::from(image.height)); + let mask = &image.bytes[w * h * 4..]; + mask[y * w.div_ceil(8) + x / 8] & (0x80 >> (x % 8)) != 0 + } + + #[test] + fn monochrome_draws_black_white_clear_and_inverts_with_its_xor_mask() { + // Four pixels on one row: (AND, XOR) = (0,0) (0,1) (1,0) (1,1). + let raw = monochrome(&[0b0011_0000], &[0b0101_0000]); + let image = convert_shape(SHAPE_MONOCHROME, 4, 2, 1, (1, 0), &raw).unwrap(); + assert_eq!((image.width, image.height, image.hot), (4, 1, (1, 0))); + assert_eq!(image.source_type, 1); + assert_eq!(pixel(&image, 0, 0), [0, 0, 0, 255], "opaque black"); + assert_eq!(pixel(&image, 1, 0), [255, 255, 255, 255], "opaque white"); + assert_eq!(pixel(&image, 2, 0), [0, 0, 0, 0], "clear"); + assert_eq!( + pixel(&image, 3, 0), + [0, 0, 0, 0], + "inverting pixels draw nothing" + ); + assert_eq!( + (0..4).map(|x| inverts(&image, x, 0)).collect::>(), + [false, false, false, true], + ); + assert_eq!( + image.bytes.len(), + pointer_shape_len(4, 1, POINTER_SHAPE_INVERT) + ); + } + + #[test] + fn monochrome_without_inverting_pixels_sends_no_mask() { + let raw = monochrome(&[0b0100_0000], &[0b1000_0000]); + let image = convert_shape(SHAPE_MONOCHROME, 2, 2, 1, (0, 0), &raw).unwrap(); + assert_eq!(image.flags, 0); + assert_eq!(image.bytes, [255, 255, 255, 255, 0, 0, 0, 0]); + } + + #[test] + fn color_is_premultiplied_and_keeps_its_alpha() { + // BGRA rows with 4 bytes of pitch padding. + let raw = [ + 200, 100, 50, 255, 200, 100, 50, 128, 0, 0, 0, 0, // + 10, 20, 30, 0, 255, 255, 255, 64, 0, 0, 0, 0, + ]; + let image = convert_shape(SHAPE_COLOR, 2, 2, 12, (0, 0), &raw).unwrap(); + assert_eq!(image.flags, 0); + assert_eq!(pixel(&image, 0, 0), [200, 100, 50, 255]); + assert_eq!(pixel(&image, 1, 0), [100, 50, 25, 128]); + assert_eq!( + pixel(&image, 0, 1), + [0, 0, 0, 0], + "fully clear has no color" + ); + assert_eq!(pixel(&image, 1, 1), [64, 64, 64, 64]); + } + + #[test] + fn masked_color_is_opaque_clear_or_inverting() { + let raw = [ + 10, 20, 30, 0, // mask 0: opaque color + 0, 0, 0, 0xFF, // XOR with black: no change + 255, 255, 255, 0xFF, // XOR with white: invert + 0, 0, 200, 0xFF, // XOR with red: drawn as invert + ]; + let image = convert_shape(SHAPE_MASKED_COLOR, 4, 1, 16, (0, 0), &raw).unwrap(); + assert_eq!(image.flags, POINTER_SHAPE_INVERT); + assert_eq!(pixel(&image, 0, 0), [10, 20, 30, 255]); + for x in 1..4 { + assert_eq!(pixel(&image, x, 0), [0, 0, 0, 0]); + } + assert_eq!( + (0..4).map(|x| inverts(&image, x, 0)).collect::>(), + [false, false, true, true], + ); + } + + #[test] + fn short_buffers_unknown_kinds_and_huge_shapes_are_safe() { + // Rows the buffer doesn't hold stay clear. + let image = convert_shape(SHAPE_COLOR, 2, 2, 8, (0, 0), &[1, 2, 3, 255]).unwrap(); + assert_eq!(pixel(&image, 0, 0), [1, 2, 3, 255]); + assert_eq!(pixel(&image, 1, 1), [0, 0, 0, 0]); + assert_eq!(convert_shape(3, 2, 2, 8, (0, 0), &[0; 16]), None); + assert_eq!(convert_shape(SHAPE_MONOCHROME, 8, 1, 1, (0, 0), &[0]), None); + let side = u32::from(POINTER_MAX_SIDE) + 8; + let raw = vec![0xFF; (side * 4 * side) as usize]; + let image = convert_shape(SHAPE_COLOR, side, side, side * 4, (0, 0), &raw).unwrap(); + assert_eq!( + (image.width, image.height), + (POINTER_MAX_SIDE, POINTER_MAX_SIDE) + ); + } + + /// Desktop Duplication reports the pointer relative to the captured + /// output, which is where it is drawn in the video. A tap on the drawn + /// hotspot, normalized as the iPad does, lands on the desktop pixel + /// Windows has the hotspot at, whatever the output's offset: a screen + /// left of the main one, a second screen right of it, the main one. + #[test] + fn a_tap_on_the_drawn_hotspot_lands_where_windows_has_it() { + use crate::input::{norm_to_output_pixel, CaptureGeometry}; + let outputs = [ + CaptureGeometry { + left: -1920, + top: 120, + width: 1920, + height: 1080, + }, + CaptureGeometry { + left: 3440, + top: 0, + width: 2388, + height: 1668, + }, + CaptureGeometry { + left: 0, + top: 0, + width: 3440, + height: 1440, + }, + ]; + let hot = (6, 9); + for output in outputs { + for (x, y) in [ + (0, 0), + (1003, 517), + (output.width as i32 - 7, output.height as i32 - 10), + ] { + // Windows: the hotspot at this desktop pixel. + let desktop = (output.left + x + hot.0, output.top + y + hot.1); + // Duplication: the image's corner, relative to the output. + let drawn = Position { + visible: true, + x, + y, + host_time_us: 0, + }; + // The iPad: the centre of the hotspot's video pixel. + let (w, h) = (output.width as f64, output.height as f64); + let norm = |p: i32, n: f64| (((f64::from(p) + 0.5) / n) * 65535.0).round() as u16; + let tapped = norm_to_output_pixel( + norm(drawn.x + hot.0, w), + norm(drawn.y + hot.1, h), + output, + ); + // Within the input mapping's own rounding: it spreads the + // video's pixels over one step fewer. + assert!( + (tapped.0 - desktop.0).abs() <= 1 && (tapped.1 - desktop.1).abs() <= 1, + "{output:?} at {x},{y}: tapped {tapped:?}, Windows {desktop:?}" + ); + } + } + } + + #[test] + fn the_feed_names_a_new_image_once_and_keeps_the_latest_position() { + let feed = PointerFeed::default(); + let image = |v: u8| PointerImage { + source_type: 2, + flags: 0, + width: 1, + height: 1, + hot: (0, 0), + bytes: vec![v; 4], + }; + feed.set_shape(image(1)); + feed.set_shape(image(1)); + assert_eq!( + feed.latest().0.unwrap().id, + 1, + "the same image keeps its id" + ); + feed.set_shape(image(2)); + assert_eq!(feed.latest().0.unwrap().id, 2); + for x in 0..3 { + feed.set_position(Position { + visible: true, + x, + y: 0, + host_time_us: 0, + }); + } + assert_eq!(feed.latest().1.unwrap().x, 2); + } +} diff --git a/host/src/capture/synthetic.rs b/host/src/capture/synthetic.rs index fadc35bb..f11cb727 100644 --- a/host/src/capture/synthetic.rs +++ b/host/src/capture/synthetic.rs @@ -9,6 +9,10 @@ //! Every frame carries its frame number encoded as a row of black/white //! squares that survives lossy H.264 encoding, so end-to-end tests can decode //! frames and assert real progression through the entire stack. +//! +//! `ETERNAL_SYNTH_POINTER=1` instead holds a still picture and moves a +//! pointer over it every frame, as when only the mouse moves on a still +//! desktop (see [`PointerScene`]). use std::sync::atomic::Ordering; use std::sync::Arc; @@ -16,7 +20,8 @@ use std::time::Instant; use tracing::info; -use super::{frame_budget_for, heartbeat, FrameSlot, RawFrame}; +use super::pointer::{convert_shape, PointerImage, Position, SHAPE_COLOR}; +use super::{frame_budget_for, heartbeat, FrameSlot, RawFrame, IDLE_KEEPALIVE}; use crate::control::SharedControl; use crate::stats::PIPELINE_STATS; @@ -78,6 +83,9 @@ pub fn run_capture_loop( // avoids normal sleep coalescing without burning an entire CPU core. let mut timer = super::timing::FrameTimer::new(); let mut next_deadline = Instant::now(); + let mut pointer = std::env::var("ETERNAL_SYNTH_POINTER") + .is_ok_and(|v| v.trim() == "1") + .then(|| PointerScene::new(&shared, width, height)); loop { if !shared.running.load(Ordering::SeqCst) @@ -92,22 +100,34 @@ pub fn run_capture_loop( heartbeat(&shared.hb_capture_loop_ms); - *frame_number += 1; - let data = render_next_frame(&mut buffers, width, height, *frame_number); - - PIPELINE_STATS.lock().record_capture(width, height); - heartbeat(&shared.hb_capture_frame_ms); - - slot.publish(RawFrame { - frame_number: *frame_number, - timestamp: frame_start, - data, - width, - height, - }); - #[cfg(debug_assertions)] - if std::mem::take(&mut proving) { - super::second_screen::note_image(&shared, generation); + let data = match pointer.as_mut() { + None => { + *frame_number += 1; + Some(render_next_frame( + &mut buffers, + width, + height, + *frame_number, + )) + } + Some(scene) => scene.tick(&shared, &mut buffers, frame_start, frame_number), + }; + + if let Some(data) = data { + PIPELINE_STATS.lock().record_capture(width, height); + heartbeat(&shared.hb_capture_frame_ms); + + slot.publish(RawFrame { + frame_number: *frame_number, + timestamp: frame_start, + data, + width, + height, + }); + #[cfg(debug_assertions)] + if std::mem::take(&mut proving) { + super::second_screen::note_image(&shared, generation); + } } next_deadline += frame_budget; @@ -123,6 +143,146 @@ pub fn run_capture_loop( Ok(()) } +/// A still picture with a pointer that moves every frame. For a client that +/// draws the pointer it goes to the pointer feed only, and a frame is sent +/// only for an IDR or the idle keepalive, as the DXGI loop does. For any +/// other client each move is a new frame with the pointer drawn in. +struct PointerScene { + width: u32, + height: u32, + picture: Vec, + image: PointerImage, + step: u32, + last_publish: Option, +} + +impl PointerScene { + fn new(shared: &SharedControl, width: u32, height: u32) -> Self { + let mut picture = super::frame_buffer((width * 4) as usize * height as usize); + render_synthetic_frame(&mut picture, width, height, 0); + let image = arrow(); + shared.pointer.set_shape(image.clone()); + info!(width, height, "Synthetic pointer over a still picture"); + Self { + width, + height, + picture, + image, + step: 0, + last_publish: None, + } + } + + fn tick( + &mut self, + shared: &SharedControl, + buffers: &mut [Arc>; 3], + now: Instant, + frame_number: &mut u64, + ) -> Option>> { + self.step = self.step.wrapping_add(1); + let (x, y) = pointer_path(self.step, self.width, self.height); + shared.pointer.set_position(Position { + visible: true, + x, + y, + host_time_us: crate::clock::host_now_us(), + }); + let draws = shared.client_draws_pointer(); + let due = !draws + || self + .last_publish + .is_none_or(|at| now - at >= IDLE_KEEPALIVE) + || shared.force_next_idr.load(Ordering::SeqCst); + if !due { + return None; + } + self.last_publish = Some(now); + *frame_number += 1; + let buffer = buffers + .iter_mut() + .find(|buffer| Arc::strong_count(buffer) == 1) + .expect("encoder and latest-frame slot retain at most two buffers"); + let buf = Arc::get_mut(buffer).unwrap(); + buf.copy_from_slice(&self.picture); + draw_frame_counter(buf, self.width, *frame_number); + if !draws { + draw_pointer(buf, self.width, self.height, &self.image, x, y); + } + Some(Arc::clone(buffer)) + } +} + +/// Where the synthetic pointer is at `step`: across the picture and back, +/// a new position every step. +pub fn pointer_path(step: u32, width: u32, height: u32) -> (i32, i32) { + let span = width.saturating_sub(64).max(2); + let along = (step * 5) % (2 * span); + let x = 16 + + if along < span { + along + } else { + 2 * span - along + } as i32; + let y = (height / 2) as i32 + ((step % 40) as i32 - 20); + (x, y) +} + +/// A 12x19 white arrow with a black edge, as Windows' color pointer is. +fn arrow() -> PointerImage { + let (w, h) = (12u32, 19u32); + let mut raw = vec![0u8; (w * h * 4) as usize]; + for y in 0..h { + for x in 0..=y.min(w - 1) { + let edge = x == 0 || x == y.min(w - 1) || y == h - 1; + let v = if edge { 0 } else { 255 }; + raw[((y * w + x) * 4) as usize..][..4].copy_from_slice(&[v, v, v, 255]); + } + } + convert_shape(SHAPE_COLOR, w, h, w * 4, (0, 0), &raw).expect("a valid arrow") +} + +/// Draws a converted pointer image over a BGRA frame: "over" for its +/// premultiplied pixels, then inverting where its mask says. +pub fn draw_pointer( + buf: &mut [u8], + width: u32, + height: u32, + image: &PointerImage, + px: i32, + py: i32, +) { + let (iw, ih) = (usize::from(image.width), usize::from(image.height)); + let mask = (image.flags & eternal_wire::v2::control::POINTER_SHAPE_INVERT != 0) + .then(|| &image.bytes[iw * ih * 4..]); + for row in 0..ih { + let sy = py + row as i32; + if sy < 0 || sy >= height as i32 { + continue; + } + for col in 0..iw { + let sx = px + col as i32; + if sx < 0 || sx >= width as i32 { + continue; + } + let src = &image.bytes[(row * iw + col) * 4..][..4]; + let at = (sy as usize * width as usize + sx as usize) * 4; + let dst = &mut buf[at..at + 3]; + let keep = 255 - u32::from(src[3]); + for c in 0..3 { + dst[c] = (u32::from(src[c]) + u32::from(dst[c]) * keep / 255).min(255) as u8; + } + if mask + .is_some_and(|mask| mask[row * iw.div_ceil(8) + col / 8] & (0x80 >> (col % 8)) != 0) + { + for c in dst.iter_mut() { + *c = !*c; + } + } + } + } +} + /// The encoder and latest-frame slot can each retain one buffer. Keep three /// allocations so a delayed consumer never discards a busy spare and causes /// a fresh full-frame allocation on the next tick. diff --git a/host/src/control.rs b/host/src/control.rs index 30ac372c..337bd97d 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -100,6 +100,8 @@ pub struct SharedControl { pub pairing: Arc>, /// Which network the next stream listens on. pub listen_on: Arc>, + /// The pointer for an iPad that draws it (see `capture::pointer`). + pub pointer: Arc, } /// Runtime state of the managed virtual extended display. @@ -162,6 +164,7 @@ impl SharedControl { .expect("OS entropy for pairing code"), )), listen_on: Arc::new(Mutex::new(Default::default())), + pointer: Arc::new(Default::default()), running: Arc::new(AtomicBool::new(false)), bitrate_bps: Arc::new(AtomicU32::new(initial_bitrate_bps)), abr_current_bps: Arc::new(AtomicU32::new(initial_bitrate_bps)), @@ -228,6 +231,12 @@ impl SharedControl { /// A client is connected and heard from: one whose session is on hold /// after a silence counts as gone for the second screen, so its display /// still goes the same time after the iPad's last packet. + /// The connected iPad draws the pointer, so capture leaves it out of + /// the video and sends it apart. + pub fn client_draws_pointer(&self) -> bool { + self.session.lock().pointer_session_id().is_some() + } + pub(crate) fn client_heard(&self) -> bool { let session = self.session.lock(); session.is_active() && !session.on_hold() @@ -316,6 +325,52 @@ mod tests { } } + /// Capture asks this before leaving the pointer out of the video. + #[test] + fn capture_draws_the_pointer_unless_the_connected_ipad_does() { + use eternal_wire::v2::control::{ControlMessage, Hello2, FEATURE_DRAWS_POINTER}; + let shared = SharedControl::new(9876, 15_000_000); + assert!(!shared.client_draws_pointer(), "nobody connected"); + for (nonce, caps, draws) in [(1, 0, false), (2, FEATURE_DRAWS_POINTER, true)] { + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: nonce, + listen_port: 50000, + decoder_caps: 1, + feature_caps: caps, + screen_px_w: 2420, + screen_px_h: 1668, + screen_pt_w: 1210, + screen_pt_h: 834, + refresh_hz: 120, + device_name: "Test iPad".into(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + struct Config; + impl crate::transport::session::ConfigSource for Config { + fn stream_config(&self) -> eternal_wire::v2::control::StreamConfig { + Default::default() + } + fn host_name(&self) -> String { + "TEST".into() + } + } + shared.session.lock().handle_control( + PeerId::udp("10.0.0.5:50000".parse().unwrap()), + 0, + hello, + &Config, + std::time::Instant::now(), + ); + assert!(shared.client_connected()); + assert_eq!(shared.client_draws_pointer(), draws, "caps {caps:#x}"); + } + } + #[test] fn client_connected_requires_a_session_not_just_an_address() { let shared = SharedControl::new(9876, 15_000_000); diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 02ccf0d6..21574acf 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -126,6 +126,7 @@ fn log_stream_summary(shared: &SharedControl) { target_fps = shared.effective_fps(), max_dgram = shared.max_dgram.load(Relaxed), input = ?*shared.encoder_input.lock(), hevc = shared.hevc_enabled.load(Relaxed), retransmits = sample.transport_retransmits, usb_drops = sample.usb_frames_dropped, + pointer_sent = sample.pointer_states_sent, pointer_shapes = sample.pointer_shapes_sent, usb_state = %sample.usb_link_state, usb_devices = sample.usb_devices, capture_heartbeat_age_ms = now_ms.saturating_sub(shared.hb_capture_frame_ms.load(Relaxed)), encode_heartbeat_age_ms = now_ms.saturating_sub(shared.hb_encode_frame_ms.load(Relaxed)), diff --git a/host/src/stats.rs b/host/src/stats.rs index 98c9b58a..c61a9c4f 100644 --- a/host/src/stats.rs +++ b/host/src/stats.rs @@ -54,6 +54,9 @@ pub struct PipelineStats { pub transport_packets_sent: u64, pub transport_fragments_sent: u64, pub transport_retransmits: u64, + /// Pointer positions and images sent to an iPad that draws the pointer. + pub pointer_states_sent: u64, + pub pointer_shapes_sent: u64, pub usb_service_reachable: bool, pub usb_devices: usize, pub usb_link_state: String, @@ -116,6 +119,8 @@ impl PipelineStats { transport_packets_sent: 0, transport_fragments_sent: 0, transport_retransmits: 0, + pointer_states_sent: 0, + pointer_shapes_sent: 0, usb_service_reachable: false, usb_devices: 0, usb_link_state: "Checking Apple device service".into(), @@ -163,6 +168,8 @@ impl PipelineStats { self.transport_packets_sent = 0; self.transport_fragments_sent = 0; self.transport_retransmits = 0; + self.pointer_states_sent = 0; + self.pointer_shapes_sent = 0; self.transport_timestamps.clear(); self.bandwidth_samples.clear(); self.bandwidth_bps = 0.0; diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 70cfb197..375d22b5 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -4,6 +4,7 @@ mod audio; pub mod fault; pub mod link; pub mod pacer; +pub mod pointer; pub mod retransmit; pub mod session; pub mod usb; @@ -15,6 +16,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::mpsc as std_mpsc; use std::time::{Duration, Instant}; +use eternal_wire::v2::control::ControlMessage; use eternal_wire::v2::media::{ media_payload_size, MediaHeader, MAX_FRAG_COUNT, MAX_MEDIA_PAYLOAD, MEDIA_HEADER_SIZE, }; @@ -125,6 +127,7 @@ impl ConfigSource for SharedConfigSource<'_> { }; numbering | eternal_wire::v2::control::HOSTCAP_KEYBOARD + | eternal_wire::v2::control::HOSTCAP_POINTER | if self.pen_available { eternal_wire::v2::control::HOSTCAP_PEN } else { @@ -269,6 +272,7 @@ pub async fn start_sender( fault, input_relay, abr, + pointer: Default::default(), }; let mut audio_sender = audio::AudioSender::default(); let pacing_clock = PacingClock::new()?; @@ -303,6 +307,13 @@ pub async fn start_sender( .keep_pen_alive(*shared.capture_geometry.lock()); } pen_spacing.arm(state.input_relay.pen_spaced_until()); + let pointer_due = { + let (shape, position) = shared.pointer.latest(); + let budget = crate::capture::frame_budget_for(shared.effective_fps()); + state + .pointer + .next_poll(budget, position, shape.map(|shape| shape.id)) + }; tokio::select! { // Service timers and queued control before another paced frame. // Random branch selection can delay a NACK by multiple frame @@ -320,6 +331,7 @@ pub async fn start_sender( } } _ = heartbeat.tick() => { + state.pointer.refresh(now()); let actions = shared.session.lock().tick(&config, true, now()); act(actions, &mut state, &links, &shared, &config, &supervisor_tx).await; } @@ -384,6 +396,13 @@ pub async fn start_sender( } } + // Before video: a position is small and the iPad draws it at once. + _ = shared.pointer.changed() => { + send_pointer(&mut state.pointer, &links, &shared).await; + } + _ = tokio::time::sleep_until(pointer_due.unwrap_or_else(Instant::now).into()), if pointer_due.is_some() => { + send_pointer(&mut state.pointer, &links, &shared).await; + } Some(packet) = audio_rx.recv() => { audio_sender.send(packet, &links, &shared, stream_epoch).await; } @@ -693,6 +712,7 @@ struct LoopState { fault: FaultInjector, input_relay: crate::input::InputRelay, abr: AbrController, + pointer: pointer::PointerSender, } /// Carries out everything the session decided. Every path that feeds the @@ -726,6 +746,12 @@ async fn act( } } } + if actions.force_idr || actions.new_target.is_some() { + state.pointer.keyframe(now()); + } + if let Some(report) = actions.report.as_ref() { + state.pointer.report(report.pointer_shape_id, now()); + } if let Some(report) = actions.report.take() { // A slow log sink must not hold the capture/encode stats lock. let retransmits = PIPELINE_STATS.lock().transport_retransmits; @@ -772,6 +798,57 @@ async fn act( } } execute_actions(actions, links, shared, supervisor_tx).await; + send_pointer(&mut state.pointer, links, shared).await; +} + +/// Sends what [`pointer::PointerSender`] says is due, to a client that +/// draws the pointer. +async fn send_pointer( + sender: &mut pointer::PointerSender, + links: &TransportLinks, + shared: &SharedControl, +) { + let session_id = shared.session.lock().pointer_session_id(); + if session_id.is_some() && sender.session_id() != session_id { + info!( + ?session_id, + "The iPad draws the pointer: sending it apart from the video" + ); + } + sender.serve(session_id); + if session_id.is_none() { + return; + } + let budget = crate::capture::frame_budget_for(shared.effective_fps()); + for send in sender.poll(now(), budget, shared.pointer.latest()) { + let messages = match send { + pointer::Send::Shape(shape) => { + let image = &shape.image; + PIPELINE_STATS.lock().pointer_shapes_sent += 1; + eternal_wire::v2::control::pointer_shape_chunks( + shape.id, + image.source_type, + image.flags, + (image.width, image.height), + image.hot, + &image.bytes, + ) + .into_iter() + .map(ControlMessage::PointerShape) + .collect() + } + pointer::Send::State(state) => { + PIPELINE_STATS.lock().pointer_states_sent += 1; + vec![ControlMessage::PointerState(state)] + } + }; + for message in messages { + let Some((peer, datagram)) = shared.session.lock().encode_for_client(&message) else { + return; + }; + let _ = links.send_to(&datagram, peer).await; + } + } } async fn execute_actions( @@ -1136,6 +1213,7 @@ mod tests { fault: FaultInjector::new(0.0, 0.0, Duration::ZERO), input_relay: crate::input::InputRelay::default(), abr: AbrController::new(15_000_000, true), + pointer: Default::default(), } } diff --git a/host/src/transport/pointer.rs b/host/src/transport/pointer.rs new file mode 100644 index 00000000..248150d1 --- /dev/null +++ b/host/src/transport/pointer.rs @@ -0,0 +1,421 @@ +//! Decides when the pointer goes to an iPad that draws it (see +//! `capture::pointer`). Pure logic with an injected clock; the transport +//! loop feeds it the latest image and position and sends what it returns. +//! +//! - A position goes at most once a frame budget, the newest one, so a +//! 1000 Hz mouse costs no more than the video would. The budgets are a +//! fixed grid from the session's first send, not counted from the last +//! send: a send that ran late would otherwise delay every later one by +//! as much. One that waits for the next budget goes the moment it starts. +//! - The last position goes once more [`REPEAT_AFTER`] later, and every +//! heartbeat, so a lost one is replaced without the pointer moving. +//! - An image goes when it changes, when a session starts, with every IDR +//! the session asks for, and again when a report the iPad sent +//! [`SHAPE_RESEND_AFTER`] or more after it went still names another one. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use eternal_wire::v2::control::{PointerState, POINTER_VISIBLE}; + +use crate::capture::pointer::{Position, Shape}; + +pub const REPEAT_AFTER: Duration = Duration::from_millis(100); +/// Long enough for the image to reach the iPad before a report it sends, +/// so a report already on its way doesn't send the image twice. +pub const SHAPE_RESEND_AFTER: Duration = Duration::from_millis(250); + +#[derive(Debug, PartialEq, Eq)] +pub enum Send { + Shape(Arc), + State(PointerState), +} + +#[derive(Default)] +pub struct PointerSender { + session_id: Option, + /// The last state sent, without its time, and when. + sent: Option<(PointerState, Instant)>, + /// Where the grid of frame budgets starts. + epoch: Option, + repeat_at: Option, + /// The image sent last, and when. + shape_sent: Option<(u32, Instant)>, + shape_due: bool, + /// The image the iPad's last report said it holds, and when it came. + client_shape: Option<(u32, Instant)>, +} + +impl PointerSender { + /// The session the pointer goes to, or `None`. A new one gets + /// everything again. + pub fn serve(&mut self, session_id: Option) { + if self.session_id != session_id { + *self = Self { + session_id, + shape_due: true, + ..Self::default() + }; + } + } + + pub fn session_id(&self) -> Option { + self.session_id + } + + /// The session's IDR: send the image with it, and the position. + pub fn keyframe(&mut self, now: Instant) { + self.shape_due = true; + self.refresh(now); + } + + /// Send the current position again at the next poll. + pub fn refresh(&mut self, now: Instant) { + self.repeat_at = Some(now); + } + + pub fn report(&mut self, pointer_shape_id: u32, now: Instant) { + self.client_shape = Some((pointer_shape_id, now)); + } + + /// What to send now, the image before the position that names it. + pub fn poll( + &mut self, + now: Instant, + budget: Duration, + (shape, position): (Option>, Option), + ) -> Vec { + let mut out = Vec::new(); + if self.session_id.is_none() { + return out; + } + if let Some(shape) = shape.as_ref() { + let changed = self.shape_sent.is_none_or(|(id, _)| id != shape.id); + let lagging = match (self.client_shape, self.shape_sent) { + (Some((held, reported)), Some((_, sent))) => { + held != shape.id && reported >= sent + SHAPE_RESEND_AFTER + } + _ => false, + }; + if changed || lagging || self.shape_due { + self.shape_due = false; + self.shape_sent = Some((shape.id, now)); + out.push(Send::Shape(Arc::clone(shape))); + } + } + let Some(position) = position else { + return out; + }; + let state = PointerState { + flags: if position.visible { POINTER_VISIBLE } else { 0 }, + shape_id: shape.as_ref().map_or(0, |shape| shape.id), + x: position.x, + y: position.y, + host_time_us: 0, + }; + let epoch = *self.epoch.get_or_insert(now); + let slot = + |at: Instant| at.saturating_duration_since(epoch).as_nanos() / budget.as_nanos().max(1); + let send = match self.sent { + None => true, + Some((sent, at)) if sent != state => slot(now) > slot(at), + Some((_, at)) => self.repeat_at.is_some_and(|due| now >= due) && slot(now) > slot(at), + }; + if send { + // A repeat follows a change, not another repeat. + let changed = self.sent.is_none_or(|(sent, _)| sent != state); + self.repeat_at = changed.then_some(now + REPEAT_AFTER); + self.sent = Some((state, now)); + out.push(Send::State(PointerState { + host_time_us: position.host_time_us, + ..state + })); + } + out + } + + /// When [`Self::poll`] next has something to send without new input: + /// the end of the budget a newer position waits for, or a repeat. + pub fn next_poll( + &self, + budget: Duration, + latest: Option, + shape_id: Option, + ) -> Option { + self.session_id?; + let (sent, at) = self.sent?; + let epoch = self.epoch?; + let waiting = latest.is_some_and(|position| { + ( + position.visible, + position.x, + position.y, + shape_id.unwrap_or(0), + ) != ( + sent.flags & POINTER_VISIBLE != 0, + sent.x, + sent.y, + sent.shape_id, + ) + }); + let budget = budget.max(Duration::from_nanos(1)); + let slot = at.saturating_duration_since(epoch).as_nanos() / budget.as_nanos(); + let next_slot = epoch + budget * (slot as u32 + 1); + let budget_end = waiting.then_some(next_slot); + [budget_end, self.repeat_at.map(|due| due.max(next_slot))] + .into_iter() + .flatten() + .min() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capture::pointer::PointerImage; + + const BUDGET: Duration = Duration::from_micros(8_333); + + fn shape(id: u32) -> Arc { + Arc::new(Shape { + id, + image: PointerImage { + source_type: 2, + flags: 0, + width: 1, + height: 1, + hot: (0, 0), + bytes: vec![0; 4], + }, + }) + } + + fn at(x: i32) -> Position { + Position { + visible: true, + x, + y: 0, + host_time_us: x as u64, + } + } + + fn states(sent: &[Send]) -> Vec { + sent.iter() + .filter_map(|send| match send { + Send::State(state) => Some(state.x), + Send::Shape(_) => None, + }) + .collect() + } + + /// Windows' updates at irregular times: a burst 1 ms apart, then gaps of + /// 30 to 60 ms, as the #183 measurements fed the host. + #[test] + fn positions_go_at_most_once_a_budget_newest_first_and_none_is_lost_at_the_end() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + let mut updates = Vec::new(); + let mut t = 0u64; + let mut x = 0; + for gap in [30, 45, 60, 38] { + for _ in 0..4 { + updates.push((t, x)); + t += 1; + x += 1; + } + t += gap; + } + // Poll at every update and every millisecond between, as the loop's + // wakeups would. + let mut sent: Vec<(u64, i32)> = Vec::new(); + let mut latest = None; + let mut next = updates.iter().peekable(); + for ms in 0..=t + 200 { + while next.peek().is_some_and(|(at_ms, _)| *at_ms == ms) { + latest = Some(at(next.next().unwrap().1)); + } + let now = start + Duration::from_millis(ms); + for x in states(&sender.poll(now, BUDGET, (Some(shape(1)), latest))) { + sent.push((ms, x)); + } + } + let slot = |ms: u64| Duration::from_millis(ms).as_nanos() / BUDGET.as_nanos(); + for pair in sent.windows(2) { + assert!( + slot(pair[1].0) > slot(pair[0].0), + "two sends in one budget: {pair:?}" + ); + } + // Each burst's last position goes once its budget ends, not at the + // next burst; the burst's first went at once. + for burst in 0..4 { + let first = updates[burst * 4]; + let last = updates[burst * 4 + 3]; + assert!( + sent.contains(&first), + "{first:?} not sent at once: {sent:?}" + ); + let went = sent.iter().find(|s| s.1 == last.1).expect("burst end sent"); + assert!(went.0 - last.0 <= 9, "{last:?} waited until {went:?}"); + } + // The very last position goes once more after REPEAT_AFTER, then + // nothing until a heartbeat refresh. + let final_x = updates.last().unwrap().1; + assert_eq!( + sent.iter().filter(|s| s.1 == final_x).count(), + 2, + "{sent:?}" + ); + } + + /// A mouse reporting about once a budget, a little early or late each + /// time (a 125 Hz mouse at 120 fps). A late send must not delay the + /// sends after it: counted from the last send, every later position + /// waited for the lag of the latest one. + #[test] + fn positions_about_once_a_budget_go_without_a_growing_lag() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + let mut seed = 0x2545_F491u32; + let mut delays = Vec::new(); + let period_us = BUDGET.as_micros() as u64; + for k in 0..600u64 { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + let arrived = k * period_us + u64::from(seed % 1500); + let latest = Some(at(k as i32)); + // Poll at arrival, then every 100 us until it went. + let mut t = arrived; + loop { + let now = start + Duration::from_micros(t); + if states(&sender.poll(now, BUDGET, (Some(shape(1)), latest))).contains(&(k as i32)) + { + delays.push(t - arrived); + break; + } + t += 100; + assert!(t < arrived + 2 * period_us, "position {k} never went"); + } + } + delays.sort_unstable(); + let mean = delays.iter().sum::() / delays.len() as u64; + let p90 = delays[delays.len() * 9 / 10]; + assert!(mean < 1_000 && p90 < 2_000, "mean {mean} us, p90 {p90} us"); + } + + #[test] + fn a_still_pointer_sends_nothing_until_a_refresh() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + let latest = (Some(shape(1)), Some(at(5))); + assert_eq!(sender.poll(start, BUDGET, latest.clone()).len(), 2); + assert_eq!( + states(&sender.poll(start + REPEAT_AFTER, BUDGET, latest.clone())), + [5] + ); + for ms in 101..2000 { + let now = start + Duration::from_millis(ms); + assert!( + sender.poll(now, BUDGET, latest.clone()).is_empty(), + "{ms} ms" + ); + assert_eq!(sender.next_poll(BUDGET, latest.1, Some(1)), None); + } + let now = start + Duration::from_secs(2); + sender.refresh(now); + assert_eq!(sender.next_poll(BUDGET, latest.1, Some(1)), Some(now)); + assert_eq!(states(&sender.poll(now, BUDGET, latest)), [5]); + } + + #[test] + fn hiding_the_pointer_is_a_change_that_goes() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + sender.poll(start, BUDGET, (None, Some(at(5)))); + let hidden = Position { + visible: false, + ..at(5) + }; + assert_eq!( + sender.next_poll(BUDGET, Some(hidden), None), + Some(start + BUDGET) + ); + let sent = sender.poll(start + BUDGET, BUDGET, (None, Some(hidden))); + assert!(matches!(sent[..], [Send::State(state)] if state.flags & POINTER_VISIBLE == 0)); + } + + #[test] + fn images_go_on_change_session_start_keyframes_and_when_a_report_lags() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + let shapes = |sent: &[Send]| { + sent.iter() + .filter_map(|send| match send { + Send::Shape(shape) => Some(shape.id), + Send::State(_) => None, + }) + .collect::>() + }; + assert!( + sender + .poll(start, BUDGET, (Some(shape(1)), None)) + .is_empty(), + "no session" + ); + sender.serve(Some(7)); + assert_eq!( + shapes(&sender.poll(start, BUDGET, (Some(shape(1)), None))), + [1] + ); + assert!(sender + .poll(start, BUDGET, (Some(shape(1)), None)) + .is_empty()); + assert_eq!( + shapes(&sender.poll(start, BUDGET, (Some(shape(2)), None))), + [2] + ); + // A report sent before image 2 could arrive names image 1: no resend. + let soon = start + SHAPE_RESEND_AFTER - Duration::from_millis(1); + sender.report(1, soon); + assert!(sender.poll(soon, BUDGET, (Some(shape(2)), None)).is_empty()); + // A later one still names it: image 2 was lost on the way. + let later = start + SHAPE_RESEND_AFTER; + sender.report(1, later); + assert_eq!( + shapes(&sender.poll(later, BUDGET, (Some(shape(2)), None))), + [2] + ); + assert!( + sender + .poll(later, BUDGET, (Some(shape(2)), None)) + .is_empty(), + "once a report" + ); + let next = later + Duration::from_millis(500); + sender.report(2, next); + assert!(sender.poll(next, BUDGET, (Some(shape(2)), None)).is_empty()); + let much_later = later + Duration::from_secs(5); + assert!(sender + .poll(much_later, BUDGET, (Some(shape(2)), None)) + .is_empty()); + sender.keyframe(much_later); + assert_eq!( + shapes(&sender.poll(much_later, BUDGET, (Some(shape(2)), None))), + [2] + ); + // A new session gets the image and position again. + sender.serve(Some(8)); + let sent = sender.poll(much_later, BUDGET, (Some(shape(2)), Some(at(1)))); + assert_eq!(shapes(&sent), [2]); + assert_eq!(states(&sent), [1]); + sender.serve(None); + assert!(sender + .poll(much_later, BUDGET, (Some(shape(3)), Some(at(2)))) + .is_empty()); + } +} diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index b6ef5504..8b3d3993 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -458,6 +458,32 @@ impl Session { .map(|session| session.session_id) } + /// The session's id while its client draws the pointer itself. A + /// client keeps its HELLO2's answer for the whole session, so capture + /// never switches mid-session. + pub fn pointer_session_id(&self) -> Option { + self.active + .as_ref() + .filter(|session| { + session.info.feature_caps & eternal_wire::v2::control::FEATURE_DRAWS_POINTER != 0 + }) + .map(|session| session.session_id) + } + + /// `message` for the connected client, numbered like every other + /// control message the host sends it. + pub fn encode_for_client(&mut self, message: &ControlMessage) -> Option<(PeerId, Vec)> { + let (peer, session_id) = self + .active + .as_ref() + .map(|session| (session.peer, session.session_id))?; + let msg_seq = self.next_msg_seq(); + Some(( + peer, + eternal_wire::v2::control::encode_control(session_id, msg_seq, message), + )) + } + pub fn last_report(&self) -> Option { self.active.as_ref().and_then(|s| s.last_report) } @@ -1681,6 +1707,47 @@ mod tests { ); } + /// Only an iPad that said it draws the pointer gets it apart from the + /// video, numbered with the session's other control messages. An older + /// iPad keeps the pointer in the video. + #[test] + fn only_an_ipad_that_draws_the_pointer_gets_it_apart() { + use eternal_wire::v2::control::{parse_control, PointerState, FEATURE_DRAWS_POINTER}; + let mut session = Session::new(1234); + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + assert_eq!(session.pointer_session_id(), None, "nobody connected"); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + assert!(session.is_active()); + assert_eq!(session.pointer_session_id(), None, "an iPad before 0.3.5"); + + let ControlMessage::Hello2(mut draws) = hello(2, 50000) else { + unreachable!() + }; + draws.feature_caps = FEATURE_DRAWS_POINTER; + let actions = + session.handle_control_authed(peer, ControlMessage::Hello2(draws), &TestConfig, now); + let (ack, _) = parse_control(&actions.replies[0].1).unwrap(); + let id = session.pointer_session_id().expect("this iPad draws it"); + assert_eq!(Some(id), session.session_id()); + let state = ControlMessage::PointerState(PointerState::default()); + let (to, datagram) = session.encode_for_client(&state).unwrap(); + assert_eq!(to, peer); + let (header, parsed) = parse_control(&datagram).unwrap(); + assert_eq!(parsed, state); + assert_eq!(header.session_id, id); + assert_eq!(header.msg_seq, ack.msg_seq + 1); + + session.handle_control_authed( + peer, + ControlMessage::Bye(ByeReason::UserDisconnect), + &TestConfig, + now, + ); + assert_eq!(session.pointer_session_id(), None); + assert!(session.encode_for_client(&state).is_none()); + } + #[test] fn duplicate_nonce_gets_identical_ack_without_new_session() { let mut session = Session::new(1234); diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index fee915b7..f5d1c388 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -3490,3 +3490,176 @@ fn quiet_clients_keep_their_session_until_they_go_silent() { let _ = supervisor_tx.send(SupervisorCommand::Shutdown); let _ = supervisor.join(); } + +/// #183: an iPad that draws the pointer gets it apart from the video. Over a +/// still picture the pointer moves every frame while the host sends only an +/// IDR and its idle keepalive; an older iPad on the same host gets a frame +/// for every move and no pointer messages, as before. +#[test] +fn the_pointer_moves_without_video_frames_for_an_ipad_that_draws_it() { + use eternal_host::capture::synthetic::pointer_path; + use eternal_wire::v2::control::{FEATURE_DRAWS_POINTER, HOSTCAP_POINTER, POINTER_VISIBLE}; + let _env = test_env(); + init_test_tracing(); + let _ = ffmpeg_next::init(); + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_POINTER", "1"); + for name in ["ETERNAL_DROP", "ETERNAL_REORDER", "ETERNAL_JITTER_MS"] { + std::env::remove_var(name); + } + + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".to_string()); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor = { + let shared = shared.clone(); + let tx = supervisor_tx.clone(); + std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + shared, + GpuInfo::software_fallback(), + tx, + supervisor_rx, + ); + }) + }; + // Positions on the synthetic path, to check every state against. + let path: std::collections::HashSet<(i32, i32)> = (0..20_000) + .map(|step| pointer_path(step, SYNTH_W, SYNTH_H)) + .collect(); + const WINDOW: Duration = Duration::from_secs(3); + + for (feature_caps, nonce) in [(FEATURE_DRAWS_POINTER, 0xE2E0_0183u32), (0, 0xE2E0_0184)] { + let draws = feature_caps != 0; + let mut receiver = + FakeReceiver::connect_as(listen_port, CAP_DECODE_H264, feature_caps, 60, nonce); + assert_ne!( + receiver.host_caps & HOSTCAP_POINTER, + 0, + "this host sends the pointer" + ); + let mut reassembler = Reassembler::new(); + let mut datagram = [0u8; 2048]; + let (mut frames, mut highest, mut states) = (0u32, 0u32, Vec::new()); + let mut shape: Option<(u32, Vec, usize)> = None; + let mut held_shape = 0; + let mut window_start: Option = None; + let deadline = Instant::now() + DEADLINE; + while window_start.is_none_or(|at| at.elapsed() < WINDOW) { + assert!(Instant::now() < deadline, "no first frame (draws={draws})"); + if receiver.last_report.elapsed() >= Duration::from_millis(400) { + receiver.last_report = Instant::now(); + receiver.send(&ControlMessage::ReceiverReport(ReceiverReport { + highest_seq: highest, + frames_complete: frames, + pointer_shape_id: held_shape, + ..Default::default() + })); + } + let Ok((len, _)) = receiver.socket.recv_from(&mut datagram) else { + continue; + }; + let bytes = &datagram[..len]; + match classify(bytes) { + Classified::Media { .. } => { + let (header, payload) = MediaHeader::decode(bytes).unwrap(); + highest = highest.max(header.frame_seq); + let done = reassembler.add_fragment( + header.frame_seq, + header.frag_index, + header.frag_count, + header.stream_epoch, + payload, + Instant::now(), + ); + if matches!(done, AddOutcome::Completed(_)) { + // The window starts after the session's first frame. + if window_start.is_some() { + frames += 1; + } + window_start.get_or_insert_with(Instant::now); + } + } + Classified::Control(_) => match parse_control(bytes).map(|(_, m)| m) { + Ok(ControlMessage::PointerState(state)) if window_start.is_some() => { + assert_ne!(state.flags & POINTER_VISIBLE, 0); + states.push((state.x, state.y)); + } + Ok(ControlMessage::PointerState(_)) => {} + Ok(ControlMessage::PointerShape(chunk)) => { + let (id, bytes, got) = shape.get_or_insert_with(|| { + (chunk.shape_id, vec![0; chunk.total_len as usize], 0) + }); + if *id == chunk.shape_id && held_shape != chunk.shape_id { + let at = chunk.offset as usize; + bytes[at..at + chunk.data.len()].copy_from_slice(&chunk.data); + *got += chunk.data.len(); + if *got >= bytes.len() { + held_shape = chunk.shape_id; + assert_eq!((chunk.width, chunk.height), (12, 19), "the arrow"); + } + } + } + _ => {} + }, + other => panic!("unexpected datagram from host: {other:?}"), + } + } + let distinct: std::collections::HashSet<_> = states.iter().collect(); + let load = std::fs::read_to_string("/proc/loadavg").unwrap_or_default(); + eprintln!( + "draws={draws}: {frames} frames, {} pointer states ({} distinct) in {WINDOW:?} {load}", + states.len(), + distinct.len() + ); + if draws { + assert_ne!(held_shape, 0, "the pointer image arrived whole"); + // 60 fps for 3 s is 180 moves; leave room for a loaded runner. + assert!( + distinct.len() >= 60, + "{} distinct positions", + distinct.len() + ); + assert!( + states.iter().all(|p| path.contains(p)), + "states off the path" + ); + // The 750 ms keepalive, and nothing for the moves. + assert!( + frames <= 6, + "{frames} video frames while only the pointer moved" + ); + } else { + assert!( + states.is_empty() && shape.is_none(), + "an older iPad gets no pointer messages" + ); + assert!( + frames >= 60, + "only {frames} frames: each move must be a frame" + ); + } + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + let gone = Instant::now() + Duration::from_secs(5); + while shared.client_connected() { + assert!(Instant::now() < gone, "the session did not end after BYE"); + std::thread::sleep(Duration::from_millis(20)); + } + } + + shared.stop(); + supervisor_tx.send(SupervisorCommand::Shutdown).unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = supervisor.join(); + let _ = done_tx.send(()); + }); + done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("supervisor must shut down within 5s"); +} diff --git a/host/tests/gui_snapshots.rs b/host/tests/gui_snapshots.rs index 2e547602..e5dcc433 100644 --- a/host/tests/gui_snapshots.rs +++ b/host/tests/gui_snapshots.rs @@ -247,6 +247,7 @@ fn connect_app( nacks_sent: 10, audio_packets_lost: 0, audio_buffer_ms: 60, + pointer_shape_id: 0, }), &Fixture, now, diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 54023198..f1897f9d 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -96,9 +96,11 @@ final class ConnectionManager: ObservableObject { /// HELLO2 feature bits. Every build says it checks for its own updates, /// so the PC tells only older iPad apps to update. + /// It draws the PC's pointer itself, so pointer motion needs no video frame. nonisolated static func featureCaps(wantsInput: Bool, wantsAudio: Bool, isUSB: Bool) -> UInt16 { (wantsInput ? Hello2.featureWantsInput : 0) | (wantsAudio ? Hello2.featureWantsAudio : 0) | (isUSB ? 0 : Hello2.featureSupportsNack) | Hello2.featureChecksUpdates + | (E2E.drawsPointer ? Hello2.featureDrawsPointer : 0) } private var awaitingPairing = false private var explicitPairingToken: Data? @@ -181,6 +183,8 @@ final class ConnectionManager: ObservableObject { private var summarizedNonces: [UInt32] = [] let frameSlot = FrameSlot() + /// The PC's pointer, drawn by `MetalView` when the PC sends it apart. + let pointerOverlay = PointerOverlay() /// Lets hot-path closures reach the CURRENT control channel without /// retaining self (rebuilt each connect). private let controlChannelBox = ControlChannelBox() @@ -553,6 +557,8 @@ final class ConnectionManager: ObservableObject { receiver?.send(data) }) controlChannelBox.value = channel + pointerOverlay.reset() + channel.pointer = pointerOverlay assembler.rttUs = { [weak channel] in channel?.clockSnapshot.withLock { $0.rttUs } ?? 10_000 } assembler.onNack = { [weak channel] nack in channel?.sendNack(nack) } assembler.onNeedsKeyframe = { [weak channel] epoch, seq in @@ -566,6 +572,7 @@ final class ConnectionManager: ObservableObject { let stats = audio?.stats ?? AudioStats() report.audioPacketsLost = UInt32(clamping: stats.lost) report.audioBufferMs = UInt16(clamping: stats.bufferMs) + report.pointerShapeId = channel?.pointer?.completeShapeId ?? 0 return report } channel.onHelloAttempt = { [weak self] attempt, total in @@ -868,6 +875,9 @@ final class ConnectionManager: ObservableObject { // neither media nor heartbeats, so skipping this when nothing // arrived is exactly when the liveness watchdog is needed. self.refreshStatsAndWatchdog() + if E2E.enabled, self.state == .connected { + E2E.pointer(self.pointerOverlay.counters, decoded: self.debugState.decodedFrames) + } if ProcessInfo.processInfo.systemUptime - lastSummary >= 10 { lastSummary = ProcessInfo.processInfo.systemUptime self.recordStreamSummary(reason: "periodic") diff --git a/ios/EternalMonitor/Core/E2ESignpost.swift b/ios/EternalMonitor/Core/E2ESignpost.swift index 675d654b..72b5455a 100644 --- a/ios/EternalMonitor/Core/E2ESignpost.swift +++ b/ios/EternalMonitor/Core/E2ESignpost.swift @@ -33,6 +33,18 @@ enum E2E { #endif } + /// `EM_DRAWS_POINTER=0` leaves the pointer in the PC's video, as apps + /// before 0.3.5 did, for the end-to-end compatibility run. + static var drawsPointer: Bool { + ProcessInfo.processInfo.environment["EM_DRAWS_POINTER"] != "0" + } + + /// The pointer counters next to the decoded frame count: with the PC + /// sending the pointer apart, it moves while no frame decodes. + static func pointer(_ counters: PointerOverlay.Counters, decoded: Int) { + emit("E2E_POINTER states=\(counters.states) stale=\(counters.staleStates) shapes=\(counters.shapes) drawn=\(counters.drawn) moves=\(counters.drawnMoves) decoded=\(decoded)") + } + /// `EM_AUTOCONNECT=host:port` makes the app connect immediately on launch, /// bypassing the connect screen — the harness's way in. static var autoconnectTarget: (host: String, port: UInt16)? { diff --git a/ios/EternalMonitor/Network/ControlChannel.swift b/ios/EternalMonitor/Network/ControlChannel.swift index a65961e1..24e1d53e 100644 --- a/ios/EternalMonitor/Network/ControlChannel.swift +++ b/ios/EternalMonitor/Network/ControlChannel.swift @@ -34,6 +34,8 @@ final class ControlChannel { var onDiagnostic: ((String) -> Void)? /// Snapshot provider for periodic receiver reports. var reportProvider: (() -> ReceiverReport)? + /// Takes the PC's pointer when the PC sends it apart from the video. + var pointer: PointerOverlay? private let queue: DispatchQueue private let send: (Data) -> Void @@ -225,6 +227,14 @@ final class ControlChannel { // traffic. guard header.sessionId == sessionId, sessionId != 0 else { return } onStreamConfig?(config) + case .pointerShape(let chunk): + guard header.sessionId == sessionId, sessionId != 0, + hostCaps & HelloAck.hostCapPointer != 0 else { return } + pointer?.handle(chunk) + case .pointerState(let state): + guard header.sessionId == sessionId, sessionId != 0, + hostCaps & HelloAck.hostCapPointer != 0 else { return } + pointer?.handle(state, msgSeq: header.msgSeq) default: break } @@ -250,6 +260,7 @@ final class ControlChannel { sessionId = ack.sessionId hostCaps = ack.hostCaps + pointer?.reset() SupportDiagnostics.notePCName(ack.hostName) onDiagnostic?("Session \(ack.sessionId) established with \(ack.hostName)") lastHeartbeatAtUs.withLock { $0 = Self.clientNowUs() } diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index a48396fa..3b39ff81 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -29,6 +29,8 @@ enum Wire { case pong = 0x17 case streamConfig = 0x18 case nack = 0x19 + case pointerShape = 0x1A + case pointerState = 0x1B case inputEvent = 0x20 case error = 0x7F } @@ -255,6 +257,9 @@ struct Hello2: Equatable { static let featureSupportsNack: UInt16 = 1 << 2 /// This app checks for its own updates, so the PC need not tell the user. 0.3.1 and older never set it. static let featureChecksUpdates: UInt16 = 1 << 4 + /// This app draws the pointer from POINTER_SHAPE and POINTER_STATE, so + /// a PC with `HelloAck.hostCapPointer` leaves it out of the video. + static let featureDrawsPointer: UInt16 = 1 << 5 static let maxNameLength = 64 var protoMin: UInt8 = 2 @@ -293,6 +298,9 @@ struct HelloAck: Equatable { /// Media frame_seq counts the frames sent in a stream epoch without gaps, /// so a forward gap is a frame lost in transit. Older iPads ignore it. static let hostCapContiguousFrameSeq: UInt16 = 1 << 5 + /// The PC sends the pointer apart from the video to an iPad that said + /// it draws it. Older iPads ignore it. + static let hostCapPointer: UInt16 = 1 << 6 var status: HelloStatus var acceptedVersion: UInt8 var clientNonce: UInt32 @@ -362,6 +370,47 @@ struct ReceiverReport: Equatable { var nacksSent: UInt32 = 0 var audioPacketsLost: UInt32 = 0 var audioBufferMs: UInt16 = 0 + /// The last pointer image held complete, 0 for none. + var pointerShapeId: UInt32 = 0 +} + +/// One chunk of a pointer image (see the Rust `PointerShapeChunk`). The +/// image is `height` rows of `width` premultiplied BGRA pixels, then, with +/// `invertFlag`, `height` rows of `(width + 7) / 8` mask bytes, most +/// significant bit first: a set bit inverts the picture under that pixel. +struct PointerShapeChunk: Equatable { + static let version: UInt8 = 1 + static let invertFlag: UInt8 = 1 << 0 + static let maxSide: UInt16 = 512 + + var shapeId: UInt32 + var sourceType: UInt8 + var flags: UInt8 + var width: UInt16 + var height: UInt16 + var hotX: UInt16 + var hotY: UInt16 + var totalLen: UInt32 + var offset: UInt32 + var data: Data + + static func imageLength(width: UInt16, height: UInt16, flags: UInt8) -> Int { + let (w, h) = (Int(width), Int(height)) + return w * h * 4 + (flags & invertFlag != 0 ? (w + 7) / 8 * h : 0) + } +} + +/// Where the PC's pointer is: the image's top-left corner in video pixels. +struct PointerState: Equatable { + static let visibleFlag: UInt8 = 1 << 0 + + var flags: UInt8 + var shapeId: UInt32 + var x: Int32 + var y: Int32 + var hostTimeUs: UInt64 + + var visible: Bool { flags & Self.visibleFlag != 0 } } struct WirePing: Equatable { @@ -404,6 +453,8 @@ enum ControlMessage: Equatable { case pong(WirePong) case streamConfig(StreamConfig) case inputEvent(WireInputEvent) + case pointerShape(PointerShapeChunk) + case pointerState(PointerState) var packetType: Wire.PacketType { switch self { @@ -418,6 +469,8 @@ enum ControlMessage: Equatable { case .pong: return .pong case .streamConfig: return .streamConfig case .inputEvent: return .inputEvent + case .pointerShape: return .pointerShape + case .pointerState: return .pointerState } } } @@ -496,12 +549,15 @@ extension Wire { body.appendLE(r.e2eLatencyMsX10) body.appendLE(r.rttMsX10) if r.fragsRepaired != 0 || r.nacksSent != 0 - || r.audioPacketsLost != 0 || r.audioBufferMs != 0 { + || r.audioPacketsLost != 0 || r.audioBufferMs != 0 || r.pointerShapeId != 0 { body.appendLE(r.fragsRepaired) body.appendLE(r.nacksSent) body.appendLE(r.audioPacketsLost) body.appendLE(r.audioBufferMs) } + if r.pointerShapeId != 0 { + body.appendLE(r.pointerShapeId) + } case .ping(let p): body.appendLE(p.t1Us) case .pong(let p): @@ -529,6 +585,28 @@ extension Wire { body.appendLE(UInt16(bitPattern: e.tiltX)) body.appendLE(UInt16(bitPattern: e.tiltY)) } + case .pointerShape(let c): + body.append(PointerShapeChunk.version) + body.append(c.sourceType) + body.append(c.flags) + body.append(0) // reserved + body.appendLE(c.shapeId) + body.appendLE(c.width) + body.appendLE(c.height) + body.appendLE(c.hotX) + body.appendLE(c.hotY) + body.appendLE(c.totalLen) + body.appendLE(c.offset) + body.appendLE(UInt16(c.data.count)) + body.append(c.data) + case .pointerState(let p): + body.append(PointerShapeChunk.version) + body.append(p.flags) + body.appendLE(UInt16(0)) // reserved + body.appendLE(p.shapeId) + body.appendLE(UInt32(bitPattern: p.x)) + body.appendLE(UInt32(bitPattern: p.y)) + body.appendLE(p.hostTimeUs) } var out = Data(capacity: ControlHeader.size + body.count) @@ -609,6 +687,16 @@ extension Wire { message = .streamConfig(cfg) case .inputEvent: message = parseInputEvent(&r) + case .pointerShape: + message = parsePointerShape(&r) + case .pointerState: + guard r.readU8() == PointerShapeChunk.version, + let flags = r.readU8(), let _ = r.readU16(), + let shapeId = r.readU32(), let x = r.readU32(), let y = r.readU32(), + let time = r.readU64() else { return nil } + message = .pointerState(PointerState( + flags: flags, shapeId: shapeId, x: Int32(bitPattern: x), y: Int32(bitPattern: y), + hostTimeUs: time)) case .media, .mediaFec, .audio, .error: return nil } @@ -699,9 +787,34 @@ extension Wire { report.audioPacketsLost = audioLost report.audioBufferMs = audioBuffer } + if r.remaining >= 4, let shape = r.readU32() { + report.pointerShapeId = shape + } return .receiverReport(report) } + /// Enforces what the Rust parser does: a nonzero id, a size within the + /// cap, the length that size implies, and a chunk inside it. + private static func parsePointerShape(_ r: inout WireReader) -> ControlMessage? { + guard r.readU8() == PointerShapeChunk.version, + let sourceType = r.readU8(), let flags = r.readU8(), let _ = r.readU8(), + let shapeId = r.readU32(), shapeId != 0, + let width = r.readU16(), let height = r.readU16(), + (1...PointerShapeChunk.maxSide).contains(width), + (1...PointerShapeChunk.maxSide).contains(height), + let hotX = r.readU16(), let hotY = r.readU16(), + let totalLen = r.readU32(), + Int(totalLen) == PointerShapeChunk.imageLength(width: width, height: height, flags: flags), + let offset = r.readU32(), + let dataLen = r.readU16(), dataLen > 0, + Int(offset) + Int(dataLen) <= Int(totalLen), + let data = r.take(Int(dataLen)) + else { return nil } + return .pointerShape(PointerShapeChunk( + shapeId: shapeId, sourceType: sourceType, flags: flags, width: width, height: height, + hotX: hotX, hotY: hotY, totalLen: totalLen, offset: offset, data: Data(data))) + } + private static func parseInputEvent(_ r: inout WireReader) -> ControlMessage? { guard let inputVer = r.readU8(), let kind = r.readU8(), diff --git a/ios/EternalMonitor/Render/MetalView.swift b/ios/EternalMonitor/Render/MetalView.swift index b20cce4a..bb11da5a 100644 --- a/ios/EternalMonitor/Render/MetalView.swift +++ b/ios/EternalMonitor/Render/MetalView.swift @@ -31,6 +31,7 @@ struct MetalView: UIViewRepresentable { renderer.pictureFit = settings.pictureFit context.coordinator.renderer = renderer context.coordinator.frameSlot = connectionManager.frameSlot + context.coordinator.pointer = connectionManager.pointerOverlay context.coordinator.attach(view: view) view.delegate = context.coordinator @@ -59,6 +60,8 @@ struct MetalView: UIViewRepresentable { var renderer: MetalRenderer? /// Direct reference to the shared FrameSlot — avoids crossing @MainActor boundary on the render thread. var frameSlot: FrameSlot? + /// The PC's pointer when this app draws it; each move redraws. + var pointer: PointerOverlay? private weak var view: MTKView? private var memoryWarning: NSObjectProtocol? @@ -71,6 +74,9 @@ struct MetalView: UIViewRepresentable { frameSlot?.onFrameStored = { [weak self] in self?.scheduleRedraw() } + pointer?.onChange = { [weak self] in + self?.scheduleRedraw() + } // The next decoded frame redraws within a frame period, or the // PC's idle keepalive; the screen keeps showing the last one. memoryWarning = NotificationCenter.default.addObserver( @@ -83,6 +89,7 @@ struct MetalView: UIViewRepresentable { func detach() { frameSlot?.onFrameStored = nil + pointer?.onChange = nil view = nil if let memoryWarning { NotificationCenter.default.removeObserver(memoryWarning) } memoryWarning = nil @@ -111,7 +118,7 @@ struct MetalView: UIViewRepresentable { func draw(in view: MTKView) { guard let renderer else { return } let pixelBuffer = frameSlot?.take() - renderer.draw(in: view, pixelBuffer: pixelBuffer) + renderer.draw(in: view, pixelBuffer: pixelBuffer, pointer: pointer) } } } @@ -152,6 +159,17 @@ private final class InFlightFrame { deinit { frame = nil } } +private struct PointerUniforms { + var rect: SIMD4 +} + +/// The current pointer image's textures, made once per image. +private struct PointerTextures { + let id: UInt32 + let image: MTLTexture + let invertMask: MTLTexture? +} + private struct DisplayUniforms { var scale: SIMD2 var matrixIndex: UInt32 @@ -163,6 +181,9 @@ final class MetalRenderer { private let device: MTLDevice private let commandQueue: MTLCommandQueue private let pipelineState: MTLRenderPipelineState + private let pointerPipeline: MTLRenderPipelineState + private let pointerInvertPipeline: MTLRenderPipelineState + private var pointerTextures: PointerTextures? private(set) var textureCache: CVMetalTextureCache? private var currentFrame: FrameTextures? /// Read on the main thread, where the paused MTKView draws. @@ -196,6 +217,10 @@ final class MetalRenderer { do { pipelineState = try device.makeRenderPipelineState(descriptor: pipelineDescriptor) + pointerPipeline = try device.makeRenderPipelineState( + descriptor: Self.pointerDescriptor(library, fragment: "pointer_fragment", view: view, invert: false)) + pointerInvertPipeline = try device.makeRenderPipelineState( + descriptor: Self.pointerDescriptor(library, fragment: "pointer_invert_fragment", view: view, invert: true)) } catch { fatalError("Failed to create pipeline state: \(error)") } @@ -209,7 +234,27 @@ final class MetalRenderer { // The held frame's textures go before the cache that made them. deinit { currentFrame = nil } - func draw(in view: MTKView, pixelBuffer: CVPixelBuffer?) { + /// Over: source + destination x (1 - source alpha). Invert: source + /// x (1 - destination) + destination x (1 - source), so white inverts + /// and black leaves the picture alone. + private static func pointerDescriptor(_ library: MTLLibrary, fragment: String, view: MTKView, + invert: Bool) -> MTLRenderPipelineDescriptor { + let descriptor = MTLRenderPipelineDescriptor() + descriptor.vertexFunction = library.makeFunction(name: "pointer_vertex") + descriptor.fragmentFunction = library.makeFunction(name: fragment) + let color = descriptor.colorAttachments[0]! + color.pixelFormat = view.colorPixelFormat + color.isBlendingEnabled = true + color.rgbBlendOperation = .add + color.alphaBlendOperation = .add + color.sourceRGBBlendFactor = invert ? .oneMinusDestinationColor : .one + color.destinationRGBBlendFactor = invert ? .oneMinusSourceColor : .oneMinusSourceAlpha + color.sourceAlphaBlendFactor = .zero + color.destinationAlphaBlendFactor = .one + return descriptor + } + + func draw(in view: MTKView, pixelBuffer: CVPixelBuffer?, pointer: PointerOverlay? = nil) { if let pixelBuffer, let frame = makeFrameTextures(from: pixelBuffer) { currentFrame = frame // Let the cache recycle textures released by completed draws. @@ -243,6 +288,10 @@ final class MetalRenderer { encoder.setFragmentTexture(frame.luma, index: 0) encoder.setFragmentTexture(frame.chroma, index: 1) encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4) + if let pointer { + drawPointer(pointer, encoder: encoder, video: CGSize(width: frame.width, height: frame.height), + drawable: view.drawableSize) + } encoder.endEncoding() // Keep the pixel buffer + texture wrappers alive until the GPU is done @@ -262,6 +311,44 @@ final class MetalRenderer { commandBuffer.commit() } + private func drawPointer(_ pointer: PointerOverlay, encoder: MTLRenderCommandEncoder, video: CGSize, + drawable: CGSize) { + let snapshot = pointer.snapshot() + guard snapshot.visible, let image = snapshot.image, let textures = textures(for: image) else { return } + let rect = PointerGeometry.rect(x: snapshot.x, y: snapshot.y, width: image.width, height: image.height, + video: video, view: drawable, fit: pictureFit) + var uniforms = PointerUniforms(rect: PointerGeometry.ndc(rect, view: drawable)) + encoder.setRenderPipelineState(pointerPipeline) + encoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 0) + encoder.setFragmentTexture(textures.image, index: 0) + encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4) + if let mask = textures.invertMask { + encoder.setRenderPipelineState(pointerInvertPipeline) + encoder.setFragmentTexture(mask, index: 0) + encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4) + } + pointer.noteDrawn(x: snapshot.x, y: snapshot.y) + } + + private func textures(for image: PointerOverlay.Image) -> PointerTextures? { + if let cached = pointerTextures, cached.id == image.id { return cached } + func texture(_ format: MTLPixelFormat, bytes: Data, bytesPerPixel: Int) -> MTLTexture? { + let descriptor = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: format, width: image.width, height: image.height, mipmapped: false) + descriptor.usage = .shaderRead + guard let texture = device.makeTexture(descriptor: descriptor) else { return nil } + bytes.withUnsafeBytes { raw in + texture.replace(region: MTLRegionMake2D(0, 0, image.width, image.height), mipmapLevel: 0, + withBytes: raw.baseAddress!, bytesPerRow: image.width * bytesPerPixel) + } + return texture + } + guard let color = texture(.bgra8Unorm, bytes: image.pixels, bytesPerPixel: 4) else { return nil } + let mask = image.invertMask.flatMap { texture(.r8Unorm, bytes: $0, bytesPerPixel: 1) } + pointerTextures = PointerTextures(id: image.id, image: color, invertMask: mask) + return pointerTextures + } + /// Whether a decoded frame is held for redraws (resize, rotation). var holdsFrame: Bool { currentFrame != nil } diff --git a/ios/EternalMonitor/Render/PointerOverlay.swift b/ios/EternalMonitor/Render/PointerOverlay.swift new file mode 100644 index 00000000..3494e274 --- /dev/null +++ b/ios/EternalMonitor/Render/PointerOverlay.swift @@ -0,0 +1,179 @@ +import CoreGraphics +import Foundation +import os + +/// The PC's pointer, which this app draws over the video when the PC sends +/// it apart (this app sets `Hello2.featureDrawsPointer`, the PC answers +/// with `HelloAck.hostCapPointer`). Pointer motion then never waits for a +/// video frame. Messages arrive on the control queue; the renderer reads a +/// snapshot on the main thread. +final class PointerOverlay { + /// A complete pointer image: premultiplied BGRA rows, and one byte per + /// pixel (0 or 255) where the picture under it is inverted. + struct Image: Equatable { + let id: UInt32 + let width: Int + let height: Int + let pixels: Data + let invertMask: Data? + } + + struct Snapshot: Equatable { + var image: Image? + var x: Int32 = 0 + var y: Int32 = 0 + var visible = false + } + + struct Counters: Equatable { + var states = 0 + var staleStates = 0 + var shapes = 0 + /// Draws with the pointer, and how many of them moved it. + var drawn = 0 + var drawnMoves = 0 + var lastDrawn: (x: Int32, y: Int32)? + + static func == (a: Counters, b: Counters) -> Bool { + (a.states, a.staleStates, a.shapes, a.drawn, a.drawnMoves) + == (b.states, b.staleStates, b.shapes, b.drawn, b.drawnMoves) + } + } + + /// A position applied or an image completed: redraw. Called on the + /// control queue. + var onChange: (() -> Void)? + + private struct Assembly { + let chunk: PointerShapeChunk + var bytes: [UInt8] + var offsets: Set = [] + var received = 0 + } + + private struct State { + var assembly: Assembly? + var snapshot = Snapshot() + var lastSeq: UInt32? + var counters = Counters() + } + + private let state = OSAllocatedUnfairLock(initialState: State()) + + /// A new session: nothing from the last one applies. + func reset() { + state.withLock { $0 = State() } + onChange?() + } + + /// The image the receiver reports name, so the PC sends it again when + /// a chunk was lost. + var completeShapeId: UInt32 { state.withLock { $0.snapshot.image?.id ?? 0 } } + + var counters: Counters { state.withLock { $0.counters } } + + func snapshot() -> Snapshot { state.withLock { $0.snapshot } } + + /// The renderer drew the pointer here. + func noteDrawn(x: Int32, y: Int32) { + state.withLock { s in + s.counters.drawn += 1 + if let last = s.counters.lastDrawn, last != (x, y) { s.counters.drawnMoves += 1 } + s.counters.lastDrawn = (x, y) + } + } + + /// Latest wins by the control header's `msgSeq`, which counts every + /// control message the PC sends. + func handle(_ pointer: PointerState, msgSeq: UInt32) { + let applied = state.withLock { s -> Bool in + if let last = s.lastSeq, Int32(bitPattern: msgSeq &- last) <= 0 { + s.counters.staleStates += 1 + return false + } + s.lastSeq = msgSeq + s.counters.states += 1 + s.snapshot.x = pointer.x + s.snapshot.y = pointer.y + s.snapshot.visible = pointer.visible + return true + } + if applied { onChange?() } + } + + func handle(_ chunk: PointerShapeChunk) { + let completed = state.withLock { s -> Bool in + if s.snapshot.image?.id == chunk.shapeId { return false } + if s.assembly?.chunk.shapeId != chunk.shapeId { + // A late chunk of an older image doesn't replace a newer one. + if let pending = s.assembly, Int32(bitPattern: chunk.shapeId &- pending.chunk.shapeId) < 0 { + return false + } + s.assembly = Assembly(chunk: chunk, bytes: [UInt8](repeating: 0, count: Int(chunk.totalLen))) + } + guard var assembly = s.assembly, + assembly.chunk.width == chunk.width, assembly.chunk.height == chunk.height, + assembly.chunk.flags == chunk.flags, + !assembly.offsets.contains(chunk.offset) + else { return false } + let start = Int(chunk.offset) + assembly.bytes.replaceSubrange(start..= assembly.bytes.count else { + s.assembly = assembly + return false + } + s.assembly = nil + s.snapshot.image = Self.image(assembly.chunk, bytes: assembly.bytes) + s.counters.shapes += 1 + return true + } + if completed { onChange?() } + } + + static func image(_ chunk: PointerShapeChunk, bytes: [UInt8]) -> Image { + let (w, h) = (Int(chunk.width), Int(chunk.height)) + let pixels = Data(bytes[0..> (x % 8)) != 0 { + expanded[y * w + x] = 255 + } + } + mask = expanded + } + return Image(id: chunk.shapeId, width: w, height: h, pixels: pixels, invertMask: mask) + } +} + +/// Where the pointer is drawn, from the same content rect the touch relay +/// normalizes against (`ContentRectMapper`), so a tap on the drawn hotspot +/// lands where the PC has the pointer. Pure and unit-tested. +enum PointerGeometry { + /// The pointer image's rect in view coordinates. `x`/`y` and the image + /// size are video pixels; the image scales with the picture. + static func rect(x: Int32, y: Int32, width: Int, height: Int, video: CGSize, view: CGSize, + fit: PictureFit) -> CGRect { + let content = ContentRectMapper(viewSize: view, videoSize: video, fit: fit).contentRect + guard content.width > 0, content.height > 0 else { return .zero } + let sx = content.width / video.width + let sy = content.height / video.height + return CGRect(x: content.minX + CGFloat(x) * sx, y: content.minY + CGFloat(y) * sy, + width: CGFloat(width) * sx, height: CGFloat(height) * sy) + } + + /// `rect` in Metal's normalized device coordinates: (minX, minY, maxX, + /// maxY) with y up, as the renderer's quad takes it. + static func ndc(_ rect: CGRect, view: CGSize) -> SIMD4 { + guard view.width > 0, view.height > 0 else { return .zero } + let x0 = rect.minX / view.width * 2 - 1 + let x1 = rect.maxX / view.width * 2 - 1 + let y0 = 1 - rect.maxY / view.height * 2 + let y1 = 1 - rect.minY / view.height * 2 + return SIMD4(Float(x0), Float(y0), Float(x1), Float(y1)) + } +} diff --git a/ios/EternalMonitor/Render/Shaders.metal b/ios/EternalMonitor/Render/Shaders.metal index 35d6b7ee..b035a183 100644 --- a/ios/EternalMonitor/Render/Shaders.metal +++ b/ios/EternalMonitor/Render/Shaders.metal @@ -72,3 +72,43 @@ fragment half4 display_fragment(VertexOut in [[stage_in]], return half4(half3(saturate(rgb)), 1.0h); } + +// The PC's pointer, drawn over the video (PointerOverlay.swift). `rect` is +// the image's quad in normalized device coordinates: minX, minY, maxX, maxY. +struct PointerUniforms { + float4 rect; +}; + +vertex VertexOut pointer_vertex(uint vid [[vertex_id]], + constant PointerUniforms &uniforms [[buffer(0)]]) { + // Texture coordinates of the strip's corners: bottom-left, bottom-right, + // top-left, top-right (texture rows run downward). + const float2 corners[4] = { + float2(0.0, 1.0), + float2(1.0, 1.0), + float2(0.0, 0.0), + float2(1.0, 0.0), + }; + float2 t = corners[vid]; + VertexOut out; + out.position = float4(mix(uniforms.rect.x, uniforms.rect.z, t.x), + mix(uniforms.rect.w, uniforms.rect.y, t.y), 0.0, 1.0); + out.texCoord = t; + return out; +} + +// Premultiplied BGRA, blended "over" the picture. +fragment half4 pointer_fragment(VertexOut in [[stage_in]], + texture2d image [[texture(0)]]) { + constexpr sampler s(filter::linear, address::clamp_to_edge); + return half4(image.sample(s, in.texCoord)); +} + +// The invert mask as white: the pipeline's blend (1 - destination) * source +// + destination * (1 - source) turns it into "invert the picture here". +fragment half4 pointer_invert_fragment(VertexOut in [[stage_in]], + texture2d mask [[texture(0)]]) { + constexpr sampler s(filter::linear, address::clamp_to_edge); + float m = mask.sample(s, in.texCoord).r; + return half4(m, m, m, m); +} diff --git a/ios/EternalMonitorTests/PointerOverlayTests.swift b/ios/EternalMonitorTests/PointerOverlayTests.swift new file mode 100644 index 00000000..cf92cece --- /dev/null +++ b/ios/EternalMonitorTests/PointerOverlayTests.swift @@ -0,0 +1,192 @@ +import XCTest +import os +@testable import EternalMonitor + +final class PointerOverlayTests: XCTestCase { + /// A 40x40 color image split as the PC splits it (1358 bytes a chunk). + private func chunks(id: UInt32, side: UInt16 = 40, fill: UInt8 = 7) -> [PointerShapeChunk] { + let total = PointerShapeChunk.imageLength(width: side, height: side, flags: 0) + let bytes = Data(repeating: fill, count: total) + return stride(from: 0, to: total, by: 1358).map { offset in + PointerShapeChunk(shapeId: id, sourceType: 2, flags: 0, width: side, height: side, hotX: 0, hotY: 0, + totalLen: UInt32(total), offset: UInt32(offset), + data: bytes.subdata(in: offset.. PointerState { + PointerState(flags: visible ? PointerState.visibleFlag : 0, shapeId: shape, x: x, y: 2, hostTimeUs: 0) + } + + func testAnImageCompletesFromChunksInAnyOrderOnce() { + let overlay = PointerOverlay() + var changes = 0 + overlay.onChange = { changes += 1 } + let parts = chunks(id: 1) + XCTAssertEqual(parts.count, 5) + for chunk in parts.dropFirst().reversed() { overlay.handle(chunk) } + overlay.handle(parts[4]) // a repeat adds nothing + XCTAssertNil(overlay.snapshot().image, "one chunk still missing") + XCTAssertEqual(overlay.completeShapeId, 0) + overlay.handle(parts[0]) + let image = try? XCTUnwrap(overlay.snapshot().image) + XCTAssertEqual(image?.id, 1) + XCTAssertEqual(image?.pixels.count, 40 * 40 * 4) + XCTAssertNil(image?.invertMask) + XCTAssertEqual(overlay.completeShapeId, 1) + XCTAssertEqual(changes, 1) + // The PC's resend of a held image changes nothing. + for chunk in parts { overlay.handle(chunk) } + XCTAssertEqual(overlay.counters.shapes, 1) + } + + func testANewImageReplacesTheOldButALateOlderChunkDoesNot() { + let overlay = PointerOverlay() + for chunk in chunks(id: 1) { overlay.handle(chunk) } + let newer = chunks(id: 2, fill: 9) + overlay.handle(newer[0]) + overlay.handle(chunks(id: 1)[1]) // a held image's late chunk + XCTAssertEqual(overlay.snapshot().image?.id, 1, "the old image stays until the new one is whole") + for chunk in newer.dropFirst() { overlay.handle(chunk) } + XCTAssertEqual(overlay.snapshot().image?.id, 2) + XCTAssertEqual(overlay.snapshot().image?.pixels.first, 9) + let third = chunks(id: 4) + overlay.handle(third[0]) + overlay.handle(chunks(id: 3)[0]) // older than the one assembling + for chunk in third.dropFirst() { overlay.handle(chunk) } + XCTAssertEqual(overlay.snapshot().image?.id, 4) + } + + func testThePositionIsLatestWinsByMessageNumber() { + let overlay = PointerOverlay() + overlay.handle(state(10), msgSeq: 5) + overlay.handle(state(9), msgSeq: 4) // reordered on the way + XCTAssertEqual(overlay.snapshot().x, 10) + overlay.handle(state(11, visible: false), msgSeq: 6) + XCTAssertFalse(overlay.snapshot().visible) + overlay.handle(state(12), msgSeq: 6 &+ 0x7FFF_FFFF) + XCTAssertEqual(overlay.snapshot().x, 12, "numbers compare across a wrap") + XCTAssertEqual(overlay.counters.states, 3) + XCTAssertEqual(overlay.counters.staleStates, 1) + overlay.reset() + overlay.handle(state(1), msgSeq: 1) + XCTAssertEqual(overlay.snapshot().x, 1, "a new session numbers from 1") + XCTAssertNil(overlay.snapshot().image) + } + + func testTheInvertMaskExpandsToABytePerPixel() { + // 9x1: two mask bytes per row, bits 0 and 8 set. + let flags = PointerShapeChunk.invertFlag + let total = PointerShapeChunk.imageLength(width: 9, height: 1, flags: flags) + XCTAssertEqual(total, 9 * 4 + 2) + var bytes = [UInt8](repeating: 0, count: total) + bytes[36] = 0x80 + bytes[37] = 0x80 + let chunk = PointerShapeChunk(shapeId: 1, sourceType: 1, flags: flags, width: 9, height: 1, hotX: 0, + hotY: 0, totalLen: UInt32(total), offset: 0, data: Data(bytes)) + let mask = [UInt8](PointerOverlay.image(chunk, bytes: bytes).invertMask ?? Data()) + XCTAssertEqual(mask, [255, 0, 0, 0, 0, 0, 0, 0, 255]) + } + + // MARK: Geometry + + /// Taps on the drawn hotspot normalize to the hotspot's video pixel, in + /// Fit and Fill, for a wider and a taller picture. + func testThePointerIsDrawnWhereTouchesMapTo() { + let view = CGSize(width: 1194, height: 834) + for video in [CGSize(width: 3440, height: 1440), CGSize(width: 1080, height: 1920)] { + for fit in [PictureFit.fit, .fill] { + let (x, y, hotX, hotY): (Int32, Int32, CGFloat, CGFloat) = (500, 300, 3, 5) + let rect = PointerGeometry.rect(x: x, y: y, width: 32, height: 32, video: video, view: view, fit: fit) + let mapper = ContentRectMapper(viewSize: view, videoSize: video, fit: fit) + let content = mapper.contentRect + XCTAssertEqual(rect.width, 32 * content.width / video.width, accuracy: 1e-9) + let hot = CGPoint(x: rect.minX + hotX * content.width / video.width, + y: rect.minY + hotY * content.height / video.height) + guard let normalized = mapper.normalize(hot) else { + // Fill trims edges: this pointer may be off screen. + XCTAssertFalse(mapper.visibleRect.contains(hot), "\(video) \(fit)") + continue + } + let expectX = (CGFloat(x) + hotX) / video.width * 65535 + let expectY = (CGFloat(y) + hotY) / video.height * 65535 + XCTAssertEqual(Double(normalized.x), Double(expectX), accuracy: 2, "\(video) \(fit)") + XCTAssertEqual(Double(normalized.y), Double(expectY), accuracy: 2, "\(video) \(fit)") + } + } + } + + /// A pointer image the size of the video covers exactly the video's + /// quad, which the renderer scales by `MetalRenderer.scale`. + func testTheQuadMatchesTheVideoQuad() { + let view = CGSize(width: 2388, height: 1668) + for video in [CGSize(width: 3440, height: 1440), CGSize(width: 1080, height: 1920)] { + for fit in [PictureFit.fit, .fill] { + let rect = PointerGeometry.rect(x: 0, y: 0, width: Int(video.width), height: Int(video.height), + video: video, view: view, fit: fit) + let ndc = PointerGeometry.ndc(rect, view: view) + let scale = MetalRenderer.scale(fit, video: video, drawable: view) + XCTAssertEqual(ndc.x, -scale.x, accuracy: 1e-4) + XCTAssertEqual(ndc.z, scale.x, accuracy: 1e-4) + XCTAssertEqual(ndc.y, -scale.y, accuracy: 1e-4) + XCTAssertEqual(ndc.w, scale.y, accuracy: 1e-4) + } + } + // Partly off the left edge: negative positions keep their offset. + let rect = PointerGeometry.rect(x: -4, y: 0, width: 8, height: 8, video: CGSize(width: 1000, height: 500), + view: CGSize(width: 1000, height: 500), fit: .fit) + XCTAssertEqual(rect.minX, -4) + XCTAssertEqual(PointerGeometry.rect(x: 0, y: 0, width: 8, height: 8, video: .zero, view: view, fit: .fit), .zero) + } + + // MARK: Channel + + /// The channel hands pointer messages on only for its session, and only + /// from a PC that said it sends the pointer. + func testPointerMessagesNeedTheSessionAndTheCapability() async throws { + for hostCaps in [UInt16(0), HelloAck.hostCapPointer] { + let queue = DispatchQueue(label: "pointer-channel-test") + let (stream, continuation) = AsyncStream.makeStream() + let channel = ControlChannel(queue: queue) { continuation.yield($0) } + let overlay = PointerOverlay() + channel.pointer = overlay + channel.startHandshake(listenPort: 50000, identity: ControlChannel.ClientIdentity( + deviceName: "Test iPad", screenPxW: 640, screenPxH: 360, screenPtW: 640, screenPtH: 360, + refreshHz: 60)) + var messages = stream.makeAsyncIterator() + let first = await messages.next() + guard case .hello2(let hello) = try XCTUnwrap(Wire.parseControl(XCTUnwrap(first))).message else { + return XCTFail("missing handshake") + } + queue.sync { + channel.handleControl(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .helloAck(HelloAck( + status: .ok, acceptedVersion: 2, clientNonce: hello.clientNonce, sessionId: 41, + heartbeatIntervalMs: 1000, reportIntervalMs: 500, livenessTimeoutMs: 3000, + streamConfig: StreamConfig(), hostName: "Test PC", authToken: Data(17...32), + hostCaps: hostCaps)))) + channel.handleControl(Wire.encodeControl(sessionId: 40, msgSeq: 2, message: .pointerState(state(1)))) + for chunk in chunks(id: 1) { + channel.handleControl(Wire.encodeControl(sessionId: 41, msgSeq: 3, message: .pointerShape(chunk))) + } + channel.handleControl(Wire.encodeControl(sessionId: 41, msgSeq: 4, message: .pointerState(state(7)))) + } + channel.stop() + continuation.finish() + let snapshot = overlay.snapshot() + if hostCaps == 0 { + XCTAssertEqual(snapshot, PointerOverlay.Snapshot(), "a PC without the bit gets no pointer drawn") + } else { + XCTAssertEqual(snapshot.x, 7, "another session's state is ignored") + XCTAssertEqual(snapshot.image?.id, 1) + XCTAssertEqual(overlay.counters.states, 1) + } + } + } + + func testThisAppAsksToDrawThePointer() { + let caps = ConnectionManager.featureCaps(wantsInput: true, wantsAudio: false, isUSB: false) + XCTAssertNotEqual(caps & Hello2.featureDrawsPointer, 0) + XCTAssertEqual(Hello2.featureDrawsPointer & (Hello2.featureWantsInput | Hello2.featureWantsAudio + | Hello2.featureSupportsNack | Hello2.featureChecksUpdates), 0) + } +} diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index 56b742d2..bbc8d610 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,7 +35,7 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 29, + Self.vectors.count, 35, "golden vector count drifted — update both test suites together" ) } @@ -243,6 +243,94 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(legacy.audioBufferMs, 0) } + // MARK: Pointer + + func testPointerCapabilityVectors() throws { + XCTAssertEqual(Hello2.featureDrawsPointer, 0x0020) + XCTAssertEqual(HelloAck.hostCapPointer, 0x0040) + let (_, hello) = try XCTUnwrap(Wire.parseControl(try vector("hello2_draws_pointer"))) + guard case .hello2(let h) = hello else { return XCTFail("wrong type") } + XCTAssertNotEqual(h.featureCaps & Hello2.featureDrawsPointer, 0) + let (_, ack) = try XCTUnwrap(Wire.parseControl(try vector("hello_ack_pointer"))) + guard case .helloAck(let a) = ack else { return XCTFail("wrong type") } + XCTAssertNotEqual(a.hostCaps & HelloAck.hostCapPointer, 0) + let (_, old) = try XCTUnwrap(Wire.parseControl(try vector("hello_ack_paired"))) + guard case .helloAck(let legacy) = old else { return XCTFail("wrong type") } + XCTAssertEqual(legacy.hostCaps & HelloAck.hostCapPointer, 0) + } + + func testPointerShapeVector() throws { + let data = try vector("pointer_shape") + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + XCTAssertEqual(header.packetType, .pointerShape) + guard case .pointerShape(let chunk) = message else { return XCTFail("wrong type") } + XCTAssertEqual(chunk.shapeId, 0x0A0B_0C0D) + XCTAssertEqual(chunk.sourceType, 1) + XCTAssertEqual(chunk.flags, PointerShapeChunk.invertFlag) + XCTAssertEqual([chunk.width, chunk.height, chunk.hotX, chunk.hotY], [3, 2, 1, 1]) + XCTAssertEqual(chunk.totalLen, 26) + XCTAssertEqual(chunk.offset, 0) + XCTAssertEqual(Array(chunk.data.suffix(2)), [0b0010_0000, 0b1000_0000]) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + let image = PointerOverlay.image(chunk, bytes: [UInt8](chunk.data)) + XCTAssertEqual([UInt8](try XCTUnwrap(image.invertMask)), [0, 0, 255, 255, 0, 0]) + XCTAssertEqual([UInt8](image.pixels.prefix(8)), [0, 0, 0, 255, 255, 255, 255, 255]) + } + + func testPointerStateVectors() throws { + for (name, visible, x, y) in [("pointer_state", true, Int32(-5), Int32(1234)), + ("pointer_hidden", false, 3000, 7)] { + let data = try vector(name) + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .pointerState(let state) = message else { return XCTFail(name) } + XCTAssertEqual(state.visible, visible, name) + XCTAssertEqual(state.x, x, name) + XCTAssertEqual(state.y, y, name) + XCTAssertEqual(state.shapeId, 0x0A0B_0C0D, name) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + } + } + + func testReceiverReportNamesThePointerShape() throws { + let data = try vector("receiver_report_pointer") + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .receiverReport(let report) = message else { return XCTFail("wrong type") } + XCTAssertEqual(report.pointerShapeId, 0x0A0B_0C0D) + XCTAssertEqual(report.fragsRepaired, 1234) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + let (_, old) = try XCTUnwrap(Wire.parseControl(try vector("receiver_report_v030"))) + guard case .receiverReport(let legacy) = old else { return XCTFail("wrong type") } + XCTAssertEqual(legacy.pointerShapeId, 0) + } + + func testPointerMessagesRejectWhatRustRejects() throws { + let good = try vector("pointer_shape") + // Body offsets: version 16, shape id 20, width 24, total length 32, + // chunk length 40. + let breaks: [(String, Range, [UInt8])] = [ + ("version", 16..<17, [2]), ("zero id", 20..<24, [0, 0, 0, 0]), ("zero width", 24..<26, [0, 0]), + ("wide", 24..<26, [0x01, 0x02]), ("total length", 32..<36, [99, 0, 0, 0]), + ("past the end", 36..<40, [1, 0, 0, 0]), ("empty chunk", 40..<42, [0, 0]), + ] + for (name, range, bytes) in breaks { + var bad = good + bad.replaceSubrange(range, with: bytes) + XCTAssertNil(Wire.parseControl(bad), name) + } + for length in 0..= 8 { + let body = UInt16(max(0, truncated.count - ControlHeader.size)) + truncated[6] = UInt8(body & 0xFF) + truncated[7] = UInt8(body >> 8) + } + XCTAssertNil(Wire.parseControl(truncated), "truncated to \(length)") + } + var state = try vector("pointer_state") + state[16] = 2 + XCTAssertNil(Wire.parseControl(state)) + } + // MARK: Control func testHello2Vector() throws { diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index f3e9f069..6a5a7b7c 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -40,6 +40,10 @@ pub const FEATURE_SUPPORTS_NACK: u16 = 1 << 2; /// The iPad app checks for its own updates (0.3.2 and later), so a HELLO2 /// without it comes from 0.3.1 or older. Older hosts ignore it. pub const FEATURE_CHECKS_UPDATES: u16 = 1 << 4; +/// The iPad draws the pointer itself from POINTER_SHAPE and POINTER_STATE. +/// A host with [`HOSTCAP_POINTER`] then leaves the pointer out of the video. +/// Bit 3 is taken by another change in review. +pub const FEATURE_DRAWS_POINTER: u16 = 1 << 5; pub const HOSTCAP_NACK: u16 = 1 << 0; pub const HOSTCAP_AUDIO: u16 = 1 << 1; pub const HOSTCAP_USB: u16 = 1 << 2; @@ -50,6 +54,9 @@ pub const HOSTCAP_PEN: u16 = 1 << 4; /// gaps, so a forward gap is a frame lost in transit. Without it, numbers /// may skip frames that were never sent. Older iPads ignore the bit. pub const HOSTCAP_CONTIGUOUS_FRAME_SEQ: u16 = 1 << 5; +/// The host sends POINTER_SHAPE and POINTER_STATE, and leaves the pointer +/// out of the video, to a client with [`FEATURE_DRAWS_POINTER`]. +pub const HOSTCAP_POINTER: u16 = 1 << 6; /// [`StreamConfig::flags`] bits. pub const STREAM_FLAG_SOFTWARE_ENCODER: u8 = 1 << 0; @@ -273,6 +280,9 @@ pub struct ReceiverReport { pub nacks_sent: u32, pub audio_packets_lost: u32, pub audio_buffer_ms: u16, + /// The last pointer shape the client holds complete, 0 for none. The + /// host sends the current shape again when this differs from it. + pub pointer_shape_id: u32, } pub const RECEIVER_REPORT_SIZE: usize = 36; @@ -318,6 +328,99 @@ pub struct InputEvent { pub const INPUT_EVENT_SIZE: usize = 30; pub const PEN_INPUT_EVENT_SIZE: usize = INPUT_EVENT_SIZE + 4; +/// The pointer messages' body layout version. +pub const POINTER_VERSION: u8 = 1; +/// [`PointerShapeChunk::flags`]: the image is followed by an invert mask. +pub const POINTER_SHAPE_INVERT: u8 = 1 << 0; +/// [`PointerState::flags`]: the pointer is on the captured screen and shown. +pub const POINTER_VISIBLE: u8 = 1 << 0; +/// Widest and tallest pointer image sent, in pixels. +pub const POINTER_MAX_SIDE: u16 = 512; +/// The fixed fields of a POINTER_SHAPE body, before its bytes. +pub const POINTER_SHAPE_FIXED_SIZE: usize = 26; +/// Image bytes in one POINTER_SHAPE datagram. +pub const POINTER_CHUNK_MAX: usize = + super::MAX_DGRAM_SIZE - CONTROL_HEADER_SIZE - POINTER_SHAPE_FIXED_SIZE; +pub const POINTER_STATE_SIZE: usize = 24; + +/// Bytes of a pointer image: `height` rows of `width` premultiplied BGRA +/// pixels, then, with [`POINTER_SHAPE_INVERT`], `height` rows of +/// `ceil(width / 8)` mask bytes, most significant bit first, where a set bit +/// inverts the picture under that pixel. +pub fn pointer_shape_len(width: u16, height: u16, flags: u8) -> usize { + let (w, h) = (usize::from(width), usize::from(height)); + let mask = if flags & POINTER_SHAPE_INVERT != 0 { + w.div_ceil(8) * h + } else { + 0 + }; + w * h * 4 + mask +} + +/// Host → client: one chunk of the pointer image with id `shape_id`. The +/// client keeps chunks until it holds all `total_len` bytes, then draws that +/// image at the position in [`PointerState`]. A new id replaces the image. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PointerShapeChunk { + /// Nonzero; changes whenever the image does. + pub shape_id: u32, + /// The Windows pointer type it came from (1 monochrome, 2 color, + /// 4 masked color). For diagnostics; the bytes are already converted. + pub source_type: u8, + pub flags: u8, + pub width: u16, + pub height: u16, + /// The click point inside the image. + pub hot_x: u16, + pub hot_y: u16, + pub total_len: u32, + pub offset: u32, + pub data: Vec, +} + +/// Splits one pointer image into the chunks that carry it. +#[allow(clippy::too_many_arguments)] +pub fn pointer_shape_chunks( + shape_id: u32, + source_type: u8, + flags: u8, + (width, height): (u16, u16), + (hot_x, hot_y): (u16, u16), + bytes: &[u8], +) -> Vec { + debug_assert_eq!(bytes.len(), pointer_shape_len(width, height, flags)); + bytes + .chunks(POINTER_CHUNK_MAX) + .enumerate() + .map(|(index, data)| PointerShapeChunk { + shape_id, + source_type, + flags, + width, + height, + hot_x, + hot_y, + total_len: bytes.len() as u32, + offset: (index * POINTER_CHUNK_MAX) as u32, + data: data.to_vec(), + }) + .collect() +} + +/// Host → client: where the pointer is, latest wins by `msg_seq`. `x`/`y` +/// place the image's top-left corner in video pixels, and may be negative +/// or past the edge when the pointer is partly off the screen. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct PointerState { + pub flags: u8, + /// The image to draw; 0 while the host has none. + pub shape_id: u32, + pub x: i32, + pub y: i32, + /// Host clock when the host read this position. + pub host_time_us: u64, +} + /// One parsed control message. #[derive(Debug, Clone, PartialEq)] pub enum ControlMessage { @@ -332,6 +435,8 @@ pub enum ControlMessage { Pong(Pong), StreamConfig(StreamConfig), InputEvent(InputEvent), + PointerShape(PointerShapeChunk), + PointerState(PointerState), } impl ControlMessage { @@ -348,6 +453,8 @@ impl ControlMessage { Self::Pong(_) => PacketType::Pong, Self::StreamConfig(_) => PacketType::StreamConfig, Self::InputEvent(_) => PacketType::InputEvent, + Self::PointerShape(_) => PacketType::PointerShape, + Self::PointerState(_) => PacketType::PointerState, } } } @@ -439,12 +546,16 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - || r.nacks_sent != 0 || r.audio_packets_lost != 0 || r.audio_buffer_ms != 0 + || r.pointer_shape_id != 0 { body.extend_from_slice(&r.frags_repaired.to_le_bytes()); body.extend_from_slice(&r.nacks_sent.to_le_bytes()); body.extend_from_slice(&r.audio_packets_lost.to_le_bytes()); body.extend_from_slice(&r.audio_buffer_ms.to_le_bytes()); } + if r.pointer_shape_id != 0 { + body.extend_from_slice(&r.pointer_shape_id.to_le_bytes()); + } } ControlMessage::Ping(p) => body.extend_from_slice(&p.t1_us.to_le_bytes()), ControlMessage::Pong(p) => { @@ -477,6 +588,25 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - body.extend_from_slice(&e.tilt_y.to_le_bytes()); } } + ControlMessage::PointerShape(c) => { + body.extend_from_slice(&[POINTER_VERSION, c.source_type, c.flags, 0]); + body.extend_from_slice(&c.shape_id.to_le_bytes()); + body.extend_from_slice(&c.width.to_le_bytes()); + body.extend_from_slice(&c.height.to_le_bytes()); + body.extend_from_slice(&c.hot_x.to_le_bytes()); + body.extend_from_slice(&c.hot_y.to_le_bytes()); + body.extend_from_slice(&c.total_len.to_le_bytes()); + body.extend_from_slice(&c.offset.to_le_bytes()); + body.extend_from_slice(&(c.data.len() as u16).to_le_bytes()); + body.extend_from_slice(&c.data); + } + ControlMessage::PointerState(p) => { + body.extend_from_slice(&[POINTER_VERSION, p.flags, 0, 0]); + body.extend_from_slice(&p.shape_id.to_le_bytes()); + body.extend_from_slice(&p.x.to_le_bytes()); + body.extend_from_slice(&p.y.to_le_bytes()); + body.extend_from_slice(&p.host_time_us.to_le_bytes()); + } } let mut datagram = vec![0u8; CONTROL_HEADER_SIZE + body.len()]; @@ -642,6 +772,9 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), report.audio_packets_lost = r.u32()?; report.audio_buffer_ms = r.u16()?; } + if r.remaining() >= 4 { + report.pointer_shape_id = r.u32()?; + } ControlMessage::ReceiverReport(report) } PacketType::Ping => ControlMessage::Ping(Ping { t1_us: r.u64()? }), @@ -677,6 +810,56 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), tilt_y: if input_ver == 2 { r.i16()? } else { 0 }, }) } + PacketType::PointerShape => { + pointer_version(r.u8()?)?; + let source_type = r.u8()?; + let flags = r.u8()?; + let _reserved = r.u8()?; + let chunk = PointerShapeChunk { + shape_id: r.u32()?, + source_type, + flags, + width: r.u16()?, + height: r.u16()?, + hot_x: r.u16()?, + hot_y: r.u16()?, + total_len: r.u32()?, + offset: r.u32()?, + data: { + let len = usize::from(r.u16()?); + r.take(len)?.to_vec() + }, + }; + if chunk.shape_id == 0 { + return Err(WireError::InvalidField("shape_id")); + } + if !(1..=POINTER_MAX_SIDE).contains(&chunk.width) + || !(1..=POINTER_MAX_SIDE).contains(&chunk.height) + { + return Err(WireError::InvalidField("pointer_size")); + } + if chunk.total_len as usize != pointer_shape_len(chunk.width, chunk.height, flags) { + return Err(WireError::InvalidField("total_len")); + } + if chunk.data.is_empty() + || chunk.offset as usize + chunk.data.len() > chunk.total_len as usize + { + return Err(WireError::InvalidField("offset")); + } + ControlMessage::PointerShape(chunk) + } + PacketType::PointerState => { + pointer_version(r.u8()?)?; + let flags = r.u8()?; + let _reserved = r.u16()?; + ControlMessage::PointerState(PointerState { + flags, + shape_id: r.u32()?, + x: r.i32()?, + y: r.i32()?, + host_time_us: r.u64()?, + }) + } PacketType::Media | PacketType::MediaFec | PacketType::Audio | PacketType::Error => { return Err(WireError::InvalidField("packet_type")); } @@ -685,6 +868,15 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), Ok((header, message)) } +/// A pointer body of another layout version is not one this parser knows. +fn pointer_version(version: u8) -> Result<(), WireError> { + if version == POINTER_VERSION { + Ok(()) + } else { + Err(WireError::InvalidField("pointer_version")) + } +} + fn truncated_name(name: &str) -> &str { if name.len() <= MAX_NAME_LEN { return name; @@ -741,6 +933,10 @@ impl<'a> Reader<'a> { Ok(u32::from_le_bytes([b[0], b[1], b[2], b[3]])) } + fn i32(&mut self) -> Result { + Ok(self.u32()? as i32) + } + fn u64(&mut self) -> Result { let b = self.take(8)?; Ok(u64::from_le_bytes([ @@ -854,6 +1050,7 @@ mod tests { assert_eq!(report.nacks_sent, 0); assert_eq!(report.audio_packets_lost, 0); assert_eq!(report.audio_buffer_ms, 0); + assert_eq!(report.pointer_shape_id, 0); } _ => {} } @@ -954,9 +1151,129 @@ mod tests { tilt_x: 0, tilt_y: 0, }), + ControlMessage::PointerShape(PointerShapeChunk { + shape_id: 7, + source_type: 1, + flags: POINTER_SHAPE_INVERT, + width: 2, + height: 2, + hot_x: 1, + hot_y: 0, + total_len: 18, + offset: 16, + data: vec![0x80, 0x40], + }), + ControlMessage::PointerState(PointerState { + flags: POINTER_VISIBLE, + shape_id: 7, + x: -3, + y: 1439, + host_time_us: 5_000_001, + }), ] } + fn shape_chunk() -> PointerShapeChunk { + let ControlMessage::PointerShape(chunk) = all_messages().remove(10) else { + unreachable!() + }; + chunk + } + + #[test] + fn pointer_shape_chunks_cover_the_image_in_order_within_a_datagram() { + let (width, height) = (64u16, 48u16); + let bytes: Vec = (0..pointer_shape_len(width, height, POINTER_SHAPE_INVERT)) + .map(|i| i as u8) + .collect(); + let chunks = + pointer_shape_chunks(9, 1, POINTER_SHAPE_INVERT, (width, height), (3, 4), &bytes); + assert_eq!(chunks.len(), bytes.len().div_ceil(POINTER_CHUNK_MAX)); + let mut joined = Vec::new(); + for chunk in &chunks { + assert_eq!(chunk.offset as usize, joined.len()); + let datagram = encode_control(1, 1, &ControlMessage::PointerShape(chunk.clone())); + assert!(datagram.len() <= super::super::MAX_DGRAM_SIZE); + let (_, parsed) = parse_control(&datagram).unwrap(); + assert_eq!(parsed, ControlMessage::PointerShape(chunk.clone())); + joined.extend_from_slice(&chunk.data); + } + assert_eq!(joined, bytes); + } + + #[test] + fn pointer_shapes_reject_a_zero_id_bad_sizes_lengths_and_versions() { + let mut cases: Vec<(PointerShapeChunk, &str)> = Vec::new(); + let mut zero = shape_chunk(); + zero.shape_id = 0; + cases.push((zero, "shape_id")); + for (w, h) in [(0, 2), (2, 0), (POINTER_MAX_SIDE + 1, 2)] { + let mut sized = shape_chunk(); + (sized.width, sized.height) = (w, h); + cases.push((sized, "pointer_size")); + } + let mut no_mask = shape_chunk(); + no_mask.flags = 0; + cases.push((no_mask, "total_len")); + let mut past_end = shape_chunk(); + past_end.offset = 17; + cases.push((past_end, "offset")); + let mut empty = shape_chunk(); + empty.data.clear(); + cases.push((empty, "offset")); + for (chunk, field) in cases { + let datagram = encode_control(1, 1, &ControlMessage::PointerShape(chunk)); + assert_eq!( + parse_control(&datagram), + Err(WireError::InvalidField(field)) + ); + } + for message in [ + ControlMessage::PointerShape(shape_chunk()), + ControlMessage::PointerState(PointerState::default()), + ] { + let mut datagram = encode_control(1, 1, &message); + datagram[CONTROL_HEADER_SIZE] = POINTER_VERSION + 1; + assert_eq!( + parse_control(&datagram), + Err(WireError::InvalidField("pointer_version")) + ); + } + } + + #[test] + fn a_report_names_the_pointer_shape_after_the_older_tail() { + let report = ReceiverReport { + stream_epoch: 3, + pointer_shape_id: 0x0102_0304, + ..Default::default() + }; + let datagram = encode_control(1, 1, &ControlMessage::ReceiverReport(report)); + assert_eq!( + datagram.len(), + CONTROL_HEADER_SIZE + RECEIVER_REPORT_SIZE + 14 + 4 + ); + assert_eq!( + parse_control(&datagram).unwrap().1, + ControlMessage::ReceiverReport(report) + ); + // A parser that stops after the older tail reads the same report + // without the id: the tail is appended, so nothing before it moved. + let mut older = datagram[..datagram.len() - 4].to_vec(); + let body_len = (older.len() - CONTROL_HEADER_SIZE) as u16; + older[6..8].copy_from_slice(&body_len.to_le_bytes()); + let ControlMessage::ReceiverReport(old) = parse_control(&older).unwrap().1 else { + unreachable!() + }; + assert_eq!( + old, + ReceiverReport { + pointer_shape_id: 0, + ..report + } + ); + } + #[test] fn every_message_round_trips() { for message in all_messages() { diff --git a/proto/src/v2/mod.rs b/proto/src/v2/mod.rs index 4b75ffe9..c0e17181 100644 --- a/proto/src/v2/mod.rs +++ b/proto/src/v2/mod.rs @@ -55,6 +55,11 @@ pub enum PacketType { Pong = 0x17, StreamConfig = 0x18, Nack = 0x19, + /// Host → client pointer image, in chunks. Sent only to clients with + /// [`control::FEATURE_DRAWS_POINTER`]. + PointerShape = 0x1A, + /// Host → client pointer position and visibility, as for `PointerShape`. + PointerState = 0x1B, InputEvent = 0x20, /// Reserved. Not sent in v2.0. Error = 0x7F, @@ -76,6 +81,8 @@ impl PacketType { 0x17 => Self::Pong, 0x18 => Self::StreamConfig, 0x19 => Self::Nack, + 0x1A => Self::PointerShape, + 0x1B => Self::PointerState, 0x20 => Self::InputEvent, 0x7F => Self::Error, _ => return None, diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 6d072d05..484ca9b2 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -32,3 +32,9 @@ receiver_report_v030 454d021500003200785634123200000003000000881300007e1300000a0 hello_ack_paired 454d02110000390000000000030000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f100100 hello_ack_contiguous_seq 454d02110000390000000000050000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f102100 hello2_v030 454d02100000410000000000040000000202efbeadde94260300010074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e20100 +hello2_draws_pointer 454d02100000410000000000060000000202efbeadde94260300210074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e20100 +pointer_shape 454d021a000034007856341233000000010101000d0c0b0a03000200010001001a000000000000001a00000000ffffffffff000000000000000010203080000000ff2080 +pointer_state 454d021b000018007856341234000000010100000d0c0b0afbffffffd2040000ab89674523010000 +pointer_hidden 454d021b000018007856341235000000010000000d0c0b0ab80b000007000000ac89674523010000 +receiver_report_pointer 454d021500003600785634123600000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00d204000037020000590000003c000d0c0b0a +hello_ack_pointer 454d02110000390000000000070000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f106100 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index e0e6c5ad..84525f8b 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -384,7 +384,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { ack.host_caps = HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ; vectors.push(( "hello_ack_contiguous_seq", - encode_control(0, 5, &ControlMessage::HelloAck(ack)), + encode_control(0, 5, &ControlMessage::HelloAck(ack.clone())), )); let (_, ControlMessage::Hello2(mut hello)) = parse_control( &vectors @@ -402,7 +402,92 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { hello.pairing_code = 123_456; vectors.push(( "hello2_v030", - encode_control(0, 4, &ControlMessage::Hello2(hello)), + encode_control(0, 4, &ControlMessage::Hello2(hello.clone())), + )); + hello.feature_caps |= FEATURE_DRAWS_POINTER; + vectors.push(( + "hello2_draws_pointer", + encode_control(0, 6, &ControlMessage::Hello2(hello)), + )); + // A 3x2 image: opaque black, opaque white, two clear pixels the mask + // after the image inverts, a half-covered pixel and opaque black. + let image: Vec = [ + [0, 0, 0, 255], + [255, 255, 255, 255], + [0, 0, 0, 0], + [0, 0, 0, 0], + [0x10, 0x20, 0x30, 0x80], + [0, 0, 0, 255], + ] + .concat(); + let mut shape = image.clone(); + shape.extend_from_slice(&[0b0010_0000, 0b1000_0000]); + vectors.push(( + "pointer_shape", + encode_control( + 0x1234_5678, + 51, + &ControlMessage::PointerShape(PointerShapeChunk { + shape_id: 0x0A0B_0C0D, + source_type: 1, + flags: POINTER_SHAPE_INVERT, + width: 3, + height: 2, + hot_x: 1, + hot_y: 1, + total_len: shape.len() as u32, + offset: 0, + data: shape, + }), + ), + )); + vectors.push(( + "pointer_state", + encode_control( + 0x1234_5678, + 52, + &ControlMessage::PointerState(PointerState { + flags: POINTER_VISIBLE, + shape_id: 0x0A0B_0C0D, + x: -5, + y: 1_234, + host_time_us: 0x0000_0123_4567_89AB, + }), + ), + )); + vectors.push(( + "pointer_hidden", + encode_control( + 0x1234_5678, + 53, + &ControlMessage::PointerState(PointerState { + flags: 0, + shape_id: 0x0A0B_0C0D, + x: 3_000, + y: 7, + host_time_us: 0x0000_0123_4567_89AC, + }), + ), + )); + let (_, ControlMessage::ReceiverReport(mut report)) = parse_control( + &vectors + .iter() + .find(|(name, _)| *name == "receiver_report_v030") + .unwrap() + .1, + ) + .unwrap() else { + unreachable!() + }; + report.pointer_shape_id = 0x0A0B_0C0D; + vectors.push(( + "receiver_report_pointer", + encode_control(0x1234_5678, 54, &ControlMessage::ReceiverReport(report)), + )); + ack.host_caps |= HOSTCAP_POINTER; + vectors.push(( + "hello_ack_pointer", + encode_control(0, 7, &ControlMessage::HelloAck(ack)), )); vectors } @@ -537,9 +622,50 @@ fn packet_type_registry_is_stable() { assert_eq!(PacketType::Pong as u8, 0x17); assert_eq!(PacketType::StreamConfig as u8, 0x18); assert_eq!(PacketType::Nack as u8, 0x19); + assert_eq!(PacketType::PointerShape as u8, 0x1A); + assert_eq!(PacketType::PointerState as u8, 0x1B); assert_eq!(PacketType::InputEvent as u8, 0x20); } +/// The pointer bits, and the fields their vectors carry. +#[test] +fn pointer_vectors_round_trip_with_their_capabilities() { + let vectors: BTreeMap<_, _> = canonical_vectors().into_iter().collect(); + assert_eq!(FEATURE_DRAWS_POINTER, 0x0020); + assert_eq!(HOSTCAP_POINTER, 0x0040); + let ControlMessage::Hello2(hello) = parse_control(&vectors["hello2_draws_pointer"]).unwrap().1 + else { + panic!() + }; + assert_ne!(hello.feature_caps & FEATURE_DRAWS_POINTER, 0); + let ControlMessage::HelloAck(ack) = parse_control(&vectors["hello_ack_pointer"]).unwrap().1 + else { + panic!() + }; + assert_ne!(ack.host_caps & HOSTCAP_POINTER, 0); + let ControlMessage::PointerShape(shape) = parse_control(&vectors["pointer_shape"]).unwrap().1 + else { + panic!() + }; + assert_eq!((shape.width, shape.height, shape.total_len), (3, 2, 26)); + let ControlMessage::PointerState(state) = parse_control(&vectors["pointer_state"]).unwrap().1 + else { + panic!() + }; + assert_eq!( + (state.x, state.y, state.flags), + (-5, 1_234, POINTER_VISIBLE) + ); + let ControlMessage::ReceiverReport(report) = parse_control(&vectors["receiver_report_pointer"]) + .unwrap() + .1 + else { + panic!() + }; + assert_eq!(report.pointer_shape_id, 0x0A0B_0C0D); + assert_eq!(report.frags_repaired, 1234); +} + /// A host that numbers frames contiguously says so in HELLO_ACK; one that /// does not leaves the bit clear, so an iPad reads its numbers as before. #[test] diff --git a/scripts/e2e_pointer.sh b/scripts/e2e_pointer.sh new file mode 100755 index 00000000..4a90a513 --- /dev/null +++ b/scripts/e2e_pointer.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# End-to-end pointer test on a Mac (#183): the real host binary with the +# synthetic pointer scene (a still picture, the pointer moving every frame) +# streams over localhost UDP to the real iPad app in the Simulator. +# +# Two runs, counted from the app's E2E_POINTER lines over EM_WINDOW seconds: +# - draws: the app draws the pointer. It must move on the iPad (positions +# applied and drawn) while almost no video frames decode: only +# the host's idle keepalive. +# - legacy: EM_DRAWS_POINTER=0, as apps before 0.3.5. No pointer messages, +# and every move is a decoded video frame, as before. +# +# Requirements as for e2e_ios.sh. Uses EM_SIM_UDID (or EM_SIM_NAME), a +# private EM_PORT and a USB listener on a free port, so it shares no fixed +# port with other simulators. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +export DEVELOPER_DIR="${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer}" +export PATH="$HOME/.cargo/bin:$PATH" +export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:-/opt/homebrew/opt/ffmpeg@7/lib/pkgconfig}" + +SIM_NAME="${EM_SIM_NAME:-iPad Pro 11-inch (M4)}" +PORT="${EM_PORT:-19936}" +WINDOW="${EM_WINDOW:-10}" +FPS="${ETERNAL_FPS:-60}" +SIZE="${EM_SIZE:-1280x720}" +OUT="${EM_OUTPUT_DIR:-$ROOT/build/e2e/pointer}" +mkdir -p "$OUT" +rm -f "$OUT/result.json" + +UDID="${EM_SIM_UDID:-}" +if [ -z "$UDID" ]; then +UDID=$(xcrun simctl list -j devices available | /usr/bin/python3 -c ' +import json, sys +for devices in json.load(sys.stdin)["devices"].values(): + for device in devices: + if device["name"] == sys.argv[1]: + print(device["udid"]); sys.exit(0) +sys.exit(1) +' "$SIM_NAME") +fi + +HOST_PID="" +cleanup() { + [ -n "$HOST_PID" ] && kill "$HOST_PID" 2>/dev/null || true + xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true +} +trap cleanup EXIT + +if [ "${EM_SKIP_BUILD:-0}" != 1 ]; then + echo "==> Building app and host" + (cd "$ROOT/ios" && xcodegen generate >/dev/null) + xcodebuild build -project "$ROOT/ios/EternalMonitor.xcodeproj" -scheme EternalMonitor \ + -configuration Release -destination "platform=iOS Simulator,id=$UDID" \ + -derivedDataPath "$ROOT/ios/build/e2e" CODE_SIGNING_ALLOWED=YES CODE_SIGN_IDENTITY=- -quiet + (cd "$ROOT" && cargo build -q --release -p eternal-host --locked) +fi +APP="$ROOT/ios/build/e2e/Build/Products/Release-iphonesimulator/EternalMonitor.app" +[ -d "$APP" ] || { echo "FAIL: app bundle not found at $APP"; exit 1; } + +xcrun simctl boot "$UDID" 2>/dev/null || true +xcrun simctl bootstatus "$UDID" -b >/dev/null +INSTALL_DIR="$(mktemp -d /tmp/em-e2e-pointer.XXXXXX)" +ditto "$APP" "$INSTALL_DIR/EternalMonitor.app" +xcrun simctl install "$UDID" "$INSTALL_DIR/EternalMonitor.app" +rm -rf "$INSTALL_DIR" + +run_mode() { + local mode=$1 draws=$2 + local dir="$OUT/$mode" + mkdir -p "$dir/state/EternalMonitor" + python3 - "$dir/state/EternalMonitor/settings.json" "$FPS" <<'PY' +import json, sys +json.dump(dict(bitrate_mbps=15.0, target_fps=int(sys.argv[2]), start_on_boot=False, + stream_audio=False, require_pairing=False), open(sys.argv[1], 'w')) +PY + xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true + local data + data=$(xcrun simctl get_app_container "$UDID" com.eternal.monitor data) + local sim_log="$data/tmp/eternal-e2e.log" + rm -f "$sim_log" + + echo "==> [$mode] host on 127.0.0.1:$PORT (synthetic pointer, ${SIZE}, ${FPS} fps)" + APPDATA="$dir/state" ETERNAL_HEADLESS=1 ETERNAL_E2E_LOG=1 ETERNAL_CAPTURE=synthetic \ + ETERNAL_SYNTH_POINTER=1 ETERNAL_SYNTH_SIZE="$SIZE" ETERNAL_ENCODER=libx264 ETERNAL_FPS="$FPS" \ + "$ROOT/target/release/eternal-host" "$PORT" >"$dir/host.log" 2>&1 & + HOST_PID=$! + sleep 2 + + echo "==> [$mode] app, EM_DRAWS_POINTER=$draws" + SIMCTL_CHILD_EM_AUTOCONNECT="127.0.0.1:$PORT" SIMCTL_CHILD_EM_E2E_LOG=1 \ + SIMCTL_CHILD_EM_USB_PORT=0 SIMCTL_CHILD_EM_DRAWS_POINTER="$draws" \ + xcrun simctl launch "$UDID" com.eternal.monitor -didSeeOnboarding YES -allowUSB NO \ + -playPCaudio NO -targetFPS "$FPS" >/dev/null + + local waited=0 + until grep -q E2E_FIRST_FRAME "$sim_log" 2>/dev/null; do + sleep 1; waited=$((waited + 1)) + [ "$waited" -lt 60 ] || { echo "FAIL [$mode]: no first frame"; exit 1; } + done + # Let the first frame's keyframe settle, then count over the window. + sleep 2 + local start end + start=$(grep E2E_POINTER "$sim_log" | tail -1) + sleep "$WINDOW" + end=$(grep E2E_POINTER "$sim_log" | tail -1) + xcrun simctl io "$UDID" screenshot "$dir/screen.png" >/dev/null 2>&1 || true + cp "$sim_log" "$dir/app.log" + xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true + kill "$HOST_PID" 2>/dev/null || true + wait "$HOST_PID" 2>/dev/null || true + HOST_PID="" + + python3 - "$mode" "$WINDOW" "$start" "$end" "$dir/host.log" "$OUT/$mode.json" <<'PY' +import json, re, sys +mode, window, start, end, host_log, out = sys.argv[1], float(sys.argv[2]), *sys.argv[3:] +field = lambda line, name: int(re.search(rf"\b{name}=(\d+)", line).group(1)) +delta = {name: field(end, name) - field(start, name) for name in ("states", "drawn", "moves", "decoded", "shapes")} +sent = [int(m) for m in re.findall(r"pointer_sent=(\d+)", re.sub(r"\x1b\[[0-9;]*m", "", open(host_log).read()))] +result = dict(mode=mode, window_s=window, **delta, pointer_states_per_s=round(delta["states"] / window, 1), + moves_per_s=round(delta["moves"] / window, 1), frames_per_s=round(delta["decoded"] / window, 2), + host_pointer_sent_last_summary=sent[-1] if sent else None, shapes_total=field(end, "shapes")) +json.dump(result, open(out, "w"), indent=1) +print(json.dumps(result)) +problems = [] +if mode == "draws": + if delta["moves"] < 15 * window: problems.append("the pointer barely moved on the iPad") + if field(end, "shapes") < 1: problems.append("no pointer image arrived") + # The 750 ms keepalive, and nothing for the moves. + if delta["decoded"] > window / 0.75 + 3: problems.append("video frames decoded while only the pointer moved") +else: + if field(end, "states") or field(end, "drawn"): problems.append("an older app got pointer messages") + if delta["decoded"] < 15 * window: problems.append("pointer moves did not arrive as video frames") +if problems: + print(f"FAIL [{mode}]: " + "; ".join(problems)); sys.exit(1) +print(f"PASS [{mode}]") +PY +} + +echo "load: $(uptime | sed 's/.*load averages*: //')" | tee "$OUT/load.txt" +run_mode draws 1 +run_mode legacy 0 +echo "load: $(uptime | sed 's/.*load averages*: //')" | tee -a "$OUT/load.txt" +python3 -c 'import json,sys; json.dump([json.load(open(p)) for p in sys.argv[2:]], open(sys.argv[1], "w"), indent=1)' \ + "$OUT/result.json" "$OUT/draws.json" "$OUT/legacy.json" +echo "PASS: $OUT/result.json" From bd84c45a64075d17c79e8ffd913164bec8435e79 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:05:40 -0500 Subject: [PATCH 017/226] Name the Wi-Fi off card as the app does in SETUP.md --- SETUP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SETUP.md b/SETUP.md index 6b0a5ba4..25ab6f21 100644 --- a/SETUP.md +++ b/SETUP.md @@ -261,7 +261,7 @@ try fewer words. If your problem isn't here, see [Known issues](https://eternalm | **That looks like the pairing code, not the PC's address** | A six-digit number was typed where the address goes. | Type the **Address** from the PC window, for example 192.168.1.20. The iPad asks for the code after it connects. | | **That looks like your router's address, not your PC's** | Nothing answered at an address ending in .1, which is usually the Wi-Fi router. | Type the **Address** shown in the PC window. | | **That address doesn't look right** | The iPad couldn't find anything by that name. | Type the **Address** exactly as the PC window shows it. | -| **This iPad isn't connected to a network** | Wi-Fi is off on the iPad, or it hasn't joined a network. | Turn on Wi-Fi and join the same network as the PC, or use a USB cable. | +| **Wi-Fi is off** | Wi-Fi is off on the iPad, or it hasn't joined a network. | Turn on Wi-Fi and join the same network as the PC, or use a USB cable. | | **Local Network is turned off for EternalMonitor** | Local Network access is off for EternalMonitor on the iPad, so it can't reach the PC over Wi-Fi. | Tap **Open Settings** and turn on **Local Network**, or go to **Settings > Privacy & Security > Local Network** and turn on **EternalMonitor**. Then tap **Try again**. A USB cable works without this setting. | | **Couldn't connect to your PC** | The connection failed for another reason. | Check that both are on the same Wi-Fi and that EternalMonitor is open on the PC, then tap **Try again**. | | **This iPad couldn't start the connection** | Something on the iPad side didn't start. | Close EternalMonitor on the iPad and open it again. | From 1319b2e6ddf277af9a818044757cea997e603334 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:08:21 -0500 Subject: [PATCH 018/226] Keep the window and the stream running when the window's graphics device is lost egui-wgpu 0.36 panicked on the frame after a device loss ("Failed to create staging buffer for index data"), which ended the app. The vendored copy now skips that frame, drops the lost device and builds a new one, and the window reloads its logo on the new device. A replacement device leaves the launch's crash-avoidance marker alone. --- host/src/gui/mod.rs | 15 +++-- host/src/gui/page_settings.rs | 2 +- host/src/gui/renderer.rs | 16 ++++- host/tests/window_device_loss.rs | 67 ++++++++++++++++++++ vendor/README.md | 23 ++++++- vendor/egui-wgpu/src/renderer.rs | 23 ++++++- vendor/egui-wgpu/src/winit.rs | 101 ++++++++++++++++++++++++++++++- 7 files changed, 236 insertions(+), 11 deletions(-) create mode 100644 host/tests/window_device_loss.rs diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 708c5a3d..53b1a294 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -363,7 +363,9 @@ pub struct AnalyzerApp { update_refused: bool, history: History, quality: QualityMeter, - logo: Option<(f32, egui::TextureHandle)>, + /// The logo for a scale and the window renderer's device it was made + /// on: textures go with a lost device. + logo: Option<(f32, u64, egui::TextureHandle)>, /// The saved language choice (see `i18n::resolve`). settings_language: Option, /// A language whose fonts were just installed; it shows from the next @@ -1271,14 +1273,19 @@ impl AnalyzerApp { fn draw_brand(&mut self, ui: &mut egui::Ui) { const LOGO: f32 = 30.0; let ppp = ui.ctx().pixels_per_point(); - if self.logo.as_ref().is_none_or(|(scale, _)| *scale != ppp) { + let device = renderer::devices(); + if self + .logo + .as_ref() + .is_none_or(|(scale, made_on, _)| (*scale, *made_on) != (ppp, device)) + { let pixels = (LOGO * ppp).round().max(1.0) as usize; - self.logo = logo_texture(ui.ctx(), pixels).map(|texture| (ppp, texture)); + self.logo = logo_texture(ui.ctx(), pixels).map(|texture| (ppp, device, texture)); } ui.horizontal(|ui| { ui.spacing_mut().item_spacing.x = 10.0; let (rect, _) = ui.allocate_exact_size(vec2(LOGO, LOGO), Sense::hover()); - if let Some((_, texture)) = &self.logo { + if let Some((_, _, texture)) = &self.logo { ui.painter().image( texture.id(), rect, diff --git a/host/src/gui/page_settings.rs b/host/src/gui/page_settings.rs index 58bfba3d..6b47d083 100644 --- a/host/src/gui/page_settings.rs +++ b/host/src/gui/page_settings.rs @@ -229,7 +229,7 @@ impl AnalyzerApp { ui.horizontal(|ui| { ui.spacing_mut().item_spacing.x = 14.0; let (rect, _) = ui.allocate_exact_size(vec2(44.0, 44.0), Sense::hover()); - if let Some((_, texture)) = &self.logo { + if let Some((_, _, texture)) = &self.logo { ui.painter().image( texture.id(), rect, diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 707042b9..15275474 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -119,6 +119,9 @@ static PRESENT: Mutex = Mutex::new(PresentTracker::new()); /// Frames whose surface texture could not be acquired, so nothing was presented. static FAILED_ACQUIRES: AtomicU64 = AtomicU64::new(0); static STATUS: Mutex> = Mutex::new(None); +/// Adapters chosen for the window this launch. After the first, each +/// replaces a lost device (`vendor/egui-wgpu`). +static SELECTIONS: AtomicU64 = AtomicU64::new(0); /// How this launch draws its window, for Settings > Troubleshooting. #[derive(Clone, Debug, Default, PartialEq)] @@ -577,6 +580,12 @@ impl PresentTracker { } } +/// How many devices the window has drawn with, or tried to: one more after +/// each loss. Its textures must be made again on each. +pub(super) fn devices() -> u64 { + SELECTIONS.load(Ordering::SeqCst) +} + fn select( adapters: &[wgpu::Adapter], surface: Option<&wgpu::Surface<'_>>, @@ -585,6 +594,10 @@ fn select( rules: &[RendererRule], overruled: Option<&'static str>, ) -> Result { + // A replacement for a lost device chooses as the start did, but this + // launch's marker stays as it is: the window started, so a later end is + // not a failed start. + let replacing = SELECTIONS.fetch_add(1, Ordering::SeqCst) > 0; let usable: Vec<&wgpu::Adapter> = adapters .iter() .filter(|adapter| surface.is_none_or(|surface| adapter.is_surface_supported(surface))) @@ -602,7 +615,7 @@ fn select( apply_rule(&candidates, &gpus, picked, rules, overruled), picked, STATE_DIR.get().map(PathBuf::as_path), - |index| start(marker(index)), + |index| replacing || start(marker(index)), ); if let Some((rule, outcome)) = ruled { info!("{}", rule_line(rule, outcome, &gpus[picked])); @@ -620,6 +633,7 @@ fn select( "remote rule" }, avoided = ?history.iter().filter(|f| gpus.contains(&f.gpu) && avoided(history, &f.gpu, &f.backend)).map(|f| (&f.gpu, &f.backend, &f.fault)).collect::>(), + replacing_lost_device = replacing, "Window renderer selected" ); let drawing = match info.backend { diff --git a/host/tests/window_device_loss.rs b/host/tests/window_device_loss.rs new file mode 100644 index 00000000..5741d8f3 --- /dev/null +++ b/host/tests/window_device_loss.rs @@ -0,0 +1,67 @@ +//! The window survives the loss of its graphics device, as in a driver +//! reset. egui-wgpu 0.36 panicked on the next frame ("Failed to create +//! staging buffer for index data"), which ended the app and the stream; the +//! patched copy in `vendor/egui-wgpu` skips the frame and replaces the device. + +use eframe::egui; +use eframe::egui_wgpu::{Renderer, RendererOptions, ScreenDescriptor}; +use eframe::wgpu; + +fn block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(future) +} + +fn device() -> (wgpu::Device, wgpu::Queue) { + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); + let adapter = block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())) + .expect("a graphics adapter"); + block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).expect("a device") +} + +/// One pass of a window with text and a button, tessellated. +fn pass(ctx: &egui::Context) -> (Vec, egui::TexturesDelta) { + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + ui.heading("EternalMonitor"); + let _ = ui.button("Start streaming"); + }); + let primitives = ctx.tessellate(output.shapes, output.pixels_per_point); + (primitives, output.textures_delta) +} + +#[test] +fn a_lost_device_skips_the_frame_instead_of_panicking() { + let (device, queue) = device(); + let mut renderer = Renderer::new( + &device, + wgpu::TextureFormat::Rgba8Unorm, + RendererOptions::default(), + ); + let ctx = egui::Context::default(); + let screen = ScreenDescriptor { + size_in_pixels: [640, 480], + pixels_per_point: 1.0, + }; + + let (primitives, mut textures) = pass(&ctx); + assert!(!primitives.is_empty()); + for (id, deltas) in &textures.set { + for delta in deltas { + renderer.update_texture(&device, &queue, *id, delta); + } + } + textures.clear(); + let mut encoder = device.create_command_encoder(&Default::default()); + renderer.update_buffers(&device, &queue, &mut encoder, &primitives, &screen); + queue.submit([encoder.finish()]); + + // wgpu now refuses every staging buffer, as after a driver reset. + device.destroy(); + let (primitives, mut textures) = pass(&ctx); + textures.clear(); + let mut encoder = device.create_command_encoder(&Default::default()); + let callbacks = renderer.update_buffers(&device, &queue, &mut encoder, &primitives, &screen); + assert!(callbacks.is_empty()); +} diff --git a/vendor/README.md b/vendor/README.md index 781290d2..f3338770 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -7,4 +7,25 @@ the workspace `Cargo.toml`). Each keeps its upstream license files. Copied from the crates.io release of egui-wgpu 0.36.2 (https://github.com/emilk/egui, MIT OR Apache-2.0), without its `Cargo.lock` -and `Cargo.toml.orig`. +and `Cargo.toml.orig`. Every change is marked `EternalMonitor patch`. + +Why: the PC window must survive the loss of its graphics device, as in a +driver reset or a device removed while another program uses the GPU. The +release panics on the next frame ("Failed to create staging buffer for index +data"), because wgpu gives no staging buffer once the device is lost, and the +panic ends the app and the stream with it. 0.37.0 still does. + +The patch: +- `Renderer::update_buffers` skips a frame whose staging buffer can't be made, + and `Renderer::render` then draws nothing. +- The winit `Painter` watches each device's lost callback, and keeps wgpu's + default error handler from panicking on what a lost device reports. +- When the device is lost, or a frame's upload fails, the `Painter` drops the + device, its surfaces and egui's renderer. eframe calls `set_window` before + each paint, which makes new ones; until that works the window skips frames + and tries again every 500 ms. The new device's first frame uploads the font + atlas again. Other textures are the app's to load again + (`host/src/gui/renderer.rs` counts devices for that). + +Drop this copy when an egui-wgpu release survives device loss, or carry the +patch to the version eframe moves to. diff --git a/vendor/egui-wgpu/src/renderer.rs b/vendor/egui-wgpu/src/renderer.rs index de8808de..ff0a36f0 100644 --- a/vendor/egui-wgpu/src/renderer.rs +++ b/vendor/egui-wgpu/src/renderer.rs @@ -258,6 +258,10 @@ pub struct Renderer { /// /// See also [`CallbackTrait`]. pub callback_resources: CallbackResources, + + /// EternalMonitor patch: the last [`Self::update_buffers`] could not upload its meshes, + /// as happens once the device is lost, so [`Self::render`] draws nothing. + pub(crate) upload_failed: bool, } impl Renderer { @@ -463,6 +467,7 @@ impl Renderer { samplers: HashMap::default(), options, callback_resources: CallbackResources::default(), + upload_failed: false, } } @@ -484,6 +489,10 @@ impl Renderer { ) { profiling::function_scope!(); + if self.upload_failed { + return; + } + let pixels_per_point = screen_descriptor.pixels_per_point; let size_in_pixels = screen_descriptor.size_in_pixels; @@ -960,6 +969,8 @@ impl Renderer { }) }; + self.upload_failed = false; + if index_count > 0 { profiling::scope!("indices", index_count.to_string().as_str()); @@ -980,12 +991,16 @@ impl Renderer { NonZeroU64::new(required_index_buffer_size).unwrap(), ); + // EternalMonitor patch: wgpu gives no staging buffer once the device is lost (or out + // of memory). Skip this frame instead of panicking; the painter replaces the device. let Some(mut index_buffer_staging) = index_buffer_staging else { - panic!( + log::warn!( "Failed to create staging buffer for index data. Index count: {index_count}. Required index buffer size: {required_index_buffer_size}. Actual size {} and capacity: {} (bytes)", self.index_buffer.buffer.size(), self.index_buffer.capacity ); + self.upload_failed = true; + return Vec::new(); }; let mut index_offset = 0; @@ -1026,12 +1041,16 @@ impl Renderer { NonZeroU64::new(required_vertex_buffer_size).unwrap(), ); + // EternalMonitor patch: wgpu gives no staging buffer once the device is lost (or out + // of memory). Skip this frame instead of panicking; the painter replaces the device. let Some(mut vertex_buffer_staging) = vertex_buffer_staging else { - panic!( + log::warn!( "Failed to create staging buffer for vertex data. Vertex count: {vertex_count}. Required vertex buffer size: {required_vertex_buffer_size}. Actual size {} and capacity: {} (bytes)", self.vertex_buffer.buffer.size(), self.vertex_buffer.capacity ); + self.upload_failed = true; + return Vec::new(); }; let mut vertex_offset = 0; diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs index 84b32927..55a922a3 100644 --- a/vendor/egui-wgpu/src/winit.rs +++ b/vendor/egui-wgpu/src/winit.rs @@ -9,9 +9,15 @@ use crate::{ capture::{CaptureReceiver, CaptureSender, CaptureState, capture_channel}, }; use core::num::NonZeroU32; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::time::Duration; use egui::{Context, Event, UserData, ViewportId, ViewportIdMap, ViewportIdSet}; use std::sync::Arc; +/// EternalMonitor patch: how often a lost device's replacement is tried again +/// while it can't be created, as during a driver reset. +const REBUILD_RETRY: Duration = Duration::from_millis(500); + struct SurfaceState { surface: wgpu::Surface<'static>, alpha_mode: wgpu::CompositeAlphaMode, @@ -43,6 +49,16 @@ pub struct Painter { surfaces: ViewportIdMap, capture_tx: CaptureSender, capture_rx: CaptureReceiver, + + // EternalMonitor patch: device-loss recovery. + /// Set when the current device is lost: by its lost callback, or when a frame's upload + /// fails. The next paint drops the device and everything made on it. + device_lost: Arc, + /// The lost device was dropped; [`Self::set_window`] builds a new one, and failing + /// that is retried instead of returned. + rebuilding: bool, + /// The new device's first paint uploads the font atlas again. + restore_fonts: bool, } impl Painter { @@ -83,6 +99,10 @@ impl Painter { capture_tx, capture_rx, + + device_lost: Arc::new(AtomicBool::new(false)), + rebuilding: false, + restore_fonts: false, } } @@ -196,8 +216,19 @@ impl Painter { if let Some(window) = window { let size = window.inner_size(); if !self.surfaces.contains_key(&viewport_id) { - let surface = self.instance.create_surface(window)?; - self.add_surface(surface, viewport_id, size).await?; + let added = match self.instance.create_surface(window) { + Ok(surface) => self.add_surface(surface, viewport_id, size).await, + Err(err) => Err(err.into()), + }; + match added { + // EternalMonitor patch: while a lost device is replaced, the window + // skips frames until a new one can be made. + Err(err) if self.rebuilding => { + log::warn!("Could not replace the lost wgpu device yet: {err}"); + self.context.request_repaint_after(REBUILD_RETRY); + } + added => added?, + } } } else { log::warn!("No window - clearing all surfaces"); @@ -245,12 +276,53 @@ impl Painter { let render_state = RenderState::create(&self.config, &self.instance, Some(&surface), self.options) .await?; + self.watch_device(&render_state.device); + if self.rebuilding { + log::info!("Replaced the lost wgpu device"); + self.rebuilding = false; + self.restore_fonts = true; + } self.render_state = Some(render_state); } self.install_surface(surface, viewport_id, size.width, size.height, false); Ok(()) } + /// EternalMonitor patch: flag the loss of `device`, and keep wgpu's default handler + /// from panicking on the errors a lost device then reports. + fn watch_device(&mut self, device: &wgpu::Device) { + let lost = Arc::new(AtomicBool::new(false)); + let flag = Arc::clone(&lost); + device.set_device_lost_callback(move |reason, message| { + log::warn!("The wgpu device was lost ({reason:?}): {message}"); + flag.store(true, Ordering::SeqCst); + }); + let flag = Arc::clone(&lost); + device.on_uncaptured_error(Arc::new(move |error| { + if flag.load(Ordering::SeqCst) { + log::debug!("wgpu error on the lost device: {error}"); + } else { + // As wgpu's default handler. + panic!("wgpu error: {error}\n"); + } + })); + self.device_lost = lost; + } + + /// EternalMonitor patch: forget a lost device and everything made on it. eframe calls + /// [`Self::set_window`] before each paint, which builds a new device and surfaces. + fn drop_lost_device(&mut self) { + log::warn!("Rebuilding the window renderer after its wgpu device was lost"); + self.render_state = None; + self.surfaces.clear(); + self.depth_texture_view.clear(); + self.msaa_texture_view.clear(); + self.screen_capture_state = None; + self.rebuilding = true; + self.restore_fonts = false; + self.context.request_repaint(); + } + /// Inserts a freshly created surface into [`Self::surfaces`] and configures it. /// /// Render state must already be initialised before calling this. @@ -510,6 +582,12 @@ impl Painter { let capture = !capture_data.is_empty(); let mut vsync_sec = 0.0; + // EternalMonitor patch: a lost device draws nothing again; replace it. + if self.device_lost.load(Ordering::SeqCst) { + self.drop_lost_device(); + return vsync_sec; + } + // If the previous frame produced `CurrentSurfaceTexture::Lost`, the action match // below set `needs_recreate`. Recreate the surface now, before re-borrowing // `self.render_state` / `self.surfaces` for the rest of the paint. @@ -565,6 +643,17 @@ impl Painter { let user_cmd_bufs = { let mut renderer = render_state.renderer.write(); + // EternalMonitor patch: the font atlas went with the lost device. Other textures + // are the app's to load again. + if core::mem::take(&mut self.restore_fonts) { + let atlas = self.context.fonts(|fonts| fonts.image()); + renderer.update_texture( + &render_state.device, + &render_state.queue, + epaint::TextureId::default(), + &epaint::ImageDelta::full(atlas, epaint::TextureAtlas::texture_options()), + ); + } #[expect(clippy::iter_over_hash_type)] // Order doesn't matter here for (id, image_deltas) in textures_delta.set.drain() { for image_delta in image_deltas { @@ -586,6 +675,14 @@ impl Painter { ) }; + // EternalMonitor patch: skip a frame whose upload failed; the next one replaces the + // device. + if render_state.renderer.read().upload_failed || self.device_lost.load(Ordering::SeqCst) { + self.device_lost.store(true, Ordering::SeqCst); + self.context.request_repaint_of(viewport_id); + return vsync_sec; + } + if surface_state.needs_reconfigure { Self::configure_surface(surface_state, render_state, &self.config.surface); surface_state.needs_reconfigure = false; From fcbc7205dc5bf82fa62455b7eb9125bb2b2891a2 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:28:13 -0500 Subject: [PATCH 019/226] Tell the iPad when the PC is locked While the lock screen or a permission prompt is up, Windows lets nothing capture the screen, and the iPad showed a frozen picture without a word (SecureDesktop: 22 events on 13 installs in a day, 116 resumes on 28). The heartbeat gets an appended notice byte, sent only while there is a notice. The PC sets it from the capture block: PC locked for SecureDesktop, nothing for the rest, and it clears when capture works again. Control bodies are append-only and both parsers ignore trailing bytes, so older iPads read the same heartbeat as before, and a plain heartbeat is byte-identical. New golden vector heartbeat_pc_locked. The iPad shows a quiet pill over the held picture, "Your PC is locked. Unlock it to continue.", until a heartbeat says capture works again. The string is English only, in the catalog and awaiting translation. --- host/src/capture/recovery.rs | 32 ++++++++++ host/src/transport/mod.rs | 63 +++++++++++++++++++ host/src/transport/session.rs | 5 ++ .../App/ConnectionManager.swift | 10 +++ ios/EternalMonitor/App/DisplayView.swift | 25 ++++++++ ios/EternalMonitor/App/UIPreview.swift | 6 +- ios/EternalMonitor/Network/WireProtocol.swift | 11 +++- .../Resources/Localizable.xcstrings | 12 ++++ ios/EternalMonitorTests/FakeUDPHost.swift | 4 +- .../LocalizationTests.swift | 9 ++- .../StreamRecoveryTests.swift | 22 +++++++ .../WireProtocolGoldenTests.swift | 19 +++++- .../DisplayStateTests.swift | 15 +++++ proto/src/v2/control.rs | 46 ++++++++++++++ proto/testdata/v2_vectors.txt | 1 + proto/tests/golden.rs | 11 ++++ 16 files changed, 285 insertions(+), 6 deletions(-) diff --git a/host/src/capture/recovery.rs b/host/src/capture/recovery.rs index 56beb0a8..61f8cf03 100644 --- a/host/src/capture/recovery.rs +++ b/host/src/capture/recovery.rs @@ -359,6 +359,18 @@ pub fn run_until_unblocked( } } +/// The heartbeat's notice while capture is blocked. A secure screen says the +/// PC is locked, so the iPad doesn't show a frozen picture without a word; it +/// clears with the block when capture works again. The other blocks keep the +/// held picture as before. +pub fn heartbeat_notice(block: Option) -> u8 { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_PC_LOCKED}; + match block { + Some(CaptureBlock::SecureDesktop) => NOTICE_PC_LOCKED, + _ => NOTICE_NONE, + } +} + /// Clears the blocked state once frames flow again. pub fn note_capture_resumed(shared: &SharedControl) { if let Some(block) = shared.capture_block.lock().take() { @@ -479,6 +491,26 @@ mod tests { assert!(shared.gpu_switch.lock().as_ref().unwrap().failed); } + #[test] + fn only_a_secure_screen_tells_the_ipad_the_pc_is_locked() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_PC_LOCKED}; + assert_eq!( + heartbeat_notice(Some(CaptureBlock::SecureDesktop)), + NOTICE_PC_LOCKED + ); + assert_eq!(heartbeat_notice(None), NOTICE_NONE); + for block in [ + CaptureBlock::DisplayChanging, + CaptureBlock::NoDisplay, + CaptureBlock::UnsupportedOnThisGpu, + CaptureBlock::TooManyCaptureApps, + CaptureBlock::SessionDisconnected, + CaptureBlock::DriverNotResponding, + ] { + assert_eq!(heartbeat_notice(Some(block)), NOTICE_NONE, "{block:?}"); + } + } + #[test] fn a_stuck_driver_call_is_waited_out_in_the_stage_not_restarted() { use crate::stall::{no_hooks, run_in, Call, Lanes, Stall, Wait}; diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 70cfb197..737b0996 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -115,6 +115,10 @@ impl ConfigSource for SharedConfigSource<'_> { } } + fn notice(&self) -> u8 { + crate::capture::recovery::heartbeat_notice(*self.shared.capture_block.lock()) + } + fn host_caps(&self) -> u16 { // `start_sender` numbers sent frames contiguously per epoch. let numbering = eternal_wire::v2::control::HOSTCAP_CONTIGUOUS_FRAME_SEQ; @@ -1424,6 +1428,65 @@ mod tests { ); } + /// While the lock screen or a permission prompt keeps Windows from + /// letting capture work, every heartbeat tells the iPad the PC is locked. + /// Once capture works again, heartbeats say nothing more. + #[test] + fn heartbeats_say_the_pc_is_locked_while_a_secure_screen_blocks_capture() { + use crate::capture::recovery::{ + note_capture_resumed, run_until_unblocked, CaptureBlock, CaptureUnavailable, + }; + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_PC_LOCKED}; + let shared = SharedControl::new(9876, 15_000_000); + shared.pairing.lock().required = false; + let now = Instant::now(); + repairable_session(&shared, "127.0.0.1:50000".parse().unwrap(), now); + let config = SharedConfigSource { + shared: &shared, + stream_epoch: 1, + pen_available: false, + }; + let notices = || { + let actions = shared.session.lock().tick(&config, true, now); + actions + .replies + .iter() + .filter_map(|(_, bytes)| match parse_control(bytes) { + Ok((_, ControlMessage::Heartbeat(heartbeat))) => Some(heartbeat.notice), + _ => None, + }) + .collect::>() + }; + assert_eq!(notices(), [NOTICE_NONE]); + // Windows answers duplication with E_ACCESSDENIED until the user is + // back on the desktop. + let mut attempts = 0; + let mut while_blocked = Vec::new(); + let result = run_until_unblocked( + &shared, + |_| { + attempts += 1; + if attempts < 3 { + Err(CaptureUnavailable { + block: CaptureBlock::from_hresult(0x8007_0005_u32 as i32).unwrap(), + detail: "Access is denied".into(), + } + .into()) + } else { + Ok(()) + } + }, + |_| { + while_blocked.extend(notices()); + true + }, + ); + assert!(result.is_ok()); + assert_eq!(while_blocked, [NOTICE_PC_LOCKED, NOTICE_PC_LOCKED]); + note_capture_resumed(&shared); + assert_eq!(notices(), [NOTICE_NONE]); + } + /// A NACK of a session that ended for liveness, while another frame is /// paced, is answered with the goodbye, as any of its late packets is. #[tokio::test] diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index b6ef5504..d0c9d688 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -288,6 +288,10 @@ pub trait ConfigSource { fn host_caps(&self) -> u16 { 0 } + /// The heartbeat's notice for the iPad (`NOTICE_*` in the wire crate). + fn notice(&self) -> u8 { + eternal_wire::v2::control::NOTICE_NONE + } } pub struct Session { @@ -964,6 +968,7 @@ impl Session { let heartbeat = ControlMessage::Heartbeat(eternal_wire::v2::control::Heartbeat { host_time_us: crate::clock::host_now_us(), stream_config: self.negotiated_config(config), + notice: config.notice(), }); actions.replies.push(( peer, diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 54023198..1829d4af 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -111,6 +111,9 @@ final class ConnectionManager: ObservableObject { /// Frames still arrive, but the picture has been held for over a second /// waiting for the keyframe that replaces a lost frame. @Published var pictureRestoring = false + /// The PC is locked or shows a permission prompt, so it can't capture and + /// the picture is held. Its heartbeats say so until capture works again. + @Published var pcLocked = false /// Nonzero while an automatic reconnect cycle is running. @Published private(set) var reconnectAttempt = 0 @@ -385,6 +388,7 @@ final class ConnectionManager: ObservableObject { linkFailure = nil versionMismatch = nil signalLost = false + pcLocked = false degradedSinceUs = nil prevCounters = FrameAssembler.Counters() diagnostics.removeAll() @@ -707,6 +711,11 @@ final class ConnectionManager: ObservableObject { Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } self.hostStreamConfig = heartbeat.streamConfig + let locked = heartbeat.notice == Heartbeat.noticePCLocked + if locked != self.pcLocked { + self.pcLocked = locked + self.record(.info, "ctrl", locked ? "PC locked: holding the picture" : "PC unlocked") + } } } channel.onStreamConfig = { [weak self, weak assembler] config in @@ -1258,6 +1267,7 @@ final class ConnectionManager: ObservableObject { stats = StreamStats() signalLost = false pictureRestoring = false + pcLocked = false degradedSinceUs = nil prevCounters = FrameAssembler.Counters() hostInfo = nil diff --git a/ios/EternalMonitor/App/DisplayView.swift b/ios/EternalMonitor/App/DisplayView.swift index ba21c5fc..9ef35f46 100644 --- a/ios/EternalMonitor/App/DisplayView.swift +++ b/ios/EternalMonitor/App/DisplayView.swift @@ -51,6 +51,10 @@ struct DisplayView: View { reconnectingCard .transition(.opacity.combined(with: .scale(scale: 0.96))) .allowsHitTesting(false) + } else if connectionManager.pcLocked { + pcLockedPill + .transition(.opacity) + .allowsHitTesting(false) } else if connectionManager.pictureRestoring { restoringPill .transition(.opacity) @@ -113,6 +117,7 @@ struct DisplayView: View { .persistentSystemOverlays(.hidden) .animation(.easeInOut(duration: 0.25), value: connectionManager.signalLost) .animation(.easeInOut(duration: 0.25), value: connectionManager.pictureRestoring) + .animation(.easeInOut(duration: 0.25), value: connectionManager.pcLocked) .animation(reduceMotion ? nil : .easeInOut(duration: 0.35), value: appeared) .onAppear { appeared = true @@ -410,6 +415,26 @@ struct DisplayView: View { .background(glass(RoundedRectangle(cornerRadius: 22, style: .continuous))) } + /// Over the held picture while the PC is locked or shows a permission + /// prompt, until it captures again. + private var pcLockedPill: some View { + HStack(spacing: 10) { + Image(systemName: "lock.fill") + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(Theme.text) + .accessibilityHidden(true) + Text(L("Your PC is locked. Unlock it to continue.")) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + } + .padding(.horizontal, 18) + .frame(minHeight: 44) + .background(glass(Capsule())) + .accessibilityElement(children: .ignore) + .accessibilityLabel(L("Your PC is locked. Unlock it to continue.")) + .accessibilityIdentifier("display.pcLocked") + } + /// The PC is still connected but a lost frame left the picture waiting /// for a fresh one; usually gone in well under a second. private var restoringPill: some View { diff --git a/ios/EternalMonitor/App/UIPreview.swift b/ios/EternalMonitor/App/UIPreview.swift index ca0e247a..f864febb 100644 --- a/ios/EternalMonitor/App/UIPreview.swift +++ b/ios/EternalMonitor/App/UIPreview.swift @@ -12,7 +12,7 @@ import SwiftUI /// Launch with `-allowUSB NO` so the USB listener stays off as well. enum UIPreview { static let mode = ProcessInfo.processInfo.environment["EM_UI_PREVIEW"] ?? "" - static var showsBackdrop: Bool { mode == "display" || mode == "quality" || mode == "restoring" } + static var showsBackdrop: Bool { mode == "display" || mode == "quality" || mode == "restoring" || mode == "locked" } static var opensQuality: Bool { mode == "quality" } static var opensSettings: Bool { mode == "settings" || mode == "settings-about" } /// Scrolls Settings to Support and About. @@ -46,6 +46,10 @@ enum UIPreview { manager.transportMode = "WiFi" manager.state = .connected manager.pictureRestoring = true + case "locked": + manager.transportMode = "WiFi" + manager.state = .connected + manager.pcLocked = true case "connecting": manager.state = .connecting case "error": diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index a48396fa..b9a85a50 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -309,6 +309,13 @@ struct HelloAck: Equatable { struct Heartbeat: Equatable { var hostTimeUs: UInt64 var streamConfig: StreamConfig + /// What to say over the picture, for as long as it lasts. PCs older than + /// the field leave it off: `noticeNone`. Unknown values are ignored. + var notice: UInt8 = Heartbeat.noticeNone + + static let noticeNone: UInt8 = 0 + /// The PC is locked or shows a permission prompt, so it can't capture. + static let noticePCLocked: UInt8 = 1 } enum ByeReason: UInt8 { @@ -469,6 +476,7 @@ extension Wire { case .heartbeat(let hb): body.appendLE(hb.hostTimeUs) hb.streamConfig.encode(into: &body) + if hb.notice != Heartbeat.noticeNone { body.append(hb.notice) } case .bye(let reason): body.append(reason.rawValue) case .keyframeRequest(let k): @@ -573,7 +581,8 @@ extension Wire { case .heartbeat: guard let hostTimeUs = r.readU64(), let cfg = StreamConfig.decode(&r) else { return nil } - message = .heartbeat(Heartbeat(hostTimeUs: hostTimeUs, streamConfig: cfg)) + message = .heartbeat(Heartbeat(hostTimeUs: hostTimeUs, streamConfig: cfg, + notice: r.readU8() ?? Heartbeat.noticeNone)) case .bye: guard let raw = r.readU8(), let reason = ByeReason(rawValue: raw) else { return nil } message = .bye(reason) diff --git a/ios/EternalMonitor/Resources/Localizable.xcstrings b/ios/EternalMonitor/Resources/Localizable.xcstrings index a4d4fa60..9d27ffc2 100644 --- a/ios/EternalMonitor/Resources/Localizable.xcstrings +++ b/ios/EternalMonitor/Resources/Localizable.xcstrings @@ -9043,6 +9043,18 @@ } } }, + "Your PC is locked. Unlock it to continue." : { + "comment" : "Over the held picture while the PC shows its lock screen or a permission prompt", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Your PC is locked. Unlock it to continue." + } + } + } + }, "⌘ key acts as" : { "comment" : "Values are Ctrl or Win", "extractionState" : "manual", diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index 273ac591..e78eb2aa 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -90,9 +90,9 @@ final class FakeUDPHost: @unchecked Sendable { } /// A session 41 heartbeat, as the PC sends one each second. - func sendHeartbeat() { + func sendHeartbeat(notice: UInt8 = Heartbeat.noticeNone) { send(Wire.encodeControl(sessionId: 41, msgSeq: 3, message: .heartbeat( - Heartbeat(hostTimeUs: 0, streamConfig: StreamConfig())))) + Heartbeat(hostTimeUs: 0, streamConfig: StreamConfig(), notice: notice)))) } /// The 64x64 H.264 keyframe fixture. diff --git a/ios/EternalMonitorTests/LocalizationTests.swift b/ios/EternalMonitorTests/LocalizationTests.swift index 821580fb..5a33593f 100644 --- a/ios/EternalMonitorTests/LocalizationTests.swift +++ b/ios/EternalMonitorTests/LocalizationTests.swift @@ -20,14 +20,21 @@ final class LocalizationTests: XCTestCase { .map { String(text[Range($0.range(at: 1), in: text)!]) }.sorted() } + /// English strings whose translations come in a separate pass. Until + /// then every language shows them in English. + private let awaitingTranslation: Set = [ + "Your PC is locked. Unlock it to continue.", + ] + /// Every language has exactly English's strings, none empty, each with /// the same placeholders. func testEveryLanguageHasEveryString() throws { let english = try table("en") XCTAssertGreaterThan(english.count, 100) + XCTAssertTrue(awaitingTranslation.isSubset(of: english.keys)) for language in AppLanguage.allCases where language != .en { let strings = try table(language.rawValue) - XCTAssertEqual(Set(strings.keys), Set(english.keys), language.rawValue) + XCTAssertEqual(Set(strings.keys), Set(english.keys).subtracting(awaitingTranslation), language.rawValue) for (key, value) in strings { XCTAssertFalse(value.trimmingCharacters(in: .whitespaces).isEmpty, "\(language.rawValue): \(key)") XCTAssertEqual(specifiers(value), specifiers(key), "\(language.rawValue): \(key)") diff --git a/ios/EternalMonitorTests/StreamRecoveryTests.swift b/ios/EternalMonitorTests/StreamRecoveryTests.swift index 3610be20..ebd60ea2 100644 --- a/ios/EternalMonitorTests/StreamRecoveryTests.swift +++ b/ios/EternalMonitorTests/StreamRecoveryTests.swift @@ -87,6 +87,28 @@ final class StreamRecoveryTests: XCTestCase { } } + /// The lock screen or a permission prompt holds the picture on the PC. + /// Its heartbeats say so, and the iPad says so over the held picture, + /// still connected, until a heartbeat says capture works again. + func testALockedPCIsShownOverTheHeldPictureUntilItCapturesAgain() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + XCTAssertFalse(manager.pcLocked) + host.sendHeartbeat(notice: Heartbeat.noticePCLocked) + try await until("the PC says it is locked") { manager.pcLocked } + XCTAssertEqual(manager.state, .connected) + host.sendHeartbeat() + try await until("the PC captures again") { !manager.pcLocked } + // A notice this iPad doesn't know says nothing. + host.sendHeartbeat(notice: 0xC8) + try await Task.sleep(nanoseconds: 500_000_000) + XCTAssertFalse(manager.pcLocked) + host.sendHeartbeat(notice: Heartbeat.noticePCLocked) + try await until("locked again") { manager.pcLocked } + manager.cancel() + XCTAssertFalse(manager.pcLocked, "the next session starts without it") + } + func testContiguousNumberingRevealsALostReopenKeyframe() async throws { // The PC reopened its encoder and its new keyframe (seq 2) was lost; // the new encoder's first P-frame continues the codec's count. A PC diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index 56b742d2..28c92618 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,7 +35,7 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 29, + Self.vectors.count, 30, "golden vector count drifted — update both test suites together" ) } @@ -302,6 +302,23 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: 42, message: message), data) } + /// The PC says it is locked in one byte after the stream config. An + /// iPad older than the field reads the same heartbeat without it. + func testPCLockedHeartbeatVector() throws { + let data = try vector("heartbeat_pc_locked") + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .heartbeat(let hb) = message else { return XCTFail("wrong type") } + XCTAssertEqual(hb.hostTimeUs, 987_654_321) + XCTAssertEqual(hb.streamConfig, Self.sampleConfig) + XCTAssertEqual(hb.notice, Heartbeat.noticePCLocked) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: 42, message: message), data) + let plain = try vector("heartbeat") + XCTAssertEqual(data.count, plain.count + 1) + XCTAssertEqual(data.dropFirst(8).dropLast(), plain.dropFirst(8)) + guard case .heartbeat(let old) = try XCTUnwrap(Wire.parseControl(plain)).message else { return XCTFail("wrong type") } + XCTAssertEqual(old.notice, Heartbeat.noticeNone) + } + func testByeVector() throws { let data = try vector("bye_background") let (_, message) = try XCTUnwrap(Wire.parseControl(data)) diff --git a/ios/EternalMonitorUITests/DisplayStateTests.swift b/ios/EternalMonitorUITests/DisplayStateTests.swift index fd82b984..1fc818c6 100644 --- a/ios/EternalMonitorUITests/DisplayStateTests.swift +++ b/ios/EternalMonitorUITests/DisplayStateTests.swift @@ -26,4 +26,19 @@ final class DisplayStateTests: UITestCase { screenshot.lifetime = .keepAlways add(screenshot) } + + func testALockedPCSaysSoOverTheHeldPicture() { + let app = XCUIApplication() + app.launchArguments = ["-didSeeOnboarding", "YES", "-allowUSB", "NO"] + app.launchEnvironment["EM_UI_PREVIEW"] = "locked" + launchCleanly(app) + let note = app.otherElements["display.pcLocked"] + XCTAssertTrue(note.waitForExistence(timeout: 10), "a locked PC must not look like a frozen picture") + XCTAssertEqual(note.label, "Your PC is locked. Unlock it to continue.") + XCTAssertFalse(app.otherElements["display.restoringPicture"].exists) + let screenshot = XCTAttachment(screenshot: app.screenshot()) + screenshot.name = "pc-locked" + screenshot.lifetime = .keepAlways + add(screenshot) + } } diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index f3e9f069..d6c6af6c 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -169,11 +169,21 @@ pub struct HelloAck { pub host_caps: u16, } +/// [`Heartbeat::notice`] values. A notice is appended to the heartbeat only +/// when it is not [`NOTICE_NONE`]: iPads older than the field ignore the byte, +/// and an iPad ignores a value it does not know. +pub const NOTICE_NONE: u8 = 0; +/// The PC is locked or shows a permission prompt, so Windows lets nothing +/// capture its screen and the picture stops until the user returns. +pub const NOTICE_PC_LOCKED: u8 = 1; + /// Host → client, every heartbeat interval while a session is active. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Heartbeat { pub host_time_us: u64, pub stream_config: StreamConfig, + /// What the iPad should say over the picture, for as long as it lasts. + pub notice: u8, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -405,6 +415,9 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - let mut cfg = [0u8; STREAM_CONFIG_SIZE]; hb.stream_config.encode_into(&mut cfg); body.extend_from_slice(&cfg); + if hb.notice != NOTICE_NONE { + body.push(hb.notice); + } } ControlMessage::Bye(reason) => body.push(*reason as u8), ControlMessage::KeyframeRequest(k) => { @@ -589,6 +602,11 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), PacketType::Heartbeat => ControlMessage::Heartbeat(Heartbeat { host_time_us: r.u64()?, stream_config: StreamConfig::decode(r.take(STREAM_CONFIG_SIZE)?)?, + notice: if r.remaining() >= 1 { + r.u8()? + } else { + NOTICE_NONE + }, }), PacketType::Bye => ControlMessage::Bye( ByeReason::from_u8(r.u8()?).ok_or(WireError::InvalidField("bye_reason"))?, @@ -908,6 +926,7 @@ mod tests { ControlMessage::Heartbeat(Heartbeat { host_time_us: 987_654_321, stream_config: sample_config(), + notice: NOTICE_NONE, }), ControlMessage::Bye(ByeReason::AppBackground), ControlMessage::KeyframeRequest(KeyframeRequest { @@ -988,6 +1007,33 @@ mod tests { } } + #[test] + fn a_heartbeat_notice_is_appended_only_when_there_is_one() { + let heartbeat = |notice| { + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: sample_config(), + notice, + }) + }; + // No notice: the bytes every earlier iPad already reads. + let plain = encode_control(7, 1, &heartbeat(NOTICE_NONE)); + assert_eq!(plain.len(), CONTROL_HEADER_SIZE + 8 + STREAM_CONFIG_SIZE); + // A notice is one byte more. Such a body, cut back to the old length + // as an older iPad reads it, is the plain heartbeat; an unknown value + // reaches the iPad as it is, for it to ignore. + for notice in [NOTICE_PC_LOCKED, 0xC8] { + let datagram = encode_control(7, 1, &heartbeat(notice)); + assert_eq!(datagram.len(), plain.len() + 1); + assert_eq!(datagram[..plain.len()][8..], plain[8..]); + assert_eq!(parse_control(&datagram).unwrap().1, heartbeat(notice)); + for len in 0..datagram.len() { + assert!(parse_control(&datagram[..len]).is_err()); + } + } + assert_eq!(parse_control(&plain).unwrap().1, heartbeat(NOTICE_NONE)); + } + #[test] fn trailing_bytes_are_ignored_for_forward_compat() { let mut datagram = encode_control(1, 1, &ControlMessage::Ping(Ping { t1_us: 5 })); diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 6d072d05..e4ca515e 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -20,6 +20,7 @@ hello2 454d02100000240000000000010000000202efbeadde94260300010074098406ba0442037 hello_ack_ok 454d02110000270000000000010000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d5043 hello_ack_busy 454d02110000270000000000020000000102efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d5043 heartbeat 454d021200001800785634122a000000b168de3a0000000003000000000aa0053c000000c0e1e400 +heartbeat_pc_locked 454d021200001900785634122a000000b168de3a0000000003000000000aa0053c000000c0e1e40001 bye_background 454d021300000100785634122b00000001 keyframe_request 454d021400000900785634122c000000030000000f10000000 receiver_report 454d021500002400785634122d00000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index e0e6c5ad..d1a649d2 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -245,6 +245,17 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { ControlMessage::Heartbeat(Heartbeat { host_time_us: 987_654_321, stream_config: stream_config(), + notice: NOTICE_NONE, + }), + ), + ( + "heartbeat_pc_locked", + 0x1234_5678, + 42, + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: stream_config(), + notice: NOTICE_PC_LOCKED, }), ), ( From e0a4f734c5254c4d609ac17166ed2344dba69bf2 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:33:35 -0500 Subject: [PATCH 020/226] Deliver the storm-brake goodbye, refuse HELLOs after a stop, and name stops in summaries Review round 1 of the stop goodbyes. - The transport no longer ends when the encoder closes its video channel. The encoder thread drops it before its failure report reaches the supervisor, so the transport used to exit before the storm brake queued its goodbye; the next transport then dropped it as stale. Now the loop stops reading video and runs until `running` goes false. - After Session::shut_down, HELLOs are refused with Busy until a transport resumes. A HELLO before the loop saw `running=false` opened a session that nothing served. One place decides it: closed_to_clients. - Summaries tell stops apart with existing tokens: Stop is end=user, the storm brake is end=error with failure=capture_failed, encoder_failed or other. A quit stays end=shutdown. SupervisorState::Failed keeps the stage. - On the iPad, the error goodbye's card is titled "The connection to your PC ended", true for a PC that restarts by itself as well as for a fault. "Your PC ended the session" stays for Stop and quit. SETUP.md lists both. - stalled_ms is a diagnostic key. The stall test checks for a free port on every address, as the transport binds, and fails fast if the transport ends early. --- SETUP.md | 3 +- host/src/diagnostics.rs | 53 ++++-- host/src/supervisor.rs | 135 +++++++++++--- host/src/telemetry/redact.rs | 5 + host/src/transport/mod.rs | 13 +- host/src/transport/session.rs | 174 +++++++++++++++--- host/tests/goodbye.rs | 43 ++++- host/tests/transport_stall.rs | 29 ++- .../App/ConnectionProblem.swift | 4 +- .../ConnectionProblemTests.swift | 5 +- ios/EternalMonitorTests/GoodbyeTests.swift | 2 +- 11 files changed, 391 insertions(+), 75 deletions(-) diff --git a/SETUP.md b/SETUP.md index 6b0a5ba4..fac7161f 100644 --- a/SETUP.md +++ b/SETUP.md @@ -274,7 +274,8 @@ try fewer words. If your problem isn't here, see [Known issues](https://eternalm | **The connection over the cable stopped** | The USB connection broke after it had started. | Keep EternalMonitor and Apple Devices open on the PC, unlock the iPad, then unplug the cable and plug it in again. | | **The USB cable was disconnected** | The cable came out, or the PC lost it. | Plug the cable back in. The iPad reconnects by itself. | | **USB connections were turned off** | **Allow USB connections** is off in the iPad app's settings. | Turn on **Allow USB connections** to use the cable again. | -| **Your PC ended the session** | Someone clicked **Stop** on the PC, changed a setting that restarts the stream, or closed EternalMonitor. Or the connection dropped, for example in a Wi-Fi stall, or the PC ran into a problem. | Tap **Try again** when the PC says **Ready for your iPad**. | +| **Your PC ended the session** | Someone clicked **Stop** on the PC, changed a setting that restarts the stream, or closed EternalMonitor. | Tap **Try again** when the PC says **Ready for your iPad**. | +| **The connection to your PC ended** | The connection dropped, for example in a Wi-Fi stall, the PC ran into a problem, or EternalMonitor restarted itself on the PC. | Tap **Try again**. With **Reconnect automatically** on, the iPad tries again by itself first. | | **Your PC is already streaming to another device** | Only one iPad can connect at a time. | Disconnect the other iPad first, then tap **Try again**. | | **Your PC is updating EternalMonitor** | The PC is installing a new version. | Wait. With **Reconnect automatically** on, the iPad connects again by itself. Otherwise tap **Try again** once EternalMonitor has opened on the PC. | | **Your PC hasn't come back from its update yet** | The update is taking longer than expected. | Check that EternalMonitor opened again on the PC. Windows may be waiting for you to click **Yes** there. Then tap **Try again**. | diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index b7009110..a2568625 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -5,8 +5,7 @@ use std::time::Duration; use crate::control::{CaptureTarget, SharedControl}; use crate::transport::link::LinkId; -use crate::transport::session::{EndReason, SessionEnd}; -use eternal_wire::v2::control::ByeReason; +use crate::transport::session::{EndReason, SessionEnd, StageFailure, StreamStop}; use tracing::info; static SYSTEM_REPORT: OnceLock = OnceLock::new(); @@ -346,21 +345,21 @@ pub fn log_session_end(end: &SessionEnd, shared: &SharedControl) { /// told the PC ended the session, or, when a restart or an update hands over /// to a new copy, to connect again. pub fn log_session_end_at_quit(shared: &SharedControl) { - let bye = if shared.handoff.holder().is_some() { - ByeReason::Error + let why = if shared.handoff.holder().is_some() { + StreamStop::Handoff } else { - ByeReason::UserDisconnect + StreamStop::Quit }; - stop_session(shared, bye); + stop_session(shared, why); } /// The stream stops for good: ends the session, which the transport tells -/// `bye` on its way out, and logs its summary. -pub fn stop_session(shared: &SharedControl, bye: ByeReason) { +/// why on its way out, and logs its summary. +pub fn stop_session(shared: &SharedControl, why: StreamStop) { let end = shared .session .lock() - .shut_down(std::time::Instant::now(), bye); + .shut_down(std::time::Instant::now(), why); if let Some(end) = end { log_session_end(&end, shared); } @@ -396,9 +395,12 @@ pub fn session_summary( _ if streamed => "streamed", EndReason::Goodbye(ByeReason::UserDisconnect) => "user_ended", EndReason::Goodbye(ByeReason::Error) | EndReason::Liveness => "no_picture", - EndReason::Goodbye(_) | EndReason::Superseded | EndReason::Shutdown => "app_ended", + EndReason::Goodbye(_) + | EndReason::Superseded + | EndReason::Shutdown + | EndReason::Stopped => "app_ended", EndReason::Revoked => "pairing", - EndReason::LinkLost => "error", + EndReason::LinkLost | EndReason::Failed(_) => "error", }; let failure = match end.reason { EndReason::Refused(status) if pairing(status) => "none", @@ -408,6 +410,9 @@ pub fn session_summary( EndReason::Liveness | EndReason::Goodbye(ByeReason::Error) if !streamed => "no_video", EndReason::Liveness => "signal_lost", EndReason::Goodbye(ByeReason::Error) => "other", + EndReason::Failed(StageFailure::Capture) => "capture_failed", + EndReason::Failed(StageFailure::Encoder) => "encoder_failed", + EndReason::Failed(StageFailure::Other) => "other", _ => "none", }; let end_token = match end.reason { @@ -415,7 +420,10 @@ pub fn session_summary( EndReason::Liveness => "liveness", EndReason::Superseded => "superseded", EndReason::Shutdown => "shutdown", - EndReason::Refused(_) | EndReason::LinkLost | EndReason::Revoked => "error", + EndReason::Stopped => "user", + EndReason::Refused(_) | EndReason::LinkLost | EndReason::Revoked | EndReason::Failed(_) => { + "error" + } }; let session_id = end.session_id.map_or("-".to_string(), |id| id.to_string()); let transport = match end.link { @@ -784,6 +792,27 @@ pub(crate) mod tests { end(Shutdown, None, 20), ["accepted", "app_ended", "none", "shutdown"], ), + // Stop on the PC and the storm brake are not a clean quit. + ( + end(Stopped, Some(250), 20_000), + ["steady", "streamed", "none", "user"], + ), + ( + end(Stopped, None, 20), + ["accepted", "app_ended", "none", "user"], + ), + ( + end(Failed(StageFailure::Capture), Some(250), 20_000), + ["steady", "streamed", "capture_failed", "error"], + ), + ( + end(Failed(StageFailure::Encoder), None, 20), + ["accepted", "error", "encoder_failed", "error"], + ), + ( + end(Failed(StageFailure::Other), Some(250), 9_000), + ["first_frame", "streamed", "other", "error"], + ), (usb, ["first_frame", "streamed", "usb_lost", "error"]), ( end(LinkLost, None, 100), diff --git a/host/src/supervisor.rs b/host/src/supervisor.rs index 78a277ad..186ffa29 100644 --- a/host/src/supervisor.rs +++ b/host/src/supervisor.rs @@ -16,6 +16,7 @@ use tracing::{error, info, warn}; use crate::control::{SharedControl, SupervisorCommand}; use crate::gpu::GpuInfo; use crate::stats::PIPELINE_STATS; +use crate::transport::session::{StageFailure, StreamStop}; /// The current pipeline generation. Stage health reports carry the generation /// they belong to; anything from an older generation is ignored (a "poisoned" @@ -88,6 +89,8 @@ pub enum SupervisorState { /// Too many automatic restarts in the window; a manual Start is required. Failed { reason: String, + /// The stage that kept failing (a wedge counts as capture). + stage: Stage, }, Stopped, ShutDown, @@ -222,7 +225,10 @@ impl Machine { STORM_WINDOW.as_secs() ); error!("{message} — giving up until a manual Start"); - self.state = SupervisorState::Failed { reason: message }; + self.state = SupervisorState::Failed { + reason: message, + stage, + }; return Effect::StopPipeline; } self.restarts.push(now); @@ -367,10 +373,7 @@ pub fn run( match effect { Effect::None => {} Effect::StopPipeline => { - if let Some(bye) = goodbye_when_stopped(&machine.state) { - crate::diagnostics::stop_session(&shared, bye); - } - shared.stop(); + stop_pipeline(&shared, &machine.state); bounded_join(&mut pipeline, generation); } Effect::Restart | Effect::Start => { @@ -412,17 +415,30 @@ pub fn run( info!("Supervisor loop ended"); } -/// What the iPad of a session is told when the pipeline stops in `state`. -/// Stopped and Failed wait for a manual Start, so the session ends now: kept, -/// it went on unanswered while its iPad lost the signal and every HELLO it -/// sent went unanswered. Stop says the PC ended the session; giving up says -/// connect again. A backoff or a held port restarts the stream by itself, -/// and its session goes on. -fn goodbye_when_stopped(state: &SupervisorState) -> Option { - use eternal_wire::v2::control::ByeReason; +/// Stops the pipeline that the machine stopped in `state`, first ending the +/// session as [`stop_when_stopped`] says: its goodbye goes out with the +/// transport, which runs until it sees `running` false. +fn stop_pipeline(shared: &SharedControl, state: &SupervisorState) { + if let Some(why) = stop_when_stopped(state) { + crate::diagnostics::stop_session(shared, why); + } + shared.stop(); +} + +/// How a session ends when the pipeline stops in `state`. Stopped and +/// Failed wait for a manual Start, so the session ends now: kept, it went on +/// unanswered while its iPad lost the signal and every HELLO it sent went +/// unanswered. Stop says the PC ended the session; giving up says connect +/// again, and names the stage in the summary. A backoff or a held port +/// restarts the stream by itself, and its session goes on. +fn stop_when_stopped(state: &SupervisorState) -> Option { match state { - SupervisorState::Stopped => Some(ByeReason::UserDisconnect), - SupervisorState::Failed { .. } => Some(ByeReason::Error), + SupervisorState::Stopped => Some(StreamStop::User), + SupervisorState::Failed { stage, .. } => Some(StreamStop::GaveUp(match stage { + Stage::Capture => StageFailure::Capture, + Stage::Encoder => StageFailure::Encoder, + Stage::Transport | Stage::Audio => StageFailure::Other, + })), SupervisorState::Running { .. } | SupervisorState::BackingOff { .. } | SupervisorState::WaitingForPort { .. } @@ -658,16 +674,77 @@ mod tests { /// a goodbye. A stream that restarts by itself keeps its session. #[test] fn only_a_stop_that_waits_for_start_ends_the_session() { - use eternal_wire::v2::control::ByeReason; + use crate::transport::link::PeerId; + use crate::transport::session::ConfigSource; + use eternal_wire::v2::control::{ + parse_control, ByeReason, ControlMessage, Hello2, StreamConfig, CAP_DECODE_H264, + }; + struct Config; + impl ConfigSource for Config { + fn stream_config(&self) -> StreamConfig { + StreamConfig { + stream_epoch: 1, + width: 1280, + height: 720, + fps: 60, + codec: 0, + flags: 0, + bitrate_bps: 10_000_000, + } + } + fn host_name(&self) -> String { + "TEST-HOST".into() + } + } + // Stops the pipeline as the run loop does, with an iPad streaming, + // and returns the goodbye left for the transport to send. + let stop = |state: &SupervisorState| { + let shared = SharedControl::new(19899, 10_000_000); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + let peer = PeerId::udp("127.0.0.1:19898".parse().unwrap()); + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 1, + listen_port: 19898, + decoder_caps: CAP_DECODE_H264, + feature_caps: 0, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "stop test".into(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + shared + .session + .lock() + .handle_control(peer, 0, hello, &Config, Instant::now()); + stop_pipeline(&shared, state); + assert!(!shared.running.load(Ordering::SeqCst)); + let goodbye = shared.session.lock().take_goodbye(); + let ended = !shared.client_connected(); + goodbye.map(|(to, bytes)| { + assert_eq!(to, peer); + assert!(ended, "a goodbye ends the session"); + match parse_control(&bytes) { + Ok((_, ControlMessage::Bye(reason))) => reason, + other => panic!("not a goodbye: {other:?}"), + } + }) + }; + let mut machine = Machine::new(1); assert_eq!( machine.on_command(&SupervisorCommand::Stop), Effect::StopPipeline ); - assert_eq!( - goodbye_when_stopped(&machine.state), - Some(ByeReason::UserDisconnect) - ); + assert_eq!(stop(&machine.state), Some(ByeReason::UserDisconnect)); let mut machine = Machine::new(1); let t0 = Instant::now(); @@ -675,17 +752,27 @@ mod tests { for _ in 0..=STORM_LIMIT { effect = machine.on_stage_exit(1, Stage::Encoder, &failed("x"), t0); if let SupervisorState::BackingOff { .. } = machine.state { - assert_eq!(goodbye_when_stopped(&machine.state), None); + assert_eq!(stop(&machine.state), None, "a backoff keeps the session"); machine.note_started(1); } } assert_eq!(effect, Effect::StopPipeline); - assert!(matches!(machine.state, SupervisorState::Failed { .. })); - assert_eq!(goodbye_when_stopped(&machine.state), Some(ByeReason::Error)); + assert!(matches!( + machine.state, + SupervisorState::Failed { + stage: Stage::Encoder, + .. + } + )); + assert_eq!( + stop_when_stopped(&machine.state), + Some(StreamStop::GaveUp(StageFailure::Encoder)) + ); + assert_eq!(stop(&machine.state), Some(ByeReason::Error)); let mut machine = Machine::new(1); machine.wait_for_port(t0); - assert_eq!(goodbye_when_stopped(&machine.state), None); + assert_eq!(stop(&machine.state), None, "a held port keeps the session"); } fn failed(reason: &str) -> StageOutcome { diff --git a/host/src/telemetry/redact.rs b/host/src/telemetry/redact.rs index 69283075..f3bf8d59 100644 --- a/host/src/telemetry/redact.rs +++ b/host/src/telemetry/redact.rs @@ -138,6 +138,7 @@ const DIAGNOSTIC_KEYS: &[&str] = &[ "ss_rule", // A silent client's session held, kept or ended. "silent_ms", + "stalled_ms", "held_ms", "last_heard", "heard", @@ -1994,6 +1995,10 @@ mod tests { "WARN Client liveness expired — tearing session down peer=192.0.2.7:50000 silent_ms=10000 \ held_ms=7000 last_heard=input late_input=0 device=iPad", ), + ( + "2750", + "INFO The stream's transport loop moved again phase=input stalled_ms=2750", + ), ]; for (name, message) in cases { let line = format!("{STAMP} {message}"); diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 20a58bc3..58f3db00 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -265,6 +265,11 @@ pub async fn start_sender( // Capture numbers skip the frames the encoder never took, and the iPad // held the frame after each skip waiting for one that never existed. let mut frame_seq: u32 = 0; + // The encoder closes its channel when it exits, before it reports why. + // The loop goes on until `running` goes false, so a goodbye the + // supervisor leaves on hearing that report (the storm brake) still goes + // out. + let mut video_open = true; let mut state = LoopState { retransmit_ring: RetransmitRing::default(), @@ -405,9 +410,13 @@ pub async fn start_sender( beat.mark(Phase::Audio); audio_sender.send(packet, &links, &shared, stream_epoch).await; } - nal_opt = nal_rx.recv() => { + nal_opt = nal_rx.recv(), if video_open => { beat.mark(Phase::Video); - let Some(nal) = nal_opt else { break; }; + let Some(nal) = nal_opt else { + info!("The encoder closed its channel; the transport runs until the stream stops"); + video_open = false; + continue; + }; let Some(session_id) = shared.session.lock().session_id() else { continue; }; let target_addr = *shared.target_addr.lock(); if target_addr.link == LinkId::Udp && target_addr.addr.is_none_or(|a| a.ip().is_unspecified() || a.port() == 0) { diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index f77c9962..5304c053 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -101,6 +101,47 @@ pub struct Actions { pub ended: Option, } +/// The stage whose failures made the PC give up, as far as the summary's +/// `failure=` names it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StageFailure { + Capture, + Encoder, + Other, +} + +/// Why the stream stops for good ([`Session::shut_down`]): what the iPad is +/// told, and how the session's summary ends. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamStop { + /// The app quits. The iPad hears that the PC ended the session. + Quit, + /// The app quits for a restart or an update that hands over to a new + /// copy: the iPad is told to connect again. + Handoff, + /// Stop on the PC. + User, + /// The PC gave up after repeated failures; the iPad may try again. + GaveUp(StageFailure), +} + +impl StreamStop { + fn bye(self) -> ByeReason { + match self { + Self::Quit | Self::User => ByeReason::UserDisconnect, + Self::Handoff | Self::GaveUp(_) => ByeReason::Error, + } + } + + fn reason(self) -> EndReason { + match self { + Self::Quit | Self::Handoff => EndReason::Shutdown, + Self::User => EndReason::Stopped, + Self::GaveUp(stage) => EndReason::Failed(stage), + } + } +} + /// How far a session got and how it ended, for its support summary. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SessionEnd { @@ -129,6 +170,10 @@ pub enum EndReason { Superseded, /// An update stopped the stream, or the app quit. Shutdown, + /// Stop on the PC. + Stopped, + /// The PC gave up restarting a stage that kept failing. + Failed(StageFailure), /// The session's link (the USB tunnel) closed. LinkLost, /// The pairing credentials changed. @@ -298,6 +343,10 @@ pub struct Session { /// either closes the host or gives up. Changed only under the session lock, /// together with the check that nobody is streaming. updating: bool, + /// The stream stopped for good ([`Self::shut_down`]) and no transport + /// has resumed since: new clients are turned away, as nothing would + /// serve them. + stream_stopped: bool, /// The goodbye for a stream an update or a stop ended, sent by the next /// tick or by the transport on its way out. goodbye: Option<(PeerId, Vec)>, @@ -320,6 +369,7 @@ impl Session { active: None, last_end: None, updating: false, + stream_stopped: false, goodbye: None, stopped: None, ended_for_liveness: None, @@ -366,15 +416,24 @@ impl Session { return false; } info!(peer = %session.peer, "Stopping the stream for an update"); - self.stopped = self.end_saying(ByeReason::HostShuttingDown, Instant::now()); + self.stopped = self.end_saying( + ByeReason::HostShuttingDown, + EndReason::Shutdown, + Instant::now(), + ); } self.updating = true; true } - /// Ends the active session for [`EndReason::Shutdown`] and leaves `bye` - /// for the transport to send before its links close. - fn end_saying(&mut self, bye: ByeReason, now: Instant) -> Option { + /// Ends the active session for `reason` and leaves `bye` for the + /// transport to send before its links close. + fn end_saying( + &mut self, + bye: ByeReason, + reason: EndReason, + now: Instant, + ) -> Option { let session = self.active.as_ref()?; let (peer, session_id) = (session.peer, session.session_id); let msg_seq = self.next_msg_seq(); @@ -386,7 +445,7 @@ impl Session { &ControlMessage::Bye(bye), ), )); - self.end(EndReason::Shutdown, now) + self.end(reason, now) } /// The goodbye left for a stopped stream ([`Self::begin_update`], @@ -400,16 +459,37 @@ impl Session { self.updating = false; } - /// The stream stops for good (the app quits, or Stop): the end of the - /// session it stops, or of one an update stopped that no tick reported - /// yet. A session it stops is told `bye` by the transport on its way - /// out; the iPad used to hear nothing, and showed a lost signal or an - /// unplugged cable. - pub fn shut_down(&mut self, now: Instant, bye: ByeReason) -> Option { - let active = self.end_saying(bye, now); + /// The stream stops for good, for `why`: the end of the session it + /// stops, or of one an update stopped that no tick reported yet. A + /// session it stops is told why by the transport on its way out; the + /// iPad used to hear nothing, and showed a lost signal or an unplugged + /// cable. Until a transport resumes, new clients are turned away. + pub fn shut_down(&mut self, now: Instant, why: StreamStop) -> Option { + self.stream_stopped = true; + let active = self.end_saying(why.bye(), why.reason(), now); self.stopped.take().or(active) } + /// Why a HELLO2 is turned away before anything else is looked at, if it + /// is: an update is about to close the host, or the stream stopped and + /// its transport is on its way out. The iPad shows Busy; a retry once + /// the stream runs again works. + fn closed_to_clients(&self) -> Option<(HelloStatus, &'static str)> { + if self.updating { + Some(( + HelloStatus::Busy, + "Client turned away while an update starts", + )) + } else if self.stream_stopped { + Some(( + HelloStatus::Busy, + "Client turned away while the stream stops", + )) + } else { + None + } + } + /// The first video went to `session_id`. Later calls change nothing. pub fn note_video_sent(&mut self, session_id: u32, now: Instant) { if let Some(session) = self.active.as_mut() { @@ -646,11 +726,10 @@ impl Session { return self.refuse(source, HelloStatus::VersionUnsupported, &hello, config, now); } - // An update is about to close the host: don't start a stream it would cut. - // The iPad shows Busy ("already streaming"); a retry after the restart works. - if self.updating { - info!(peer = %source, "Client turned away while an update starts"); - return self.refuse(source, HelloStatus::Busy, &hello, config, now); + // Don't start a stream nothing will serve. + if let Some((status, why)) = self.closed_to_clients() { + info!(peer = %source, "{why}"); + return self.refuse(source, status, &hello, config, now); } // Keep credentials locked through authorization, rotation and ACK creation: @@ -902,6 +981,7 @@ impl Session { /// count against it: 0.3.1 expired live iPads the moment a slow restart /// finished and made them reconnect. pub fn transport_resumed(&mut self, now: Instant) { + self.stream_stopped = false; if let Some(session) = self.active.as_mut() { session.liveness_deadline = session.liveness_deadline.max(now + LIVENESS_TIMEOUT); } @@ -2664,37 +2744,83 @@ mod tests { (4, EndReason::Shutdown) ); assert_eq!( - session.shut_down(now, ByeReason::UserDisconnect), + session.shut_down(now, StreamStop::Quit), None, "reported once" ); assert_eq!(session.take_goodbye(), None, "nobody left to tell"); session.end_update(); + // The stream runs again: clients are let in. + session.transport_resumed(now); session.handle_control(usb, 0, identified_hello(5, 77), &TestConfig, now); let quit = session - .shut_down(now + Duration::from_secs(3), ByeReason::UserDisconnect) + .shut_down(now + Duration::from_secs(3), StreamStop::Quit) .unwrap(); assert_eq!((quit.client_nonce, quit.reason), (5, EndReason::Shutdown)); assert_eq!(quit.duration, Duration::from_secs(3)); assert!(!session.is_active()); } + /// A HELLO2 that arrives after the stream stopped, before its transport + /// noticed (up to a liveness tick), used to open a session nothing would + /// serve: no heartbeats, and the PC counted it as connected. It is turned + /// away until a transport resumes. + #[test] + fn a_hello_after_the_stream_stopped_is_turned_away_until_it_runs_again() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(500); + assert_eq!(session.shut_down(now, StreamStop::User), None); + let refused = session.handle_control(peer, 0, identified_hello(7, 77), &TestConfig, now); + assert_eq!(parse_ack(&refused.replies[0].1).status, HelloStatus::Busy); + assert!(!session.is_active()); + + session.transport_resumed(now); + let admitted = session.handle_control(peer, 0, identified_hello(8, 77), &TestConfig, now); + assert_eq!(admitted.new_target, Some(peer)); + assert!(session.is_active()); + } + /// Quitting or Stop ends the stream for good: the iPad hears why, from /// the transport on its way out, instead of waiting out a lost signal. /// A restart tells it to connect again. #[test] fn a_stopped_stream_tells_its_ipad() { let now = Instant::now(); - for (peer, bye) in [ - (PeerId::usb(17), ByeReason::UserDisconnect), - (addr([10, 0, 0, 5], 50000), ByeReason::Error), + let usb = PeerId::usb(17); + let wifi = addr([10, 0, 0, 5], 50000); + for (peer, why, bye, reason) in [ + ( + usb, + StreamStop::Quit, + ByeReason::UserDisconnect, + EndReason::Shutdown, + ), + ( + wifi, + StreamStop::Handoff, + ByeReason::Error, + EndReason::Shutdown, + ), + ( + wifi, + StreamStop::User, + ByeReason::UserDisconnect, + EndReason::Stopped, + ), + ( + usb, + StreamStop::GaveUp(StageFailure::Encoder), + ByeReason::Error, + EndReason::Failed(StageFailure::Encoder), + ), ] { let mut session = Session::new(500); session.handle_control(peer, 0, identified_hello(6, 77), &TestConfig, now); let id = session.session_id().unwrap(); - let end = session.shut_down(now, bye).unwrap(); - assert_eq!(end.reason, EndReason::Shutdown); + let end = session.shut_down(now, why).unwrap(); + assert_eq!(end.reason, reason); let (to, bytes) = session.take_goodbye().expect("a goodbye"); let (header, message) = eternal_wire::v2::control::parse_control(&bytes).unwrap(); assert_eq!( diff --git a/host/tests/goodbye.rs b/host/tests/goodbye.rs index 32eb3c21..2a0d9a0d 100644 --- a/host/tests/goodbye.rs +++ b/host/tests/goodbye.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant}; use eternal_host::control::SharedControl; use eternal_host::transport::link::{read_datagram, read_preamble, write_datagram, PeerId}; -use eternal_host::transport::session::ConfigSource; +use eternal_host::transport::session::{ConfigSource, StageFailure, StreamStop}; use eternal_host::transport::{self}; use eternal_wire::v2::control::{ encode_control, parse_control, ByeReason, ControlMessage, Hello2, StreamConfig, CAP_DECODE_H264, @@ -62,8 +62,9 @@ fn is_bye(bytes: &[u8], reason: ByeReason) -> bool { matches!(parse_control(bytes), Ok((_, ControlMessage::Bye(said))) if said == reason) } +/// The transport binds every address, so the port must be free on all. fn free_udp_port() -> u16 { - std::net::UdpSocket::bind("127.0.0.1:0") + std::net::UdpSocket::bind("0.0.0.0:0") .and_then(|s| s.local_addr()) .map(|a| a.port()) .unwrap() @@ -78,13 +79,17 @@ enum End { Quit, /// The app quits for "Restart EternalMonitor". Restart, + /// The encoder kept failing: it closed its channel as it exited, and the + /// supervisor, on hearing why, gave up (the storm brake). + GaveUp, } /// How the transport stops after the stream ended. enum Stop { /// The host is closing: `running` goes false. Shutdown, - /// The encoder stopped first and closed its channel. + /// The encoder stopped first and closed its channel, then the host + /// closes. EncoderEof, } @@ -114,6 +119,7 @@ fn udp_goodbyes(end: End, stop: Stop, reason: ByeReason) -> usize { .handle_control(peer, 0, hello(), &Config, Instant::now()); assert_eq!(admitted.new_target, Some(peer)); let (nal_tx, nal_rx) = mpsc::channel(4); + let mut nal_tx = Some(nal_tx); let (_audio_tx, audio_rx) = mpsc::channel(4); let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); let control = shared.clone(); @@ -125,6 +131,17 @@ fn udp_goodbyes(end: End, stop: Stop, reason: ByeReason) -> usize { // Let the loop take its first ticks. tokio::time::sleep(Duration::from_millis(40)).await; match end { + End::GaveUp => { + // The encoder thread drops its sender before its exit report + // reaches the supervisor. Longer than a liveness tick: the + // transport used to end here, before the goodbye existed. + drop(nal_tx.take()); + tokio::time::sleep(Duration::from_millis(400)).await; + eternal_host::diagnostics::stop_session( + &shared, + StreamStop::GaveUp(StageFailure::Encoder), + ); + } End::Update => assert!(shared.session.lock().begin_update(true)), End::Quit => eternal_host::diagnostics::log_session_end_at_quit(&shared), End::Restart => { @@ -137,7 +154,10 @@ fn udp_goodbyes(end: End, stop: Stop, reason: ByeReason) -> usize { } match stop { Stop::Shutdown => shared.stop(), - Stop::EncoderEof => drop(nal_tx), + Stop::EncoderEof => { + drop(nal_tx.take()); + shared.stop(); + } } // Only a backstop for a transport that never stops. It stops at its // next liveness tick, 250 ms away, but a loaded machine can leave @@ -189,6 +209,17 @@ fn quitting_tells_the_ipad_the_pc_ended_the_session() { ); } +/// The storm brake gives up after the encoder kept failing. The encoder +/// closes its channel before the supervisor hears why, so the transport must +/// still be there to send the goodbye the supervisor leaves. +#[test] +fn giving_up_on_a_failing_encoder_tells_the_ipad_to_connect_again() { + assert_eq!( + udp_goodbyes(End::GaveUp, Stop::Shutdown, ByeReason::Error), + 3 + ); +} + /// A restart hands over to a new copy that is back in seconds: the iPad is /// told to connect again, at once. #[test] @@ -261,7 +292,9 @@ fn usb_goodbye(end: End, reason: ByeReason) { tokio::time::sleep(Duration::from_millis(40)).await; match end { End::Update => assert!(shared.session.lock().begin_update(true)), - End::Quit | End::Restart => eternal_host::diagnostics::log_session_end_at_quit(&shared), + End::Quit | End::Restart | End::GaveUp => { + eternal_host::diagnostics::log_session_end_at_quit(&shared) + } } shared.stop(); let stopping = Instant::now(); diff --git a/host/tests/transport_stall.rs b/host/tests/transport_stall.rs index fe20e731..4bafae14 100644 --- a/host/tests/transport_stall.rs +++ b/host/tests/transport_stall.rs @@ -11,8 +11,9 @@ use eternal_host::transport::beat::{Phase, Verdict, Watch}; use eternal_host::transport::{self}; use tokio::sync::mpsc; +/// The transport binds every address, so the port must be free on all. fn free_udp_port() -> u16 { - std::net::UdpSocket::bind("127.0.0.1:0") + std::net::UdpSocket::bind("0.0.0.0:0") .and_then(|s| s.local_addr()) .map(|a| a.port()) .unwrap() @@ -23,9 +24,17 @@ fn now_ms() -> u64 { } /// Checks the beat as the supervisor does, every 100 ms, until `until`. -fn watch_until(watch: &mut Watch, shared: &SharedControl, until: Instant) -> Vec { +/// Fails at once if the transport ended (it could not bind, say), which +/// would otherwise read as a loop that never marks. +fn watch_until( + watch: &mut Watch, + shared: &SharedControl, + transport: &std::thread::JoinHandle<()>, + until: Instant, +) -> Vec { let mut verdicts = Vec::new(); while Instant::now() < until { + assert!(!transport.is_finished(), "the transport ended early"); verdicts.extend(watch.check(shared.transport_beat.read(), now_ms())); std::thread::sleep(Duration::from_millis(100)); } @@ -65,6 +74,7 @@ fn a_stuck_transport_loop_is_named_with_its_step() { started.elapsed() < Duration::from_secs(30), "the loop starts" ); + assert!(!transport.is_finished(), "the transport ended early"); std::thread::sleep(Duration::from_millis(10)); } // Running freely, it ticks every 250 ms. @@ -72,13 +82,19 @@ fn a_stuck_transport_loop_is_named_with_its_step() { watch_until( &mut watch, &shared, + &transport, Instant::now() + Duration::from_millis(1500) ), [] ); let held = shared.session.lock(); - let stuck = watch_until(&mut watch, &shared, Instant::now() + Duration::from_secs(4)); + let stuck = watch_until( + &mut watch, + &shared, + &transport, + Instant::now() + Duration::from_secs(4), + ); drop(held); assert!( matches!( @@ -87,7 +103,12 @@ fn a_stuck_transport_loop_is_named_with_its_step() { ), "{stuck:?}" ); - let moved = watch_until(&mut watch, &shared, Instant::now() + Duration::from_secs(1)); + let moved = watch_until( + &mut watch, + &shared, + &transport, + Instant::now() + Duration::from_secs(1), + ); assert!( matches!( moved.as_slice(), diff --git a/ios/EternalMonitor/App/ConnectionProblem.swift b/ios/EternalMonitor/App/ConnectionProblem.swift index d476ad3c..cd4e262f 100644 --- a/ios/EternalMonitor/App/ConnectionProblem.swift +++ b/ios/EternalMonitor/App/ConnectionProblem.swift @@ -75,7 +75,9 @@ enum ConnectionProblem: Equatable { case .lost(.usbTurnedOff): return "USB connections were turned off" case .reconnecting: return "Reconnecting to your PC…" case .reconnectFailed: return "Couldn't reconnect to your PC" - case .pcEndedSession, .pcEndedSessionOnError: return "Your PC ended the session" + case .pcEndedSession: return "Your PC ended the session" + // Also what a PC that restarts by itself sends: true for both. + case .pcEndedSessionOnError: return "The connection to your PC ended" case .pcBusy: return "Your PC is already streaming to another device" case .pcUpdating(.reconnecting), .pcUpdating(.manual): return "Your PC is updating EternalMonitor" case .pcUpdating(.notBack): return "Your PC hasn't come back from its update yet" diff --git a/ios/EternalMonitorTests/ConnectionProblemTests.swift b/ios/EternalMonitorTests/ConnectionProblemTests.swift index 7450d989..deb9dff7 100644 --- a/ios/EternalMonitorTests/ConnectionProblemTests.swift +++ b/ios/EternalMonitorTests/ConnectionProblemTests.swift @@ -248,7 +248,10 @@ final class ConnectionProblemTests: XCTestCase { XCTAssertEqual(problem.detail, expected[reason]?.1, "\(reason)") } let error = ConnectionProblem.pcEndedSessionOnError - XCTAssertEqual(error.title, ConnectionProblem.pcEndedSession.title, "one title, as SETUP.md lists it") + // A PC that restarts by itself also says error: the title must be + // true for a restart as well as a fault, and never blame the PC's user. + XCTAssertEqual(error.title, "The connection to your PC ended") + XCTAssertEqual(ConnectionProblem.pcEndedSession.title, "Your PC ended the session") XCTAssertFalse(error.detail.contains("Stop") || error.detail.contains("setting"), error.detail) XCTAssertEqual(error.actions, [.retry]) XCTAssertTrue(error.canReport) diff --git a/ios/EternalMonitorTests/GoodbyeTests.swift b/ios/EternalMonitorTests/GoodbyeTests.swift index 61564338..210c861e 100644 --- a/ios/EternalMonitorTests/GoodbyeTests.swift +++ b/ios/EternalMonitorTests/GoodbyeTests.swift @@ -246,7 +246,7 @@ final class GoodbyeTests: XCTestCase { host.sendBye(.error) try await until("disconnected") { manager.state == .disconnected } XCTAssertEqual(manager.problem, .pcEndedSessionOnError) - XCTAssertEqual(manager.problem?.title, "Your PC ended the session") + XCTAssertEqual(manager.problem?.title, "The connection to your PC ended") XCTAssertTrue(manager.problem?.detail.hasPrefix("The connection to your PC dropped") == true, manager.problem?.detail ?? "") XCTAssertEqual(manager.problem?.actions, [.retry]) From 190fbba0f30231d38de2e5a00e37ca72442ba3b6 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:43:44 -0500 Subject: [PATCH 021/226] Repair a keyframe until it completes on lossy WiFi At 4-22% WiFi loss each replacement keyframe arrived incomplete: one round of requests and repairs leaves some of its 80-300 fragments missing, and the iPad dropped it at its 25 ms deadline because the small frames after it, which need it, were already complete. The picture then froze until the iPad reconnected. An incomplete keyframe is now held at its deadline while the frames behind it complete, until it completes, a newer keyframe completes or a second passes. It asks again every RTT + 5 ms once something has arrived since its last request. Swift and the Rust mirror change together, with shared traces. The WiFi repair simulation at 10% loss each way now completes 397 of 399 keyframes instead of 148 of 390, and no freeze lasts a second (35 did, up to 3.4 s). A measurement test runs the same loss through the real encoder and transport. --- docs/decisions.md | 28 +++ host/tests/e2e_synthetic.rs | 195 ++++++++++++++++++ .../Network/FrameAssembler.swift | 35 +++- proto/src/reassembly.rs | 28 ++- proto/testdata/repair_vectors.txt | 55 +++++ proto/tests/wifi_repair_sim.rs | 76 +++++++ 6 files changed, 407 insertions(+), 10 deletions(-) diff --git a/docs/decisions.md b/docs/decisions.md index 8ab265e0..09700fe7 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -225,6 +225,34 @@ such frames then completed from late or repaired fragments. The iPad now requests a wide gap in several NACKs and asks for a keyframe only when a frame is dropped. Evidence: `EternalMonitor-Handoff/evidence/rc-hardware-20260924/`. +### Repair a keyframe until it completes + +On WiFi with 4-22% loss, iPads held a frozen picture for 1-4 s and then +reconnected. The PC answered every keyframe request at once, but each +replacement keyframe arrived incomplete. A keyframe at 1080p is 80-300 +fragments, so at 10% loss one round of requests and repairs leaves some +missing. The iPad asked again once, and dropped the keyframe at its 25 ms +deadline because the small frames sent after it were already complete. Those +frames need the keyframe, so the decoder held the picture and asked for +another, which failed the same way. + +A keyframe now keeps its place while the frames after it complete. At its +deadline it is held and asked for again, like a frame with nothing behind it, +until it completes, a newer keyframe completes, a second has passed, or the +32 frames the assembler holds are full. It +also asks again every RTT + 5 ms, but only once something has arrived since +its last request. The repair simulation (`proto/tests/wifi_repair_sim.rs`, +10% loss each way, 150-fragment keyframes at 60 fps) completed 148 of 390 +keyframes before, with 35 freezes over a second and the longest 3.4 s. After, +it completes 397 of 399, the longest freeze is 0.61 s, and no freeze lasts a +second. The 1% loss rows also lost fewer frames. + +Delta frames keep the 25 ms rule and their single retry. Holding them too, +for at most 100 ms, with every frame asking again, removed nearly every +remaining drop in the simulation, but raised the p50 latency at 10% loss from +6.6 to 10.7 ms. It would also undo the repair-window rules above, which were +tuned on a physical iPad, so it waits for a device run. + ### Use Apple's usbmuxd tunnel for USB Apple's installed device service handles the cable/trust relationship. EMLINK diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index fee915b7..4187355c 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -789,6 +789,201 @@ fn slow_usb_link_measurement() { assert!(run.tunnel_open, "the tunnel must stay open"); } +/// Measures recovery from lossy WiFi through the real encoder and transport: +/// how long the picture stands still and how many keyframes reach the fake +/// iPad whole. `ETERNAL_TEST_LOSS` (default 0.10) is the loss each way. +/// The host's fault injection drops first transmissions only, so this +/// receiver also loses that share of the repairs and of its own requests, +/// as WiFi does. `ETERNAL_TEST_ENCODER` picks the encoder (default libx264), +/// `ETERNAL_TEST_SECONDS` the length (default 30). Timing on a busy machine +/// changes the numbers, so this only reports them; `wifi_repair_sim.rs` +/// holds the assertions. +#[test] +#[ignore = "measurement; run with --ignored --nocapture"] +fn lossy_wifi_recovery_measurement() { + let _env = test_env(); + init_test_tracing(); + let _ = ffmpeg_next::init(); + let env = |name: &str| std::env::var(name).ok().filter(|v| !v.is_empty()); + let loss: f64 = env("ETERNAL_TEST_LOSS").map_or(0.10, |v| v.parse().unwrap()); + let seconds: u64 = env("ETERNAL_TEST_SECONDS").map_or(30, |v| v.parse().unwrap()); + let encoder = env("ETERNAL_TEST_ENCODER").unwrap_or_else(|| "libx264".into()); + if env("ETERNAL_SYNTH_SIZE").is_none() { + std::env::set_var("ETERNAL_SYNTH_SIZE", "1920x1080"); + } + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_DROP", loss.to_string()); + if env("ETERNAL_REORDER").is_none() { + std::env::set_var("ETERNAL_REORDER", "0.02"); + } + if env("ETERNAL_JITTER_MS").is_none() { + std::env::set_var("ETERNAL_JITTER_MS", "4"); + } + let port = free_udp_port(); + let shared = SharedControl::new(port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some(encoder.clone()); + let (tx, rx) = mpsc::channel(); + let pipeline_shared = shared.clone(); + let pipeline_tx = tx.clone(); + let gpu = if encoder == "libx264" { + GpuInfo::software_fallback() + } else { + GpuInfo::detect() + }; + let supervisor = std::thread::spawn(move || { + eternal_host::supervisor::run(port, pipeline_shared, gpu, pipeline_tx, rx); + }); + let mut receiver = FakeReceiver::connect_full( + port, + CAP_DECODE_H264, + eternal_wire::v2::control::FEATURE_SUPPORTS_NACK, + ); + receiver + .socket + .set_read_timeout(Some(Duration::from_millis(1))) + .unwrap(); + let mut assembler = Reassembler::new(); + assembler.configure_repair( + true, + Duration::from_micros(16_667), + Duration::from_millis(5), + ); + // The iPad's clock, as in `lossy_stream_recovers_and_adapts`: a pass + // counts at most MAX_PASS, so a descheduled test thread expires nothing. + const MAX_PASS: Duration = Duration::from_millis(20); + let mut ipad_now = Instant::now(); + let mut pass_started = ipad_now; + let mut rng = 0x2545_F491_4F6C_DD1Du64; + let mut lost = move || { + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + ((rng >> 11) as f64 / (1u64 << 53) as f64) < loss + }; + // The app's request spacing: the control channel sends one keyframe + // request per 500 ms, and a held picture asks again every 600 ms. + let mut last_request: Option = None; + let mut wants_keyframe = false; + let mut keyframes_seen = std::collections::HashSet::new(); + let mut keyframes_complete = 0u64; + let mut previous: Option<(u32, u32, bool)> = None; + let mut last_decodable: Option = None; + let mut freezes = Vec::new(); + let mut decodable = 0u64; + let mut delivered = 0u64; + let mut requests = 0u64; + let mut bitrate = 0; + let mut bytes = [0; 2048]; + let started = Instant::now(); + while started.elapsed() < Duration::from_secs(seconds) { + let woke = Instant::now(); + ipad_now += (woke - pass_started).min(MAX_PASS); + pass_started = woke; + if let Ok((len, _)) = receiver.socket.recv_from(&mut bytes) { + if let Ok((header, payload)) = MediaHeader::decode(&bytes[..len]) { + if header.session_id == receiver.session_id && !(header.is_retransmit && lost()) { + if header.is_keyframe { + keyframes_seen.insert((header.stream_epoch, header.frame_seq)); + } + assembler.add_media(header, payload, ipad_now); + } + } else if let Ok((_, ControlMessage::Heartbeat(hb))) = parse_control(&bytes[..len]) { + bitrate = hb.stream_config.bitrate_bps; + } + } + assembler.tick(ipad_now); + for nack in assembler.take_nacks() { + if !lost() { + receiver.send(&ControlMessage::Nack(nack)); + } + } + wants_keyframe |= assembler.take_keyframe_request().is_some(); + let held = last_decodable.is_some_and(|at| ipad_now - at >= Duration::from_millis(600)); + let spacing = if wants_keyframe { 500 } else { 600 }; + if (wants_keyframe || held) + && last_request.is_none_or(|at| ipad_now - at >= Duration::from_millis(spacing)) + { + wants_keyframe = false; + last_request = Some(ipad_now); + requests += 1; + let c = assembler.counters(); + if !lost() { + receiver.send(&ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: c.stream_epoch, + last_complete_seq: assembler.latest_completed_seq(), + reason: KeyframeReason::GapLoss, + })); + } + } + for frame in assembler.take_ready() { + delivered += 1; + keyframes_complete += u64::from(frame.is_keyframe); + let epoch = assembler.counters().stream_epoch; + // The host numbers sent frames contiguously per epoch, so a + // delta decodes only right after the frame before it decoded. + let decodes = frame.is_keyframe + || previous.is_some_and(|(e, seq, ok)| ok && e == epoch && seq + 1 == frame.seq); + previous = Some((epoch, frame.seq, decodes)); + if decodes { + decodable += 1; + if let Some(at) = last_decodable { + freezes.push(ipad_now - at); + } + last_decodable = Some(ipad_now); + } + } + if receiver.last_report.elapsed() >= Duration::from_millis(500) { + receiver.last_report = Instant::now(); + let c = assembler.counters(); + receiver.send(&ControlMessage::ReceiverReport(ReceiverReport { + stream_epoch: c.stream_epoch, + highest_seq: c.highest_seq, + frames_complete: c.frames_complete as u32, + frames_dropped: c.frames_dropped as u32, + frags_received: c.frags_received as u32, + frags_lost: c.frags_lost as u32, + frags_repaired: c.frags_repaired as u32, + nacks_sent: c.nacks_sent as u32, + assembler_depth: c.assembler_depth, + jitter_us: c.jitter_us, + ..Default::default() + })); + } + } + let (sent, retransmits) = { + let stats = eternal_host::stats::PIPELINE_STATS.lock(); + (stats.transport_packets_sent, stats.transport_retransmits) + }; + shared.stop(); + tx.send(SupervisorCommand::Shutdown).unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = supervisor.join(); + let _ = done_tx.send(()); + }); + let _ = done_rx.recv_timeout(Duration::from_secs(5)); + freezes.sort(); + let over = |ms: u64| { + freezes + .iter() + .filter(|f| **f > Duration::from_millis(ms)) + .count() + }; + let longest = freezes.last().copied().unwrap_or_default(); + eprintln!( + "LOSSY_E2E encoder={encoder} loss={loss} seconds={seconds} keyframes={keyframes_complete}/{} \ + decodable={decodable}/{delivered} sent={sent} retransmits={retransmits} requests={requests} \ + longest_freeze_ms={} freezes_over_250ms={} freezes_over_1s={} end_bitrate={bitrate} counters={:?}", + keyframes_seen.len(), + longest.as_millis(), + over(250), + over(1000), + assembler.counters() + ); + assert!(delivered > 0, "the stream must deliver frames"); +} + /// Accepts the host's USB tunnel on `listener` within `within`, past its /// EMLINK preamble, as `read_preamble` reads it. Other programs dial a /// test's loopback listener too: a development tool that probes each new diff --git a/ios/EternalMonitor/Network/FrameAssembler.swift b/ios/EternalMonitor/Network/FrameAssembler.swift index e75f7e74..824e16b4 100644 --- a/ios/EternalMonitor/Network/FrameAssembler.swift +++ b/ios/EternalMonitor/Network/FrameAssembler.swift @@ -133,11 +133,14 @@ final class FrameAssembler { var deadline: UInt64? var retryAt: UInt64? var requestedAt: UInt64? + /// The latest request for this frame; see `asksKeyframeAgain(_:at:)`. + var askedAt: UInt64? var completedAt: UInt64? var stallAllowanceUsed: UInt64 = 0 var stallCountedUntil: UInt64 = 0 /// Indices already requested; each is requested when first seen - /// missing and at most once more by the frame's single retry. + /// missing and at most once more by the frame's single retry. A + /// keyframe retries until it completes; see `asksKeyframeAgain(_:at:)`. var requested = Set() var retried = false var keyframeRequested = false @@ -266,7 +269,7 @@ final class FrameAssembler { // A held frame now delays a newer one: one more repair window // once a complete frame waits. let latest = now + repairWindow() - for held in pending.keys where pending[held]?.holdRetryUs ?? 0 > 0 { + for held in pending.keys where pending[held]?.holdRetryUs ?? 0 > 0 && pending[held]?.isKeyframe == false { let dropBy = min(pending[held]?.dropBy ?? latest, latest) pending[held]?.dropBy = dropBy } @@ -359,7 +362,13 @@ final class FrameAssembler { // Retire in sequence order. Expiring a later partial // first must not make an older repair look stale. if pending.keys.min() == seq { - let blocking = pending.values.contains { $0.isComplete } + // Every frame up to the next keyframe refers to this + // keyframe, so a complete one behind it is no reason + // to drop it: the decoder would hold the picture + // until another keyframe, and on lossy WiFi each + // replacement arrived incomplete too, for seconds. + // It is held, as a frame alone is. + let blocking = !frame.isKeyframe && pending.values.contains { $0.isComplete } if !blocking, let gap = frame.gapAt, now < gap + Self.maxHoldUs { hold(seq, now: now) } else if blocking { @@ -381,8 +390,9 @@ final class FrameAssembler { } } else if frame.gapAt == nil, now >= frame.createdAt + framePeriodUs { beginGap(seq, through: frame.fragmentCount - 1, now: now) - } else if let retry = frame.retryAt, now >= retry, !frame.retried { + } else if let retry = frame.retryAt, now >= retry, !frame.retried || asksKeyframeAgain(frame, at: now) { pending[seq]?.retried = true + pending[seq]?.retryAt = now + min(rttUs(), 1_000_000) + 5_000 request(seq, through: frame.fragmentCount - 1, again: true, now: now) } } else if now >= frame.lastFragmentAt + Self.staleFrameUs { @@ -437,6 +447,17 @@ final class FrameAssembler { request(seq, through: index, again: false, now: now) } + /// A keyframe has many fragments, and at 10% loss one round of requests + /// and repairs leaves some missing, so a keyframe asks again every RTT + /// plus 5 ms, not once. It asks only once something has arrived since it + /// last asked: while WiFi holds traffic, its request and the repair are + /// still queued. A fragment arriving now may be the missing one, so a + /// keyframe asks again only on a later tick. + private func asksKeyframeAgain(_ frame: PendingFrame, at now: UInt64) -> Bool { + guard frame.isKeyframe, let asked = frame.askedAt, let arrival = lastArrivalUs else { return false } + return arrival > asked && arrival < now + } + private func repairWindow() -> UInt64 { max(8_000, min(25_000, 2 * min(rttUs(), 1_000_000) + framePeriodUs * 3 / 2)) } @@ -449,14 +470,15 @@ final class FrameAssembler { /// keyframes in an afternoon. A starved PC can also stop sending for /// several hundred milliseconds with two frames incomplete and then send /// both repairs together. Ask again, less often each time, until a newer - /// frame completes or the frame has waited `maxHoldUs`. + /// frame completes or the frame has waited `maxHoldUs`. A keyframe waits + /// for a newer keyframe instead of a newer frame. private func hold(_ seq: UInt32, now: UInt64) { guard let frame = pending[seq] else { return } let wait = frame.holdRetryUs == 0 ? repairWindow() : min(frame.holdRetryUs * 2, Self.maxHoldRetryUs) pending[seq]?.holdRetryUs = wait pending[seq]?.deadline = now + wait - if pending.count > 1 { pending[seq]?.dropBy = min(frame.dropBy ?? now, now) } + if pending.count > 1 && !frame.isKeyframe { pending[seq]?.dropBy = min(frame.dropBy ?? now, now) } pending[seq]?.stallAllowanceUsed = Self.maxStallAllowanceUs pending[seq]?.retried = true request(seq, through: frame.fragmentCount - 1, again: true, now: now) @@ -473,6 +495,7 @@ final class FrameAssembler { guard !missing.isEmpty else { return } pending[seq]?.requested.formUnion(missing) if pending[seq]?.requestedAt == nil { pending[seq]?.requestedAt = now } + pending[seq]?.askedAt = now for start in stride(from: 0, to: missing.count, by: 64) { counters.withLock { $0.nacksSent += 1 } onNack?(Nack(streamEpoch: currentEpoch ?? 0, frameSeq: seq, fragCount: frame.fragmentCount, diff --git a/proto/src/reassembly.rs b/proto/src/reassembly.rs index 4a1d7b8d..35af19d3 100644 --- a/proto/src/reassembly.rs +++ b/proto/src/reassembly.rs @@ -72,6 +72,8 @@ struct PendingFrame { deadline: Option, retry_at: Option, requested_at: Option, + /// The latest request for this frame (see `FrameAssembler.PendingFrame.askedAt`). + asked_at: Option, completed_at: Option, stall_used: Duration, stall_counted_until: Option, @@ -313,7 +315,7 @@ impl Reassembler { // once a complete frame waits. let latest = now + self.repair_window(); for frame in self.pending.values_mut() { - if !frame.hold_retry.is_zero() { + if !frame.hold_retry.is_zero() && !frame.header.is_keyframe { frame.drop_by = Some(frame.drop_by.map_or(latest, |by| by.min(latest))); } } @@ -331,6 +333,7 @@ impl Reassembler { deadline: None, retry_at: None, requested_at: None, + asked_at: None, completed_at: None, stall_used: Duration::ZERO, stall_counted_until: None, @@ -438,7 +441,10 @@ impl Reassembler { .is_some_and(|deadline| now >= deadline + self.stall(frame, now)) { if self.pending.first_key_value().map(|(&s, _)| s) == Some(seq) { - let blocking = self.pending.values().any(PendingFrame::complete); + // The frames behind a keyframe need it, so they never + // make it go (see `FrameAssembler.tick(at:)`). + let blocking = !frame.header.is_keyframe + && self.pending.values().any(PendingFrame::complete); if !blocking && frame.gap_at.is_some_and(|gap| now < gap + MAX_HOLD) { self.hold(seq, now); } else if blocking { @@ -474,10 +480,14 @@ impl Reassembler { { let through = frame.header.frag_count - 1; self.begin_gap(seq, through, now); - } else if !frame.retried && frame.retry_at.is_some_and(|retry| now >= retry) { + } else if (!frame.retried || self.asks_keyframe_again(frame, now)) + && frame.retry_at.is_some_and(|retry| now >= retry) + { let through = frame.header.frag_count - 1; + let next = now + self.rtt + Duration::from_millis(5); if let Some(frame) = self.pending.get_mut(&seq) { frame.retried = true; + frame.retry_at = Some(next); } self.request(seq, through, true, now); } @@ -511,6 +521,15 @@ impl Reassembler { self.request(seq, through, false, now); } + /// A keyframe asks again every RTT + 5 ms while anything has arrived + /// since it last asked (see `FrameAssembler.asksKeyframeAgain(_:at:)`). + fn asks_keyframe_again(&self, frame: &PendingFrame, now: Instant) -> bool { + frame.header.is_keyframe + && self.last_arrival.is_some_and(|arrival| { + arrival < now && frame.asked_at.is_some_and(|asked| arrival > asked) + }) + } + fn repair_window(&self) -> Duration { let window_us = (2 * self.rtt.as_micros() + self.frame_period.as_micros() * 3 / 2).clamp(8_000, 25_000); @@ -533,7 +552,7 @@ impl Reassembler { }; frame.hold_retry = wait; frame.deadline = Some(now + wait); - if newer_waiting { + if newer_waiting && !frame.header.is_keyframe { frame.drop_by = Some(frame.drop_by.map_or(now, |by| by.min(now))); } frame.stall_used = MAX_STALL_ALLOWANCE; @@ -602,6 +621,7 @@ impl Reassembler { } frame.requested.extend(&missing); frame.requested_at.get_or_insert(now); + frame.asked_at = Some(now); let frag_count = frame.header.frag_count; for chunk in missing.chunks(64) { self.counters.nacks_sent += 1; diff --git a/proto/testdata/repair_vectors.txt b/proto/testdata/repair_vectors.txt index fd0c7faa..7b0d1014 100644 --- a/proto/testdata/repair_vectors.txt +++ b/proto/testdata/repair_vectors.txt @@ -429,3 +429,58 @@ reset 16667 10000 add 0 1 1 0 1 0 1 add 1000 1 3 0 1 0 3 1 expect 2 0 2 0 0 0 0 1,3 0 - + +# Every frame up to the next keyframe refers to the keyframe, so a complete +# frame behind an incomplete keyframe is no reason to drop it: at its +# deadline the keyframe is held and asked for again, and its repair then +# delivers both. On lossy WiFi, dropping it froze the picture for seconds, +# because each replacement keyframe arrived incomplete too. +reset 16667 10000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +add 16667 1 2 0 1 0 4 +add 17000 1 1 0 3 1 1 1 +expect 0 0 3 0 0 2 1 - 0 1:1|1:1 +tick 40000 +expect 0 0 3 0 0 3 1 - 0 1:1|1:1|1:1 +add 45000 1 1 1 3 1 2 1 +expect 2 0 3 0 1 3 0 1,2 0 1:1|1:1|1:1 + +# A held keyframe still goes a second after its gap, with the frames that +# need it, under one keyframe request. +reset 16667 10000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +add 16667 1 2 0 1 0 4 +add 17000 1 1 0 3 1 1 1 +tick 40000 +tick 65000 +tick 115000 +tick 215000 +tick 415000 +tick 615000 +tick 815000 +tick 1014999 +expect 0 0 3 0 0 9 1 - 0 1:1|1:1|1:1|1:1|1:1|1:1|1:1|1:1|1:1 +tick 1015000 +expect 0 2 3 1 0 9 0 - 1 1:1|1:1|1:1|1:1|1:1|1:1|1:1|1:1|1:1 + +# A keyframe has many fragments, and one round of requests and repairs can +# leave some missing, so it asks again every RTT + 5 ms instead of once. It +# asks only after something arrived since it last asked: while WiFi holds +# traffic, the request and the repair are still queued. +reset 16667 10000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +tick 16000 +expect 0 0 2 0 0 2 1 - 0 1:1|1:1 +tick 40000 +expect 0 0 2 0 0 2 1 - 0 1:1|1:1 +add 41000 1 1 0 3 1 1 1 +expect 0 0 2 0 0 2 1 - 0 1:1|1:1 +tick 41001 +expect 0 0 2 0 0 3 1 - 0 1:1|1:1|1:1 +tick 50000 +expect 0 0 2 0 0 3 1 - 0 1:1|1:1|1:1 +add 50000 1 1 1 3 1 2 1 +expect 1 0 2 0 1 3 0 1 0 1:1|1:1|1:1 diff --git a/proto/tests/wifi_repair_sim.rs b/proto/tests/wifi_repair_sim.rs index e57a4709..d54da059 100644 --- a/proto/tests/wifi_repair_sim.rs +++ b/proto/tests/wifi_repair_sim.rs @@ -52,6 +52,13 @@ struct Outcome { nacks: u64, latency_p50_ms: f64, latency_p90_ms: f64, + /// Keyframes that reached the decoder whole. + keyframes_complete: u64, + /// The longest time the picture stood still between decodable frames. + longest_freeze_ms: f64, + /// Freezes the user notices: over 250 ms, and over a second. + freezes_over_250ms: u64, + freezes_over_1s: u64, } struct Rng(u64); @@ -181,6 +188,7 @@ fn simulate(s: Scenario) -> Outcome { let mut resends: HashMap<(u32, u16), u64> = HashMap::new(); let mut previous_sent: HashMap = HashMap::new(); let mut nacks_sent: HashMap = HashMap::new(); + let mut last_decoded = 0u64; while let Some(((now, _), event)) = events.pop_first() { if now > end { @@ -268,7 +276,13 @@ fn simulate(s: Scenario) -> Outcome { let decodes = frame.is_keyframe || decodes_from == previous_sent.get(&frame.seq).copied(); decodes_from = decodes.then_some(frame.seq); + outcome.keyframes_complete += u64::from(frame.is_keyframe); if decodes { + let freeze = (now - last_decoded) as f64 / 1000.0; + last_decoded = now; + outcome.longest_freeze_ms = outcome.longest_freeze_ms.max(freeze); + outcome.freezes_over_250ms += u64::from(freeze > 250.0); + outcome.freezes_over_1s += u64::from(freeze > 1000.0); outcome.decodable += 1; latencies.push((now - sent_at[&frame.seq].0) as f64 / 1000.0); } else { @@ -371,3 +385,65 @@ fn wifi_repair_table() { } } } + +/// A 1080p desktop at the bitrate ABR falls to on lossy WiFi: a keyframe is +/// about 200 KB, a delta about 16 KB. iPads in the field froze for 1-4 s at +/// 4-22% loss: every replacement keyframe arrived incomplete, and was dropped +/// as soon as the small frames behind it completed. +const LOSSY: Scenario = Scenario { + name: "lossy WiFi, 60 fps", + fps: 60, + delta_fragments: 12, + keyframe_fragments: 150, + loss: 0.10, + late_uplink: 0.3, + stall_every_s: 5, + seconds: 120, + capture_skips: false, +}; + +/// At 10% loss each way, recovery keyframes complete and the picture never +/// stands still for a second. 0.3.4's assembler completed 148 of 390 +/// keyframes here and froze 35 times for over a second, up to 3.4 s. +#[test] +fn keyframes_recover_the_picture_at_ten_percent_loss() { + let outcome = simulate(LOSSY); + eprintln!("WIFI_SIM {} {outcome:?}", LOSSY.name); + assert!( + outcome.keyframes_complete * 100 >= outcome.keyframes_sent * 95, + "{outcome:?}" + ); + assert_eq!(outcome.freezes_over_1s, 0, "{outcome:?}"); + assert!(outcome.longest_freeze_ms < 1_000.0, "{outcome:?}"); +} + +/// Prints the loss table used in the PR: 5-20% loss, keyframes of 80, 150 +/// and 300 fragments. +#[test] +fn lossy_wifi_table() { + for loss in [0.05, 0.10, 0.15, 0.20] { + for keyframe_fragments in [80, 150, 300] { + let outcome = simulate(Scenario { + loss, + keyframe_fragments, + ..LOSSY + }); + eprintln!( + "LOSSY loss={loss} kf={keyframe_fragments} keyframes={}/{} drops={} keyframe_requests={} \ + longest_freeze_ms={:.0} freezes_over_250ms={} freezes_over_1s={} decodable={}/{} \ + latency_p50_ms={:.1} latency_p90_ms={:.1}", + outcome.keyframes_complete, + outcome.keyframes_sent, + outcome.dropped, + outcome.keyframe_requests, + outcome.longest_freeze_ms, + outcome.freezes_over_250ms, + outcome.freezes_over_1s, + outcome.decodable, + outcome.frames, + outcome.latency_p50_ms, + outcome.latency_p90_ms + ); + } + } +} From 99ba31af71b3afde9ceef6f85deae4381620a3e1 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:49:48 -0500 Subject: [PATCH 022/226] Move a saved PC only after it shows it is the same PC Before reconnecting at a new address, the iPad now checks that it is the same PC. It sends a token-less IDENTIFY with a random nonce, and a PC on 0.3.5 or later answers with HMAC-SHA256 keyed with its pairing token over the nonce and a fixed context. Only a matching answer lets the normal HELLO2 go to the new address. A PC that doesn't answer, as before 0.3.5, means no move. The messages are versioned (0x1A, 0x1B) and older apps and PCs ignore them. A saved PC moves only if all of these hold: - the iPad holds a pairing for it; - its entry came from Bonjour's list (a new found mark; older, typed and scanned entries never move); - both the old and the new address are home network addresses. Bonjour services still resolving count when two PCs claim one name. Try again after a refused move goes back to the saved address. The new address becomes the target only once the PC is accepted. The old entry and pairing stay until the new pairing is saved and read back. The e2e now pairs for real at the Mac's home network address (read at run time) and moves the PC to another port. Test fixtures use documentation addresses. --- host/src/pairing.rs | 40 ++ host/src/transport/session.rs | 85 +++- ios/EternalMonitor/App/ConnectView.swift | 35 +- .../App/ConnectionManager.swift | 146 ++++--- .../App/EternalMonitorApp.swift | 3 +- ios/EternalMonitor/App/MovedPC.swift | 99 ++++- ios/EternalMonitor/Core/E2ESignpost.swift | 15 +- .../Network/NetworkScanner.swift | 13 + ios/EternalMonitor/Network/WireProtocol.swift | 41 ++ .../ConnectionProblemTests.swift | 2 +- ios/EternalMonitorTests/FakeUDPHost.swift | 29 ++ ios/EternalMonitorTests/MovedPCTests.swift | 376 +++++++++++------- .../WireProtocolGoldenTests.swift | 21 +- proto/src/v2/control.rs | 61 +++ proto/src/v2/mod.rs | 6 + proto/testdata/v2_vectors.txt | 2 + proto/tests/golden.rs | 46 +++ scripts/e2e_moved_pc.sh | 126 +++--- 18 files changed, 861 insertions(+), 285 deletions(-) diff --git a/host/src/pairing.rs b/host/src/pairing.rs index 61c6696b..757b870f 100644 --- a/host/src/pairing.rs +++ b/host/src/pairing.rs @@ -41,6 +41,18 @@ pub fn parse_token(text: &str) -> Option<[u8; 16]> { (token != [0; 16]).then_some(token) } +fn identity_proof(token: &[u8; 16], nonce: &[u8; 16]) -> [u8; 32] { + let key = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, token); + let mut context = ring::hmac::Context::with_key(&key); + context.update(nonce); + context.update(eternal_wire::v2::control::IDENTIFY_CONTEXT); + context + .sign() + .as_ref() + .try_into() + .expect("SHA-256 is 32 bytes") +} + fn random_code(previous: u32) -> Result { // Rejection sampling avoids modulo bias. Exclude 000000 (wire sentinel) // and the preceding code so a rotation always changes what the user sees. @@ -141,6 +153,13 @@ impl Pairing { Ok(()) } + /// The answer to an iPad checking that this is the PC it paired with: + /// HMAC-SHA256 keyed with the pairing token over `nonce ‖ context`. Only + /// a PC holding the token can make it, and the token never leaves. + pub fn identity_proof(&self, nonce: &[u8; 16]) -> [u8; 32] { + identity_proof(&self.token, nonce) + } + pub fn regenerate_token(&mut self) -> Result<(), getrandom::Error> { let token = new_token()?; self.rotate_code()?; @@ -355,6 +374,27 @@ impl RateLimiter { mod tests { use super::*; + /// The known answer the wire golden vector and the iPad's tests share. + #[test] + fn identity_proof_is_the_shared_known_answer() { + let token = std::array::from_fn(|index| index as u8 + 1); + let nonce = std::array::from_fn(|index| 0xA0 + index as u8); + let pairing = Pairing::new(true, token).unwrap(); + let hex: String = pairing + .identity_proof(&nonce) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + assert_eq!( + hex, + "126dc93467af3b0621c4f9b420e0d323143259d743f85703b9fec83989eba353" + ); + // Another token, or another nonce, gives another proof. + let other = Pairing::new(true, [9; 16]).unwrap(); + assert_ne!(other.identity_proof(&nonce), pairing.identity_proof(&nonce)); + assert_ne!(pairing.identity_proof(&[0; 16]), pairing.identity_proof(&nonce)); + } + fn guess(nonce: u32) -> FailedRequest { FailedRequest { nonce, diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index b6ef5504..7704e6a8 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -532,7 +532,7 @@ impl Session { // INPUT_EVENT moved the mouse and clicked, a stray RECEIVER_REPORT // steered the bitrate, and a late BYE from a superseded session tore // down the one that replaced it. - if !matches!(message, ControlMessage::Hello2(_)) { + if !matches!(message, ControlMessage::Hello2(_) | ControlMessage::Identify(_)) { match self.active.as_ref() { Some(session) if session.session_id == header_session_id && !session.awaiting_takeover => {} @@ -542,6 +542,7 @@ impl Session { } match message { ControlMessage::Hello2(hello) => self.handle_hello(source, hello, config, now), + ControlMessage::Identify(request) => Self::answer_identify(source, request, config), ControlMessage::Bye(reason) => self.handle_bye(source, reason, now), ControlMessage::KeyframeRequest(request) => self.handle_keyframe(source, request, now), ControlMessage::ReceiverReport(report) => self.handle_report(source, report, now), @@ -851,6 +852,33 @@ impl Session { actions } + /// An iPad checking that this is the PC it paired with, before it + /// reconnects at a new address. Stateless: it starts no session and + /// works while another device streams. + fn answer_identify( + source: PeerId, + request: eternal_wire::v2::control::Identify, + config: &impl ConfigSource, + ) -> Actions { + let mut actions = Actions::default(); + let Some(pairing) = config.pairing() else { + return actions; + }; + if request.version != eternal_wire::v2::control::IDENTIFY_VERSION { + return actions; + } + let identity = ControlMessage::Identity(eternal_wire::v2::control::Identity { + version: eternal_wire::v2::control::IDENTIFY_VERSION, + nonce: request.nonce, + proof: pairing.lock().identity_proof(&request.nonce), + }); + actions.replies.push(( + source, + eternal_wire::v2::control::encode_control(0, 1, &identity), + )); + actions + } + fn handle_ping( &mut self, source: PeerId, @@ -2253,6 +2281,61 @@ mod tests { Some(&self.0) } } + fn identity_reply(actions: &Actions) -> eternal_wire::v2::control::Identity { + assert_eq!(actions.replies.len(), 1); + let (header, ControlMessage::Identity(identity)) = + eternal_wire::v2::control::parse_control(&actions.replies[0].1).unwrap() + else { + panic!("expected IDENTITY") + }; + assert_eq!(header.session_id, 0); + identity + } + + fn identify(version: u8, nonce: [u8; 16]) -> ControlMessage { + ControlMessage::Identify(eternal_wire::v2::control::Identify { version, nonce }) + } + + /// The PC proves it holds its pairing token, with pairing on or off, + /// without starting a session, and while another iPad streams. + #[test] + fn identify_is_answered_with_the_token_proof_and_starts_nothing() { + let config = pair_config(true); + let mut session = Session::new(100); + let now = Instant::now(); + let asker = addr([192, 0, 2, 9], 50100); + let actions = session.handle_control(asker, 0, identify(1, [3; 16]), &config, now); + let identity = identity_reply(&actions); + assert_eq!(actions.replies[0].0, asker); + assert_eq!(identity.nonce, [3; 16]); + assert_eq!(identity.proof, config.0.lock().identity_proof(&[3; 16])); + assert_ne!(identity.proof, crate::pairing::Pairing::new(true, [8; 16]).unwrap().identity_proof(&[3; 16])); + assert!(session.session_id().is_none()); + assert!(actions.new_target.is_none()); + + let streaming = addr([192, 0, 2, 5], 50000); + session.handle_control(streaming, 0, ControlMessage::Hello2(pair_hello(1)), &pair_config(false), now); + let live = session.session_id(); + assert!(live.is_some()); + let actions = session.handle_control(asker, 0, identify(1, [4; 16]), &pair_config(false), now); + assert_eq!(identity_reply(&actions).proof, config.0.lock().identity_proof(&[4; 16])); + assert_eq!(session.session_id(), live, "the stream goes on"); + assert!(actions.new_target.is_none() && !actions.client_lost); + } + + /// A layout this PC doesn't know gets no answer, as from a PC before + /// 0.3.5; so does a host without pairing state. + #[test] + fn identify_of_another_version_is_not_answered() { + let mut session = Session::new(100); + let now = Instant::now(); + let asker = addr([192, 0, 2, 9], 50100); + let actions = session.handle_control(asker, 0, identify(2, [3; 16]), &pair_config(true), now); + assert!(actions.replies.is_empty()); + let actions = session.handle_control(asker, 0, identify(1, [3; 16]), &TestConfig, now); + assert!(actions.replies.is_empty()); + } + fn pair_config(required: bool) -> PairConfig { PairConfig(parking_lot::Mutex::new( crate::pairing::Pairing::new(required, [7; 16]).unwrap(), diff --git a/ios/EternalMonitor/App/ConnectView.swift b/ios/EternalMonitor/App/ConnectView.swift index 7b8aaea6..90cefdca 100644 --- a/ios/EternalMonitor/App/ConnectView.swift +++ b/ios/EternalMonitor/App/ConnectView.swift @@ -20,9 +20,6 @@ struct ConnectView: View { // track whether we've already pre-filled from lastHost so we // don't clobber user edits on every re-render. @State private var didPrefillFromLastHost = false - /// The address the app last put in the field: the last PC, or one - /// found, scanned or picked from Recent. - @State private var filledAddress: String? @State private var showQRScanner = false @State private var showDetails = false @State private var feedbackRequest: FeedbackRequest? @@ -65,12 +62,6 @@ struct ConnectView: View { && connectionManager.feedbackAsk.due(streaming: connectionManager.state == .connected) } - /// The address field holds something the app didn't put there: the - /// user typed it. - private var typedAddress: Bool { - normalizedHostIP != filledAddress - } - private var canConnect: Bool { !normalizedHostIP.isEmpty && parsedPort != nil } @@ -216,7 +207,7 @@ struct ConnectView: View { if UIPreview.opensSetup && !didSeeOnboarding { showSetup = true; return } #endif if !didPrefillFromLastHost && hostIP.isEmpty && !settings.lastHost.isEmpty { - fill(settings.lastHost) + hostIP = settings.lastHost if settings.lastPort != 0 { port = String(settings.lastPort) } @@ -282,9 +273,10 @@ struct ConnectView: View { setupOutcome = .finished switch outcome { case .connect(let host, let port): - fill(host) + hostIP = host self.port = String(port) - connectionManager.connect(host: host, port: port) + // Setup offers only the PCs Bonjour found. + connectionManager.connect(host: host, port: port, found: true) return case .scanQR: showQRScanner = true @@ -456,7 +448,7 @@ struct ConnectView: View { if !settings.lastHost.isEmpty && settings.lastHost != normalizedHostIP { Button { - fill(settings.lastHost) + hostIP = settings.lastHost if settings.lastPort != 0 { port = String(settings.lastPort) } } label: { Label(L("Use last PC: \(settings.lastHost)"), systemImage: "clock.arrow.circlepath") @@ -476,7 +468,7 @@ struct ConnectView: View { focusedField = nil guard let p = parsedPort else { return } withAnimation(.easeInOut(duration: 0.2)) { - connectionManager.connect(host: normalizedHostIP, port: p, typed: typedAddress) + connectionManager.connect(host: normalizedHostIP, port: p) } } label: { Text(L("Connect")) @@ -549,7 +541,7 @@ struct ConnectView: View { focusedField = .port } else if canConnect, let p = parsedPort { focusedField = nil - connectionManager.connect(host: normalizedHostIP, port: p, typed: typedAddress) + connectionManager.connect(host: normalizedHostIP, port: p) } } .focused($focusedField, equals: field) @@ -558,11 +550,6 @@ struct ConnectView: View { } } - private func fill(_ address: String) { - hostIP = address - filledAddress = address.trimmingCharacters(in: .whitespacesAndNewlines) - } - // MARK: - Connecting private var connectingCard: some View { @@ -634,7 +621,7 @@ struct ConnectView: View { private func handleScannedQR(_ value: String) { switch PairingCode.parse(value) { case .success(let target): - fill(target.host) + hostIP = target.host port = String(target.port) connectionManager.connect(host: target.host, port: target.port, token: target.token) case .failure(.wrongScheme): @@ -711,9 +698,9 @@ struct ConnectView: View { Button { // Only an IPv4 address reaches the PC; see DiscoveredHost. guard let address = host.address else { return } - fill(address) + hostIP = address port = "\(host.port)" - connectionManager.connect(host: address, port: host.port) + connectionManager.connect(host: address, port: host.port, found: true) } label: { hostRow( icon: "desktopcomputer", @@ -757,7 +744,7 @@ struct ConnectView: View { if conn.isUSB { connectionManager.resumeUSBConnections() } else { - fill(conn.host) + hostIP = conn.host port = "\(conn.port)" connectionManager.connect(host: conn.host, port: conn.port) } diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 5d3c3b77..2c677f84 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -102,7 +102,7 @@ final class ConnectionManager: ObservableObject { } private var awaitingPairing = false private var explicitPairingToken: Data? - private let pairingStore = PairingStore.shared + private let pairingStore: PairingStore @Published private(set) var diagnostics: [DiagnosticEntry] = [] /// Live stream health for the HUD, recomputed at the 4 Hz flush. @Published private(set) var stats = StreamStats() @@ -221,24 +221,29 @@ final class ConnectionManager: ObservableObject { /// Looks for PCs with Bonjour, given the service name wanted. Tests /// replace it. var findPCs: (String) async -> [DiscoveredHost] = { await MovedPC.find($0) } - /// "host:port" of each address the user typed in this launch. Saved - /// entries keep the mark once they show a picture. - private var typedAddresses = Set() - /// A saved PC that didn't answer at its address and was found at another. + /// Which addresses a saved PC may move from and to. Tests on loopback + /// widen it. + var homeAddress: (String) -> Bool = MovedPC.isHomeAddress + /// "host:port" of each address picked from Bonjour's list in this + /// launch. Saved entries keep the mark once they show a picture. + private var foundAddresses = Set() + /// A saved PC that didn't answer at its address and showed it is the + /// same PC at another. private struct Move { let from: (host: String, port: UInt16) let name: String - /// Its pairing token, or zeros: the PC at the new address must give - /// the same one back. + /// Its pairing token, which the PC proved it holds. let token: Data } /// The move the next attempt makes, and the one the current attempt made. private var pendingMove: Move? private var attemptMove: Move? - init(timing: Timing = Timing(), udpReceiver: @escaping (UInt16) -> any MediaLink = { UDPReceiver(port: $0) }) { + init(timing: Timing = Timing(), udpReceiver: @escaping (UInt16) -> any MediaLink = { UDPReceiver(port: $0) }, + pairingStore: PairingStore? = nil) { self.timing = timing self.makeUDPLink = udpReceiver + self.pairingStore = pairingStore ?? .shared } func refreshUSBAvailability() { @@ -318,11 +323,12 @@ final class ConnectionManager: ObservableObject { // MARK: - Connect / Disconnect - /// `typed`: the user typed this address, so it is never looked up by name. - func connect(host: String, port: UInt16, token: Data? = nil, typed: Bool = false) { + /// `found`: the address is the one Bonjour listed for the PC, so the PC + /// may be looked up by name again if the address changes. + func connect(host: String, port: UInt16, token: Data? = nil, found: Bool = false) { guard state == .disconnected else { return } explicitPairingToken = token - if typed { typedAddresses.insert(PairingStore.address(host: host, port: port)) } + if found { foundAddresses.insert(PairingStore.address(host: host, port: port)) } if let token { do { try pairingStore.save(token: token, hostName: "", address: PairingStore.address(host: host, port: port)) } catch { record(.warning, "pairing", "Could not save the scanned pairing on this iPad.") } @@ -378,7 +384,9 @@ final class ConnectionManager: ObservableObject { SupportDiagnostics.noteTypedAddress(normalizedHost) port = targetPort receiver = makeUDPLink(port) - lastTarget = (normalizedHost, port) + // A moved attempt becomes the target to retry only once the PC + // there is the saved one (see onSessionEstablished). + if pendingMove == nil { lastTarget = (normalizedHost, port) } case .usb(let link): normalizedHost = "USB" port = 9877 @@ -486,20 +494,15 @@ final class ConnectionManager: ObservableObject { UIApplication.shared.isIdleTimerDisabled = UserDefaults.standard.flag(forKey: "keepScreenAwake") E2E.firstFrame(width: frameWidth, height: frameHeight, link: isUSB ? "usb" : "udp") + var saved = true if let move = self.attemptMove { - // The same PC by its name and pairing: its new address - // replaces the old one, and so does its pairing. - RecentConnectionStore.shared.moved(from: move.from, to: (normalizedHost, port), - name: self.hostInfo?.hostName ?? move.name) - if MovedPC.replacesSavedEntry(move.from.host) { - try? self.pairingStore.remove(address: PairingStore.address(host: move.from.host, port: move.from.port)) - } + saved = self.savedMove(move, to: (normalizedHost, port), pairingAddress: pairingAddress) } else { RecentConnectionStore.shared.add(host: isUSB ? (self.hostInfo?.hostName ?? "USB host") : normalizedHost, port: port, isUSB: isUSB, name: self.hostInfo?.hostName, - typed: self.typedAddresses.contains(pairingAddress)) + found: self.foundAddresses.contains(pairingAddress)) } - if !isUSB { + if !isUSB && saved { UserDefaults.standard.set(normalizedHost, forKey: "lastHost") UserDefaults.standard.set(Int(port), forKey: "lastPort") } @@ -622,11 +625,14 @@ final class ConnectionManager: ObservableObject { let acceptedAt = ProcessInfo.processInfo.systemUptime Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } - if let move = self.attemptMove, !Self.isSamePC(move, info) { - self.record(.warning, "connection", "The PC at the new address isn't the saved one") - self.disconnect(.otherPC) - self.problem = .noReply(usb: false) - return + if let move = self.attemptMove { + guard Self.isSamePC(move, info) else { + self.record(.warning, "connection", "The PC at the new address isn't the saved one") + self.disconnect(.otherPC) + self.problem = .noReply(usb: false) + return + } + self.lastTarget = (normalizedHost, port) } self.sessionAcceptedAt = acceptedAt self.acceptedAddresses.insert("\(normalizedHost):\(port)") @@ -711,7 +717,7 @@ final class ConnectionManager: ObservableObject { self.disconnect(.legacyPC) } else { self.record(.warning, "ctrl", "HELLO2 handshake got no reply yet") - if !isUSB { self.lookForMovedPC(connectionID, host: normalizedHost, port: port, token: pairingToken) } + if !isUSB { self.lookForMovedPC(connectionID, host: normalizedHost, port: port) } } } } @@ -1048,39 +1054,75 @@ final class ConnectionManager: ObservableObject { } } - /// The attempt's HELLO2 got no answer at `host:port`. A saved PC whose - /// address may have changed is looked for by name; where Bonjour has it - /// at another address, the attempt moves there with the PC's pairing. - private func lookForMovedPC(_ connectionID: UUID, host: String, port: UInt16, token: Data) { + /// The attempt's HELLO2 got no answer at `host:port`. A paired PC that + /// this iPad found with Bonjour is looked for by name. Where Bonjour has + /// it at another home network address, the iPad checks that it is the + /// same PC, and only then moves the attempt there with its pairing. + /// Otherwise the attempt goes on as before. + private func lookForMovedPC(_ connectionID: UUID, host: String, port: UInt16) { let address = PairingStore.address(host: host, port: port) let saved = RecentConnectionStore.shared.entry(host: host, port: port) let name = saved?.name ?? pairingStore.pcName(address: address) - guard attemptMove == nil, let name, - MovedPC.canLookUp(host: host, pcName: name, typed: typedAddresses.contains(address) || saved?.typed == true) + let token = (try? pairingStore.token(address: address)) ?? nil + guard attemptMove == nil, let name, let token, + MovedPC.canLookUp(host: host, pcName: name, found: foundAddresses.contains(address) || saved?.found == true, + paired: PairingStore.valid(token), homeAddress: homeAddress) else { return } record(.info, "connection", "Looking for the saved PC by name: no answer at its address") + let current = { [weak self] in + // The PC may have answered meanwhile, or the attempt ended. + guard let self else { return false } + return self.currentAttempt.withLock { $0 == connectionID } && self.state == .connecting + && self.hostInfo == nil && !self.awaitingPairing + } Task { @MainActor [weak self] in guard let self else { return } let found = await self.findPCs(MovedPC.serviceName(pcName: name)) - // The PC may have answered meanwhile, or the attempt ended. - guard self.currentAttempt.withLock({ $0 == connectionID }), self.state == .connecting, - self.hostInfo == nil, !self.awaitingPairing else { return } - guard let moved = MovedPC.newAddress(pcName: name, host: host, port: port, found: found) else { + guard current() else { return } + guard let moved = MovedPC.newAddress(pcName: name, host: host, port: port, found: found, + homeAddress: self.homeAddress) else { self.record(.info, "connection", "The saved PC wasn't found at another address") return } - self.record(.info, "connection", "Found the saved PC at \(moved.host):\(moved.port)") + self.record(.info, "connection", "Checking that the PC at \(moved.host):\(moved.port) is the saved one") + let same = await MovedPC.confirm(host: moved.host, token: token, link: self.makeUDPLink(moved.port)) + guard current() else { return } + guard same else { + self.record(.info, "connection", "The PC there didn't show it is the saved one: staying at the saved address") + return + } + self.record(.info, "connection", "It is the saved PC: connecting there") self.disconnect(.pcMoved) self.pendingMove = Move(from: (host, port), name: name, token: token) - self.explicitPairingToken = PairingStore.valid(token) ? token : nil + self.explicitPairingToken = token self.startAttempt(.udp(host: moved.host, port: moved.port)) } } /// The PC that accepted a moved attempt is the saved one: it has the - /// saved name, and gave back the saved pairing token if there was one. + /// saved name, and gives back the saved pairing token, or none if its + /// pairing is off now. private static func isSamePC(_ move: Move, _ info: ControlChannel.SessionInfo) -> Bool { - info.hostName == move.name && (!PairingStore.valid(move.token) || info.authToken == move.token) + info.hostName == move.name && (!PairingStore.valid(info.authToken) || info.authToken == move.token) + } + + /// The moved PC showed a picture: its pairing is saved at the new + /// address, then the new entry replaces the old one. If the pairing + /// couldn't be saved, the old entry and pairing stay as they were. + private func savedMove(_ move: Move, to new: (host: String, port: UInt16), pairingAddress: String) -> Bool { + do { + try pairingStore.save(token: move.token, hostName: hostInfo?.hostName ?? move.name, address: pairingAddress) + guard try pairingStore.token(address: pairingAddress) == move.token else { throw CocoaError(.fileWriteUnknown) } + } catch { + record(.warning, "pairing", "The PC's new address wasn't saved on this iPad; keeping the saved one") + return false + } + RecentConnectionStore.shared.moved(from: move.from, to: new, name: hostInfo?.hostName ?? move.name, + replacesOld: homeAddress(move.from.host)) + if homeAddress(move.from.host) { + try? pairingStore.remove(address: PairingStore.address(host: move.from.host, port: move.from.port)) + } + return true } /// Retires the automatic reconnect cycle, if one is running: its task @@ -1632,8 +1674,9 @@ struct RecentConnection: Codable, Identifiable { let isUSB: Bool /// The name the PC gave. Entries saved before 0.3.5 have none. var name: String? = nil - /// The user typed this address. It is never looked up by name again. - var typed: Bool? = nil + /// The address is the one Bonjour listed for the PC. Only such an + /// entry may move to a new address by itself; nil is unknown. + var found: Bool? = nil } extension RecentConnection { @@ -1658,14 +1701,14 @@ final class RecentConnectionStore: ObservableObject { } /// Saves a connection that showed a picture. A Wi-Fi entry keeps its - /// name and typed mark when this connection gives none. - func add(host: String, port: UInt16, isUSB: Bool, name: String? = nil, typed: Bool = false) { + /// name and found mark when this connection gives none. + func add(host: String, port: UInt16, isUSB: Bool, name: String? = nil, found: Bool = false) { let previous = isUSB ? nil : entry(host: host, port: port) connections.removeAll { $0.host == host && $0.port == port && $0.isUSB == isUSB } connections.insert( RecentConnection(host: host, port: port, lastUsed: Date(), isUSB: isUSB, name: isUSB ? nil : name.flatMap { $0.isEmpty ? nil : $0 } ?? previous?.name, - typed: isUSB ? nil : (typed || previous?.typed == true)), + found: isUSB ? nil : (found || previous?.found == true)), at: 0 ) if connections.count > 10 { connections = Array(connections.prefix(10)) } @@ -1677,13 +1720,14 @@ final class RecentConnectionStore: ObservableObject { connections.first { !$0.isUSB && $0.host == host && $0.port == port } } - /// The PC moved from `old` to `new`: the old entry gives way where - /// `MovedPC.replacesSavedEntry` says so. - func moved(from old: (host: String, port: UInt16), to new: (host: String, port: UInt16), name: String) { - if MovedPC.replacesSavedEntry(old.host) { + /// The PC moved from `old` to `new`, an address Bonjour gave. A home + /// network entry gives way to it (`replacesOld`). + func moved(from old: (host: String, port: UInt16), to new: (host: String, port: UInt16), name: String, + replacesOld: Bool) { + if replacesOld { connections.removeAll { !$0.isUSB && $0.host == old.host && $0.port == old.port } } - add(host: new.host, port: new.port, isUSB: false, name: name) + add(host: new.host, port: new.port, isUSB: false, name: name, found: true) } private func load() { diff --git a/ios/EternalMonitor/App/EternalMonitorApp.swift b/ios/EternalMonitor/App/EternalMonitorApp.swift index 2d4146f3..2f778928 100644 --- a/ios/EternalMonitor/App/EternalMonitorApp.swift +++ b/ios/EternalMonitor/App/EternalMonitorApp.swift @@ -33,7 +33,8 @@ struct EternalMonitorApp: App { // Automated end-to-end harness: EM_AUTOCONNECT=host:port // connects immediately, bypassing the connect screen. if let target = E2E.autoconnectTarget { - connectionManager.connect(host: target.host, port: target.port) + connectionManager.connect(host: target.host, port: target.port, token: target.token, + found: target.found) } #if DEBUG UIPreview.apply(to: connectionManager) diff --git a/ios/EternalMonitor/App/MovedPC.swift b/ios/EternalMonitor/App/MovedPC.swift index 368e0352..cdc80ef9 100644 --- a/ios/EternalMonitor/App/MovedPC.swift +++ b/ios/EternalMonitor/App/MovedPC.swift @@ -1,39 +1,42 @@ +import CryptoKit import Foundation import Network +import os /// Finding a saved PC again after its address changed: the router gave it a -/// new one, or it moved between a cable and Wi-Fi. The protocol has no PC id, -/// so a PC is the name it gives in its HELLO_ACK, which is also the name in -/// its Bonjour service. Its pairing token then proves it is the same PC. +/// new one, or it moved between a cable and Wi-Fi. Only a paired PC that this +/// iPad found with Bonjour, on a home network address, moves by itself; the +/// iPad checks that it is the same PC before reconnecting. enum MovedPC { /// The Bonjour service a PC named `pcName` advertises. static func serviceName(pcName: String) -> String { "EternalMonitor (\(pcName))" } /// Whether an attempt at `host` that got no answer may look for its PC - /// by name. Not an address the user typed, which may be a VPN address - /// they chose on purpose, and not a host name, which DNS keeps current. - static func canLookUp(host: String, pcName: String?, typed: Bool) -> Bool { - guard !typed, let pcName, !pcName.isEmpty else { return false } - return IPv4Address(host) != nil && host.split(separator: ".").count == 4 + /// by name: the PC has a name and a pairing, and its address came from + /// Bonjour (`found`), not from typing, a QR code or an older version. + static func canLookUp(host: String, pcName: String?, found: Bool, paired: Bool, + homeAddress: (String) -> Bool = isHomeAddress) -> Bool { + guard found, paired, let pcName, !pcName.isEmpty else { return false } + return homeAddress(host) } /// Where Bonjour found the PC named `pcName` instead of `host:port`, or - /// nil: not found, found at the same address, or not one PC. Another PC - /// with the same name makes Bonjour rename one of the two services, so - /// any service named for it besides the one found makes it ambiguous. - static func newAddress(pcName: String, host: String, port: UInt16, - found: [DiscoveredHost]) -> (host: String, port: UInt16)? { + /// nil: not found, found at the same address, not on a home network + /// address, or not one PC. Another PC with the same name makes Bonjour + /// rename one of the two services, so any other service named for it, + /// resolved or not, makes it ambiguous. + static func newAddress(pcName: String, host: String, port: UInt16, found: [DiscoveredHost], + homeAddress: (String) -> Bool = isHomeAddress) -> (host: String, port: UInt16)? { let name = serviceName(pcName: pcName) let claims = found.filter { $0.name == name || $0.name.hasPrefix(name + " (") } guard claims.count == 1, let pc = claims.first, pc.name == name, let address = pc.address, - address != host || pc.port != port else { return nil } + homeAddress(address), address != host || pc.port != port else { return nil } return (address, pc.port) } - /// Whether the saved entry at `host` gives way to the PC's new address: - /// only a home network address, which the router hands to another - /// device in time. A VPN or Tailscale address stays for use away from home. - static func replacesSavedEntry(_ host: String) -> Bool { + /// A home network address, which the router hands out and may hand to + /// another device in time. VPN and Tailscale addresses never move. + static func isHomeAddress(_ host: String) -> Bool { guard host.split(separator: ".").count == 4, let bytes = IPv4Address(host)?.rawValue, bytes.count == 4 else { return false } return bytes[0] == 10 || (bytes[0] == 172 && (16...31).contains(bytes[1])) || (bytes[0] == 192 && bytes[1] == 168) || (bytes[0] == 169 && bytes[1] == 254) @@ -41,7 +44,7 @@ enum MovedPC { /// Looks for EternalMonitor PCs for up to `seconds`, stopping soon after /// the service `name` has an address. The wait after it lets a second - /// service with the same name show up. + /// service with the same name show up. Services still resolving count. @MainActor static func find(_ name: String, seconds: Double = 4) async -> [DiscoveredHost] { let scanner = NetworkScanner() @@ -54,8 +57,62 @@ enum MovedPC { if foundAt == nil, scanner.hosts.contains(where: { $0.name == name && $0.address != nil }) { foundAt = Date() } if let foundAt, Date().timeIntervalSince(foundAt) >= 0.75 { break } } - let hosts = scanner.hosts + let claims = scanner.claims scanner.stopScan() - return hosts + return claims + } + + /// What the PC holding `token` answers to `nonce`. + static func proof(token: Data, nonce: Data) -> Data { + Data(HMAC.authenticationCode(for: nonce + WireIdentify.context, using: SymmetricKey(data: token))) + } + + /// Asks the PC at `host:port`, over `link` and nothing else, to show it + /// is the PC that `token` paired with. True once its answer checks out; + /// false after `attempts` requests go unanswered, as from a PC before + /// 0.3.5, or a wrong answer. The token itself is never sent. + static func confirm(host: String, token: Data, link: any MediaLink, attempts: Int = 8, + interval: Double = 0.25) async -> Bool { + var generator = SystemRandomNumberGenerator() + let nonce = Data((0..<16).map { _ in UInt8.random(in: .min ... .max, using: &generator) }) + let request = Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identify(WireIdentify(nonce: nonce))) + let key = SymmetricKey(data: token) + let box = LinkBox(link) + return await withCheckedContinuation { continuation in + let done = OSAllocatedUnfairLock(initialState: false) + let finish: @Sendable (Bool) -> Void = { result in + guard done.withLock({ finished in defer { finished = true }; return !finished }) else { return } + box.link.controlQueue.async { + box.link.onControlDatagram = nil + box.link.onListenerReady = nil + box.link.onError = nil + box.link.stop() + } + continuation.resume(returning: result) + } + link.onControlDatagram = { data in + guard case .identity(let identity)? = Wire.parseControl(data)?.message, + identity.version == WireIdentify.version, identity.nonce == nonce else { return } + finish(HMAC.isValidAuthenticationCode(identity.proof, authenticating: nonce + WireIdentify.context, + using: key)) + } + link.onError = { _ in finish(false) } + link.onListenerReady = { _ in + for attempt in 0..() /// Services whose IPv4 address is being looked up by host name, each /// with the lookup whose answer still counts. A removed service, a new /// scan or a newer address drops its entry, so a late answer can't list @@ -106,6 +108,7 @@ final class NetworkScanner: NSObject, ObservableObject { searchProblem = nil hosts = [] resolvingServices.removeAll() + serviceNames.removeAll() ipv4Lookups.removeAll() status = .looking @@ -127,6 +130,14 @@ final class NetworkScanner: NSObject, ObservableObject { } } + /// Every PC this search heard of: those listed, and those whose address + /// never came or is still coming, without one. A lookup that must not + /// mistake one PC for another counts them all. + var claims: [DiscoveredHost] { + hosts + serviceNames.filter { name in !hosts.contains { $0.name == name } }.sorted() + .map { DiscoveredHost(name: $0, address: nil, port: 0, finding: true) } + } + func stopScan() { browser.stop() resolvingServices.values.forEach { service in @@ -170,6 +181,7 @@ extension NetworkScanner: NetServiceBrowserDelegate, NetServiceDelegate { ) { Task { @MainActor in let key = Self.serviceKey(service) + serviceNames.insert(service.name) guard resolvingServices[key] == nil else { return } service.delegate = self @@ -195,6 +207,7 @@ extension NetworkScanner: NetServiceBrowserDelegate, NetServiceDelegate { resolving.stop() } ipv4Lookups.removeValue(forKey: service.name) + serviceNames.remove(service.name) hosts.removeAll { $0.name == service.name } if !moreComing { diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index a48396fa..28a57d98 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -30,6 +30,8 @@ enum Wire { case streamConfig = 0x18 case nack = 0x19 case inputEvent = 0x20 + case identify = 0x1A + case identity = 0x1B case error = 0x7F } @@ -374,6 +376,25 @@ struct WirePong: Equatable { var t3Us: UInt64 } +/// iPad → PC, session 0: before reconnecting to a PC it found at a new +/// address, the iPad checks that it is the same PC. The PC answers with a +/// `WireIdentity`. 32 reserved bytes keep the request as long as the answer. +struct WireIdentify: Equatable { + static let version: UInt8 = 1 + /// What the proof signs after the nonce. + static let context = Data("EternalMonitor identify v1".utf8) + var version = WireIdentify.version + var nonce: Data +} + +/// PC → iPad, session 0: HMAC-SHA256 keyed with the pairing token over +/// `nonce ‖ WireIdentify.context`. +struct WireIdentity: Equatable { + var version = WireIdentify.version + var nonce: Data + var proof: Data +} + struct WireInputEvent: Equatable { var inputVer: UInt8 = 1 var kind: UInt8 @@ -404,6 +425,8 @@ enum ControlMessage: Equatable { case pong(WirePong) case streamConfig(StreamConfig) case inputEvent(WireInputEvent) + case identify(WireIdentify) + case identity(WireIdentity) var packetType: Wire.PacketType { switch self { @@ -418,6 +441,8 @@ enum ControlMessage: Equatable { case .pong: return .pong case .streamConfig: return .streamConfig case .inputEvent: return .inputEvent + case .identify: return .identify + case .identity: return .identity } } } @@ -510,6 +535,16 @@ extension Wire { body.appendLE(p.t3Us) case .streamConfig(let cfg): cfg.encode(into: &body) + case .identify(let i): + precondition(i.nonce.count == 16) + body.append(i.version) + body.append(i.nonce) + body.append(Data(repeating: 0, count: 32)) + case .identity(let i): + precondition(i.nonce.count == 16 && i.proof.count == 32) + body.append(i.version) + body.append(i.nonce) + body.append(i.proof) case .inputEvent(let e): body.append(e.inputVer) body.append(e.kind) @@ -609,6 +644,12 @@ extension Wire { message = .streamConfig(cfg) case .inputEvent: message = parseInputEvent(&r) + case .identify: + guard let version = r.readU8(), let nonce = r.take(16), r.take(32) != nil else { return nil } + message = .identify(WireIdentify(version: version, nonce: Data(nonce))) + case .identity: + guard let version = r.readU8(), let nonce = r.take(16), let proof = r.take(32) else { return nil } + message = .identity(WireIdentity(version: version, nonce: Data(nonce), proof: Data(proof))) case .media, .mediaFec, .audio, .error: return nil } diff --git a/ios/EternalMonitorTests/ConnectionProblemTests.swift b/ios/EternalMonitorTests/ConnectionProblemTests.swift index cb8dadae..dcf85314 100644 --- a/ios/EternalMonitorTests/ConnectionProblemTests.swift +++ b/ios/EternalMonitorTests/ConnectionProblemTests.swift @@ -205,7 +205,7 @@ final class ConnectionProblemTests: XCTestCase { func testEveryProblemSaysWhatToDoWithoutTransportJargon() { let all: [ConnectionProblem] = [ - .noReply(usb: false), .noReply(usb: true), .noReplyAtNewAddress("192.168.1.31"), .noPicture, .receiverNotReady, + .noReply(usb: false), .noReply(usb: true), .noReplyAtNewAddress("192.0.2.31"), .noPicture, .receiverNotReady, .badAddress("pc"), .noNetwork, .localNetworkBlocked, .couldNotConnect, .usbFailed, .lost(.pcStoppedResponding), .lost(.cableUnplugged), .lost(.usbTurnedOff), .reconnecting(attempt: 2, of: 6), .reconnectFailed, .pcEndedSession, .pcEndedSessionOnError, .pcBusy, .pairingCodeAsAddress, .routerAddress, diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index cfc29ed9..c355def5 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -16,6 +16,8 @@ final class FakeUDPHost: @unchecked Sendable { private(set) var lastNonce: UInt32 = 0 /// The pairing token the HELLO2 answered last carried. private(set) var lastAuthToken = Data() + /// A HELLO2 came before the identify request was answered. + private(set) var helloBeforeIdentify = false init() { var address = sockaddr_in() @@ -89,6 +91,33 @@ final class FakeUDPHost: @unchecked Sendable { } } + /// Answers the next identify request as the PC holding `token` would, + /// or with `proof` instead. False after thirty quiet seconds. + func answerIdentify(token: Data, proof: Data? = nil) -> Bool { + for _ in 0..<100 { + guard let (data, from, length) = receive() else { continue } + let message = Wire.parseControl(data)?.message + if case .hello2? = message { helloBeforeIdentify = true } + guard case .identify(let request)? = message else { continue } + client = (from, length) + send(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identity(WireIdentity( + nonce: request.nonce, proof: proof ?? MovedPC.proof(token: token, nonce: request.nonce))))) + return true + } + return false + } + + /// The control messages received within `seconds`, by type. + func received(within seconds: Double) -> [Wire.PacketType] { + var types: [Wire.PacketType] = [] + let deadline = Date().addingTimeInterval(seconds) + while Date() < deadline { + guard let (data, _, _) = receive(), let (header, _) = Wire.parseControl(data) else { continue } + types.append(header.packetType) + } + return types + } + /// Ends session 41 from the PC's side. func sendBye(_ reason: ByeReason) { send(Wire.encodeControl(sessionId: 41, msgSeq: 2, message: .bye(reason))) diff --git a/ios/EternalMonitorTests/MovedPCTests.swift b/ios/EternalMonitorTests/MovedPCTests.swift index a1246b48..011e36de 100644 --- a/ios/EternalMonitorTests/MovedPCTests.swift +++ b/ios/EternalMonitorTests/MovedPCTests.swift @@ -2,21 +2,36 @@ import Combine import XCTest @testable import EternalMonitor +/// Pairings kept in memory. `failSaves` makes every save fail, as a full +/// or locked Keychain does. +private final class MemoryPairingStorage: PairingStorage { + var values: [String: Data] = [:] + var failSaves = false + func records() throws -> [PairingRecord] { values.map { PairingRecord(account: $0.key, token: $0.value) } } + func save(account: String, token: Data) throws { + if failSaves { throw KeychainPairingStorage.Failure(status: errSecNotAvailable) } + values[account] = token + } + func remove(account: String?) throws { + if let account { values.removeValue(forKey: account) } else { values.removeAll() } + } +} + /// A saved PC whose address changed: a new lease from the router, or the /// PC moved between its cable and Wi-Fi. @MainActor final class MovedPCTests: XCTestCase { - private static let token = Data(17...32) - private var hostsToForget: [String] = [] + nonisolated private static let token = Data(17...32) + /// The choice tests' home network: documentation addresses stand in. + private static let home: (String) -> Bool = { $0.hasPrefix("192.0.2.") || $0.hasPrefix("198.51.100.") } override func setUp() async throws { let store = RecentConnectionStore.shared let saved = store.connections - let defaults = UserDefaults.standard.object(forKey: "recentConnections") + let defaults = ["recentConnections", "lastHost", "lastPort"].map { ($0, UserDefaults.standard.object(forKey: $0)) } addTeardownBlock { @MainActor in store.connections = saved - UserDefaults.standard.set(defaults, forKey: "recentConnections") - for address in self.hostsToForget { try? PairingStore.shared.remove(address: address) } + for (key, value) in defaults { UserDefaults.standard.set(value, forKey: key) } } } @@ -27,93 +42,133 @@ final class MovedPCTests: XCTestCase { // MARK: - The choice func testAStaleSavedAddressMovesToTheFreshBonjourAddress() { - XCTAssertTrue(MovedPC.canLookUp(host: "192.168.1.20", pcName: "Studio PC", typed: false)) - let fresh = [found("EternalMonitor (Office PC)", "192.168.1.40"), found("EternalMonitor (Studio PC)", "192.168.1.31")] - let moved = MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: fresh) - XCTAssertEqual(moved?.host, "192.168.1.31") + let fresh = [found("EternalMonitor (Office PC)", "192.0.2.40"), found("EternalMonitor (Studio PC)", "192.0.2.31")] + let moved = MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.20", port: 9876, found: fresh, homeAddress: Self.home) + XCTAssertEqual(moved?.host, "192.0.2.31") XCTAssertEqual(moved?.port, 9876) // A new port is a new address too. - let port = MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.31", port: 9876, - found: [found("EternalMonitor (Studio PC)", "192.168.1.31", port: 9900)]) + let port = MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.31", port: 9876, + found: [found("EternalMonitor (Studio PC)", "192.0.2.31", port: 9900)], homeAddress: Self.home) XCTAssertEqual(port?.port, 9900) // Still at the saved address: it is just not answering. - XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.31", port: 9876, found: fresh)) - XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: [])) + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.31", port: 9876, found: fresh, homeAddress: Self.home)) + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.20", port: 9876, found: [], homeAddress: Self.home)) } - func testAManualAddressIsNeverLookedUpOrReplaced() { - XCTAssertFalse(MovedPC.canLookUp(host: "192.168.1.20", pcName: "Studio PC", typed: true)) - // No name, and host names that DNS keeps current, aren't looked up. - XCTAssertFalse(MovedPC.canLookUp(host: "192.168.1.20", pcName: nil, typed: false)) - XCTAssertFalse(MovedPC.canLookUp(host: "192.168.1.20", pcName: "", typed: false)) - XCTAssertFalse(MovedPC.canLookUp(host: "studio-pc.local", pcName: "Studio PC", typed: false)) - XCTAssertFalse(MovedPC.canLookUp(host: "fe80::1", pcName: "Studio PC", typed: false)) + /// Only a paired PC that Bonjour listed, on a home network address, + /// moves by itself. Typed, scanned and older entries, and VPN or + /// Tailscale addresses on either side, never do. + func testOnlyAPairedPCFoundWithBonjourOnAHomeAddressIsLookedUp() throws { + XCTAssertTrue(MovedPC.canLookUp(host: "192.0.2.20", pcName: "Studio PC", found: true, paired: true, homeAddress: Self.home)) + XCTAssertFalse(MovedPC.canLookUp(host: "192.0.2.20", pcName: "Studio PC", found: false, paired: true, homeAddress: Self.home)) + XCTAssertFalse(MovedPC.canLookUp(host: "192.0.2.20", pcName: "Studio PC", found: true, paired: false, homeAddress: Self.home)) + XCTAssertFalse(MovedPC.canLookUp(host: "192.0.2.20", pcName: nil, found: true, paired: true, homeAddress: Self.home)) + XCTAssertFalse(MovedPC.canLookUp(host: "203.0.113.20", pcName: "Studio PC", found: true, paired: true, homeAddress: Self.home)) - for home in ["192.168.1.20", "10.0.0.5", "172.20.1.2", "169.254.3.4"] { - XCTAssertTrue(MovedPC.replacesSavedEntry(home), home) + for home in ["10.0.0.1", "10.255.255.254", "172.16.0.1", "172.31.255.254", "192.168.0.1", "192.168.255.254", "169.254.0.1"] { + XCTAssertTrue(MovedPC.isHomeAddress(home), home) } - for kept in ["100.101.102.103", "127.0.0.1", "8.8.8.8", "studio-pc.local", "fe80::1", "192.168.1"] { - XCTAssertFalse(MovedPC.replacesSavedEntry(kept), kept) + for other in ["100.64.0.1", "100.127.255.254", "172.15.255.254", "172.32.0.1", "127.0.0.1", "192.0.2.1", + "studio-pc.local", "fe80::1", "192.168.1"] { + XCTAssertFalse(MovedPC.isHomeAddress(other), other) + XCTAssertFalse(MovedPC.canLookUp(host: other, pcName: "Studio PC", found: true, paired: true), other) } + // Bonjour giving a VPN address is no move either. + XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "10.255.255.254", port: 9876, + found: [found("EternalMonitor (Studio PC)", "100.64.0.9")])) + + // An entry saved before this version doesn't say where it came from. + let legacy = #"[{"host":"192.0.2.20","port":9876,"lastUsed":0,"isUSB":false}]"# + let decoded = try JSONDecoder().decode([RecentConnection].self, from: Data(legacy.utf8)) + XCTAssertNil(decoded.first?.found) let store = RecentConnectionStore.shared store.connections = [] - store.add(host: "192.168.1.20", port: 9876, isUSB: false, name: "Studio PC", typed: true) - store.add(host: "192.168.1.20", port: 9876, isUSB: false) - XCTAssertEqual(store.entry(host: "192.168.1.20", port: 9876)?.typed, true, "picked from Recent, still typed") - XCTAssertEqual(store.entry(host: "192.168.1.20", port: 9876)?.name, "Studio PC") - - // A Tailscale address stays for use away from home; a home address gives way. - store.add(host: "100.101.102.103", port: 9876, isUSB: false, name: "Studio PC") - store.moved(from: ("100.101.102.103", 9876), to: ("192.168.1.31", 9876), name: "Studio PC") - XCTAssertNotNil(store.entry(host: "100.101.102.103", port: 9876)) - store.add(host: "192.168.1.22", port: 9876, isUSB: false, name: "Studio PC") - store.moved(from: ("192.168.1.22", 9876), to: ("192.168.1.32", 9876), name: "Studio PC") - XCTAssertNil(store.entry(host: "192.168.1.22", port: 9876)) - XCTAssertEqual(store.connections.first?.host, "192.168.1.32") - XCTAssertEqual(store.connections.first?.name, "Studio PC") + store.add(host: "192.0.2.20", port: 9876, isUSB: false, name: "Studio PC", found: true) + store.add(host: "192.0.2.20", port: 9876, isUSB: false) + XCTAssertEqual(store.entry(host: "192.0.2.20", port: 9876)?.found, true, "picked again from Recent, still found") + XCTAssertEqual(store.entry(host: "192.0.2.20", port: 9876)?.name, "Studio PC") + store.add(host: "192.0.2.21", port: 9876, isUSB: false, name: "Studio PC") + XCTAssertEqual(store.entry(host: "192.0.2.21", port: 9876)?.found, false) + + store.moved(from: ("192.0.2.20", 9876), to: ("192.0.2.31", 9876), name: "Studio PC", replacesOld: false) + XCTAssertNotNil(store.entry(host: "192.0.2.20", port: 9876)) + store.moved(from: ("192.0.2.20", 9876), to: ("192.0.2.32", 9876), name: "Studio PC", replacesOld: true) + XCTAssertNil(store.entry(host: "192.0.2.20", port: 9876)) + XCTAssertEqual(store.connections.first?.host, "192.0.2.32") + XCTAssertEqual(store.connections.first?.found, true) } func testTwoPCsWithTheSameNameAreNeverConfused() { - let ours = found("EternalMonitor (Studio PC)", "192.168.1.31") + let ours = found("EternalMonitor (Studio PC)", "192.0.2.31") // Bonjour renamed the second service: either one may be the saved PC. - let renamed = found("EternalMonitor (Studio PC) (2)", "192.168.1.40") - XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: [ours, renamed])) - XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: [renamed])) - // A second claim still resolving counts as well. - XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, - found: [ours, found("EternalMonitor (Studio PC) (2)", nil)])) + let renamed = found("EternalMonitor (Studio PC) (2)", "192.0.2.40") + let choose = { MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.20", port: 9876, found: $0, homeAddress: Self.home) } + XCTAssertNil(choose([ours, renamed])) + XCTAssertNil(choose([renamed])) + XCTAssertNil(choose([ours, found("EternalMonitor (Studio PC) (2)", nil)])) // Other PCs, however alike their names, are no claim. - let others = [found("EternalMonitor (Studio PC 2)", "192.168.1.41"), found("EternalMonitor (Studio)", "192.168.1.42")] - XCTAssertNil(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: others)) - XCTAssertEqual(MovedPC.newAddress(pcName: "Studio PC", host: "192.168.1.20", port: 9876, found: others + [ours])?.host, - "192.168.1.31") + let others = [found("EternalMonitor (Studio PC 2)", "192.0.2.41"), found("EternalMonitor (Studio)", "192.0.2.42")] + XCTAssertNil(choose(others)) + XCTAssertEqual(choose(others + [ours])?.host, "192.0.2.31") + } + + /// The real scanner: a second service still resolving when the lookup + /// ends has no address yet, and still makes the choice ambiguous. + func testAServiceStillResolvingCountsAsAClaim() async throws { + let scanner = NetworkScanner() + defer { scanner.stopScan() } + let choose = { MovedPC.newAddress(pcName: "Studio PC", host: "192.0.2.20", port: 9876, found: scanner.claims, + homeAddress: Self.home) } + scanner.merge(found("EternalMonitor (Studio PC)", "192.0.2.31")) + XCTAssertEqual(choose()?.host, "192.0.2.31") + scanner.netServiceBrowser(NetServiceBrowser(), didFind: NetService(domain: "local.", type: "_eternaldisplay._udp.", + name: "EternalMonitor (Studio PC) (2)"), moreComing: false) + let deadline = Date().addingTimeInterval(5) + while scanner.claims.count < 2, Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertEqual(scanner.hosts.map(\.name), ["EternalMonitor (Studio PC)"], "not listed: no address yet") + XCTAssertEqual(scanner.claims.map(\.name), ["EternalMonitor (Studio PC)", "EternalMonitor (Studio PC) (2)"]) + XCTAssertNil(choose()) + } + + /// The same known answer as the host's pairing tests and the wire's + /// golden `identity` vector. + func testTheIdentityProofIsTheSharedKnownAnswer() { + let proof = MovedPC.proof(token: Data(1...16), nonce: Data(0xA0...0xAF)) + XCTAssertEqual(proof.map { String(format: "%02x", $0) }.joined(), + "126dc93467af3b0621c4f9b420e0d323143259d743f85703b9fec83989eba353") + XCTAssertNotEqual(MovedPC.proof(token: Data(2...17), nonce: Data(0xA0...0xAF)), proof) } // MARK: - Connecting - /// A saved PC named "Test PC" at `old`'s address, paired when `token` is set. - private func savePC(at old: FakeUDPHost, token: Data?) throws { - RecentConnectionStore.shared.connections = [ - RecentConnection(host: "127.0.0.1", port: old.port, lastUsed: .now, isUSB: false, name: "Test PC"), - ] - let address = PairingStore.address(host: "127.0.0.1", port: old.port) - hostsToForget.append(address) - if let token { try PairingStore.shared.save(token: token, hostName: "Test PC", address: address) } + private struct Setup { + let manager: ConnectionManager + let storage: MemoryPairingStorage + let pairings: PairingStore + let asked: () -> [String] } - /// A manager whose Bonjour finds "Test PC" at `new`'s port, and the - /// service names it was asked for. - private func manager(finding new: FakeUDPHost, connectTimeout: Double) -> (ConnectionManager, () -> [String]) { + /// A saved PC named "Test PC" at `old`, Bonjour-found unless `found` says + /// otherwise, paired with `token`; Bonjour has it at `new`. Loopback + /// counts as a home address here. + private func setUp(old: FakeUDPHost, new: FakeUDPHost, token: Data?, found: Bool? = true, + connectTimeout: Double) throws -> Setup { + RecentConnectionStore.shared.connections = [ + RecentConnection(host: "127.0.0.1", port: old.port, lastUsed: .now, isUSB: false, name: "Test PC", found: found), + ] + let storage = MemoryPairingStorage() + let pairings = PairingStore(storage: storage) + if let token { try pairings.save(token: token, hostName: "Test PC", address: PairingStore.address(host: "127.0.0.1", port: old.port)) } let manager = ConnectionManager(timing: .init(connectTimeoutSeconds: connectTimeout, reconnectGraceSeconds: 0.1, - reconnectBackoffSeconds: { _ in 0.05 })) - hostsToForget.append(PairingStore.address(host: "127.0.0.1", port: new.port)) + reconnectBackoffSeconds: { _ in 0.05 }), pairingStore: pairings) + manager.homeAddress = { _ in true } var asked: [String] = [] manager.findPCs = { name in asked.append(name) return [DiscoveredHost(name: "EternalMonitor (Test PC)", address: "127.0.0.1", port: new.port)] } - return (manager, { asked }) + return Setup(manager: manager, storage: storage, pairings: pairings, asked: { asked }) } private func until(_ description: String, seconds: Double, _ condition: () -> Bool) async throws { @@ -122,103 +177,156 @@ final class MovedPCTests: XCTestCase { XCTAssertTrue(condition(), description) } - private func logged(_ manager: ConnectionManager, _ prefix: String) -> String? { - manager.diagnostics.first { $0.message.hasPrefix(prefix) }?.message + private func address(_ host: FakeUDPHost) -> String { PairingStore.address(host: "127.0.0.1", port: host.port) } + + private func stream(_ setup: Setup, from new: FakeUDPHost) async throws { + let accepted = await Task.detached { new.answerIdentify(token: Self.token) && new.acceptSession(authToken: Self.token) }.value + XCTAssertTrue(accepted, "the iPad never checked and tried the new address") + try await until("handshake", seconds: 60) { setup.manager.hostInfo != nil } + new.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming", seconds: 60) { setup.manager.state == .connected } } - /// The saved address stays silent; Bonjour has the same PC at another - /// port, and the iPad streams from there with its pairing, no tap needed. - func testASilentSavedAddressReconnectsWhereBonjourFoundThePC() async throws { + /// The saved address stays silent. Bonjour has the same PC at another + /// port; it shows it holds the pairing before the token goes anywhere, + /// and the iPad streams from there with no tap. + func testASilentSavedAddressReconnectsOnceThePCShowsItIsTheSame() async throws { let old = FakeUDPHost(), new = FakeUDPHost() - try savePC(at: old, token: Self.token) - let (manager, asked) = manager(finding: new, connectTimeout: 60) - defer { manager.cancel() } + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 60) + defer { setup.manager.cancel() } // Every line logged: the new attempt clears Connection details. var lines: [String] = [] - let log = manager.$diagnostics.sink { if let line = $0.last?.message { lines.append(line) } } + let log = setup.manager.$diagnostics.sink { if let line = $0.last?.message { lines.append(line) } } defer { log.cancel() } - manager.connect(host: "127.0.0.1", port: old.port) - let accepted = await Task.detached { new.acceptSession(authToken: Self.token) }.value - XCTAssertTrue(accepted, "the iPad never tried the new address") - XCTAssertEqual(asked(), ["EternalMonitor (Test PC)"]) - XCTAssertEqual(new.lastAuthToken, Self.token, "the saved pairing went along") - try await until("handshake", seconds: 60) { manager.hostInfo != nil } - new.sendKeyframe(try FakeUDPHost.keyframe()) - try await until("streaming", seconds: 60) { manager.state == .connected } + setup.manager.connect(host: "127.0.0.1", port: old.port) + try await stream(setup, from: new) + XCTAssertEqual(setup.asked(), ["EternalMonitor (Test PC)"]) + XCTAssertFalse(new.helloBeforeIdentify, "nothing with the token before the PC showed it is the same") + XCTAssertEqual(new.lastAuthToken, Self.token, "then the saved pairing went along") XCTAssertTrue(lines.contains("Disconnect state=connecting reason=pcMoved"), lines.joined(separator: "\n")) - // The saved address counts as not answering, in the closed set the backend accepts. let summary = lines.first { $0.hasPrefix("session_summary ") } ?? "" XCTAssertTrue(summary.contains(" stage=hello_sent outcome=no_reply failure=no_reply fallback=none end=superseded "), summary) let saved = RecentConnectionStore.shared.connections.first XCTAssertEqual(saved?.port, new.port) XCTAssertEqual(saved?.name, "Test PC") - XCTAssertNotNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: old.port), - "loopback is no home address, so its entry stays") - XCTAssertEqual(try PairingStore.shared.token(address: PairingStore.address(host: "127.0.0.1", port: new.port)), Self.token) + XCTAssertEqual(saved?.found, true) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: old.port)) + XCTAssertEqual(try setup.pairings.token(address: address(new)), Self.token) + XCTAssertNil(try setup.pairings.token(address: address(old))) + XCTAssertEqual(UserDefaults.standard.integer(forKey: "lastPort"), Int(new.port)) } - func testATypedAddressIsNeverLookedUp() async throws { - let old = FakeUDPHost(), new = FakeUDPHost() - try savePC(at: old, token: nil) - let (manager, asked) = manager(finding: new, connectTimeout: 4) - defer { manager.cancel() } - manager.connect(host: "127.0.0.1", port: old.port, typed: true) - try await until("timed out", seconds: 15) { manager.state == .disconnected } - XCTAssertEqual(asked(), []) - XCTAssertEqual(manager.problem, .noReply(usb: false)) - } - - /// A PC with the saved name at the new address that doesn't give the - /// saved pairing back is another PC: the iPad leaves it and keeps its pairing. - func testAnotherPCWithTheSameNameIsLeftAndThePairingKept() async throws { - for answer in ["pairing off", "asks for a code"] { + /// A PC that gives a wrong answer, or none (a PC before 0.3.5), gets + /// nothing more: no HELLO2, no token. The attempt ends as it always did. + func testAPCThatDoesNotShowItIsTheSameIsLeftAlone() async throws { + for answer in ["wrong proof", "no answer"] { let old = FakeUDPHost(), new = FakeUDPHost() - try savePC(at: old, token: Self.token) - let (manager, _) = manager(finding: new, connectTimeout: 60) - defer { manager.cancel() } - manager.connect(host: "127.0.0.1", port: old.port) - let accepted = await Task.detached { - answer == "pairing off" ? new.acceptSession(authToken: Data(repeating: 0, count: 16)) - : new.acceptSession(status: .unauthorized) + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 7) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let types = await Task.detached { () -> [Wire.PacketType] in + if answer == "wrong proof", !new.answerIdentify(token: Data(2...17)) { return [] } + return new.received(within: 6) }.value - XCTAssertTrue(accepted, answer) - try await until("left it: \(answer)", seconds: 30) { manager.state == .disconnected } - XCTAssertEqual(manager.problem, .noReply(usb: false), answer) - XCTAssertNil(manager.pairingPrompt, answer) - XCTAssertNotNil(logged(manager, "The PC at the new address isn't the saved one"), answer) - XCTAssertEqual(try PairingStore.shared.token(address: PairingStore.address(host: "127.0.0.1", port: old.port)), - Self.token, answer) - XCTAssertNil(try PairingStore.shared.token(address: PairingStore.address(host: "127.0.0.1", port: new.port)), answer) - XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: new.port), answer) + XCTAssertFalse(types.contains(.hello2), answer) + XCTAssertFalse(new.helloBeforeIdentify, answer) + if answer == "no answer" { XCTAssertEqual(Set(types), [.identify], answer) } + try await until("timed out: \(answer)", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.asked().count, 1, answer) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false), answer) + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token, answer) + XCTAssertNotNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: old.port), answer) + } + } + + /// Unpaired, nothing proves which PC is which: with the saved PC gone, a + /// same-named PC with pairing off isn't even looked for. + func testAnUnpairedSavedPCIsNeverMoved() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: nil, connectTimeout: 5) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let types = await Task.detached { new.received(within: 4) }.value + XCTAssertEqual(types, []) + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.asked(), []) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: new.port)) + } + + /// Entries that didn't come from Bonjour's list: typed, scanned, or saved + /// by an older version, which doesn't say. + func testAnEntryNotFoundWithBonjourIsNeverLookedUp() async throws { + for found: Bool? in [nil, false] { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, found: found, connectTimeout: 4) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.asked(), [], "\(String(describing: found))") + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) } } - /// Unpaired, the name is what tells PCs apart. - func testAnUnpairedPCWithAnotherNameIsLeft() async throws { + /// The PC at the new address showed it holds the pairing, then wasn't + /// the saved PC by name. Try again goes back to the saved address and + /// checks again; it never goes straight to the other PC. + func testTryAgainAfterARefusalGoesBackToTheSavedAddress() async throws { let old = FakeUDPHost(), new = FakeUDPHost() - try savePC(at: old, token: nil) - let (manager, _) = manager(finding: new, connectTimeout: 60) - defer { manager.cancel() } - manager.connect(host: "127.0.0.1", port: old.port) - let accepted = await Task.detached { new.acceptSession(hostName: "Other PC", authToken: Data(repeating: 0, count: 16)) }.value + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 60) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let accepted = await Task.detached { + new.answerIdentify(token: Self.token) && new.acceptSession(hostName: "Other PC", authToken: Data(repeating: 0, count: 16)) + }.value XCTAssertTrue(accepted) - try await until("left it", seconds: 30) { manager.state == .disconnected } - XCTAssertEqual(manager.problem, .noReply(usb: false)) + try await until("left it", seconds: 30) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + + setup.manager.retry() + let (atOld, atNew) = await Task.detached { + (old.helloAttempts(within: 1.5), new.received(within: 1)) + }.value + XCTAssertGreaterThan(atOld, 0, "Try again goes to the saved address") + XCTAssertFalse(atNew.contains(.hello2), "and not straight to the other PC") + XCTAssertNil(setup.manager.pairingPrompt) + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token) + } + + /// The new address's pairing can't be saved: the stream goes on, and the + /// saved entry and pairing stay where they were. + func testTheSavedEntryStaysWhenTheNewPairingCantBeSaved() async throws { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 60) + defer { setup.manager.cancel() } + UserDefaults.standard.set("127.0.0.1", forKey: "lastHost") + UserDefaults.standard.set(Int(old.port), forKey: "lastPort") + setup.storage.failSaves = true + setup.manager.connect(host: "127.0.0.1", port: old.port) + try await stream(setup, from: new) + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token) + XCTAssertNil(try setup.pairings.token(address: address(new))) + XCTAssertNotNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: old.port)) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: new.port)) + XCTAssertEqual(UserDefaults.standard.integer(forKey: "lastPort"), Int(old.port)) + XCTAssertTrue(setup.manager.diagnostics.contains { $0.message.hasPrefix("The PC's new address wasn't saved") }) } - /// Found at a new address that doesn't answer either: the card says so. + /// The PC showed it is the same at its new address, then didn't answer + /// HELLO2 there either: the card says where the iPad found it. func testNoAnswerAtTheNewAddressSaysWhereItLooked() async throws { let old = FakeUDPHost(), new = FakeUDPHost() - try savePC(at: old, token: nil) - let (manager, asked) = manager(finding: new, connectTimeout: 5) - defer { manager.cancel() } - manager.connect(host: "127.0.0.1", port: old.port) - let tried = await Task.detached { new.helloAttempts(within: 6) }.value + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 7) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let answered = await Task.detached { new.answerIdentify(token: Self.token) }.value + XCTAssertTrue(answered) + let tried = await Task.detached { new.helloAttempts(within: 3) }.value XCTAssertGreaterThan(tried, 0) - try await until("timed out", seconds: 15) { manager.state == .disconnected } - XCTAssertEqual(asked().count, 1, "looked once, not again from the new address") - XCTAssertEqual(manager.problem, .noReplyAtNewAddress("127.0.0.1")) - XCTAssertEqual(manager.problem?.actions, [.retry]) + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.asked().count, 1, "looked once, not again from the new address") + XCTAssertEqual(setup.manager.problem, .noReplyAtNewAddress("127.0.0.1")) + XCTAssertEqual(setup.manager.problem?.actions, [.retry]) } } diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index 56b742d2..24f2a7fd 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,11 +35,30 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 29, + Self.vectors.count, 31, "golden vector count drifted — update both test suites together" ) } + /// The identify request and its answer: nonce A0..AF, and the proof of + /// pairing token 01..10, which the iPad checks as the PC's own. + func testIdentifyAndIdentityVectors() throws { + let nonce = Data(0xA0...0xAF) + let request = try vector("identify") + let (requestHeader, asked) = try XCTUnwrap(Wire.parseControl(request)) + XCTAssertEqual(asked, .identify(WireIdentify(version: 1, nonce: nonce))) + XCTAssertEqual(requestHeader.sessionId, 0) + XCTAssertEqual(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: asked), request) + + let answer = try vector("identity") + XCTAssertEqual(answer.count, request.count, "never longer than the request") + let (_, message) = try XCTUnwrap(Wire.parseControl(answer)) + guard case .identity(let identity) = message else { return XCTFail("wrong type") } + XCTAssertEqual(identity.nonce, nonce) + XCTAssertEqual(identity.proof, MovedPC.proof(token: Data(1...16), nonce: nonce)) + XCTAssertEqual(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: message), answer) + } + func testPairingRejectionVectors() throws { for (name, status): (String, HelloStatus) in [("hello_ack_unauthorized", .unauthorized), ("hello_ack_rate_limited", .rateLimited)] { let data = try vector(name) diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index f3e9f069..684fa306 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -315,6 +315,33 @@ pub struct InputEvent { pub tilt_y: i16, } +/// [`Identify`] and [`Identity`] layout version. +pub const IDENTIFY_VERSION: u8 = 1; +/// What an [`Identity`] proof signs after the nonce. +pub const IDENTIFY_CONTEXT: &[u8] = b"EternalMonitor identify v1"; + +/// Client → host, session_id 0: before reconnecting to a PC it found at a new +/// address, the iPad checks that it is the same PC. The host answers with an +/// [`Identity`] whose proof only the PC holding the pairing token can make. +/// 32 reserved bytes keep the request as long as the answer. Hosts before +/// 0.3.5 don't know the type and drop it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Identify { + pub version: u8, + pub nonce: [u8; 16], +} + +/// Host → client, session_id 0: `proof` is HMAC-SHA256 keyed with the +/// pairing token over `nonce ‖ IDENTIFY_CONTEXT`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Identity { + pub version: u8, + pub nonce: [u8; 16], + pub proof: [u8; 32], +} + +pub const IDENTIFY_BODY_SIZE: usize = 49; + pub const INPUT_EVENT_SIZE: usize = 30; pub const PEN_INPUT_EVENT_SIZE: usize = INPUT_EVENT_SIZE + 4; @@ -332,6 +359,8 @@ pub enum ControlMessage { Pong(Pong), StreamConfig(StreamConfig), InputEvent(InputEvent), + Identify(Identify), + Identity(Identity), } impl ControlMessage { @@ -348,6 +377,8 @@ impl ControlMessage { Self::Pong(_) => PacketType::Pong, Self::StreamConfig(_) => PacketType::StreamConfig, Self::InputEvent(_) => PacketType::InputEvent, + Self::Identify(_) => PacketType::Identify, + Self::Identity(_) => PacketType::Identity, } } } @@ -457,6 +488,16 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - cfg.encode_into(&mut block); body.extend_from_slice(&block); } + ControlMessage::Identify(i) => { + body.push(i.version); + body.extend_from_slice(&i.nonce); + body.extend_from_slice(&[0; 32]); + } + ControlMessage::Identity(i) => { + body.push(i.version); + body.extend_from_slice(&i.nonce); + body.extend_from_slice(&i.proof); + } ControlMessage::InputEvent(e) => { body.push(e.input_ver); body.push(e.kind); @@ -677,6 +718,17 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), tilt_y: if input_ver == 2 { r.i16()? } else { 0 }, }) } + PacketType::Identify => { + let version = r.u8()?; + let nonce = r.take(16)?.try_into().unwrap(); + r.take(32)?; + ControlMessage::Identify(Identify { version, nonce }) + } + PacketType::Identity => ControlMessage::Identity(Identity { + version: r.u8()?, + nonce: r.take(16)?.try_into().unwrap(), + proof: r.take(32)?.try_into().unwrap(), + }), PacketType::Media | PacketType::MediaFec | PacketType::Audio | PacketType::Error => { return Err(WireError::InvalidField("packet_type")); } @@ -937,6 +989,15 @@ mod tests { t3_us: 44, }), ControlMessage::StreamConfig(sample_config()), + ControlMessage::Identify(Identify { + version: IDENTIFY_VERSION, + nonce: [7; 16], + }), + ControlMessage::Identity(Identity { + version: IDENTIFY_VERSION, + nonce: [7; 16], + proof: [9; 32], + }), ControlMessage::InputEvent(InputEvent { input_ver: 1, kind: 0, diff --git a/proto/src/v2/mod.rs b/proto/src/v2/mod.rs index 4b75ffe9..aa7ddc1f 100644 --- a/proto/src/v2/mod.rs +++ b/proto/src/v2/mod.rs @@ -56,6 +56,10 @@ pub enum PacketType { StreamConfig = 0x18, Nack = 0x19, InputEvent = 0x20, + /// Client → host: prove you are the PC this iPad paired with (0.3.5+). + Identify = 0x1A, + /// Host → client: the answer to [`PacketType::Identify`]. + Identity = 0x1B, /// Reserved. Not sent in v2.0. Error = 0x7F, } @@ -77,6 +81,8 @@ impl PacketType { 0x18 => Self::StreamConfig, 0x19 => Self::Nack, 0x20 => Self::InputEvent, + 0x1A => Self::Identify, + 0x1B => Self::Identity, 0x7F => Self::Error, _ => return None, }) diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 6d072d05..cda185d1 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -32,3 +32,5 @@ receiver_report_v030 454d021500003200785634123200000003000000881300007e1300000a0 hello_ack_paired 454d02110000390000000000030000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f100100 hello_ack_contiguous_seq 454d02110000390000000000050000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f102100 hello2_v030 454d02100000410000000000040000000202efbeadde94260300010074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e20100 +identify 454d021a00003100000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf0000000000000000000000000000000000000000000000000000000000000000 +identity 454d021b00003100000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf126dc93467af3b0621c4f9b420e0d323143259d743f85703b9fec83989eba353 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index e0e6c5ad..bdf567a4 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -404,9 +404,40 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { "hello2_v030", encode_control(0, 4, &ControlMessage::Hello2(hello)), )); + let nonce = std::array::from_fn(|index| 0xA0 + index as u8); + vectors.push(( + "identify", + encode_control( + 0, + 1, + &ControlMessage::Identify(Identify { + version: IDENTIFY_VERSION, + nonce, + }), + ), + )); + // The proof for pairing token 01..10: the same known answer the host's + // pairing tests and the iPad's identity tests check. + vectors.push(( + "identity", + encode_control( + 0, + 1, + &ControlMessage::Identity(Identity { + version: IDENTIFY_VERSION, + nonce, + proof: IDENTITY_PROOF, + }), + ), + )); vectors } +const IDENTITY_PROOF: [u8; 32] = [ + 0x12, 0x6d, 0xc9, 0x34, 0x67, 0xaf, 0x3b, 0x06, 0x21, 0xc4, 0xf9, 0xb4, 0x20, 0xe0, 0xd3, 0x23, + 0x14, 0x32, 0x59, 0xd7, 0x43, 0xf8, 0x57, 0x03, 0xb9, 0xfe, 0xc8, 0x39, 0x89, 0xeb, 0xa3, 0x53, +]; + fn render(vectors: &[(&'static str, Vec)]) -> String { let mut out = String::new(); out.push_str( @@ -538,6 +569,21 @@ fn packet_type_registry_is_stable() { assert_eq!(PacketType::StreamConfig as u8, 0x18); assert_eq!(PacketType::Nack as u8, 0x19); assert_eq!(PacketType::InputEvent as u8, 0x20); + assert_eq!(PacketType::Identify as u8, 0x1A); + assert_eq!(PacketType::Identity as u8, 0x1B); +} + +/// The request is as long as the answer, so answering never sends more +/// bytes than arrived; a request cut short of its reserved bytes is refused. +#[test] +fn identify_request_is_as_long_as_its_answer() { + let vectors: BTreeMap<_, _> = canonical_vectors().into_iter().collect(); + assert_eq!(vectors["identify"].len(), vectors["identity"].len()); + assert_eq!(vectors["identify"].len(), 16 + IDENTIFY_BODY_SIZE); + match parse_control(&vectors["identity"]).unwrap().1 { + ControlMessage::Identity(identity) => assert_eq!(identity.proof, IDENTITY_PROOF), + other => panic!("{other:?}"), + } } /// A host that numbers frames contiguously says so in HELLO_ACK; one that diff --git a/scripts/e2e_moved_pc.sh b/scripts/e2e_moved_pc.sh index 368eac47..fc2ab65f 100755 --- a/scripts/e2e_moved_pc.sh +++ b/scripts/e2e_moved_pc.sh @@ -1,14 +1,17 @@ #!/usr/bin/env bash # A saved PC changes address, as after a new lease from the router or a move -# between its cable and Wi-Fi. The real app streams from the real host at -# 127.0.0.1. The host is then killed and started again, and a silent socket -# takes 127.0.0.1:PORT: the saved address stops answering, while the PC -# still answers at its network addresses and advertises them with Bonjour -# under the same name. Success = the app streams again from another address -# by itself, after its HELLO2s to the old one went unanswered. +# between its cable and Wi-Fi. The real app pairs with the real host at this +# Mac's home network address, connecting as a tap on the PC in Bonjour's list +# would. The host is then killed and started on another port, which it +# advertises with Bonjour under the same name, and a silent socket takes the +# old address. The app should check that it is the same PC and stream again +# from the new port by itself (EM_EXPECT=move, the default), or keep trying +# the saved address and never contact the new one (EM_EXPECT=stay): +# EM_PAIRING=off the PC has pairing off, so nothing proves it +# EM_MOVED_HOST_BIN= the PC comes back as a build before 0.3.5 # # On a shared Mac this is an iOS e2e run: hold the machine's e2e lock, and use -# your own simulator (EM_SIM_UDID) and a private port (EM_PORT). +# your own simulator (EM_SIM_UDID) and a private port (EM_PORT, and the next). set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" @@ -18,19 +21,36 @@ export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:-/opt/homebrew/opt/ffmpeg@7/lib/pkgcon UDID="${EM_SIM_UDID:?set EM_SIM_UDID to your own simulator}" PORT="${EM_PORT:-19893}" +NEW_PORT=$((PORT + 1)) +EXPECT="${EM_EXPECT:-move}" +PAIRING="${EM_PAIRING:-on}" TIMEOUT_SECS="${EM_TIMEOUT:-90}" +STAY_SECS="${EM_STAY_SECS:-45}" OUT="${EM_OUTPUT_DIR:-$ROOT/build/e2e/moved-pc}" HOST_BIN="${EM_HOST_BIN:-$ROOT/target/release/eternal-host}" +MOVED_HOST_BIN="${EM_MOVED_HOST_BIN:-$HOST_BIN}" # The PC's name: its Bonjour service and HELLO_ACK both carry it. PC_NAME="EMMovedPC$$" +# A test pairing token, written to the host's settings and handed to the +# app in a pairing link, as the PC's QR code would. +TOKEN="0123456789abcdef0123456789abcdef" +case "$EXPECT" in move|stay) ;; *) echo "Invalid EM_EXPECT: $EXPECT" >&2; exit 2;; esac mkdir -p "$OUT" rm -f "$OUT/result.json" "$OUT/old-address.count" APP_LOG="$OUT/app.log" HOST_LOG="$OUT/host.log" +# This Mac's address on its home network, read now: the move needs one. +IFACE=$(route -n get default 2>/dev/null | awk '/interface:/ {print $2}') +LAN=$(ipconfig getifaddr "$IFACE" 2>/dev/null || true) +python3 - "$LAN" <<'PY' || { echo "FAIL: no home network address on the default interface" >&2; exit 1; } +import ipaddress,sys +a=ipaddress.ip_address(sys.argv[1]) +sys.exit(0 if a.version==4 and a.is_private and not a.is_loopback and a not in ipaddress.ip_network('100.64.0.0/10') else 1) +PY + HOST_PID="" SILENT_PID="" -APP_PID="" cleanup() { status=$? trap - EXIT @@ -67,12 +87,14 @@ if [ "${EM_SKIP_BUILD:-0}" != 1 ]; then fi APP="$ROOT/ios/build/e2e/Build/Products/Release-iphonesimulator/EternalMonitor.app" [ -d "$APP" ] || { echo "FAIL: app bundle not found at $APP"; exit 1; } -[ -x "$HOST_BIN" ] || { echo "FAIL: host binary not found at $HOST_BIN"; exit 1; } +[ -x "$HOST_BIN" ] && [ -x "$MOVED_HOST_BIN" ] || { echo "FAIL: host binary not found"; exit 1; } echo "==> Installing app with nothing saved" xcrun simctl bootstatus "$UDID" -b >/dev/null xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true xcrun simctl uninstall "$UDID" com.eternal.monitor 2>/dev/null || true +# Pairings live in the simulator's Keychain and outlast the app. +xcrun simctl keychain "$UDID" reset STAGE="$(mktemp -d /tmp/em-moved-pc-install.XXXXXX)" ditto "$APP" "$STAGE/EternalMonitor.app" xcrun simctl install "$UDID" "$STAGE/EternalMonitor.app" @@ -85,12 +107,13 @@ touch "$SIM_LOG" ln "$SIM_LOG" "$APP_LOG" mkdir -p "$OUT/state/EternalMonitor" -echo '{"bitrate_mbps": 8, "target_fps": 30, "start_on_boot": false, "stream_audio": false, "require_pairing": false}' \ +REQUIRE=$([ "$PAIRING" = on ] && echo true || echo false) +echo "{\"bitrate_mbps\": 8, \"target_fps\": 30, \"start_on_boot\": false, \"stream_audio\": false, \"require_pairing\": $REQUIRE, \"auth_token_hex\": \"$TOKEN\"}" \ > "$OUT/state/EternalMonitor/settings.json" start_host() { APPDATA="$OUT/state" COMPUTERNAME="$PC_NAME" ETERNAL_HEADLESS=1 ETERNAL_E2E_LOG=1 ETERNAL_USB_DIRECT= \ ETERNAL_CAPTURE=synthetic ETERNAL_AUDIO=synthetic ETERNAL_SYNTH_SIZE=640x360 ETERNAL_ENCODER=libx264 ETERNAL_FPS=30 \ - "$HOST_BIN" "$PORT" >>"$HOST_LOG" 2>&1 & + "$1" "$2" >>"$HOST_LOG" 2>&1 & HOST_PID=$! } frames_at_least() { [ "$(grep -c 'E2E_FIRST_FRAME' "$APP_LOG" || true)" -ge "$1" ]; } @@ -102,66 +125,77 @@ wait_for() { elapsed=$((elapsed + 1)) if [ "$elapsed" -ge "$limit" ]; then echo "FAIL: $what after ${limit}s" - echo "----- app milestones -----"; tail -40 "$APP_LOG" + echo "----- app milestones -----"; grep -v E2E_STATS "$APP_LOG" | tail -40 exit 1 fi done } : > "$HOST_LOG" -echo "==> Starting host $PC_NAME on 0.0.0.0:$PORT" -start_host -echo "==> Launching app, streaming from 127.0.0.1:$PORT" -LAUNCH=$(SIMCTL_CHILD_EM_AUTOCONNECT="127.0.0.1:$PORT" SIMCTL_CHILD_EM_E2E_LOG=1 SIMCTL_CHILD_EM_USB_PORT=0 \ - xcrun simctl launch "$UDID" com.eternal.monitor -didSeeOnboarding YES -allowUSB NO -autoReconnect YES -playPCaudio NO) -APP_PID="${LAUNCH##*: }" +echo "==> Starting host $PC_NAME on port $PORT (pairing $PAIRING)" +start_host "$HOST_BIN" "$PORT" +echo "==> Launching app: pairs and streams from the home network address, port $PORT" +LINK="$LAN:$PORT" +[ "$PAIRING" = on ] && LINK="eternaldisplay://$LINK?t=$TOKEN" +SIMCTL_CHILD_EM_AUTOCONNECT="$LINK" SIMCTL_CHILD_EM_AUTOCONNECT_FOUND=1 SIMCTL_CHILD_EM_E2E_LOG=1 SIMCTL_CHILD_EM_USB_PORT=0 \ + xcrun simctl launch "$UDID" com.eternal.monitor -didSeeOnboarding YES -allowUSB NO -autoReconnect YES -playPCaudio NO >/dev/null wait_for "no first stream" "$TIMEOUT_SECS" frames_at_least 1 -echo "==> Moving the PC: kill, restart, and silence 127.0.0.1:$PORT" +echo "==> Moving the PC to port $NEW_PORT; the old address goes silent" MOVED_AT=$(wc -l < "$APP_LOG") kill -KILL "$HOST_PID" wait "$HOST_PID" 2>/dev/null || true -start_host -wait_for "restarted host never bound UDP $PORT" 20 sh -c "lsof -nP -a -p $HOST_PID -iUDP:$PORT >/dev/null" -# More specific than the host's wildcard socket, so it takes every datagram -# sent to 127.0.0.1:PORT, and answers none. -python3 -u - "$PORT" "$OUT/old-address.count" <<'PY' & +python3 -u - "$LAN" "$PORT" "$OUT/old-address.count" <<'PY' & import signal,socket,sys s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) -s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) -s.bind(('127.0.0.1',int(sys.argv[1]))) +s.bind((sys.argv[1],int(sys.argv[2]))) count=0 def done(*_): - open(sys.argv[2],'w').write(str(count)); sys.exit(0) + open(sys.argv[3],'w').write(str(count)); sys.exit(0) signal.signal(signal.SIGTERM,done) while True: s.recv(4096); count+=1 PY SILENT_PID=$! +start_host "$MOVED_HOST_BIN" "$NEW_PORT" -echo "==> Waiting for the app to stream again by itself (timeout ${TIMEOUT_SECS}s)" -wait_for "no stream after the move" "$TIMEOUT_SECS" frames_at_least 2 -sleep 2 +if [ "$EXPECT" = move ]; then + echo "==> Waiting for the app to stream again by itself (timeout ${TIMEOUT_SECS}s)" + wait_for "no stream after the move" "$TIMEOUT_SECS" frames_at_least 2 + sleep 2 +else + echo "==> Watching for ${STAY_SECS}s: the app must not move" + sleep "$STAY_SECS" +fi kill "$SILENT_PID"; wait "$SILENT_PID" 2>/dev/null || true; SILENT_PID="" -python3 - "$APP_LOG" "$MOVED_AT" "$PORT" "$OUT/old-address.count" "$OUT/result.json" <<'PY' +python3 - "$APP_LOG" "$MOVED_AT" "$PORT" "$NEW_PORT" "$OUT/old-address.count" "$OUT/result.json" "$EXPECT" "$HOST_LOG" <<'PY' import json,pathlib,re,sys -lines=pathlib.Path(sys.argv[1]).read_text(errors='replace').splitlines()[int(sys.argv[2]):] -port=sys.argv[3] -starts=[m.group(1) for l in lines if (m:=re.search(r'Starting connection attempt=\S+ transport=WiFi to (\S+):'+port+' ', l))] -frame=next(i for i,l in enumerate(lines) if 'E2E_FIRST_FRAME' in l) -last=[m.group(1) for l in lines[:frame] if (m:=re.search(r'Starting connection attempt=\S+ transport=WiFi to (\S+):'+port+' ', l))][-1] -old=int(pathlib.Path(sys.argv[4]).read_text()) +log,moved_at,port,new_port,count,result,expect,host_log=sys.argv[1:9] +lines=pathlib.Path(log).read_text(errors='replace').splitlines()[int(moved_at):] +starts=[m.group(1) for l in lines if (m:=re.search(r'Starting connection attempt=\S+ transport=WiFi to \S+:(\d+) ', l))] +frames=[i for i,l in enumerate(lines) if 'E2E_FIRST_FRAME' in l] +old=int(pathlib.Path(count).read_text()) +host=pathlib.Path(host_log).read_text(errors='replace') errors=[] -if '127.0.0.1' not in starts: errors.append('never retried the saved address') +if port not in starts: errors.append('never retried the saved address') if old==0: errors.append('the saved address got no HELLO2') -if last=='127.0.0.1': errors.append('streamed again from the saved address, not a new one') -if not any('Found the saved PC at ' in l for l in lines[:frame]): errors.append('no Bonjour lookup') -result=dict(scenario='moved-pc', status='FAIL' if errors else 'PASS', errors=errors, - attempts_after_move=len(starts), datagrams_at_old_address=old, streamed_from_loopback=last=='127.0.0.1') -pathlib.Path(sys.argv[5]).write_text(json.dumps(result, indent=2)+'\n') -print(json.dumps(result)) +if expect=='move': + before=[m.group(1) for l in lines[:frames[0]] if (m:=re.search(r'Starting connection attempt=\S+ transport=WiFi to \S+:(\d+) ', l))] if frames else [] + if not frames: errors.append('no stream after the move') + elif before[-1]!=new_port: errors.append('streamed again, but not from the new port') + if not any('It is the saved PC: connecting there' in l for l in lines): errors.append('no identity check before the move') +else: + if frames: errors.append('streamed again although it must not move') + if new_port in starts: errors.append('connected to the new port') +result_data=dict(scenario='moved-pc', expect=expect, status='FAIL' if errors else 'PASS', errors=errors, + attempts_after_move=len(starts), attempts_at_new_port=starts.count(new_port), + datagrams_at_old_address=old, + identity_checks=sum('Checking that the PC at' in l for l in lines), + lookups=sum('Looking for the saved PC by name' in l for l in lines)) +pathlib.Path(result).write_text(json.dumps(result_data, indent=2)+'\n') +print(json.dumps(result_data)) sys.exit(1 if errors else 0) PY -xcrun simctl io "$UDID" screenshot "$OUT/streaming-after-move.png" >/dev/null 2>&1 || true -echo "PASS: streamed again after the move. Evidence: $OUT" +xcrun simctl io "$UDID" screenshot "$OUT/after-move.png" >/dev/null 2>&1 || true +echo "PASS ($EXPECT). Evidence: $OUT" From 15d4b6f59b86875beca868c225244716a13577fc Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:52:09 -0500 Subject: [PATCH 023/226] Keep a stop's goodbye when it lands while the transport starts Startup reopened to clients and then, in a second lock, dropped any queued goodbye as stale. A Stop between the two ended the session and queued a current goodbye, which startup then dropped: the iPad timed out instead of showing the card. Session::transport_resumed now retires a stale goodbye and reopens to clients in one call, under the lock the transport reads `running` with. stop_session queues its goodbye and clears `running` under that same lock. A transport that starts after a stop and sees the stream stopping keeps the goodbye and sends it on its way out. One that starts first leaves later goodbyes alone. --- host/src/diagnostics.rs | 15 +++++--- host/src/transport/mod.rs | 14 +++++--- host/src/transport/session.rs | 66 ++++++++++++++++++++++++++++++++--- host/tests/goodbye.rs | 58 ++++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 14 deletions(-) diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index a2568625..999b6698 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -354,12 +354,17 @@ pub fn log_session_end_at_quit(shared: &SharedControl) { } /// The stream stops for good: ends the session, which the transport tells -/// why on its way out, and logs its summary. +/// why on its way out, stops the stream, and logs the summary. The goodbye +/// and `running` change under one session lock, so a transport starting +/// meanwhile never takes the goodbye for a stale one +/// (`Session::transport_resumed`). pub fn stop_session(shared: &SharedControl, why: StreamStop) { - let end = shared - .session - .lock() - .shut_down(std::time::Instant::now(), why); + let end = { + let mut session = shared.session.lock(); + let end = session.shut_down(std::time::Instant::now(), why); + shared.stop(); + end + }; if let Some(end) = end { log_session_end(&end, shared); } diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 58f3db00..5ef05ce2 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -281,10 +281,16 @@ pub async fn start_sender( let pacing_clock = PacingClock::new()?; let mut links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); - shared.session.lock().transport_resumed(now()); - // A goodbye left while no transport ran (a Stop while the port was held) - // is for a session that is over; its iPad may be in a new one by now. - if shared.session.lock().take_goodbye().is_some() { + // One lock, with `running` read under it: a stop + // (`diagnostics::stop_session`) queues its goodbye and clears `running` + // under the same lock, so this either sees that stop and keeps its + // goodbye, or comes first and the stop's goodbye is kept anyway. + let retired = { + let mut session = shared.session.lock(); + let stopping = !shared.running.load(Ordering::SeqCst); + session.transport_resumed(now(), stopping) + }; + if retired { debug!("Dropped a goodbye left while the stream was stopped"); } let beat = std::sync::Arc::clone(&shared.transport_beat); diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 5304c053..bab672de 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -980,11 +980,25 @@ impl Session { /// and reports during the restart gap went unheard, so the gap must not /// count against it: 0.3.1 expired live iPads the moment a slow restart /// finished and made them reconnect. - pub fn transport_resumed(&mut self, now: Instant) { - self.stream_stopped = false; + /// + /// `stopping`: the stream was stopped for good (`running` false, read + /// under the session lock, which [`Self::shut_down`]'s caller holds while + /// it clears it) before this transport got going. Its goodbye is + /// current, and this transport sends it on its way out; clients stay + /// turned away. Otherwise the stream runs again: clients are let in, and + /// a goodbye a stop left while no transport ran is retired, as its + /// session is long over. Both happen in this one call, so a stop that + /// comes after it keeps its goodbye. Returns whether one was retired. + pub fn transport_resumed(&mut self, now: Instant, stopping: bool) -> bool { if let Some(session) = self.active.as_mut() { session.liveness_deadline = session.liveness_deadline.max(now + LIVENESS_TIMEOUT); } + if stopping { + return false; + } + let stale = self.stream_stopped && self.goodbye.take().is_some(); + self.stream_stopped = false; + stale } /// Periodic tick: expire the session when the client has been silent past @@ -1948,7 +1962,7 @@ mod tests { session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); // The transport was down for 11 s (a capture restart with backoff). let resumed = now + Duration::from_secs(11); - session.transport_resumed(resumed); + session.transport_resumed(resumed, false); assert!(!session.tick(&TestConfig, false, resumed).client_lost); assert!(session.is_active()); // A client that stays silent after the restart still expires. @@ -2752,7 +2766,7 @@ mod tests { session.end_update(); // The stream runs again: clients are let in. - session.transport_resumed(now); + session.transport_resumed(now, false); session.handle_control(usb, 0, identified_hello(5, 77), &TestConfig, now); let quit = session .shut_down(now + Duration::from_secs(3), StreamStop::Quit) @@ -2776,12 +2790,54 @@ mod tests { assert_eq!(parse_ack(&refused.replies[0].1).status, HelloStatus::Busy); assert!(!session.is_active()); - session.transport_resumed(now); + session.transport_resumed(now, false); let admitted = session.handle_control(peer, 0, identified_hello(8, 77), &TestConfig, now); assert_eq!(admitted.new_target, Some(peer)); assert!(session.is_active()); } + /// A transport starting after a stop retires the stop's goodbye only if + /// the stream runs again; one starting while the stream is stopping keeps + /// it, and keeps clients out. A stop after the start keeps its goodbye. + #[test] + fn a_starting_transport_retires_only_a_goodbye_from_before_the_stream_ran_again() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let admit = |session: &mut Session, nonce| { + session.handle_control(peer, 0, identified_hello(nonce, 77), &TestConfig, now) + }; + + // Stopped while no transport ran; Start brought the stream back. + let mut session = Session::new(500); + admit(&mut session, 1); + session.shut_down(now, StreamStop::User).unwrap(); + assert!( + session.transport_resumed(now, false), + "a stale goodbye is retired" + ); + assert_eq!(session.take_goodbye(), None); + assert_eq!(admit(&mut session, 2).new_target, Some(peer)); + + // Stopped while this transport was starting: the goodbye is current. + let mut session = Session::new(500); + admit(&mut session, 3); + session.shut_down(now, StreamStop::User).unwrap(); + assert!(!session.transport_resumed(now, true)); + assert!( + session.take_goodbye().is_some(), + "the current goodbye is kept" + ); + let refused = admit(&mut session, 4); + assert_eq!(parse_ack(&refused.replies[0].1).status, HelloStatus::Busy); + + // Stopped once the transport ran: kept, for the transport to send. + let mut session = Session::new(500); + admit(&mut session, 5); + assert!(!session.transport_resumed(now, false), "nothing to retire"); + session.shut_down(now, StreamStop::User).unwrap(); + assert!(session.take_goodbye().is_some()); + } + /// Quitting or Stop ends the stream for good: the iPad hears why, from /// the transport on its way out, instead of waiting out a lost signal. /// A restart tells it to connect again. diff --git a/host/tests/goodbye.rs b/host/tests/goodbye.rs index 2a0d9a0d..b01c53df 100644 --- a/host/tests/goodbye.rs +++ b/host/tests/goodbye.rs @@ -220,6 +220,64 @@ fn giving_up_on_a_failing_encoder_tells_the_ipad_to_connect_again() { ); } +/// Stop lands while a pipeline generation is starting: after the supervisor +/// re-armed `running`, before the new transport reopened to clients. The +/// stopped session's goodbye is current, and the transport going up must +/// send it on its way out. Startup used to take it for one left while no +/// transport ran and drop it, and the iPad timed out instead of showing the +/// card. +#[test] +fn a_stop_during_startup_still_says_goodbye() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + eternal_host::clock::init(); + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let goodbyes = runtime.block_on(async { + let ipad = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let port = free_udp_port(); + let shared = SharedControl::new(port, 10_000_000); + shared.pairing.lock().required = false; + // The new generation's `running`, re-armed before its threads start. + shared.running.store(true, Ordering::SeqCst); + let peer = PeerId::udp(ipad.local_addr().unwrap()); + // The session carried over from the previous generation. + shared + .session + .lock() + .handle_control(peer, 0, hello(), &Config, Instant::now()); + // Stop, as the supervisor's stop_pipeline does it. + eternal_host::diagnostics::stop_session(&shared, StreamStop::User); + shared.stop(); + let (_nal_tx, nal_rx) = mpsc::channel(4); + let (_audio_tx, audio_rx) = mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + let control = shared.clone(); + tokio::time::timeout( + Duration::from_secs(30), + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx), + ) + .await + .expect("the transport stops") + .unwrap(); + assert!( + shared.session.lock().take_goodbye().is_none(), + "the goodbye was not left behind" + ); + let mut bytes = [0; 2048]; + let mut goodbyes = 0; + while let Ok(Ok((len, _))) = + tokio::time::timeout(Duration::from_millis(100), ipad.recv_from(&mut bytes)).await + { + goodbyes += usize::from(is_bye(&bytes[..len], ByeReason::UserDisconnect)); + } + goodbyes + }); + assert_eq!(goodbyes, 3); +} + /// A restart hands over to a new copy that is back in seconds: the iPad is /// told to connect again, at once. #[test] From c9a17ab7129ace1e78b03dbbff0d83b6873fa47d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:53:51 -0500 Subject: [PATCH 024/226] Say Wi-Fi is off only for an unsatisfied path, keep the Apple Devices note to its PC, and wait out a keyframe still arriving Review fixes: - A waiting connection now carries whether iPadOS called its path unsatisfied with no usable network. Only that is the new wifiOff problem ("Wi-Fi is off", failure=no_network). Every other path failure, no route to the PC or its network included, keeps the card and failure=other it had before. - The Apple Devices note is kept with the PC's target and the time it was seen, dropped when an attempt goes to another PC, and stops applying after 30 minutes. The card gives it as what happened last time and what to try if the cable doesn't connect. - The no-first-frame watchdog stays quiet while a frame has been arriving in order for over 100 ms and still is, as a large keyframe does over the cable. --- SETUP.md | 3 +- ios/EternalMonitor/App/ConnectView.swift | 2 +- .../App/ConnectionManager.swift | 23 ++++-- .../App/ConnectionProblem.swift | 26 ++++--- ios/EternalMonitor/App/SessionSummary.swift | 3 +- ios/EternalMonitor/App/UIPreview.swift | 2 +- .../Network/FrameAssembler.swift | 13 +++- ios/EternalMonitor/Network/MediaLink.swift | 4 ++ ios/EternalMonitor/Network/UDPReceiver.swift | 15 +++- .../ConnectionProblemTests.swift | 45 +++++++++--- ios/EternalMonitorTests/FeedbackTests.swift | 2 +- .../FrameAssemblerTests.swift | 38 ++++++++++ ios/EternalMonitorTests/PCNoticeTests.swift | 71 +++++++++++++++---- .../SessionSummaryTests.swift | 25 ++++++- 14 files changed, 222 insertions(+), 50 deletions(-) diff --git a/SETUP.md b/SETUP.md index 25ab6f21..45aa2300 100644 --- a/SETUP.md +++ b/SETUP.md @@ -261,7 +261,8 @@ try fewer words. If your problem isn't here, see [Known issues](https://eternalm | **That looks like the pairing code, not the PC's address** | A six-digit number was typed where the address goes. | Type the **Address** from the PC window, for example 192.168.1.20. The iPad asks for the code after it connects. | | **That looks like your router's address, not your PC's** | Nothing answered at an address ending in .1, which is usually the Wi-Fi router. | Type the **Address** shown in the PC window. | | **That address doesn't look right** | The iPad couldn't find anything by that name. | Type the **Address** exactly as the PC window shows it. | -| **Wi-Fi is off** | Wi-Fi is off on the iPad, or it hasn't joined a network. | Turn on Wi-Fi and join the same network as the PC, or use a USB cable. | +| **Wi-Fi is off** | iPadOS has no network for the connection: Wi-Fi is off on the iPad, or it hasn't joined a network. | Turn on Wi-Fi and join the same network as the PC, or use a USB cable. | +| **This iPad isn't connected to a network** | Wi-Fi is off on the iPad, or it hasn't joined a network. | Turn on Wi-Fi and join the same network as the PC, or use a USB cable. | | **Local Network is turned off for EternalMonitor** | Local Network access is off for EternalMonitor on the iPad, so it can't reach the PC over Wi-Fi. | Tap **Open Settings** and turn on **Local Network**, or go to **Settings > Privacy & Security > Local Network** and turn on **EternalMonitor**. Then tap **Try again**. A USB cable works without this setting. | | **Couldn't connect to your PC** | The connection failed for another reason. | Check that both are on the same Wi-Fi and that EternalMonitor is open on the PC, then tap **Try again**. | | **This iPad couldn't start the connection** | Something on the iPad side didn't start. | Close EternalMonitor on the iPad and open it again. | diff --git a/ios/EternalMonitor/App/ConnectView.swift b/ios/EternalMonitor/App/ConnectView.swift index c1506e73..abafb723 100644 --- a/ios/EternalMonitor/App/ConnectView.swift +++ b/ios/EternalMonitor/App/ConnectView.swift @@ -597,7 +597,7 @@ struct ConnectView: View { .font(.app(15, .medium, relativeTo: .subheadline)) .foregroundStyle(Theme.text) Text(USBStatusText.describe(connectionManager.usbStatus, - pcLacksAppleDevices: connectionManager.pcLacksAppleDevices)) + appleDevicesUnreachable: connectionManager.appleDevicesUnreachable?.applies(now: Date()) == true)) .font(.app(13.5, relativeTo: .footnote)) .foregroundStyle(Theme.textMuted) .fixedSize(horizontal: false, vertical: true) diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 755f027b..04eefd52 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -140,9 +140,18 @@ final class ConnectionManager: ObservableObject { @Published private(set) var hostStreamConfig: StreamConfig? @Published private(set) var usbStatus = "USB: waiting for a cable" - /// The PC of the last session could not reach Apple's USB service; see - /// `USBStatusText.describe`. - @Published private(set) var pcLacksAppleDevices = false + /// A PC left the USB capability out of its HELLO_ACK: when it was + /// sampled, that PC couldn't reach Apple's USB service. Kept only for + /// the PC at that target, and only for a while; see `USBStatusText`. + struct AppleDevicesSighting: Equatable { + static let lifetime: TimeInterval = 30 * 60 + /// "host:port" of the PC over Wi-Fi, or "USB". + let target: String + let at: Date + + func applies(now: Date) -> Bool { now >= at && now.timeIntervalSince(at) < Self.lifetime } + } + @Published private(set) var appleDevicesUnreachable: AppleDevicesSighting? private var mediaLink: (any MediaLink)? private var usbListener: USBListener? private var isForeground = true @@ -374,6 +383,9 @@ final class ConnectionManager: ObservableObject { } let isUSB = receiver.isUSB let pairingAddress = isUSB ? "USB" : PairingStore.address(host: normalizedHost, port: port) + let pcTarget = isUSB ? "USB" : "\(normalizedHost):\(port)" + // What one PC said about Apple Devices says nothing about another. + if appleDevicesUnreachable?.target != pcTarget { appleDevicesUnreachable = nil } let pairingToken = explicitPairingToken ?? ((try? pairingStore.token(address: pairingAddress)) ?? Data(repeating: 0, count: 16)) explicitPairingToken = nil let connectionID = UUID() @@ -608,7 +620,8 @@ final class ConnectionManager: ObservableObject { at: isUSB ? nil : "\(normalizedHost):\(port)") self.livenessTimeoutUs = UInt64(info.livenessTimeoutMs) * 1000 self.hostInfo = info - self.pcLacksAppleDevices = info.hostCaps & HelloAck.hostCapUSB == 0 + self.appleDevicesUnreachable = info.hostCaps & HelloAck.hostCapUSB == 0 + ? AppleDevicesSighting(target: pcTarget, at: Date()) : nil if self.awaitingPairing { self.awaitingPairing = false self.pairingPrompt = nil @@ -792,7 +805,7 @@ final class ConnectionManager: ObservableObject { return } guard isUSB else { - self.disconnect(failure == .noNetwork ? .noNetwork : .linkFailed) + self.disconnect(failure == .wifiOff ? .noNetwork : .linkFailed) self.problem = failure return } diff --git a/ios/EternalMonitor/App/ConnectionProblem.swift b/ios/EternalMonitor/App/ConnectionProblem.swift index 2cd8e27d..35ad5da9 100644 --- a/ios/EternalMonitor/App/ConnectionProblem.swift +++ b/ios/EternalMonitor/App/ConnectionProblem.swift @@ -19,6 +19,9 @@ enum ConnectionProblem: Equatable { /// usually the router. case routerAddress case noNetwork + /// iPadOS called the path unsatisfied for want of any usable network: + /// Wi-Fi is off, or joined to no network. + case wifiOff case localNetworkBlocked case couldNotConnect /// The cable connection failed after it had opened. @@ -66,7 +69,8 @@ enum ConnectionProblem: Equatable { case .badAddress: return "That address doesn't look right" case .pairingCodeAsAddress: return "That looks like the pairing code, not the PC's address" case .routerAddress: return "That looks like your router's address, not your PC's" - case .noNetwork: return "Wi-Fi is off" + case .noNetwork: return "This iPad isn't connected to a network" + case .wifiOff: return "Wi-Fi is off" case .localNetworkBlocked: return "Local Network is turned off for EternalMonitor" case .couldNotConnect: return "Couldn't connect to your PC" case .usbFailed: return "The connection over the cable stopped" @@ -103,7 +107,7 @@ enum ConnectionProblem: Equatable { case .routerAddress: return "Addresses ending in .1 usually belong to the Wi-Fi router. Type the Address shown in the " + "EternalMonitor window on the PC, for example 192.168.1.20." - case .noNetwork: + case .noNetwork, .wifiOff: return "Turn on Wi-Fi and join the same network as your PC, or connect with a USB cable." case .localNetworkBlocked: return "iPadOS is stopping this app from reaching your PC over Wi-Fi. " @@ -199,7 +203,7 @@ enum ConnectionProblem: Equatable { /// never answered. static func afterReconnecting(lastAttempt: ConnectionProblem?) -> ConnectionProblem { switch lastAttempt { - case .localNetworkBlocked, .noNetwork, .badAddress, .usbFailed, .noPicture, .pcBusy, .pcEndedSession, + case .localNetworkBlocked, .noNetwork, .wifiOff, .badAddress, .usbFailed, .noPicture, .pcBusy, .pcEndedSession, .pcEndedSessionOnError, .pcUpdating: return lastAttempt ?? .reconnectFailed default: @@ -247,6 +251,7 @@ enum ConnectionProblem: Equatable { // The cable is a local tunnel: Wi-Fi, DNS and Local Network don't apply. if isUSB { return .usbFailed } if error.kind == .localNetworkDenied { return .localNetworkBlocked } + if error.pathUnsatisfied { return .wifiOff } if let nw = error.underlying as? NWError { switch nw { case .dns(let code): @@ -346,16 +351,17 @@ struct NoPictureStreak: Equatable { /// What the USB cable card says for each listener status. enum USBStatusText { - /// `pcLacksAppleDevices`: the PC said in its last session that it can't - /// reach Apple's USB service, so Apple Devices isn't running there or - /// isn't installed. Only a cable session would show the PC more. - static func describe(_ status: String, pcLacksAppleDevices: Bool = false) -> String { + /// `appleDevicesUnreachable`: when this iPad last connected, the PC + /// couldn't reach Apple's USB service. That was one sample, and a host + /// may leave the capability out for other reasons, so the card gives + /// it as what happened and what to try, not as the current state. + static func describe(_ status: String, appleDevicesUnreachable: Bool = false) -> String { switch status { case "USB: off": return L("Off. Turn on to connect with a cable.") - case "USB: waiting for a cable" where pcLacksAppleDevices: + case "USB: waiting for a cable" where appleDevicesUnreachable: // A problem: stays English, like the other problem text. - return "Apple Devices isn't running on your PC, so the cable can't connect. " - + "Open it, or install it from the Microsoft Store. Wi-Fi still works." + return "Last time this iPad connected, your PC couldn't reach Apple Devices. If the cable doesn't connect, " + + "open Apple Devices on the PC, or install it from the Microsoft Store. Wi-Fi still works." case "USB: waiting for a cable": return L("Plug this iPad into your PC. Keep Apple Devices open on the PC.") case "USB: disconnected": return L("Paused. Tap Connect USB to use the cable again.") case "USB: connecting": return L("Connecting over the cable…") diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index 587ae9fd..a212e1fb 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -27,7 +27,8 @@ enum AttemptEnd: Equatable { case closedBeforePicture /// This iPad's network side failed, for example Local Network denied. case linkFailed - /// This iPad has no network: Wi-Fi is off or joined to none. + /// iPadOS found no usable network for the path: Wi-Fi is off or + /// joined to none (`ConnectionProblem.wifiOff`). case noNetwork case receiverFailed /// The PC asked for the pairing code, and a new HELLO2 sends it: the diff --git a/ios/EternalMonitor/App/UIPreview.swift b/ios/EternalMonitor/App/UIPreview.swift index e6f7242e..f122ea61 100644 --- a/ios/EternalMonitor/App/UIPreview.swift +++ b/ios/EternalMonitor/App/UIPreview.swift @@ -61,7 +61,7 @@ enum UIPreview { case "error-local-network": manager.problem = .localNetworkBlocked case "error-wifi-off": - manager.problem = .noNetwork + manager.problem = .wifiOff case "mismatch": manager.versionMismatch = .iPadOlder manager.connectionError = VersionMismatch.iPadOlder.message diff --git a/ios/EternalMonitor/Network/FrameAssembler.swift b/ios/EternalMonitor/Network/FrameAssembler.swift index 83644991..3c8f3301 100644 --- a/ios/EternalMonitor/Network/FrameAssembler.swift +++ b/ios/EternalMonitor/Network/FrameAssembler.swift @@ -18,8 +18,8 @@ final class FrameAssembler { var onNack: ((Nack) -> Void)? var onNeedsKeyframe: ((_ epoch: UInt32, _ lastCompleteSeq: UInt32) -> Void)? /// Video has arrived for `noFrameYetUs` and no frame has completed, and - /// again after each further `noFrameYetUs` while video still arrives. - /// `missing` names the fragments the latest incomplete frames lack. + /// again after each further `noFrameYetUs` while video still arrives, + /// but never while a frame is still arriving in order. `missing` names the fragments the latest incomplete frames lack. var onNoFrameYet: ((_ epoch: UInt32, _ missing: String) -> Void)? var repairEnabled = false var framePeriodUs: UInt64 = 16_667 @@ -421,6 +421,15 @@ final class FrameAssembler { private func reportNoFrameYet(at now: UInt64) { guard !deliveredAny, let since = noFrameSince, now >= since + Self.noFrameYetUs, let arrived = lastArrivalUs, arrived >= since else { return } + // A frame that has been arriving in order for a while and still is, + // as a large keyframe does over a slow cable, isn't lost: another + // keyframe would only queue behind it. A frame with a hole, or one + // whose fragments all came at once, doesn't count. + let stillArriving = pending.values.contains { frame in + !frame.isComplete && Int(frame.contiguous) == frame.fragments.count + && now < frame.lastFragmentAt + Self.staleFrameUs && now >= frame.createdAt + Self.staleFrameUs + } + guard !stillArriving else { return } noFrameSince = now let waiting = pending.keys.sorted().compactMap { seq in pending[seq].flatMap { $0.isComplete ? nil : Self.describe(seq, $0, dropped: false) } diff --git a/ios/EternalMonitor/Network/MediaLink.swift b/ios/EternalMonitor/Network/MediaLink.swift index 75c7a626..1dc99caf 100644 --- a/ios/EternalMonitor/Network/MediaLink.swift +++ b/ios/EternalMonitor/Network/MediaLink.swift @@ -21,6 +21,10 @@ struct LinkError: Error, CustomStringConvertible { let kind: Kind let message: String var underlying: Error? + /// The path was unsatisfied for want of any usable network when this + /// was reported. Only that is "Wi-Fi is off"; a path that is up but + /// can't reach the PC (no route to it) is not. + var pathUnsatisfied = false init(_ kind: Kind, _ message: String, underlying: Error? = nil) { self.kind = kind diff --git a/ios/EternalMonitor/Network/UDPReceiver.swift b/ios/EternalMonitor/Network/UDPReceiver.swift index 8b8dac30..462de2a1 100644 --- a/ios/EternalMonitor/Network/UDPReceiver.swift +++ b/ios/EternalMonitor/Network/UDPReceiver.swift @@ -112,9 +112,8 @@ final class UDPReceiver: MediaLink { case .waiting(let error): guard !self.reportedWaiting else { return } self.reportedWaiting = true - let denied = connection.currentPath?.unsatisfiedReason == .localNetworkDenied - self.onError?(LinkError(denied ? .localNetworkDenied : .pathUnavailable, - "UDP path not ready: \(error.localizedDescription)", underlying: error)) + let path = connection.currentPath + self.onError?(Self.waitingError(error, status: path?.status, reason: path?.unsatisfiedReason)) case .failed(let error): SupportDiagnostics.shared.record("error", "udp", "Network.framework failed: \(error)") self.onError?(LinkError(.connectionFailed, "UDP connection failed: \(error.localizedDescription)", underlying: error)) @@ -130,6 +129,16 @@ final class UDPReceiver: MediaLink { connection.start(queue: queue) } + /// What a connection waiting for its path reports: Local Network + /// denied, or a path that isn't ready. Only a path iPadOS calls + /// unsatisfied with no usable network is `pathUnsatisfied`. + static func waitingError(_ error: NWError, status: NWPath.Status?, reason: NWPath.UnsatisfiedReason?) -> LinkError { + var link = LinkError(reason == .localNetworkDenied ? .localNetworkDenied : .pathUnavailable, + "UDP path not ready: \(error.localizedDescription)", underlying: error) + link.pathUnsatisfied = status == .unsatisfied && reason == .notAvailable + return link + } + private func startBSD(host: String) { var hints = addrinfo() hints.ai_family = AF_UNSPEC diff --git a/ios/EternalMonitorTests/ConnectionProblemTests.swift b/ios/EternalMonitorTests/ConnectionProblemTests.swift index 48bc68f6..5d5cf9f5 100644 --- a/ios/EternalMonitorTests/ConnectionProblemTests.swift +++ b/ios/EternalMonitorTests/ConnectionProblemTests.swift @@ -162,8 +162,9 @@ final class ConnectionProblemTests: XCTestCase { XCTAssertEqual(wifi(LinkError(.connectionFailed, "UDP connection failed: PolicyDenied", underlying: NWError.dns(-65570))), .localNetworkBlocked) // Offline. XCTAssertEqual(wifi(LinkError(.pathUnavailable, "UDP path not ready: Network is down", underlying: NWError.posix(.ENETDOWN))), .noNetwork) - XCTAssertEqual(ConnectionProblem.noNetwork.title, "Wi-Fi is off") - XCTAssertTrue(ConnectionProblem.noNetwork.detail.contains("Turn on Wi-Fi"), ConnectionProblem.noNetwork.detail) + XCTAssertEqual(ConnectionProblem.noNetwork.title, "This iPad isn't connected to a network") + XCTAssertEqual(ConnectionProblem.wifiOff.title, "Wi-Fi is off") + XCTAssertTrue(ConnectionProblem.wifiOff.detail.contains("Turn on Wi-Fi"), ConnectionProblem.wifiOff.detail) // Unknown failures get general advice, never the raw error. let unknown = LinkError(.connectionFailed, "UDP socket failed: Address already in use", underlying: POSIXError(.EADDRINUSE)) XCTAssertEqual(wifi(unknown), .couldNotConnect) @@ -193,10 +194,38 @@ final class ConnectionProblemTests: XCTestCase { } } + /// A connection waiting for its path. Only a path iPadOS calls + /// unsatisfied with no usable network is "Wi-Fi is off". No route to + /// the PC or its network, on a path that is up or not yet known, keeps + /// the advice it had before. + func testOnlyAnUnsatisfiedPathIsWiFiOff() { + let wifi = { (error: LinkError) in ConnectionProblem.linkFailure(error, isUSB: false, host: "studio-pc") } + let off = UDPReceiver.waitingError(.posix(.ENETDOWN), status: .unsatisfied, reason: .notAvailable) + XCTAssertEqual(off.kind, .pathUnavailable) + XCTAssertTrue(off.pathUnsatisfied) + XCTAssertEqual(wifi(off), .wifiOff) + for code: POSIXErrorCode in [.EHOSTUNREACH, .ENETUNREACH] { + for status: NWPath.Status? in [.satisfied, .requiresConnection, nil] { + let waiting = UDPReceiver.waitingError(.posix(code), status: status, reason: .notAvailable) + XCTAssertFalse(waiting.pathUnsatisfied, "\(code) \(String(describing: status))") + XCTAssertNotEqual(wifi(waiting), .wifiOff, "\(code) \(String(describing: status))") + XCTAssertEqual(wifi(waiting), .noNetwork, "unchanged: \(code) \(String(describing: status))") + } + } + // Unsatisfied for a reason other than no network: not Wi-Fi off. + XCTAssertFalse(UDPReceiver.waitingError(.posix(.ENETDOWN), status: .unsatisfied, reason: .wifiDenied).pathUnsatisfied) + let denied = UDPReceiver.waitingError(.posix(.EPERM), status: .unsatisfied, reason: .localNetworkDenied) + XCTAssertEqual(denied.kind, .localNetworkDenied) + XCTAssertFalse(denied.pathUnsatisfied) + XCTAssertEqual(wifi(denied), .localNetworkBlocked) + // Over the cable it is always the cable. + XCTAssertEqual(ConnectionProblem.linkFailure(off, isUSB: true, host: "USB"), .usbFailed) + } + func testGivingUpOnReconnectingKeepsASpecificLastFailure() { // A classified failure in the last attempt is more useful than // "Couldn't reconnect"; a PC that just never answered is not. - for specific: ConnectionProblem in [.localNetworkBlocked, .noNetwork, .badAddress("pc"), .usbFailed, .noPicture, + for specific: ConnectionProblem in [.localNetworkBlocked, .noNetwork, .wifiOff, .badAddress("pc"), .usbFailed, .noPicture, .pcBusy, .pcEndedSession, .pcEndedSessionOnError] { XCTAssertEqual(ConnectionProblem.afterReconnecting(lastAttempt: specific), specific) } @@ -208,7 +237,7 @@ final class ConnectionProblemTests: XCTestCase { func testEveryProblemSaysWhatToDoWithoutTransportJargon() { let all: [ConnectionProblem] = [ .noReply(usb: false), .noReply(usb: true), .noPicture, .receiverNotReady, .badAddress("pc"), - .noNetwork, .localNetworkBlocked, .couldNotConnect, .usbFailed, .lost(.pcStoppedResponding), .lost(.cableUnplugged), + .noNetwork, .wifiOff, .localNetworkBlocked, .couldNotConnect, .usbFailed, .lost(.pcStoppedResponding), .lost(.cableUnplugged), .lost(.usbTurnedOff), .reconnecting(attempt: 2, of: 6), .reconnectFailed, .pcEndedSession, .pcEndedSessionOnError, .pcBusy, .pairingCodeAsAddress, .routerAddress, .pcUpdating(.reconnecting), .pcUpdating(.manual), .pcUpdating(.notBack), @@ -307,12 +336,12 @@ final class ConnectionProblemTests: XCTestCase { /// waiting card says why the cable won't connect. Other states don't /// change, and a cable session (which reaches it) clears the flag. func testUSBCardSaysWhenThePCLacksAppleDevices() { - let waiting = USBStatusText.describe("USB: waiting for a cable", pcLacksAppleDevices: true) - XCTAssertEqual(waiting, "Apple Devices isn't running on your PC, so the cable can't connect. " - + "Open it, or install it from the Microsoft Store. Wi-Fi still works.") + let waiting = USBStatusText.describe("USB: waiting for a cable", appleDevicesUnreachable: true) + XCTAssertEqual(waiting, "Last time this iPad connected, your PC couldn't reach Apple Devices. If the cable doesn't connect, " + + "open Apple Devices on the PC, or install it from the Microsoft Store. Wi-Fi still works.") XCTAssertNotEqual(waiting, USBStatusText.describe("USB: waiting for a cable")) for status in ["USB: off", "USB: disconnected", "USB: connecting", "USB: unavailable", "USB: connected to PC"] { - XCTAssertEqual(USBStatusText.describe(status, pcLacksAppleDevices: true), USBStatusText.describe(status), status) + XCTAssertEqual(USBStatusText.describe(status, appleDevicesUnreachable: true), USBStatusText.describe(status), status) } } diff --git a/ios/EternalMonitorTests/FeedbackTests.swift b/ios/EternalMonitorTests/FeedbackTests.swift index 2cb8d9ff..501eb69f 100644 --- a/ios/EternalMonitorTests/FeedbackTests.swift +++ b/ios/EternalMonitorTests/FeedbackTests.swift @@ -68,7 +68,7 @@ final class FeedbackTests: XCTestCase { } let failures: [ConnectionProblem] = [ .noReply(usb: false), .noReply(usb: true), .noPicture, .receiverNotReady, .badAddress("pc"), - .routerAddress, .noNetwork, .localNetworkBlocked, .couldNotConnect, .usbFailed, + .routerAddress, .noNetwork, .wifiOff, .localNetworkBlocked, .couldNotConnect, .usbFailed, .lost(.pcStoppedResponding), .reconnectFailed, .pcEndedSession, .pcEndedSessionOnError, .pcUpdating(.notBack), ] diff --git a/ios/EternalMonitorTests/FrameAssemblerTests.swift b/ios/EternalMonitorTests/FrameAssemblerTests.swift index 9780e58a..6c390acb 100644 --- a/ios/EternalMonitorTests/FrameAssemblerTests.swift +++ b/ios/EternalMonitorTests/FrameAssemblerTests.swift @@ -716,6 +716,44 @@ final class FrameAssemblerTests: XCTestCase { XCTAssertTrue(completed.isEmpty) } + /// Over the cable a large first keyframe can take longer than 2 s to + /// arrive, in order and without loss. That isn't a frame that never + /// completes: no report, so no request for another keyframe that would + /// only queue behind it. + func testAKeyframeStillArrivingInOrderIsNotReported() { + now = 0 + assembler = FrameAssembler(nowUs: { [weak self] in self?.now ?? 0 }) + assembler.onFrameAssembled = { [weak self] data, _, _, _ in self?.completed.append(data) } + var reports: [String] = [] + assembler.onNoFrameYet = { _, missing in reports.append(missing) } + for index in UInt16(0)..<350 { + now = UInt64(index) * 10_000 + add(seq: 1, index: index, count: 350, byte: 1, keyframe: true) + now += 5_000 + assembler.tick() + } + XCTAssertEqual(reports, []) + XCTAssertEqual(completed.count, 1, "the keyframe arrived whole after 3.5 s") + } + + /// The same keyframe stalls with its tail missing for good: it is + /// reported as soon as it has stopped arriving and is dropped. + func testAKeyframeThatStopsArrivingIsReported() { + now = 0 + assembler = FrameAssembler(nowUs: { [weak self] in self?.now ?? 0 }) + var reports: [String] = [] + assembler.onNoFrameYet = { _, missing in reports.append(missing) } + for index in UInt16(0)..<250 { + now = UInt64(index) * 10_000 + add(seq: 1, index: index, count: 350, byte: 1, keyframe: true) + assembler.tick() + } + XCTAssertEqual(reports, [], "still arriving at 2.49 s") + now = 2_600_000 + assembler.tick() + XCTAssertEqual(reports, ["seq=1 keyframe dropped with 250/350 fragments, missing 250-349; largest fragment 1 bytes"]) + } + func testMissingFragmentsAreListedAsRanges() { XCTAssertEqual(FrameAssembler.ranges([0, 1, 2, 5, 7, 8]), "0-2,5,7-8") XCTAssertEqual(FrameAssembler.ranges(stride(from: 0, to: 40, by: 4).map { UInt16($0) }), diff --git a/ios/EternalMonitorTests/PCNoticeTests.swift b/ios/EternalMonitorTests/PCNoticeTests.swift index 3c2fecda..bd1c6616 100644 --- a/ios/EternalMonitorTests/PCNoticeTests.swift +++ b/ios/EternalMonitorTests/PCNoticeTests.swift @@ -10,21 +10,64 @@ final class PCNoticeTests: XCTestCase { XCTAssertTrue(condition(), description) } - /// A PC that can't reach Apple's USB service (Apple Devices not running - /// or not installed) leaves the bit out of its HELLO_ACK. The USB card - /// then says so; a PC that reaches it clears that. - func testThePCsAppleDevicesStateReachesTheUSBCard() async throws { + /// One session with `host`, whose HELLO_ACK carries `caps`. + private func session(_ manager: ConnectionManager, _ host: FakeUDPHost, caps: UInt16) async throws { + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession(hostCaps: caps) }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + try await until("accepted") { manager.hostInfo?.hostCaps == caps } + manager.cancel() + } + + /// A PC that couldn't reach Apple's USB service (Apple Devices not + /// running or not installed) leaves the bit out of its HELLO_ACK. It is + /// kept for that PC after the session, and cleared by the same PC's + /// next session once the service is back. + func testTheSightingClearsWhenThePCsServiceRecovers() async throws { let manager = ConnectionManager() addTeardownBlock { await MainActor.run { manager.cancel() } } - XCTAssertFalse(manager.pcLacksAppleDevices, "nothing known before a session") - for (caps, lacks) in [(UInt16(0), true), (HelloAck.hostCapUSB, false)] { - let host = FakeUDPHost() - manager.connect(host: "127.0.0.1", port: host.port) - let accepted = await Task.detached { host.acceptSession(hostCaps: caps) }.value - XCTAssertTrue(accepted, "the iPad never sent HELLO2") - try await until("accepted") { manager.hostInfo?.hostCaps == caps && manager.pcLacksAppleDevices == lacks } - manager.cancel() - XCTAssertEqual(manager.pcLacksAppleDevices, lacks, "kept after the session for the Connect screen") - } + XCTAssertNil(manager.appleDevicesUnreachable, "nothing known before a session") + let host = FakeUDPHost() + let before = Date() + try await session(manager, host, caps: 0) + let sighting = try XCTUnwrap(manager.appleDevicesUnreachable) + XCTAssertEqual(sighting.target, "127.0.0.1:\(host.port)") + XCTAssertTrue(sighting.applies(now: Date())) + XCTAssertGreaterThanOrEqual(sighting.at, before) + + try await session(manager, host, caps: HelloAck.hostCapUSB) + XCTAssertNil(manager.appleDevicesUnreachable) + } + + /// Connecting to another PC drops what the last one said, at once and + /// whether or not the new one answers. + func testSwitchingPCsDropsTheOtherPCsSighting() async throws { + let manager = ConnectionManager() + addTeardownBlock { await MainActor.run { manager.cancel() } } + let first = FakeUDPHost() + try await session(manager, first, caps: 0) + XCTAssertNotNil(manager.appleDevicesUnreachable) + + let second = FakeUDPHost() + manager.connect(host: "127.0.0.1", port: second.port) + XCTAssertNil(manager.appleDevicesUnreachable, "another PC's sighting") + manager.cancel() + XCTAssertNil(manager.appleDevicesUnreachable) + + // The same PC again keeps it until that PC answers. + try await session(manager, first, caps: 0) + manager.connect(host: "127.0.0.1", port: first.port) + XCTAssertNotNil(manager.appleDevicesUnreachable) + manager.cancel() + } + + /// A sighting is one sample: it stops applying after a while. + func testASightingGoesStale() { + let at = Date(timeIntervalSince1970: 1_000_000) + let sighting = ConnectionManager.AppleDevicesSighting(target: "192.168.1.20:9876", at: at) + XCTAssertTrue(sighting.applies(now: at)) + XCTAssertTrue(sighting.applies(now: at + ConnectionManager.AppleDevicesSighting.lifetime - 1)) + XCTAssertFalse(sighting.applies(now: at + ConnectionManager.AppleDevicesSighting.lifetime)) + XCTAssertFalse(sighting.applies(now: at - 60), "a clock set back") } } diff --git a/ios/EternalMonitorTests/SessionSummaryTests.swift b/ios/EternalMonitorTests/SessionSummaryTests.swift index 0a4a5ef8..e173f11d 100644 --- a/ios/EternalMonitorTests/SessionSummaryTests.swift +++ b/ios/EternalMonitorTests/SessionSummaryTests.swift @@ -174,10 +174,9 @@ final class SessionSummaryTests: XCTestCase { let link = DeniedLink() let manager = ConnectionManager(udpReceiver: { _ in link }) manager.connect(host: "192.168.1.20", port: 9876) - link.onError?(LinkError(.pathUnavailable, "UDP path not ready: The operation couldn’t be completed. (Network.NWError error 50 - Network is down)", - underlying: NWError.posix(.ENETDOWN))) + link.onError?(UDPReceiver.waitingError(.posix(.ENETDOWN), status: .unsatisfied, reason: .notAvailable)) try await until("ended") { manager.state == .disconnected } - XCTAssertEqual(manager.problem, .noNetwork) + XCTAssertEqual(manager.problem, .wifiOff) XCTAssertEqual(manager.problem?.title, "Wi-Fi is off") XCTAssertEqual(logged(manager, "connection").filter { $0.hasPrefix("Disconnect ") }, ["Disconnect state=connecting reason=noNetwork"]) @@ -186,6 +185,26 @@ final class SessionSummaryTests: XCTestCase { Self.assertContract(line) } + /// No route to the PC or its network while the path is up: the PC is + /// asleep or elsewhere, not this iPad's Wi-Fi off. It keeps the card + /// and the `other` failure it had before Wi-Fi off got its own. + func testUnreachableWhileWaitingIsNotWiFiOff() async throws { + for code: POSIXErrorCode in [.EHOSTUNREACH, .ENETUNREACH] { + let link = DeniedLink() + let manager = ConnectionManager(udpReceiver: { _ in link }) + manager.connect(host: "192.168.1.20", port: 9876) + link.onError?(UDPReceiver.waitingError(.posix(code), status: .satisfied, reason: nil)) + try await until("ended") { manager.state == .disconnected } + XCTAssertNotEqual(manager.problem, .wifiOff, "\(code)") + XCTAssertNotEqual(manager.problem?.title, "Wi-Fi is off", "\(code)") + XCTAssertEqual(logged(manager, "connection").filter { $0.hasPrefix("Disconnect ") }, + ["Disconnect state=connecting reason=linkFailed"], "\(code)") + let line = logged(manager, "summary").first ?? "" + XCTAssertTrue(line.contains(" failure=other "), line) + Self.assertContract(line) + } + } + /// Try again opens a new connection, which asks iPadOS again: once /// Local Network is on, the old denial isn't shown or remembered. func testTryAgainAfterLocalNetworkDeniedAsksIPadOSAgain() async throws { From dcc207398c6d617feca31e556f016172505551de Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 13:04:20 -0500 Subject: [PATCH 025/226] Give a keyframe's requests one schedule apart from its hold A held keyframe asked for its missing fragments at each hold expiry as well as on its retry timer, and the hold neither moved the timer nor waited for traffic. At 1 ms RTT the same fragment went out 1 ms after a retry, inside the PC's 5 ms resend limit; at 200 ms RTT the holds asked at 26 and 51 ms, before any repair could return. A keyframe's hold now only moves its deadline. Every repeated request goes through one retry: due RTT + 5 ms after the last one, and after the first only once something arrived since the last request, on a later tick. Shared traces cover a hold and retry on the same tick, silence, and 1 ms and 200 ms RTT. --- docs/decisions.md | 20 ++--- .../Network/FrameAssembler.swift | 43 +++++++---- proto/src/reassembly.rs | 52 +++++++++---- proto/testdata/repair_vectors.txt | 76 ++++++++++++++++--- 4 files changed, 144 insertions(+), 47 deletions(-) diff --git a/docs/decisions.md b/docs/decisions.md index 09700fe7..8f425469 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -237,15 +237,17 @@ frames need the keyframe, so the decoder held the picture and asked for another, which failed the same way. A keyframe now keeps its place while the frames after it complete. At its -deadline it is held and asked for again, like a frame with nothing behind it, -until it completes, a newer keyframe completes, a second has passed, or the -32 frames the assembler holds are full. It -also asks again every RTT + 5 ms, but only once something has arrived since -its last request. The repair simulation (`proto/tests/wifi_repair_sim.rs`, -10% loss each way, 150-fragment keyframes at 60 fps) completed 148 of 390 -keyframes before, with 35 freezes over a second and the longest 3.4 s. After, -it completes 397 of 399, the longest freeze is 0.61 s, and no freeze lasts a -second. The 1% loss rows also lost fewer frames. +deadline it is held, like a frame with nothing behind it, until it completes, +a newer keyframe completes, a second has passed, or the 32 frames the +assembler holds are full. The hold only moves its deadline. The keyframe's +requests follow one schedule: after its first retry it asks again every RTT + +5 ms, sooner than which no repair can return, and only once something has +arrived since its last request. The repair simulation +(`proto/tests/wifi_repair_sim.rs`, 10% loss each way, 150-fragment keyframes +at 60 fps) completed 148 of 390 keyframes before, with 35 freezes over a +second and the longest 3.4 s. After, it completes 394 of 399, the longest +freeze is 0.61 s, and no freeze lasts a second. The 1% loss rows also lost +fewer frames. Delta frames keep the 25 ms rule and their single retry. Holding them too, for at most 100 ms, with every frame asking again, removed nearly every diff --git a/ios/EternalMonitor/Network/FrameAssembler.swift b/ios/EternalMonitor/Network/FrameAssembler.swift index 824e16b4..d690fe5f 100644 --- a/ios/EternalMonitor/Network/FrameAssembler.swift +++ b/ios/EternalMonitor/Network/FrameAssembler.swift @@ -133,14 +133,14 @@ final class FrameAssembler { var deadline: UInt64? var retryAt: UInt64? var requestedAt: UInt64? - /// The latest request for this frame; see `asksKeyframeAgain(_:at:)`. + /// The latest request for this frame; see `keyframeRetryDue(_:at:)`. var askedAt: UInt64? var completedAt: UInt64? var stallAllowanceUsed: UInt64 = 0 var stallCountedUntil: UInt64 = 0 /// Indices already requested; each is requested when first seen /// missing and at most once more by the frame's single retry. A - /// keyframe retries until it completes; see `asksKeyframeAgain(_:at:)`. + /// keyframe retries until it completes; see `keyframeRetryDue(_:at:)`. var requested = Set() var retried = false var keyframeRequested = false @@ -390,11 +390,12 @@ final class FrameAssembler { } } else if frame.gapAt == nil, now >= frame.createdAt + framePeriodUs { beginGap(seq, through: frame.fragmentCount - 1, now: now) - } else if let retry = frame.retryAt, now >= retry, !frame.retried || asksKeyframeAgain(frame, at: now) { - pending[seq]?.retried = true - pending[seq]?.retryAt = now + min(rttUs(), 1_000_000) + 5_000 - request(seq, through: frame.fragmentCount - 1, again: true, now: now) + } else if !frame.isKeyframe, let retryAt = frame.retryAt, now >= retryAt, !frame.retried { + retry(seq, now: now) } + // A keyframe's requests follow their own schedule, apart from + // its hold: one request per retry, never two in one interval. + if keyframeRetryDue(seq, at: now) { retry(seq, now: now) } } else if now >= frame.lastFragmentAt + Self.staleFrameUs { // Timed from the latest fragment: over a slow USB tunnel a // big keyframe is still arriving long after its first one. @@ -449,15 +450,29 @@ final class FrameAssembler { /// A keyframe has many fragments, and at 10% loss one round of requests /// and repairs leaves some missing, so a keyframe asks again every RTT - /// plus 5 ms, not once. It asks only once something has arrived since it - /// last asked: while WiFi holds traffic, its request and the repair are - /// still queued. A fragment arriving now may be the missing one, so a - /// keyframe asks again only on a later tick. - private func asksKeyframeAgain(_ frame: PendingFrame, at now: UInt64) -> Bool { - guard frame.isKeyframe, let asked = frame.askedAt, let arrival = lastArrivalUs else { return false } + /// plus 5 ms, not once: sooner, its repair can't have come back, and the + /// PC resends a fragment at most every 5 ms. This one schedule covers all + /// its repeated requests; its hold only moves its deadline. After its + /// first retry it asks only once something has arrived since it last + /// asked: while WiFi holds traffic, its request and the repair are still + /// queued, and with nothing arriving it does not ask again. A fragment + /// arriving now may be the missing one, so it asks only on a later tick. + private func keyframeRetryDue(_ seq: UInt32, at now: UInt64) -> Bool { + guard let frame = pending[seq], frame.isKeyframe, !frame.isComplete, let retryAt = frame.retryAt, + now >= retryAt else { return false } + guard frame.retried else { return true } + guard let asked = frame.askedAt, let arrival = lastArrivalUs else { return false } return arrival > asked && arrival < now } + /// Asks again for every missing fragment and schedules the next retry. + private func retry(_ seq: UInt32, now: UInt64) { + guard let frame = pending[seq] else { return } + pending[seq]?.retried = true + pending[seq]?.retryAt = now + min(rttUs(), 1_000_000) + 5_000 + request(seq, through: frame.fragmentCount - 1, again: true, now: now) + } + private func repairWindow() -> UInt64 { max(8_000, min(25_000, 2 * min(rttUs(), 1_000_000) + framePeriodUs * 3 / 2)) } @@ -471,7 +486,8 @@ final class FrameAssembler { /// several hundred milliseconds with two frames incomplete and then send /// both repairs together. Ask again, less often each time, until a newer /// frame completes or the frame has waited `maxHoldUs`. A keyframe waits - /// for a newer keyframe instead of a newer frame. + /// for a newer keyframe instead of a newer frame, and its hold only moves + /// its deadline: its requests follow `keyframeRetryDue(_:at:)`. private func hold(_ seq: UInt32, now: UInt64) { guard let frame = pending[seq] else { return } let wait = frame.holdRetryUs == 0 @@ -480,6 +496,7 @@ final class FrameAssembler { pending[seq]?.deadline = now + wait if pending.count > 1 && !frame.isKeyframe { pending[seq]?.dropBy = min(frame.dropBy ?? now, now) } pending[seq]?.stallAllowanceUsed = Self.maxStallAllowanceUs + guard !frame.isKeyframe else { return } pending[seq]?.retried = true request(seq, through: frame.fragmentCount - 1, again: true, now: now) } diff --git a/proto/src/reassembly.rs b/proto/src/reassembly.rs index 35af19d3..56ef19df 100644 --- a/proto/src/reassembly.rs +++ b/proto/src/reassembly.rs @@ -480,16 +480,16 @@ impl Reassembler { { let through = frame.header.frag_count - 1; self.begin_gap(seq, through, now); - } else if (!frame.retried || self.asks_keyframe_again(frame, now)) + } else if !frame.header.is_keyframe + && !frame.retried && frame.retry_at.is_some_and(|retry| now >= retry) { - let through = frame.header.frag_count - 1; - let next = now + self.rtt + Duration::from_millis(5); - if let Some(frame) = self.pending.get_mut(&seq) { - frame.retried = true; - frame.retry_at = Some(next); - } - self.request(seq, through, true, now); + self.retry(seq, now); + } + // A keyframe's requests follow their own schedule, apart + // from its hold (see `FrameAssembler.tick(at:)`). + if self.keyframe_retry_due(seq, now) { + self.retry(seq, now); } } else if now.saturating_duration_since(frame.last_fragment_at) >= STALE_FRAME_TIMEOUT { // Timed from the latest fragment: over a slow USB tunnel a @@ -521,13 +521,32 @@ impl Reassembler { self.request(seq, through, false, now); } - /// A keyframe asks again every RTT + 5 ms while anything has arrived - /// since it last asked (see `FrameAssembler.asksKeyframeAgain(_:at:)`). - fn asks_keyframe_again(&self, frame: &PendingFrame, now: Instant) -> bool { + /// A keyframe asks again every RTT + 5 ms; after its first retry, only + /// once something arrived since it last asked (see + /// `FrameAssembler.keyframeRetryDue(_:at:)`). + fn keyframe_retry_due(&self, seq: u32, now: Instant) -> bool { + let Some(frame) = self.pending.get(&seq) else { + return false; + }; frame.header.is_keyframe - && self.last_arrival.is_some_and(|arrival| { - arrival < now && frame.asked_at.is_some_and(|asked| arrival > asked) - }) + && !frame.complete() + && frame.retry_at.is_some_and(|retry| now >= retry) + && (!frame.retried + || self.last_arrival.is_some_and(|arrival| { + arrival < now && frame.asked_at.is_some_and(|asked| arrival > asked) + })) + } + + /// Asks again for every missing fragment and schedules the next retry. + fn retry(&mut self, seq: u32, now: Instant) { + let next = now + self.rtt + Duration::from_millis(5); + let Some(frame) = self.pending.get_mut(&seq) else { + return; + }; + frame.retried = true; + frame.retry_at = Some(next); + let through = frame.header.frag_count - 1; + self.request(seq, through, true, now); } fn repair_window(&self) -> Duration { @@ -538,7 +557,7 @@ impl Reassembler { /// No complete frame waits behind this frame, so dropping it gains /// nothing (see `FrameAssembler.hold(_:now:)`): ask again, less often - /// each time. + /// each time. A keyframe's hold only moves its deadline. fn hold(&mut self, seq: u32, now: Instant) { let window = self.repair_window(); let newer_waiting = self.pending.len() > 1; @@ -556,6 +575,9 @@ impl Reassembler { frame.drop_by = Some(frame.drop_by.map_or(now, |by| by.min(now))); } frame.stall_used = MAX_STALL_ALLOWANCE; + if frame.header.is_keyframe { + return; + } frame.retried = true; let through = frame.header.frag_count - 1; self.request(seq, through, true, now); diff --git a/proto/testdata/repair_vectors.txt b/proto/testdata/repair_vectors.txt index 7b0d1014..02d9c027 100644 --- a/proto/testdata/repair_vectors.txt +++ b/proto/testdata/repair_vectors.txt @@ -432,9 +432,11 @@ expect 2 0 2 0 0 0 0 1,3 0 - # Every frame up to the next keyframe refers to the keyframe, so a complete # frame behind an incomplete keyframe is no reason to drop it: at its -# deadline the keyframe is held and asked for again, and its repair then -# delivers both. On lossy WiFi, dropping it froze the picture for seconds, -# because each replacement keyframe arrived incomplete too. +# deadline the keyframe is held, and its repair then delivers both. On lossy +# WiFi, dropping it froze the picture for seconds, because each replacement +# keyframe arrived incomplete too. The hold only moves the deadline; the +# request at 40 ms is the keyframe's retry, due since 31.667 ms and sent +# because a resend arrived after its last request. reset 16667 10000 add 0 1 1 0 3 0 1 1 add 1000 1 1 2 3 0 3 1 @@ -446,8 +448,9 @@ expect 0 0 3 0 0 3 1 - 0 1:1|1:1|1:1 add 45000 1 1 1 3 1 2 1 expect 2 0 3 0 1 3 0 1,2 0 1:1|1:1|1:1 -# A held keyframe still goes a second after its gap, with the frames that -# need it, under one keyframe request. +# With nothing arriving, a held keyframe asks no more: its request and the +# repair would wait in the same queue. It still goes a second after its gap, +# with the frames that need it, under one keyframe request. reset 16667 10000 add 0 1 1 0 3 0 1 1 add 1000 1 1 2 3 0 3 1 @@ -461,14 +464,14 @@ tick 415000 tick 615000 tick 815000 tick 1014999 -expect 0 0 3 0 0 9 1 - 0 1:1|1:1|1:1|1:1|1:1|1:1|1:1|1:1|1:1 +expect 0 0 3 0 0 3 1 - 0 1:1|1:1|1:1 tick 1015000 -expect 0 2 3 1 0 9 0 - 1 1:1|1:1|1:1|1:1|1:1|1:1|1:1|1:1|1:1 +expect 0 2 3 1 0 3 0 - 1 1:1|1:1|1:1 # A keyframe has many fragments, and one round of requests and repairs can -# leave some missing, so it asks again every RTT + 5 ms instead of once. It -# asks only after something arrived since it last asked: while WiFi holds -# traffic, the request and the repair are still queued. +# leave some missing, so it retries every RTT + 5 ms instead of once. After +# its first retry it asks only once something arrived since it last asked, +# and never on the tick that takes a datagram, which may be the missing one. reset 16667 10000 add 0 1 1 0 3 0 1 1 add 1000 1 1 2 3 0 3 1 @@ -484,3 +487,56 @@ tick 50000 expect 0 0 2 0 0 3 1 - 0 1:1|1:1|1:1 add 50000 1 1 1 3 1 2 1 expect 1 0 2 0 1 3 0 1 0 1:1|1:1|1:1 + +# A keyframe's hold expiring on the tick its retry falls due sends one +# request, and the next waits a full RTT + 5 ms. +reset 16667 10000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +add 16667 1 2 0 1 0 4 +add 17000 1 1 0 3 1 1 1 +tick 40000 +expect 0 0 3 0 0 3 1 - 0 1:1|1:1|1:1 +add 50000 1 3 0 1 0 5 +tick 65000 +expect 0 0 4 0 0 4 1 - 0 1:1|1:1|1:1|1:1 +tick 79999 +expect 0 0 4 0 0 4 1 - 0 1:1|1:1|1:1|1:1 + +# At 1 ms RTT a keyframe retries every 6 ms, within the PC's 5 ms resend +# limit. Its hold expiring at 26 ms, 1 ms after a retry, asks nothing more. +reset 16667 1000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +add 6000 1 1 0 3 1 1 1 +tick 7000 +add 12000 1 1 0 3 1 1 1 +tick 13000 +add 18000 1 1 0 3 1 1 1 +tick 19000 +add 24000 1 1 0 3 1 1 1 +tick 25000 +expect 0 0 2 0 0 5 1 - 0 1:1|1:1|1:1|1:1|1:1 +tick 26000 +expect 0 0 2 0 0 5 1 - 0 1:1|1:1|1:1|1:1|1:1 +tick 30999 +expect 0 0 2 0 0 5 1 - 0 1:1|1:1|1:1|1:1|1:1 +add 31000 1 1 1 3 1 2 1 +expect 1 0 2 0 1 5 0 1 0 1:1|1:1|1:1|1:1|1:1 + +# At 200 ms RTT a repair can't return sooner than that, so the holds at 26 +# and 51 ms ask nothing; the retry follows at RTT + 5 ms. +reset 16667 200000 +add 0 1 1 0 3 0 1 1 +add 1000 1 1 2 3 0 3 1 +add 20000 1 1 0 3 1 1 1 +tick 26000 +expect 0 0 2 0 0 1 1 - 0 1:1 +tick 51000 +expect 0 0 2 0 0 1 1 - 0 1:1 +tick 205999 +expect 0 0 2 0 0 1 1 - 0 1:1 +tick 206000 +expect 0 0 2 0 0 2 1 - 0 1:1|1:1 +add 210000 1 1 1 3 1 2 1 +expect 1 0 2 0 1 2 0 1 0 1:1|1:1 From 0a237520c28debf9a6267d897eb483068f461552 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:31:25 -0500 Subject: [PATCH 026/226] Hand iPad input to Windows on its own thread, never on the transport loop Windows' input calls wait while it reconfigures displays. On a test PC, every virtual display change held SendInput for 50-130 ms and pen injection for about 40 ms, on the transport loop. In the field one such wait lasted about 53 s: the iPad heard nothing, new HELLOs went unanswered, and the stopped stream kept its port from the next one. - The transport queues input; an input thread owns the relay and hands it to Windows in order. It also sends pen frames held back for their spacing and refreshes a still contact, which the loop did before. - The queue is bounded. While Windows takes nothing, a pointer or Pencil move replaces the move queued just before it. Input that waited over a second lands only as a release of what the PC holds, as late input from the iPad does. Ends, key releases and resets are never dropped. - A stopped stream drops its handle at once. The thread then releases what the PC holds whenever Windows answers. - A Windows input call over 250 ms is logged, and so is a backlog once it clears. - input_soak, a fake iPad that streams harmless input while connecting and leaving, reproduces the waits on a PC set to Second screen. --- host/examples/input_soak.rs | 322 ++++++++++++++ host/src/input/injector.rs | 789 ++++++++++++++++++++++++++++++++++ host/src/input/mod.rs | 91 +++- host/src/input/pen_contact.rs | 10 +- host/src/transport/beat.rs | 3 +- host/src/transport/mod.rs | 161 +------ host/src/transport/pacer.rs | 67 --- host/tests/input_stall.rs | 235 ++++++++++ 8 files changed, 1454 insertions(+), 224 deletions(-) create mode 100644 host/examples/input_soak.rs create mode 100644 host/src/input/injector.rs create mode 100644 host/tests/input_stall.rs diff --git a/host/examples/input_soak.rs b/host/examples/input_soak.rs new file mode 100644 index 00000000..0c393924 --- /dev/null +++ b/host/examples/input_soak.rs @@ -0,0 +1,322 @@ +//! A fake iPad that streams input at a host while it connects and leaves +//! over and over, so a host set to Second screen turns the virtual display +//! on and off under the input. It logs every gap in what the host sends and +//! how long each HELLO waits for its answer: a stuck transport loop shows up +//! as both. It also logs its own send errors and late wake-ups, so a stall +//! of its own is not mistaken for the host's. +//! +//! The input is harmless on a real desktop: pointer and Pencil hover moves +//! (no clicks, no ink) and taps of Right Ctrl alone. +//! +//! `input_soak ` +use std::io::Write; +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use eternal_wire::v2::control::{ + encode_control, parse_control, ByeReason, ControlMessage, Hello2, HelloStatus, InputEvent, + Ping, ReceiverReport, CAP_DECODE_H264, CAP_DECODE_HEVC, FEATURE_SUPPORTS_NACK, + FEATURE_WANTS_INPUT, +}; +use eternal_wire::v2::{classify, Classified}; + +const KIND_PENCIL: u8 = 1; +const KIND_KEY: u8 = 4; +const KIND_HOVER: u8 = 6; +const PHASE_BEGAN: u8 = 0; +const PHASE_MOVED: u8 = 1; +const PHASE_ENDED: u8 = 2; +const RIGHT_CTRL: u16 = 0xe4; +/// Connected and away times, cycled. The short ones leave while the +/// display is still coming up. +const CYCLE: [(u64, u64); 4] = [(25, 15), (4, 14), (25, 15), (8, 3)]; +const GAP_LOGGED: Duration = Duration::from_millis(1500); +const INPUT_EVERY: Duration = Duration::from_millis(8); +/// A wake-up this late is the soak's own stall. +const LATE_WAKE: Duration = Duration::from_millis(100); + +struct Log { + start: Instant, + file: Mutex, +} + +impl Log { + fn note(&self, line: String) { + let line = format!("{{\"t_ms\":{},{line}}}", self.start.elapsed().as_millis()); + println!("{line}"); + let _ = writeln!(self.file.lock().unwrap(), "{line}"); + } +} + +/// What the receiving thread heard. +#[derive(Default)] +struct Heard { + session: AtomicU32, + frames: AtomicU64, + connected: AtomicBool, + cycle: AtomicU32, + last: Mutex>, + worst_gap_ms: AtomicU64, + recv_errors: AtomicU64, +} + +fn receive(socket: &UdpSocket, heard: &Heard, log: &Log) { + let mut buffer = [0u8; 2048]; + loop { + let (len, _) = match socket.recv_from(&mut buffer) { + Ok(received) => received, + Err(error) => { + if !matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) { + heard.recv_errors.fetch_add(1, Ordering::Relaxed); + } + continue; + } + }; + let bytes = &buffer[..len]; + let now = Instant::now(); + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(bytes) { + if ack.status == HelloStatus::Ok + && ack.client_nonce == heard.cycle.load(Ordering::SeqCst) + && heard.session.load(Ordering::SeqCst) == 0 + { + *heard.last.lock().unwrap() = Some(now); + heard.session.store(ack.session_id, Ordering::SeqCst); + } + continue; + } + if !heard.connected.load(Ordering::SeqCst) { + continue; + } + if let Classified::Media { .. } = classify(bytes) { + heard.frames.fetch_add(1, Ordering::Relaxed); + } + let last = heard.last.lock().unwrap().replace(now); + if let Some(gap) = last.map(|last| now - last) { + heard + .worst_gap_ms + .fetch_max(gap.as_millis() as u64, Ordering::Relaxed); + if gap >= GAP_LOGGED { + log.note(format!( + "\"event\":\"gap\",\"cycle\":{},\"gap_ms\":{}", + heard.cycle.load(Ordering::SeqCst), + gap.as_millis() + )); + } + } + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + let host: SocketAddr = args[1].parse().unwrap(); + let seconds: u64 = args[2].parse().unwrap(); + let start = Instant::now(); + let log = Arc::new(Log { + start, + file: Mutex::new(std::fs::File::create(&args[3]).unwrap()), + }); + let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").unwrap()); + socket + .set_read_timeout(Some(Duration::from_millis(100))) + .unwrap(); + let heard = Arc::new(Heard::default()); + { + let (socket, heard, log) = (socket.clone(), heard.clone(), log.clone()); + std::thread::spawn(move || receive(&socket, &heard, &log)); + } + let mut msg_seq = 0u32; + let send_errors = AtomicU64::new(0); + let send = |msg_seq: &mut u32, session_id: u32, message: &ControlMessage| { + *msg_seq += 1; + if socket + .send_to(&encode_control(session_id, *msg_seq, message), host) + .is_err() + { + send_errors.fetch_add(1, Ordering::Relaxed); + } + }; + let mut event_id = 0u32; + let (mut worst_ack, mut worst_wake, mut cycles) = (Duration::ZERO, Duration::ZERO, 0u32); + 'run: for (on, off) in CYCLE.iter().cycle().copied() { + if start.elapsed() >= Duration::from_secs(seconds) { + break; + } + cycles += 1; + heard.cycle.store(cycles, Ordering::SeqCst); + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: cycles, + listen_port: socket.local_addr().unwrap().port(), + decoder_caps: CAP_DECODE_H264 | CAP_DECODE_HEVC, + feature_caps: FEATURE_WANTS_INPUT | FEATURE_SUPPORTS_NACK, + screen_px_w: 2732, + screen_px_h: 2048, + screen_pt_w: 1366, + screen_pt_h: 1024, + refresh_hz: 60, + device_name: "Input soak".into(), + device_id: 0x50a6, + preferred_fps: 60, + auth_token: [0; 16], + pairing_code: 0, + }); + heard.session.store(0, Ordering::SeqCst); + let asked = Instant::now(); + let mut next_hello = asked; + let session_id = loop { + if start.elapsed() >= Duration::from_secs(seconds) { + break 'run; + } + let session_id = heard.session.load(Ordering::SeqCst); + if session_id != 0 { + break session_id; + } + if Instant::now() >= next_hello { + send(&mut msg_seq, 0, &hello); + next_hello += Duration::from_millis(500); + } + std::thread::sleep(Duration::from_millis(2)); + }; + let waited = asked.elapsed(); + worst_ack = worst_ack.max(waited); + heard.connected.store(true, Ordering::SeqCst); + log.note(format!( + "\"event\":\"ack\",\"cycle\":{cycles},\"waited_ms\":{}", + waited.as_millis() + )); + let connected_until = Instant::now() + Duration::from_secs(on); + let (mut next_report, mut next_ping) = (Instant::now(), Instant::now()); + let mut step = 0u32; + let mut key_down = false; + let mut due = Instant::now(); + while Instant::now() < connected_until { + let late = Instant::now().saturating_duration_since(due); + if late >= LATE_WAKE { + worst_wake = worst_wake.max(late); + log.note(format!( + "\"event\":\"late_wake\",\"cycle\":{cycles},\"late_ms\":{}", + late.as_millis() + )); + } + due = Instant::now() + INPUT_EVERY; + step += 1; + event_id += 1; + let angle = f64::from(step) / 60.0; + let mut event = InputEvent { + input_ver: 1, + kind: KIND_HOVER, + phase: PHASE_MOVED, + buttons: 0, + event_id, + x_norm: (32768.0 + 20000.0 * angle.cos()) as u16, + y_norm: (32768.0 + 20000.0 * angle.sin()) as u16, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: 0, + modifiers: 0, + client_time_us: start.elapsed().as_micros() as u64, + tilt_x: 0, + tilt_y: 0, + }; + match step % 4 { + // Pencil hover: native pen frames without contact. + 1 | 3 => { + event.input_ver = 2; + event.kind = KIND_PENCIL; + event.tilt_x = (step % 90) as i16 - 45; + event.tilt_y = 30; + if step % 400 == 399 { + event.phase = PHASE_ENDED; // leave + } + } + 2 if step % 32 == 2 => { + event.kind = KIND_KEY; + event.keycode = RIGHT_CTRL; + event.phase = if key_down { PHASE_ENDED } else { PHASE_BEGAN }; + key_down = !key_down; + } + _ => {} + } + send(&mut msg_seq, session_id, &ControlMessage::InputEvent(event)); + let now = Instant::now(); + if now >= next_report { + next_report += Duration::from_millis(500); + let complete = heard.frames.load(Ordering::Relaxed) as u32; + send( + &mut msg_seq, + session_id, + &ControlMessage::ReceiverReport(ReceiverReport { + stream_epoch: 1, + highest_seq: complete, + frames_complete: complete, + ..ReceiverReport::default() + }), + ); + } + if now >= next_ping { + next_ping += Duration::from_secs(2); + send( + &mut msg_seq, + session_id, + &ControlMessage::Ping(Ping { + t1_us: start.elapsed().as_micros() as u64, + }), + ); + } + std::thread::sleep(due.saturating_duration_since(Instant::now())); + } + if key_down { + event_id += 1; + send( + &mut msg_seq, + session_id, + &ControlMessage::InputEvent(InputEvent { + input_ver: 1, + kind: KIND_KEY, + phase: PHASE_ENDED, + buttons: 0, + event_id, + x_norm: 0, + y_norm: 0, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: RIGHT_CTRL, + modifiers: 0, + client_time_us: start.elapsed().as_micros() as u64, + tilt_x: 0, + tilt_y: 0, + }), + ); + } + for _ in 0..3 { + send( + &mut msg_seq, + session_id, + &ControlMessage::Bye(ByeReason::UserDisconnect), + ); + } + heard.connected.store(false, Ordering::SeqCst); + *heard.last.lock().unwrap() = None; + log.note(format!( + "\"event\":\"bye\",\"cycle\":{cycles},\"frames\":{},\"send_errors\":{},\"recv_errors\":{}", + heard.frames.swap(0, Ordering::Relaxed), + send_errors.swap(0, Ordering::Relaxed), + heard.recv_errors.swap(0, Ordering::Relaxed) + )); + std::thread::sleep(Duration::from_secs(off)); + } + log.note(format!( + "\"event\":\"summary\",\"cycles\":{cycles},\"worst_gap_ms\":{},\"worst_ack_ms\":{},\"worst_late_wake_ms\":{}", + heard.worst_gap_ms.load(Ordering::Relaxed), + worst_ack.as_millis(), + worst_wake.as_millis() + )); +} diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs new file mode 100644 index 00000000..feb8572a --- /dev/null +++ b/host/src/input/injector.rs @@ -0,0 +1,789 @@ +//! Input injection on its own thread. +//! +//! Windows' input calls (`SendInput`, `InjectSyntheticPointerInput`, +//! `GetSystemMetrics`, creating the pen device) can wait while Windows +//! reconfigures its displays. They ran on the transport loop, which then +//! answered nobody: no heartbeats, no HELLO answers, and a stopped stream +//! kept its port. Now the transport only queues input, and this thread +//! hands it to Windows in order. +//! +//! The queue is bounded. While Windows takes nothing, a pointer or Pencil +//! move replaces the move queued just before it, so a backlog never replays +//! a path the iPad already left. Input that waited longer than +//! [`INPUT_LATE`] goes to the relay as late, which lands only a release of +//! what the PC holds, as for input that arrived late from the iPad. Ends, +//! key releases and session resets are never dropped or merged. + +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use eternal_wire::v2::control::InputEvent; +use parking_lot::{Condvar, Mutex}; +use tracing::{info, warn}; + +use super::{ + CaptureGeometry, InputRelay, KIND_HOVER, KIND_MOUSE_ABS, KIND_PENCIL, KIND_TOUCH, PHASE_MOVED, +}; +use crate::transport::session::INPUT_LATE; + +/// Commands kept at most. A second of a Pencil at 240 Hz with every other +/// kind of input; while Windows is stuck, moves merge and the queue grows +/// only by presses and releases. +const CAPACITY: usize = 1024; + +/// From this many waiting commands, a move replaces the move before it. +/// Below it every sample goes to Windows, as the Pencil's ink needs. +const MERGE_FROM: usize = 32; + +/// How often a native pen contact is refreshed; Windows expires a still one. +const PEN_TICK: Duration = Duration::from_millis(50); + +/// A Windows call this slow is logged: it would have stalled the stream. +const SLOW_CALL: Duration = Duration::from_millis(250); + +/// Waits this short are timed precisely, for pen frames held back to keep +/// them `FRAME_SPACING` apart; a condition variable wakes in whole +/// milliseconds or worse on Windows. +const PRECISE_WAIT: Duration = Duration::from_millis(2); + +/// How long the transport waits at its start for the thread to say whether +/// Windows gave it a pen device. +const START_WAIT: Duration = Duration::from_secs(1); + +/// What the injection thread drives: the input relay, or a test's fake. +pub(crate) trait Sink { + fn relay(&mut self, session_id: u32, event: &InputEvent, output: CaptureGeometry, late: bool); + fn reset(&mut self); + fn keep_alive(&mut self, output: Option); + fn pen_available(&self) -> bool; + fn spaced_until(&self) -> Option; + fn tick_needed(&self) -> bool; +} + +impl Sink for InputRelay { + fn relay(&mut self, session_id: u32, event: &InputEvent, output: CaptureGeometry, late: bool) { + if late { + self.relay_late(session_id, event, output); + } else { + InputRelay::relay(self, session_id, event, output); + } + } + fn reset(&mut self) { + InputRelay::reset(self); + } + fn keep_alive(&mut self, output: Option) { + self.keep_pen_alive(output); + } + fn pen_available(&self) -> bool { + InputRelay::pen_available(self) + } + fn spaced_until(&self) -> Option { + self.pen_spaced_until() + } + fn tick_needed(&self) -> bool { + self.pen_tick_needed() + } +} + +#[derive(Debug, Clone, Copy)] +enum Command { + Relay { + session_id: u32, + event: InputEvent, + output: CaptureGeometry, + late: bool, + }, + /// Release what the PC holds: the session ended, expired or is on hold. + Reset, +} + +impl Command { + /// A pointer or Pencil move or hover, which a later one can stand in for. + fn merges_with(&self, earlier: &Command) -> bool { + let ( + Command::Relay { + session_id, + event, + late, + .. + }, + Command::Relay { + session_id: earlier_session, + event: earlier_event, + late: earlier_late, + .. + }, + ) = (self, earlier) + else { + return false; + }; + let moves = |event: &InputEvent| { + event.phase == PHASE_MOVED + && matches!( + event.kind, + KIND_TOUCH | KIND_MOUSE_ABS | KIND_HOVER | KIND_PENCIL + ) + }; + session_id == earlier_session + && late == earlier_late + && moves(event) + && moves(earlier_event) + && event.kind == earlier_event.kind + && event.input_ver == earlier_event.input_ver + && event.buttons == earlier_event.buttons + } + + /// An end or reset: what keeps the PC from holding a key or button. + fn must_land(&self) -> bool { + match self { + Command::Reset => true, + Command::Relay { event, .. } => event.phase > PHASE_MOVED, + } + } +} + +#[derive(Debug)] +struct Queued { + command: Command, + at: Instant, +} + +#[derive(Debug, Default)] +struct State { + commands: VecDeque, + closed: bool, + merged: u64, + dropped: u64, +} + +#[derive(Debug)] +struct Shared { + state: Mutex, + wake: Condvar, + capacity: usize, + merge_from: usize, + stale_after: Duration, +} + +impl Shared { + fn push(&self, command: Command, now: Instant) { + let mut state = self.state.lock(); + let waiting = state.commands.len(); + if waiting >= self.merge_from { + if let Some(last) = state.commands.back_mut() { + if command.merges_with(&last.command) { + // The newer sample, in the older one's place: it has + // waited as long. + last.command = command; + state.merged += 1; + return; + } + } + } + // Ends and resets may pass the bound, by as many as were pressed. + if waiting >= self.capacity && !(command.must_land() && waiting < 2 * self.capacity) { + state.dropped += 1; + return; + } + state.commands.push_back(Queued { command, at: now }); + drop(state); + self.wake.notify_one(); + } +} + +/// The transport's handle on the injection thread. Dropping it lets the +/// thread hand over what is queued and release what the PC holds, without +/// waiting for Windows. +#[derive(Debug)] +pub struct Injector { + shared: Arc, + pen_available: bool, +} + +impl Injector { + /// Starts the thread with a new relay. `geometry` is the capture rect a + /// still Pencil follows. + pub fn start(geometry: Arc>>) -> Self { + #[cfg(all(test, not(windows)))] + let recorder = super::recorder::current(); + Self::spawn(geometry, Limits::default(), move || { + #[cfg(all(test, not(windows)))] + super::recorder::adopt(recorder); + InputRelay::default() + }) + } + + fn spawn( + geometry: Arc>>, + limits: Limits, + sink: impl FnOnce() -> S + Send + 'static, + ) -> Self { + let shared = Arc::new(Shared { + state: Mutex::new(State::default()), + wake: Condvar::new(), + capacity: limits.capacity, + merge_from: limits.merge_from, + stale_after: limits.stale_after, + }); + let (ready_tx, ready) = std::sync::mpsc::channel(); + let worker = Arc::clone(&shared); + let spawned = std::thread::Builder::new() + .name("input-inject".into()) + .spawn(move || { + let mut sink = sink(); + let _ = ready_tx.send(sink.pen_available()); + run(&worker, &mut sink, &geometry); + }); + let pen_available = match spawned { + Ok(_) => ready.recv_timeout(START_WAIT).unwrap_or_else(|_| { + warn!( + wait_ms = START_WAIT.as_millis() as u64, + "Windows did not set up input in time; the Pencil uses the mouse this stream" + ); + false + }), + Err(error) => { + warn!(%error, "Could not start the input thread; input is ignored this stream"); + shared.state.lock().closed = true; + false + } + }; + Self { + shared, + pen_available, + } + } + + /// Whether the relay has a native pen device, for the HELLO answer. + pub fn pen_available(&self) -> bool { + self.pen_available + } + + /// Queue one session-validated event. Never waits for Windows. + pub fn relay(&self, session_id: u32, event: InputEvent, output: CaptureGeometry, late: bool) { + self.shared.push( + Command::Relay { + session_id, + event, + output, + late, + }, + Instant::now(), + ); + } + + /// Queue a release of every key, button and pen contact the PC holds. + pub fn reset(&self) { + self.shared.push(Command::Reset, Instant::now()); + } +} + +impl Drop for Injector { + fn drop(&mut self) { + self.shared.state.lock().closed = true; + self.shared.wake.notify_one(); + } +} + +#[derive(Debug, Clone, Copy)] +struct Limits { + capacity: usize, + merge_from: usize, + stale_after: Duration, +} + +impl Default for Limits { + fn default() -> Self { + Self { + capacity: CAPACITY, + merge_from: MERGE_FROM, + stale_after: INPUT_LATE, + } + } +} + +enum Next { + Command(Queued), + /// A pen frame's spacing or the refresh tick is due. + Due, + Closed, +} + +/// Input that waited for Windows, counted until the queue empties. +#[derive(Debug, Default)] +struct Backlog { + stale: u64, + longest: Duration, +} + +fn run(shared: &Shared, sink: &mut S, geometry: &Mutex>) { + let mut timer = crate::capture::timing::FrameTimer::new(); + let mut next_tick = Instant::now() + PEN_TICK; + let mut backlog = Backlog::default(); + loop { + let next = { + let mut state = shared.state.lock(); + loop { + if let Some(queued) = state.commands.pop_front() { + break Next::Command(queued); + } + if state.closed { + break Next::Closed; + } + let (merged, dropped) = ( + std::mem::take(&mut state.merged), + std::mem::take(&mut state.dropped), + ); + if backlog.stale > 0 || merged > 0 || dropped > 0 { + info!( + stale = backlog.stale, + merged, + dropped, + longest_ms = backlog.longest.as_millis() as u64, + "Input waited for Windows: moves merged, and only releases sent of what waited too long" + ); + backlog = Backlog::default(); + } + let now = Instant::now(); + let spaced = sink.spaced_until(); + let due = spaced.or_else(|| sink.tick_needed().then_some(next_tick)); + match due { + Some(at) if at <= now => break Next::Due, + Some(at) if spaced.is_some() && at - now <= PRECISE_WAIT => { + drop(state); + timer.wait_until(at); + state = shared.state.lock(); + if state.commands.is_empty() { + break Next::Due; + } + } + Some(at) => { + if shared.wake.wait_until(&mut state, at).timed_out() { + break Next::Due; + } + } + None => shared.wake.wait(&mut state), + } + } + }; + match next { + Next::Command(Queued { command, at }) => { + let started = Instant::now(); + let waited = started.saturating_duration_since(at); + backlog.longest = backlog.longest.max(waited); + match command { + Command::Relay { + session_id, + event, + output, + late, + } => { + let stale = waited > shared.stale_after; + backlog.stale += u64::from(stale && !late); + sink.relay(session_id, &event, output, late || stale); + } + Command::Reset => sink.reset(), + } + note_slow(started, "input"); + } + // The relay goes with the thread, releasing what the PC holds. + Next::Closed => return, + Next::Due => {} + } + // Pen frames held back for their spacing, and the refresh a still + // contact needs, go between commands. + let now = Instant::now(); + let spaced_due = sink.spaced_until().is_some_and(|at| at <= now); + let tick_due = sink.spaced_until().is_none() && sink.tick_needed() && next_tick <= now; + if spaced_due || tick_due { + if tick_due { + next_tick = now + PEN_TICK; + } + let started = Instant::now(); + sink.keep_alive(*geometry.lock()); + note_slow(started, "pen_refresh"); + } + } +} + +fn note_slow(started: Instant, what: &'static str) { + let took = started.elapsed(); + if took >= SLOW_CALL { + warn!( + what, + took_ms = took.as_millis() as u64, + "Windows took a long time to take input" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::input::{KIND_KEY, PHASE_BEGAN, PHASE_ENDED}; + use std::sync::mpsc; + + const OUTPUT: CaptureGeometry = CaptureGeometry { + left: 0, + top: 0, + width: 1920, + height: 1080, + }; + + #[derive(Debug, Clone, PartialEq)] + enum Call { + Relay { + kind: u8, + phase: u8, + id: u32, + late: bool, + }, + Reset, + KeepAlive, + } + + /// Records calls, and can hold the first relay until told to go on, + /// as Windows does while it reconfigures displays. + struct Fake { + calls: Arc>>, + hold: Option>, + entered: Option>, + } + + impl Sink for Fake { + fn relay(&mut self, _: u32, event: &InputEvent, _: CaptureGeometry, late: bool) { + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(hold) = self.hold.take() { + let _ = hold.recv(); + } + self.calls.lock().push(Call::Relay { + kind: event.kind, + phase: event.phase, + id: event.event_id, + late, + }); + } + fn reset(&mut self) { + self.calls.lock().push(Call::Reset); + } + fn keep_alive(&mut self, _: Option) { + self.calls.lock().push(Call::KeepAlive); + } + fn pen_available(&self) -> bool { + true + } + fn spaced_until(&self) -> Option { + None + } + fn tick_needed(&self) -> bool { + false + } + } + + struct Harness { + injector: Option, + calls: Arc>>, + go: mpsc::Sender<()>, + entered: mpsc::Receiver<()>, + } + + fn harness(limits: Limits) -> Harness { + let calls = Arc::new(Mutex::new(Vec::new())); + let (go, hold) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + let recorded = Arc::clone(&calls); + let injector = Injector::spawn(Arc::new(Mutex::new(Some(OUTPUT))), limits, move || Fake { + calls: recorded, + hold: Some(hold), + entered: Some(entered_tx), + }); + Harness { + injector: Some(injector), + calls, + go, + entered, + } + } + + impl Harness { + fn send(&self, kind: u8, phase: u8, id: u32) { + self.injector + .as_ref() + .unwrap() + .relay(1, event(kind, phase, id), OUTPUT, false); + } + + /// Holds Windows in the first event's call until `go`. + fn stuck(&self) { + self.send(KIND_KEY, PHASE_BEGAN, 0); + self.entered + .recv_timeout(Duration::from_secs(5)) + .expect("the thread takes the first event"); + } + + fn wait_for(&self, count: usize) -> Vec { + let started = Instant::now(); + while self.calls.lock().len() < count { + assert!( + started.elapsed() < Duration::from_secs(5), + "{:?}", + self.calls.lock() + ); + std::thread::sleep(Duration::from_millis(1)); + } + self.calls.lock().clone() + } + } + + fn event(kind: u8, phase: u8, id: u32) -> InputEvent { + InputEvent { + input_ver: 1, + kind, + phase, + buttons: 0, + event_id: id, + x_norm: id as u16, + y_norm: 0, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: 0x04, + modifiers: 0, + client_time_us: 0, + tilt_x: 0, + tilt_y: 0, + } + } + + fn relayed(kind: u8, phase: u8, id: u32) -> Call { + Call::Relay { + kind, + phase, + id, + late: false, + } + } + + #[test] + fn queueing_never_waits_for_a_stuck_windows() { + let harness = harness(Limits::default()); + harness.stuck(); + let started = Instant::now(); + for id in 1..=5000 { + harness.send(KIND_MOUSE_ABS, PHASE_MOVED, id); + } + harness.injector.as_ref().unwrap().reset(); + assert!(started.elapsed() < Duration::from_millis(500)); + harness.go.send(()).unwrap(); + } + + #[test] + fn every_command_reaches_windows_in_the_order_it_came() { + let harness = harness(Limits::default()); + harness.stuck(); + let sent = [ + (KIND_MOUSE_ABS, PHASE_BEGAN, 1), + (KIND_MOUSE_ABS, PHASE_MOVED, 2), + (KIND_PENCIL, PHASE_MOVED, 3), + (KIND_MOUSE_ABS, PHASE_MOVED, 4), + (KIND_KEY, PHASE_ENDED, 5), + (KIND_MOUSE_ABS, PHASE_ENDED, 6), + ]; + for (kind, phase, id) in sent { + harness.send(kind, phase, id); + } + harness.injector.as_ref().unwrap().reset(); + harness.send(KIND_KEY, PHASE_BEGAN, 7); + harness.go.send(()).unwrap(); + let mut expected = vec![relayed(KIND_KEY, PHASE_BEGAN, 0)]; + expected.extend(sent.map(|(kind, phase, id)| relayed(kind, phase, id))); + expected.push(Call::Reset); + expected.push(relayed(KIND_KEY, PHASE_BEGAN, 7)); + assert_eq!(harness.wait_for(expected.len()), expected); + } + + #[test] + fn a_backed_up_queue_merges_moves_but_keeps_every_edge_in_place() { + let harness = harness(Limits { + merge_from: 4, + ..Limits::default() + }); + harness.stuck(); + // Four wait as they came; then each move stands in for the one + // before it, up to the next edge. + let mut id = 0; + for _ in 0..4 { + id += 1; + harness.send(KIND_HOVER, PHASE_MOVED, id); + } + for _ in 0..50 { + id += 1; + harness.send(KIND_HOVER, PHASE_MOVED, id); + } + harness.send(KIND_MOUSE_ABS, PHASE_BEGAN, 100); + for move_id in 101..150 { + harness.send(KIND_MOUSE_ABS, PHASE_MOVED, move_id); + } + // A Pencil move never stands in for a mouse move. + harness.send(KIND_PENCIL, PHASE_MOVED, 150); + harness.send(KIND_MOUSE_ABS, PHASE_ENDED, 151); + harness.injector.as_ref().unwrap().reset(); + harness.go.send(()).unwrap(); + let expected = vec![ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_HOVER, PHASE_MOVED, 1), + relayed(KIND_HOVER, PHASE_MOVED, 2), + relayed(KIND_HOVER, PHASE_MOVED, 3), + relayed(KIND_HOVER, PHASE_MOVED, 54), + relayed(KIND_MOUSE_ABS, PHASE_BEGAN, 100), + relayed(KIND_MOUSE_ABS, PHASE_MOVED, 149), + relayed(KIND_PENCIL, PHASE_MOVED, 150), + relayed(KIND_MOUSE_ABS, PHASE_ENDED, 151), + Call::Reset, + ]; + assert_eq!(harness.wait_for(expected.len()), expected); + std::thread::sleep(Duration::from_millis(20)); + assert_eq!(harness.calls.lock().len(), expected.len()); + } + + #[test] + fn input_that_waited_too_long_goes_as_late_and_releases_still_land() { + let harness = harness(Limits { + stale_after: Duration::from_millis(50), + ..Limits::default() + }); + harness.stuck(); + harness.send(KIND_KEY, PHASE_ENDED, 1); + harness.send(KIND_MOUSE_ABS, PHASE_BEGAN, 2); + std::thread::sleep(Duration::from_millis(100)); + harness.go.send(()).unwrap(); + // Fresh input after Windows came back is not late. + harness.wait_for(3); + harness.send(KIND_MOUSE_ABS, PHASE_ENDED, 3); + let late = |kind, phase, id| Call::Relay { + kind, + phase, + id, + late: true, + }; + assert_eq!( + harness.wait_for(4), + [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + late(KIND_KEY, PHASE_ENDED, 1), + late(KIND_MOUSE_ABS, PHASE_BEGAN, 2), + relayed(KIND_MOUSE_ABS, PHASE_ENDED, 3), + ] + ); + } + + #[test] + fn a_full_queue_drops_new_moves_and_presses_but_keeps_releases() { + let harness = harness(Limits { + capacity: 3, + merge_from: 100, + stale_after: INPUT_LATE, + }); + harness.stuck(); + for id in 1..=3 { + harness.send(KIND_KEY, PHASE_BEGAN, id); + } + harness.send(KIND_KEY, PHASE_BEGAN, 4); + harness.send(KIND_MOUSE_ABS, PHASE_MOVED, 5); + harness.send(KIND_KEY, PHASE_ENDED, 6); + harness.injector.as_ref().unwrap().reset(); + harness.go.send(()).unwrap(); + assert_eq!( + harness.wait_for(6), + [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_KEY, PHASE_BEGAN, 1), + relayed(KIND_KEY, PHASE_BEGAN, 2), + relayed(KIND_KEY, PHASE_BEGAN, 3), + relayed(KIND_KEY, PHASE_ENDED, 6), + Call::Reset, + ] + ); + } + + #[test] + fn a_dropped_handle_returns_at_once_and_its_input_still_lands() { + let mut harness = harness(Limits::default()); + harness.stuck(); + harness.send(KIND_KEY, PHASE_ENDED, 1); + let started = Instant::now(); + drop(harness.injector.take()); + assert!(started.elapsed() < Duration::from_millis(100)); + harness.go.send(()).unwrap(); + assert_eq!( + harness.wait_for(2), + [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_KEY, PHASE_ENDED, 1), + ] + ); + } + + /// The real relay, with a native pen device that takes every frame + /// (Windows' own off Windows). A stall delivers 64 samples at once; the + /// spacing holds back all but the first burst, and the thread sends the + /// rest within a few times the spacing, not at the 50 ms refresh. + #[cfg(not(windows))] + #[test] + fn the_thread_sends_a_held_pen_backlog_at_its_spacing() { + use crate::input::{native_pen, PenPhase, PenSample}; + const SAMPLES: usize = 64; + let (taken_tx, taken_rx) = mpsc::channel(); + let delivered = Arc::new(Mutex::new(None)); + let at = Arc::clone(&delivered); + let injector = Injector::spawn( + Arc::new(Mutex::new(Some(OUTPUT))), + Limits::default(), + move || { + let taken = native_pen::install(); + *at.lock() = Some(Instant::now()); + native_pen::with(|device| { + for x in 0..SAMPLES as u16 { + let sample = PenSample { + phase: if x == 0 { + PenPhase::Down + } else { + PenPhase::Move + }, + x: i32::from(x), + y: 10, + x_norm: x, + y_norm: 10, + pressure: 512, + tilt_x: 0, + tilt_y: 0, + }; + device.deliver(sample, native_pen::SCREEN, Some(OUTPUT), Instant::now()); + } + }); + assert!( + taken.lock().len() < SAMPLES / 2, + "the spacing must hold the backlog back" + ); + taken_tx.send(taken).unwrap(); + InputRelay::default() + }, + ); + let taken = taken_rx.recv().unwrap(); + let started = Instant::now(); + while taken.lock().len() < SAMPLES && started.elapsed() < Duration::from_secs(2) { + std::thread::sleep(Duration::from_millis(1)); + } + drop(injector); + let taken = taken.lock(); + let order: Vec<_> = taken.iter().map(|(frame, _)| frame.x_norm).collect(); + assert_eq!(order, (0..SAMPLES as u16).collect::>()); + let drained = taken.last().unwrap().1 - delivered.lock().unwrap(); + // 58 held frames 250 µs apart take 15 ms; the 50 ms tick alone + // would take 500 ms. + assert!(drained < Duration::from_millis(250), "{drained:?}"); + } +} diff --git a/host/src/input/mod.rs b/host/src/input/mod.rs index d1edea10..85126401 100644 --- a/host/src/input/mod.rs +++ b/host/src/input/mod.rs @@ -10,6 +10,9 @@ use eternal_wire::v2::control::InputEvent; mod hid; pub use hid::hid_to_scancode; +mod injector; +pub use injector::Injector; + #[cfg(windows)] pub mod windows_inject; @@ -367,8 +370,9 @@ pub fn resolve( pub mod recorder { use super::Injection; - /// The end-to-end tests record on the transport's thread and read on - /// their own. Unit tests run in parallel, so each keeps its own record. + /// The end-to-end tests record on the input thread and read on their + /// own. Unit tests run in parallel, so each keeps its own record, which + /// the input threads it starts write to. fn recorded(f: impl FnOnce(&mut Vec) -> R) -> R { #[cfg(not(test))] { @@ -377,14 +381,39 @@ pub mod recorder { } #[cfg(test)] { - thread_local! { - static RECORDED: std::cell::RefCell> = - const { std::cell::RefCell::new(Vec::new()) }; - } - RECORDED.with_borrow_mut(f) + current().with(f) } } + /// A unit test's own record, which the threads it starts share. + #[cfg(test)] + #[derive(Debug, Clone, Default)] + pub struct Record(std::sync::Arc>>); + + #[cfg(test)] + impl Record { + fn with(&self, f: impl FnOnce(&mut Vec) -> R) -> R { + f(&mut self.0.lock().unwrap()) + } + } + + #[cfg(test)] + thread_local! { + static RECORD: std::cell::RefCell = std::cell::RefCell::default(); + } + + /// This thread's record, for a thread it starts to [`adopt`]. + #[cfg(test)] + pub fn current() -> Record { + RECORD.with_borrow(Record::clone) + } + + /// Record on this thread into `record`. + #[cfg(test)] + pub fn adopt(record: Record) { + RECORD.set(record); + } + pub fn record(injections: &[Injection]) { recorded(|recorded| recorded.extend_from_slice(injections)); } @@ -406,8 +435,9 @@ pub(crate) mod native_pen { use super::pen_contact::{Outcome, PointerKind}; use super::pen_device::{PenApi, PenDevice}; use super::{CaptureGeometry, Injection, PenSample, VirtualScreen}; + use parking_lot::Mutex; use std::cell::RefCell; - use std::rc::Rc; + use std::sync::Arc; use std::time::Instant; pub(crate) const SCREEN: VirtualScreen = VirtualScreen { @@ -418,7 +448,7 @@ pub(crate) mod native_pen { }; /// Frames Windows took, with when. - pub(crate) type Taken = Rc>>; + pub(crate) type Taken = Arc>>; #[derive(Debug)] pub(crate) struct Windows(Taken); @@ -444,7 +474,7 @@ pub(crate) mod native_pen { _: VirtualScreen, _: Option, ) -> Outcome { - self.0.borrow_mut().push((frame, Instant::now())); + self.0.lock().push((frame, Instant::now())); Outcome::Injected } fn send_mouse(&mut self, inputs: &[Injection]) -> usize { @@ -463,7 +493,7 @@ pub(crate) mod native_pen { /// Give this thread's relays a device, and return what Windows takes. pub(crate) fn install() -> Taken { let taken = Taken::default(); - DEVICE.set(Some(PenDevice::new(Windows(Rc::clone(&taken))))); + DEVICE.set(Some(PenDevice::new(Windows(Arc::clone(&taken))))); taken } @@ -473,6 +503,43 @@ pub(crate) mod native_pen { } } +/// Lets an end-to-end test hold every injection where Windows' calls would +/// be, as Windows held them while it reconfigured displays. +#[cfg(feature = "test-hooks")] +pub mod hold { + use parking_lot::{Condvar, Mutex}; + + /// Whether injections wait, and how many are waiting. + static HELD: Mutex<(bool, usize)> = Mutex::new((false, 0)); + static RELEASED: Condvar = Condvar::new(); + + pub fn hold() { + HELD.lock().0 = true; + } + + pub fn release() { + HELD.lock().0 = false; + RELEASED.notify_all(); + } + + /// Injections waiting now. + pub fn waiting() -> usize { + HELD.lock().1 + } + + pub(super) fn pass() { + let mut held = HELD.lock(); + if !held.0 { + return; + } + held.1 += 1; + while held.0 { + RELEASED.wait(&mut held); + } + held.1 -= 1; + } +} + fn log_injections(injections: &[Injection]) { if std::env::var("ETERNAL_INPUT_RECORDER_LOG").is_ok_and(|value| value == "1") { for injection in injections { @@ -732,6 +799,8 @@ impl InputRelay { } fn inject(&mut self, injections: &[Injection]) { + #[cfg(feature = "test-hooks")] + hold::pass(); #[cfg(windows)] windows_inject::inject(injections, &mut self.pen_device, self.output); #[cfg(not(windows))] diff --git a/host/src/input/pen_contact.rs b/host/src/input/pen_contact.rs index 074d772d..56beb608 100644 --- a/host/src/input/pen_contact.rs +++ b/host/src/input/pen_contact.rs @@ -129,7 +129,7 @@ const NOT_READY_WINDOW: Duration = Duration::from_millis(2); /// Retries ERROR_NOT_READY briefly while Windows is accepting frames. A frame /// still refused after that means Windows' input queue is stuck; later /// frames get one attempt each until one is accepted, so a stuck queue costs -/// one call per sample or refresh tick rather than stalling the transport. +/// one call per sample or refresh tick rather than stalling the input thread. #[derive(Debug, Default)] pub struct NotReadyRetry { stalled: bool, @@ -173,8 +173,7 @@ pub(super) const FRAME_SPACING: Duration = Duration::from_micros(250); pub(super) const FRAME_BURST: u32 = 6; /// The most frames one `flush` sends before it yields. Injection runs on the -/// transport loop, so a backlog goes out in chunks with NACK answers, input -/// and video pacing in between. +/// input thread, so a backlog goes out in chunks with newer input in between. const PASS_FRAMES: u32 = FRAME_BURST; /// The most time one `flush` spends sending frames before it yields, should @@ -385,9 +384,8 @@ impl PenContact { self.abandoned.push((mapped(sample, output), now)); return; } - // Behind frames held back, the sample waits for the flush at the - // top of the next transport pass, so one pass never sends more than - // one budget. + // Behind frames held back, the sample waits for the input thread's + // next flush, so one pass never sends more than one budget. let held = self.spaced_until().is_some(); self.enqueue(sample, now); if !held { diff --git a/host/src/transport/beat.rs b/host/src/transport/beat.rs index f1911b20..b140725d 100644 --- a/host/src/transport/beat.rs +++ b/host/src/transport/beat.rs @@ -24,7 +24,8 @@ pub enum Phase { Tick, /// Handling a control datagram from an iPad, or a USB tunnel event. Control, - /// Injecting the iPad's touch, pen, mouse or keyboard input into Windows. + /// Queueing the iPad's touch, pen, mouse or keyboard input for the input + /// thread, which hands it to Windows. Input, /// Sending what the session decided: replies, goodbyes, summaries. Actions, diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 20a58bc3..b73aa90d 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -32,7 +32,7 @@ use abr::AbrController; use beat::Phase; use fault::FaultInjector; use link::{LinkId, PeerId}; -use pacer::{Deadline, FramePacer, PacingClock}; +use pacer::{FramePacer, PacingClock}; use retransmit::RetransmitRing; use session::{Actions, ConfigSource, HEARTBEAT_INTERVAL}; use usb::{LinkEvent, TransportLinks}; @@ -222,11 +222,11 @@ pub async fn start_sender( .lock() .set_target_addr(shared.target_addr.lock().to_string()); - let input_relay = crate::input::InputRelay::default(); + let input = crate::input::Injector::start(std::sync::Arc::clone(&shared.capture_geometry)); let config = SharedConfigSource { shared: &shared, stream_epoch, - pen_available: input_relay.pen_available(), + pen_available: input.pen_available(), }; let abr_enabled = !std::env::var("ETERNAL_ABR").is_ok_and(|v| v.trim() == "0"); @@ -269,7 +269,7 @@ pub async fn start_sender( let mut state = LoopState { retransmit_ring: RetransmitRing::default(), fault, - input_relay, + input, abr, }; let mut audio_sender = audio::AudioSender::default(); @@ -294,37 +294,14 @@ pub async fn start_sender( // starts. It is looked at again while nobody streams. let mut network_check = tokio::time::interval(Duration::from_secs(10)); network_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let mut pen_tick = tokio::time::interval(Duration::from_millis(50)); - pen_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - let mut pen_spacing = Deadline::new("pen-spacing")?; loop { - // A pen backlog sends one bounded chunk at the top of every pass, - // whatever the select below keeps ready, so the loop serves one - // branch between chunks and the backlog never waits behind them. - if state - .input_relay - .pen_spaced_until() - .is_some_and(|at| at <= Instant::now()) - { - beat.mark(Phase::Input); - state - .input_relay - .keep_pen_alive(*shared.capture_geometry.lock()); - } - pen_spacing.arm(state.input_relay.pen_spaced_until()); beat.mark(Phase::Waiting); tokio::select! { // Service timers and queued control before another paced frame. // Random branch selection can delay a NACK by multiple frame // bursts on platforms with coarse sleep granularity. biased; - // Not while a backlog is held: its chunk went at the top of - // this pass, and a second one would leave nothing between them. - _ = pen_tick.tick(), if state.input_relay.pen_tick_needed() && state.input_relay.pen_spaced_until().is_none() => { - beat.mark(Phase::Input); - state.input_relay.keep_pen_alive(*shared.capture_geometry.lock()); - } _ = tokio::time::sleep_until(state.fault.next_deadline().unwrap_or_else(|| Instant::now() + Duration::from_secs(3600)).into()), if state.fault.enabled() => { beat.mark(Phase::Video); let destination = *shared.target_addr.lock(); @@ -535,10 +512,6 @@ pub async fn start_sender( "Frame sent" ); } - // Only wakes an otherwise idle loop for the next chunk, which - // goes at the top of the pass. Last, because any branch ready - // before it starts that pass anyway. - _ = pen_spacing.wait(), if pen_spacing.armed() => {} } } @@ -721,7 +694,9 @@ async fn repair_while_pacing( struct LoopState { retransmit_ring: RetransmitRing, fault: FaultInjector, - input_relay: crate::input::InputRelay, + /// Hands input to Windows on its own thread: Windows' input calls can + /// wait seconds while it reconfigures displays. + input: crate::input::Injector, abr: AbrController, } @@ -742,7 +717,7 @@ async fn act( state.fault.clear(); } if actions.new_target.is_some() || actions.client_lost || actions.release_input { - state.input_relay.reset(); + state.input.reset(); } if let Some(nack) = actions.retransmit.take() { send_repairs(&links.udp.socket, &mut state.retransmit_ring, shared, &nack).await; @@ -750,11 +725,9 @@ async fn act( if let Some((session_id, event)) = actions.input.take() { shared.transport_beat.mark(Phase::Input); if let Some(geometry) = *shared.capture_geometry.lock() { - if actions.input_late { - state.input_relay.relay_late(session_id, &event, geometry); - } else { - state.input_relay.relay(session_id, &event, geometry); - } + state + .input + .relay(session_id, event, geometry, actions.input_late); } } if let Some(report) = actions.report.take() { @@ -1167,7 +1140,7 @@ mod tests { LoopState { retransmit_ring: RetransmitRing::default(), fault: FaultInjector::new(0.0, 0.0, Duration::ZERO), - input_relay: crate::input::InputRelay::default(), + input: crate::input::Injector::start(Default::default()), abr: AbrController::new(15_000_000, true), } } @@ -1266,8 +1239,16 @@ mod tests { ) .await; } + // Windows takes input on the input thread. + let injected = |count: usize| { + let started = Instant::now(); + while recorder::peek().len() < count && started.elapsed() < Duration::from_secs(5) { + std::thread::sleep(Duration::from_millis(1)); + } + recorder::take() + }; assert!(matches!( - recorder::take().as_slice(), + injected(3).as_slice(), [ Injection::KeyDown { .. }, Injection::MoveAbs { .. }, @@ -1288,7 +1269,7 @@ mod tests { .await; assert!(shared.session.lock().session_id().is_none()); assert!(matches!( - recorder::take().as_slice(), + injected(2).as_slice(), [Injection::KeyUp { .. }, Injection::LeftUp { .. }] )); } @@ -1497,104 +1478,6 @@ mod tests { ); } - /// `start_sender` itself, with a native pen device that takes every - /// frame (Windows' own off Windows). A stall delivers 64 samples at - /// once while the encoder always has a frame ready, which the loop takes - /// in a branch ahead of the pen's wake-up. The spacing holds back all - /// but the first burst, and the loop sends the rest within a few times - /// the spacing, one chunk at the top of each pass. - #[cfg(not(windows))] - #[test] - fn the_loop_sends_a_held_pen_backlog_while_video_is_always_ready() { - use crate::input::{native_pen, CaptureGeometry, PenPhase, PenSample}; - const SAMPLES: usize = 64; - const VIDEO_QUEUE: usize = 1 << 16; - let output = CaptureGeometry { - left: 0, - top: 0, - width: 1920, - height: 1080, - }; - let taken = native_pen::install(); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let (delivered, queued) = runtime.block_on(async { - let shared = SharedControl::new(0, 15_000_000); - shared.running.store(true, Ordering::SeqCst); - *shared.capture_geometry.lock() = Some(output); - let (nal_tx, nal_rx) = mpsc::channel(VIDEO_QUEUE); - let (_audio_tx, audio_rx) = mpsc::channel(1); - let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); - let frame = |sequence| NALUnit { - data: vec![0, 0, 0, 1, 0x41], - sequence, - timestamp: Instant::now(), - encode_duration_us: 0, - is_keyframe: false, - }; - // A full queue, kept full until the loop stops and drops it. - let mut sequence = 0; - while nal_tx.try_send(frame(sequence)).is_ok() { - sequence += 1; - } - let video = std::thread::spawn(move || { - while nal_tx.blocking_send(frame(sequence)).is_ok() { - sequence += 1; - } - sequence - }); - let delivered = Instant::now(); - native_pen::with(|device| { - for x in 0..SAMPLES as u16 { - let sample = PenSample { - phase: if x == 0 { - PenPhase::Down - } else { - PenPhase::Move - }, - x: i32::from(x), - y: 10, - x_norm: x, - y_norm: 10, - pressure: 512, - tilt_x: 0, - tilt_y: 0, - }; - device.deliver(sample, native_pen::SCREEN, Some(output), Instant::now()); - } - }); - assert!( - taken.borrow().len() < SAMPLES / 2, - "the spacing must hold the backlog back" - ); - let stop = shared.clone(); - let watch = async { - while taken.borrow().len() < SAMPLES && delivered.elapsed() < Duration::from_secs(2) - { - tokio::time::sleep(Duration::from_millis(1)).await; - } - stop.stop(); - }; - let (sent, ()) = tokio::join!( - start_sender(nal_rx, audio_rx, 0, shared, supervisor_tx), - watch - ); - sent.unwrap(); - (delivered, video.join().unwrap()) - }); - let taken = taken.borrow(); - let order: Vec<_> = taken.iter().map(|(frame, _)| frame.x_norm).collect(); - assert_eq!(order, (0..SAMPLES as u16).collect::>()); - let drained = taken.last().unwrap().1 - delivered; - // 58 held frames 250 µs apart take 15 ms; the 50 ms tick alone - // would take 500 ms. - assert!(drained < Duration::from_millis(250), "{drained:?}"); - // The loop kept taking video. - assert!(queued > (VIDEO_QUEUE + SAMPLES) as u64, "{queued}"); - } - #[test] fn a_failed_send_names_the_link_and_repeats_identically() { let out = std::sync::Arc::new(parking_lot::Mutex::new(Vec::::new())); diff --git a/host/src/transport/pacer.rs b/host/src/transport/pacer.rs index a44643c9..e0a70c97 100644 --- a/host/src/transport/pacer.rs +++ b/host/src/transport/pacer.rs @@ -45,58 +45,6 @@ impl PacingClock { } } -/// Wakes the transport at a deadline from a dedicated thread, as -/// `PacingClock` does, for pen frames held back to keep them -/// `FRAME_SPACING` apart. The transport arms it at the top of each loop and -/// selects on `wait`; arming the same deadline again sends nothing, and a -/// superseded deadline only costs an extra flush. -pub struct Deadline { - tx: std::sync::mpsc::Sender, - due: std::sync::Arc, - armed: Option, -} - -impl Deadline { - pub fn new(name: &str) -> std::io::Result { - let (tx, rx) = std::sync::mpsc::channel::(); - let due = std::sync::Arc::new(tokio::sync::Notify::new()); - let notify = std::sync::Arc::clone(&due); - std::thread::Builder::new() - .name(name.into()) - .spawn(move || { - let mut timer = crate::capture::timing::FrameTimer::new(); - while let Ok(deadline) = rx.recv() { - timer.wait_until(deadline); - notify.notify_one(); - } - })?; - Ok(Self { - tx, - due, - armed: None, - }) - } - - pub fn arm(&mut self, deadline: Option) { - if deadline == self.armed { - return; - } - self.armed = deadline; - if let Some(deadline) = deadline { - let _ = self.tx.send(deadline); - } - } - - pub fn armed(&self) -> bool { - self.armed.is_some() - } - - pub async fn wait(&mut self) { - self.due.notified().await; - self.armed = None; - } -} - /// Frames at or below this many fragments go out unpaced. pub const PACE_THRESHOLD_FRAGS: usize = 16; /// Datagrams per burst between gaps. @@ -165,21 +113,6 @@ mod tests { ready.await.unwrap(); // dropping the clock never strands a pending wait } - #[tokio::test] - async fn a_deadline_wakes_the_runtime_once_it_has_passed() { - let mut deadline = Deadline::new("test-deadline").unwrap(); - let started = Instant::now(); - let at = started + Duration::from_micros(250); - deadline.arm(Some(at)); - // Arming it again does not queue a second wake. - deadline.arm(Some(at)); - assert!(deadline.armed()); - deadline.wait().await; - assert!(Instant::now() >= at); - assert!(started.elapsed() < Duration::from_secs(1)); - assert!(!deadline.armed()); - } - #[test] fn small_frames_are_never_paced() { let now = Instant::now(); diff --git a/host/tests/input_stall.rs b/host/tests/input_stall.rs new file mode 100644 index 00000000..f1199476 --- /dev/null +++ b/host/tests/input_stall.rs @@ -0,0 +1,235 @@ +//! Windows' input calls can wait for seconds while it reconfigures displays. +//! In the field the transport loop sat in one for about 53 s during a +//! virtual display change: its iPad heard nothing, new HELLOs went +//! unanswered, and the stopped stream kept its port from the next one. +//! +//! The real transport loop on loopback, with every injection held where +//! Windows' calls are. The input it takes is a key with no Windows scan +//! code, so nothing reaches the desktop on Windows either. + +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use eternal_host::control::SharedControl; +use eternal_host::input::{hold, CaptureGeometry, KIND_KEY, PHASE_BEGAN}; +use eternal_host::transport; +use eternal_host::transport::beat::Watch; +use eternal_wire::v2::control::{ + encode_control, parse_control, ControlMessage, Hello2, HelloStatus, InputEvent, Ping, + CAP_DECODE_H264, FEATURE_WANTS_INPUT, +}; +use tokio::sync::mpsc; + +/// A HID usage Windows has no scan code for: the relay injects nothing. +const NO_KEY: u16 = 0x01; + +struct Ipad { + socket: UdpSocket, + host: SocketAddr, + device_id: u64, + msg_seq: u32, +} + +impl Ipad { + fn new(host: SocketAddr, device_id: u64) -> Self { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + Self { + socket, + host, + device_id, + msg_seq: 0, + } + } + + fn send(&mut self, session_id: u32, message: &ControlMessage) { + self.msg_seq += 1; + self.socket + .send_to( + &encode_control(session_id, self.msg_seq, message), + self.host, + ) + .unwrap(); + } + + fn hello(&mut self) { + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: self.device_id as u32, + listen_port: self.socket.local_addr().unwrap().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_WANTS_INPUT, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "Input stall iPad".into(), + device_id: self.device_id, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + self.send(0, &hello); + } + + /// Control messages heard until `until`. + fn listen(&mut self, until: Instant) -> Vec { + let mut heard = Vec::new(); + let mut bytes = [0u8; 2048]; + while Instant::now() < until { + if let Ok((len, _)) = self.socket.recv_from(&mut bytes) { + if let Ok((_, message)) = parse_control(&bytes[..len]) { + heard.push(message); + } + } + } + heard + } + + fn accepted(&mut self) -> u32 { + let started = Instant::now(); + loop { + assert!( + started.elapsed() < Duration::from_secs(10), + "no HELLO answer" + ); + self.hello(); + let heard = self.listen(Instant::now() + Duration::from_millis(500)); + if let Some(id) = heard.iter().find_map(|message| match message { + ControlMessage::HelloAck(ack) if ack.status == HelloStatus::Ok => { + Some(ack.session_id) + } + _ => None, + }) { + return id; + } + } + } +} + +fn now_ms() -> u64 { + eternal_host::clock::host_now_us() / 1000 +} + +#[test] +fn input_windows_holds_never_stops_the_transport_loop() { + eternal_host::clock::init(); + // Never reach a real iPad over USB. + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + let port = UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap(); + let shared = SharedControl::new(port, 10_000_000); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + *shared.capture_geometry.lock() = Some(CaptureGeometry { + left: 0, + top: 0, + width: 1280, + height: 720, + }); + let (nal_tx, nal_rx) = mpsc::channel(4); + let (_audio_tx, audio_rx) = mpsc::channel(4); + let control = shared.clone(); + let transport = std::thread::spawn(move || { + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(transport::start_sender( + nal_rx, + audio_rx, + port, + control, + supervisor_tx, + )) + .unwrap(); + }); + let host = SocketAddr::from(([127, 0, 0, 1], port)); + let mut ipad = Ipad::new(host, 1); + let session_id = ipad.accepted(); + + // Windows stops taking input, and the iPad presses a key. + hold::hold(); + ipad.send( + session_id, + &ControlMessage::InputEvent(InputEvent { + input_ver: 1, + kind: KIND_KEY, + phase: PHASE_BEGAN, + buttons: 0, + event_id: 1, + x_norm: 0, + y_norm: 0, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: NO_KEY, + modifiers: 0, + client_time_us: 0, + tilt_x: 0, + tilt_y: 0, + }), + ); + let started = Instant::now(); + while hold::waiting() == 0 { + assert!( + started.elapsed() < Duration::from_secs(5), + "the key reaches Windows" + ); + std::thread::sleep(Duration::from_millis(5)); + } + + // While Windows holds it, the iPad still hears its heartbeats, another + // iPad's HELLO is answered, and the watch sees the loop moving. + let mut watch = Watch::default(); + let mut verdicts = Vec::new(); + let mut other = Ipad::new(host, 2); + other.hello(); + let mut heartbeats = 0; + let mut answered = false; + let until = Instant::now() + Duration::from_secs(3); + while Instant::now() < until { + ipad.send(session_id, &ControlMessage::Ping(Ping { t1_us: 1 })); + let step = Instant::now() + Duration::from_millis(100); + heartbeats += ipad + .listen(step) + .iter() + .filter(|message| matches!(message, ControlMessage::Heartbeat(_))) + .count(); + answered |= other + .listen(Instant::now() + Duration::from_millis(5)) + .iter() + .any(|message| matches!(message, ControlMessage::HelloAck(_))); + verdicts.extend(watch.check(shared.transport_beat.read(), now_ms())); + } + assert_eq!(hold::waiting(), 1, "Windows still holds the key"); + assert!( + heartbeats >= 2, + "heartbeats while Windows held input: {heartbeats}" + ); + assert!(answered, "the other iPad's HELLO went unanswered"); + assert_eq!(verdicts, []); + + // A stream stopped meanwhile ends at once, and frees its port. + shared.stop(); + drop(nal_tx); + let stopping = Instant::now(); + while !transport.is_finished() { + assert!( + stopping.elapsed() < Duration::from_secs(3), + "the stopped loop waits for Windows" + ); + std::thread::sleep(Duration::from_millis(10)); + } + transport.join().unwrap(); + assert!(UdpSocket::bind(("127.0.0.1", port)).is_ok()); + hold::release(); +} From 01bb4d2dd3de05c64335cf6e0263966916ad9305 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 13:04:27 -0500 Subject: [PATCH 027/226] Clear the Apple Devices hint when it expires, even on an idle screen Review fix: the 30-minute limit was only checked when SwiftUI redrew the USB card, so an idle Connect screen kept the hint. The sighting now carries its expiry, a cancellable task clears that sighting and publishes the change when it comes, and coming back to the foreground checks it against the clock again. The clock and the lifetime can be injected for tests. --- ios/EternalMonitor/App/ConnectView.swift | 2 +- .../App/ConnectionManager.swift | 44 +++++++++++++-- ios/EternalMonitorTests/PCNoticeTests.swift | 54 +++++++++++++++++-- 3 files changed, 90 insertions(+), 10 deletions(-) diff --git a/ios/EternalMonitor/App/ConnectView.swift b/ios/EternalMonitor/App/ConnectView.swift index abafb723..fd39f919 100644 --- a/ios/EternalMonitor/App/ConnectView.swift +++ b/ios/EternalMonitor/App/ConnectView.swift @@ -597,7 +597,7 @@ struct ConnectView: View { .font(.app(15, .medium, relativeTo: .subheadline)) .foregroundStyle(Theme.text) Text(USBStatusText.describe(connectionManager.usbStatus, - appleDevicesUnreachable: connectionManager.appleDevicesUnreachable?.applies(now: Date()) == true)) + appleDevicesUnreachable: connectionManager.appleDevicesUnreachable != nil)) .font(.app(13.5, relativeTo: .footnote)) .foregroundStyle(Theme.textMuted) .fixedSize(horizontal: false, vertical: true) diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 04eefd52..e1a5d84b 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -142,16 +142,21 @@ final class ConnectionManager: ObservableObject { @Published private(set) var usbStatus = "USB: waiting for a cable" /// A PC left the USB capability out of its HELLO_ACK: when it was /// sampled, that PC couldn't reach Apple's USB service. Kept only for - /// the PC at that target, and only for a while; see `USBStatusText`. + /// the PC at that target, and only until `until`; see `USBStatusText`. struct AppleDevicesSighting: Equatable { static let lifetime: TimeInterval = 30 * 60 /// "host:port" of the PC over Wi-Fi, or "USB". let target: String let at: Date + let until: Date - func applies(now: Date) -> Bool { now >= at && now.timeIntervalSince(at) < Self.lifetime } + func applies(now: Date) -> Bool { now >= at && now < until } } + /// Cleared when it expires, so an idle Connect screen drops the hint too. @Published private(set) var appleDevicesUnreachable: AppleDevicesSighting? + private var appleDevicesExpiry: Task? + /// The wall clock for the Apple Devices sighting. Tests replace it. + var now: () -> Date = { Date() } private var mediaLink: (any MediaLink)? private var usbListener: USBListener? private var isForeground = true @@ -225,6 +230,7 @@ final class ConnectionManager: ObservableObject { ?? Int(ConnectionManager.connectionTimeoutSeconds)) var reconnectGraceSeconds = 2.0 var reconnectBackoffSeconds: (Int) -> Double = { attempt in Double(min(1 << (attempt - 1), 15)) } + var appleDevicesNoteSeconds = AppleDevicesSighting.lifetime } var timing: Timing /// The Wi-Fi link to a host's port. Tests pass a fake one, or replace it @@ -385,7 +391,7 @@ final class ConnectionManager: ObservableObject { let pairingAddress = isUSB ? "USB" : PairingStore.address(host: normalizedHost, port: port) let pcTarget = isUSB ? "USB" : "\(normalizedHost):\(port)" // What one PC said about Apple Devices says nothing about another. - if appleDevicesUnreachable?.target != pcTarget { appleDevicesUnreachable = nil } + if appleDevicesUnreachable?.target != pcTarget { keepAppleDevicesSighting(nil) } let pairingToken = explicitPairingToken ?? ((try? pairingStore.token(address: pairingAddress)) ?? Data(repeating: 0, count: 16)) explicitPairingToken = nil let connectionID = UUID() @@ -620,8 +626,11 @@ final class ConnectionManager: ObservableObject { at: isUSB ? nil : "\(normalizedHost):\(port)") self.livenessTimeoutUs = UInt64(info.livenessTimeoutMs) * 1000 self.hostInfo = info - self.appleDevicesUnreachable = info.hostCaps & HelloAck.hostCapUSB == 0 - ? AppleDevicesSighting(target: pcTarget, at: Date()) : nil + let seen = self.now() + self.keepAppleDevicesSighting(info.hostCaps & HelloAck.hostCapUSB == 0 + ? AppleDevicesSighting(target: pcTarget, at: seen, + until: seen.addingTimeInterval(self.timing.appleDevicesNoteSeconds)) + : nil) if self.awaitingPairing { self.awaitingPairing = false self.pairingPrompt = nil @@ -1209,10 +1218,35 @@ final class ConnectionManager: ObservableObject { } } + /// Shows `sighting` until it expires, then clears it, as long as no + /// newer one has replaced it. + private func keepAppleDevicesSighting(_ sighting: AppleDevicesSighting?) { + appleDevicesExpiry?.cancel() + appleDevicesExpiry = nil + guard let sighting, sighting.applies(now: now()) else { + appleDevicesUnreachable = nil + return + } + appleDevicesUnreachable = sighting + let wait = sighting.until.timeIntervalSince(now()) + appleDevicesExpiry = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(wait)) + guard !Task.isCancelled, let self, self.appleDevicesUnreachable == sighting else { return } + self.appleDevicesUnreachable = nil + } + } + + /// A suspended app's timer may not have fired on time: check the + /// sighting against the clock again, and wait out what is left of it. + func recheckAppleDevicesSighting() { + if let sighting = appleDevicesUnreachable { keepAppleDevicesSighting(sighting) } + } + /// Resume the session that backgrounding interrupted (opt-out toggle). func handleAppForegrounded() { endBackgroundGoodbye() isForeground = true + recheckAppleDevicesSighting() usbPaused = usbStoppedForNoPicture refreshUSBAvailability() guard resumeOnForeground else { return } diff --git a/ios/EternalMonitorTests/PCNoticeTests.swift b/ios/EternalMonitorTests/PCNoticeTests.swift index bd1c6616..ef994bd3 100644 --- a/ios/EternalMonitorTests/PCNoticeTests.swift +++ b/ios/EternalMonitorTests/PCNoticeTests.swift @@ -1,3 +1,4 @@ +import Combine import XCTest @testable import EternalMonitor @@ -61,13 +62,58 @@ final class PCNoticeTests: XCTestCase { manager.cancel() } - /// A sighting is one sample: it stops applying after a while. + /// A sighting is one sample: it applies from when it was seen until it + /// expires, and not under a clock set back. func testASightingGoesStale() { let at = Date(timeIntervalSince1970: 1_000_000) - let sighting = ConnectionManager.AppleDevicesSighting(target: "192.168.1.20:9876", at: at) + let until = at + ConnectionManager.AppleDevicesSighting.lifetime + let sighting = ConnectionManager.AppleDevicesSighting(target: "192.168.1.20:9876", at: at, until: until) XCTAssertTrue(sighting.applies(now: at)) - XCTAssertTrue(sighting.applies(now: at + ConnectionManager.AppleDevicesSighting.lifetime - 1)) - XCTAssertFalse(sighting.applies(now: at + ConnectionManager.AppleDevicesSighting.lifetime)) + XCTAssertTrue(sighting.applies(now: until - 1)) + XCTAssertFalse(sighting.applies(now: until)) XCTAssertFalse(sighting.applies(now: at - 60), "a clock set back") } + + /// The Connect screen may sit idle after the session: the published + /// hint goes by itself when it expires, with nothing else happening. + func testThePublishedHintExpiresByItself() async throws { + let seen = Date(timeIntervalSince1970: 2_000_000) + var timing = ConnectionManager.Timing() + timing.appleDevicesNoteSeconds = 0.5 + let manager = ConnectionManager(timing: timing) + manager.now = { seen } + addTeardownBlock { await MainActor.run { manager.cancel() } } + var published: [ConnectionManager.AppleDevicesSighting?] = [] + let watch = manager.$appleDevicesUnreachable.dropFirst().sink { published.append($0) } + defer { watch.cancel() } + + try await session(manager, FakeUDPHost(), caps: 0) + let sighting = try XCTUnwrap(manager.appleDevicesUnreachable) + XCTAssertEqual(sighting.at, seen) + XCTAssertEqual(sighting.until, seen + 0.5) + try await until("expired", seconds: 10) { manager.appleDevicesUnreachable == nil } + XCTAssertEqual(published.last, .some(nil), "the change was published: \(published)") + XCTAssertEqual(USBStatusText.describe("USB: waiting for a cable", + appleDevicesUnreachable: manager.appleDevicesUnreachable != nil), + USBStatusText.describe("USB: waiting for a cable")) + } + + /// Back in the foreground the sighting is checked against the clock, + /// since a suspended app's timer may have missed its time. + func testComingBackChecksTheClock() async throws { + var clock = Date(timeIntervalSince1970: 3_000_000) + let manager = ConnectionManager() + manager.now = { clock } + addTeardownBlock { await MainActor.run { manager.cancel() } } + try await session(manager, FakeUDPHost(), caps: 0) + let sighting = try XCTUnwrap(manager.appleDevicesUnreachable) + XCTAssertEqual(sighting.until, clock + ConnectionManager.AppleDevicesSighting.lifetime) + + clock = sighting.until - 60 + manager.recheckAppleDevicesSighting() + XCTAssertEqual(manager.appleDevicesUnreachable, sighting, "a minute left") + clock = sighting.until + manager.recheckAppleDevicesSighting() + XCTAssertNil(manager.appleDevicesUnreachable) + } } From 62b6942a48dbf54323eaf494bf5b1e77b7edb561 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 13:11:11 -0500 Subject: [PATCH 028/226] Run no Disable task at startup when Windows reports the display disabled The host ran the Disable task at every start, even with the virtual display already off. In the field that startup Disable was the task that timed out most often: 10 of 90 starts on 5 installs, against 1 of 55 later Disables and 2 of 79 Enables. Its instance then stayed in Task Scheduler for minutes, and every later Enable waited 10 s behind it and failed ("Previous VDD task timed out"), so Second screen fell back to the mirror for the whole session. The host now reads the device through Configuration Manager first. One that reports problem 22 (disabled) is off, and is recorded as off without running a task. Any other state, including a display left on by a crash, runs the Disable as before. The task runner's timeout messages now say whether the instance it gave up on was queued (never started) or in progress (the toggle did not finish). --- host/src/main.rs | 7 ++-- host/src/vdd.rs | 63 ++++++++++++++++++++++++++++++++++++ host/src/vdd_task.ps1 | 9 ++++-- scripts/win/Test-VddTask.ps1 | 17 +++++++--- 4 files changed, 85 insertions(+), 11 deletions(-) diff --git a/host/src/main.rs b/host/src/main.rs index 0dbd39cb..8ee7606e 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -98,9 +98,10 @@ fn main() -> Result<(), Box> { // Never leave a virtual display attached across runs. If a previous run crashed or was // force-killed while streaming the virtual extended display, the VDD device is still enabled - // and shows up as a phantom monitor. Disabling it unconditionally at startup guarantees the - // managed display only ever exists while EternalMonitor is actively using it. It runs beside - // startup: a slow or hung task used to hold a Mirror stream's first frame for up to 30 s. + // and shows up as a phantom monitor. Disabling it at startup, unless Windows already reports it + // disabled, guarantees the managed display only ever exists while EternalMonitor is actively + // using it. It runs beside startup: a slow or hung task used to hold a Mirror stream's first + // frame for up to 30 s. vdd::disable_in_background(); // Belt-and-suspenders: a panic anywhere (GUI, supervisor, pipeline threads) must still tear diff --git a/host/src/vdd.rs b/host/src/vdd.rs index 23e3cb38..d0f2b429 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -214,6 +214,17 @@ impl DriverTasks { self.gate.lock().health } + /// Before any request, with the device as Configuration Manager reports + /// it: one Windows reports disabled (problem 22) is off already, so no + /// Disable needs to run. True when so. + fn found_off(&self, device: crate::vdd_device::DeviceState) -> bool { + let off = device == crate::vdd_device::DeviceState::Problem(22); + if off { + self.gate.lock().known_enabled = Some(false); + } + off + } + #[cfg(test)] fn set_enabled( &self, @@ -499,7 +510,16 @@ pub fn disable() -> bool { /// runs on its own thread, and this returns once it holds the turn, so /// every later request (an Enable, the quit's Disable) still runs after it. /// A hung task no longer keeps a Mirror stream black for up to 30 s. +/// +/// A display Windows already reports disabled, as at almost every start, +/// runs no task. That Disable was the task that hung most often in the +/// field, and every later request waited behind it and failed. pub fn disable_in_background() { + #[cfg(windows)] + if TASKS.found_off(crate::vdd_device::query()) { + info!("Virtual display already off — not running the Disable task"); + return; + } let (held, turn) = std::sync::mpsc::channel(); let spawned = std::thread::Builder::new() .name("vdd-startup-off".into()) @@ -572,6 +592,49 @@ mod tests { assert_eq!(gate.before(false, Purpose::Attempt, now), Gate::Run); } + #[test] + fn a_display_windows_reports_disabled_at_startup_runs_no_disable_task() { + use crate::vdd_device::DeviceState; + use std::cell::Cell; + let tasks = DriverTasks::new(); + assert!(tasks.found_off(DeviceState::Problem(22))); + // Neither the startup cleanup nor a quit before any stream runs one. + assert!(tasks.set_enabled(false, Purpose::Cleanup, || {}, |_, _| { + panic!("ran the Disable task for a display that is off") + })); + // The first Enable runs, and so does the Disable after it. + let ran = Cell::new(Vec::new()); + let run = |enabled: bool, _: &mut dyn FnMut()| { + let mut calls = ran.take(); + calls.push(enabled); + ran.set(calls); + TaskOutcome::Done + }; + assert!(tasks.set_enabled(true, Purpose::Attempt, || {}, run)); + assert!(tasks.set_enabled(false, Purpose::Cleanup, || {}, run)); + assert_eq!(ran.take(), vec![true, false]); + + // Any other state may be a display left on, or is unknown: the + // Disable runs as before. + for device in [ + DeviceState::Ready, + DeviceState::NoMonitor, + DeviceState::NotStarted, + DeviceState::Problem(43), + DeviceState::NoDriver, + DeviceState::Missing, + ] { + let tasks = DriverTasks::new(); + assert!(!tasks.found_off(device), "{device:?}"); + let ran = Cell::new(false); + tasks.set_enabled(false, Purpose::Cleanup, || {}, |_, _| { + ran.set(true); + TaskOutcome::Done + }); + assert!(ran.get(), "no Disable for {device:?}"); + } + } + #[test] fn a_hung_task_is_not_waited_out_again_until_the_cooldown_ends() { let mut gate = TaskGate::new(); diff --git a/host/src/vdd_task.ps1 b/host/src/vdd_task.ps1 index 49acd03d..f9fffb6a 100644 --- a/host/src/vdd_task.ps1 +++ b/host/src/vdd_task.ps1 @@ -19,10 +19,13 @@ try { if ($_.Exception.GetBaseException().HResult -ne -2147024894) { throw } } } + # A timeout says whether the instance was queued (Task Scheduler never + # started it) or in progress (the toggle itself did not finish). + $stateNames = @{ 2 = 'queued'; 4 = 'in progress' } $deadline = (Get-Date).AddSeconds(10) foreach ($name in $waitFor) { - while ($service.GetFolder('\').GetTask($name).State -in @(2,4)) { - if ((Get-Date) -gt $deadline) { throw "Previous VDD task timed out: $name" } + while (($state = $service.GetFolder('\').GetTask($name).State) -in @(2,4)) { + if ((Get-Date) -gt $deadline) { throw "Previous VDD task timed out: $name ($($stateNames[[int]$state]))" } Start-Sleep -Milliseconds 50 } } @@ -34,7 +37,7 @@ try { if (!$running) { throw "VDD task did not start: $Task" } $deadline = (Get-Date).AddSeconds(15) while ($true) { - if ((Get-Date) -gt $deadline) { throw "VDD task timed out: $Task" } + if ((Get-Date) -gt $deadline) { throw "VDD task timed out: $Task ($($stateNames[[int]$running.State]))" } try { $running.Refresh() } catch { # SCHED_E_TASK_NOT_RUNNING means the recorded instance completed. if ($_.Exception.GetBaseException().HResult -eq -2147216629) { break } diff --git a/scripts/win/Test-VddTask.ps1 b/scripts/win/Test-VddTask.ps1 index 4d223a4a..1c3f8cef 100644 --- a/scripts/win/Test-VddTask.ps1 +++ b/scripts/win/Test-VddTask.ps1 @@ -6,6 +6,7 @@ $script = Get-Content -Raw (Join-Path $PSScriptRoot '..\..\host\src\vdd_task.ps1 $mock = @' # $EMBusy[name]: how many more state reads report that task as running. # $EMMissing: tasks that do not exist, as Task Scheduler reports them. +# $EMRunState: the state of the instance Run starts (3: it has finished). function New-Object { param([string]$ComObject, [Parameter(ValueFromRemainingArguments=$true)]$Rest) if ($ComObject -ne 'Schedule.Service') { throw "Unexpected COM object $ComObject" } @@ -29,7 +30,7 @@ function New-Object { param($arguments) $busy = ($global:EMBusy.Keys | Where-Object { $global:EMBusy[$_] -gt 0 }) -join ',' [Console]::Out.WriteLine("RUN $($this.Name) busy=[$busy]") - $running = [pscustomobject]@{ State = 3 } + $running = [pscustomobject]@{ State = $global:EMRunState } $running | Add-Member ScriptMethod Refresh {} $running } @@ -41,11 +42,11 @@ function New-Object { } '@ -function Invoke-Case([hashtable]$busy, [string]$arguments, [string[]]$missing = @()) { +function Invoke-Case([hashtable]$busy, [string]$arguments, [string[]]$missing = @(), [int]$runState = 3) { $file = Join-Path $env:TEMP ('em-vdd-task-test-' + [guid]::NewGuid().ToString('N') + '.ps1') $busyText = ($busy.GetEnumerator() | ForEach-Object { "'$($_.Key)' = $($_.Value)" }) -join '; ' $missingText = ($missing | ForEach-Object { "'$_'" }) -join ', ' - $body = "`$global:EMBusy = @{ $busyText }`n`$global:EMMissing = @($missingText)`n$mock`n& {`n$script`n} $arguments`n" + $body = "`$global:EMBusy = @{ $busyText }`n`$global:EMMissing = @($missingText)`n`$global:EMRunState = $runState`n$mock`n& {`n$script`n} $arguments`n" Set-Content -Path $file -Value $body -Encoding UTF8 # The child's stderr is expected output here, not a failure of this script. $ErrorActionPreference = 'Continue' @@ -70,13 +71,19 @@ if ($case.Code -ne 0 -or $case.Output -notmatch 'RUN Disable busy=\[\]') { $started = Get-Date $case = Invoke-Case @{ Enable = 1000000 } "-Task 'Disable' -After 'Enable'" $took = ((Get-Date) - $started).TotalSeconds -if ($case.Code -ne 1 -or $case.Output -notmatch 'Previous VDD task timed out: Enable' -or $case.Output -match 'RUN') { +if ($case.Code -ne 1 -or $case.Output -notmatch 'Previous VDD task timed out: Enable \(in progress\)' -or $case.Output -match 'RUN') { throw "A hung Enable was not reported as a timeout: $($case.Output)" } # The script's own wait is bounded at 10 s; a cold PowerShell start for the # child can take tens of seconds more on a busy PC. A wait that never ended # would take hours with this mock. if ($took -gt 120) { throw "The wait took $took s" } +# An instance Task Scheduler never starts reads as queued, so the host's log +# tells it apart from a toggle that hangs. +$case = Invoke-Case @{} "-Task 'Disable' -After 'Enable'" @() 2 +if ($case.Code -ne 1 -or $case.Output -notmatch 'VDD task timed out: Disable \(queued\)') { + throw "A queued instance was not reported as one: $($case.Output)" +} # Without -After, as older hosts call it, only this task is waited for. $case = Invoke-Case @{ Enable = 1000000 } "-Task 'Disable'" if ($case.Code -ne 0 -or $case.Output -notmatch 'RUN Disable') { @@ -92,7 +99,7 @@ $case = Invoke-Case @{} "-Task 'Disable' -After 'Enable'" @('Disable') if ($case.Code -ne 1 -or $case.Output -notmatch '0x80070002' -or $case.Output -match 'RUN') { throw "A missing requested task was not reported: $($case.Output)" } -Write-Output 'VDD task checks passed: waits for the other direction, earlier runs, hung Enable bound, no -After, missing tasks' +Write-Output 'VDD task checks passed: waits for the other direction, earlier runs, hung Enable bound, queued instance, no -After, missing tasks' # Some cases end their child with exit code 1 on purpose. Without this the # last one stays in $LASTEXITCODE, and a CI step that ends with it fails. # Every failure above throws, which exits 1. From e7c0ee7ef735c9fe4c6957fc2667fa77d0c3d688 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 13:11:11 -0500 Subject: [PATCH 029/226] Try the second screen again once the desktop is back after it failed while away A second screen that showed no picture while the PC was locked, a UAC prompt was up, or the displays were off is excused and not saved. Its viewer still stayed on the mirror for the rest of the app run, after the user came back, until they clicked Try again. Now, when the desktop was away at the failure, a connected viewer gets the second screen again once it is back. Windows is asked at most once a second. A failure with the desktop there, such as a display that keeps dropping out, is not retried by itself, so it can't turn the display on and off in a loop. --- host/src/capture/second_screen.rs | 111 +++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 2 deletions(-) diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 785d4fdd..401c79b3 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -154,6 +154,10 @@ struct Proving { /// a secure screen or with the displays off don't count. const LOSS_LIMIT: u32 = 8; +/// How often [`State::retry_if_back`] asks Windows whether the desktop is +/// back. +const DESKTOP_CHECK: Duration = Duration::from_secs(1); + #[derive(Debug, Default)] pub(crate) struct State { /// The display attached; the stream has not moved onto it yet. @@ -180,6 +184,10 @@ pub(crate) struct State { /// `blocked_retry_at` when that is set. retry_blocked: bool, blocked_retry_at: Option, + /// The last failure came while the user's desktop was away (see + /// [`desktop_away`]): once it is back, a connected viewer gets the + /// second screen again. When to look at the desktop next. + retry_when_back: Option, /// The graphics cards and the screens' wiring, as last read off the /// transport loop: see [`facts`]. facts: facts::Cache, @@ -223,6 +231,7 @@ impl State { if let Some(reason) = self.fail_unproven(status, now, blocked) { return Step::Fallback(reason); } + self.retry_if_back(status, connected, now); match *status { VddStatus::WaitingForClient if connected => { let hold = if blocked { @@ -332,13 +341,39 @@ impl State { Some(reason) } + /// After a failure while the desktop was away, try the second screen + /// again for a connected viewer once the desktop is back. Without this + /// the app run stayed on the mirror after the PC was unlocked or its + /// displays woke, until the user clicked Try again. Windows is asked at + /// most once a second: this runs on the transport loop. + fn retry_if_back(&mut self, status: &mut VddStatus, connected: bool, now: Instant) { + let due = self.retry_when_back.is_some_and(|at| now >= at); + if !due || !connected || *status != VddStatus::Failed { + return; + } + if self.desktop_away() { + self.retry_when_back = Some(now + DESKTOP_CHECK); + return; + } + self.retry_when_back = None; + tracing::info!( + reason = self.failure.unwrap_or("no_frames"), + "The desktop is back after the second screen failed while it was away — trying \ + the second screen again" + ); + *status = VddStatus::WaitingForClient; + self.failure = None; + } + /// Save the failure for later connections and app runs, unless /// something that passes may explain it: then only this app run keeps - /// mirroring, as before. Nor while the mirror is `blocked`, as + /// mirroring, as before, or, when the desktop is away now, until it is + /// back. Nor while the mirror is `blocked`, as /// [`mirror_blocked_now`] read it for this failure: then the next /// connection tries the second screen again. fn remember(&mut self, reason: &'static str, now: Instant, interrupted: bool, blocked: bool) { self.clear_blocked_retry(); + self.retry_when_back = None; if blocked { tracing::info!( reason, @@ -348,13 +383,18 @@ impl State { self.retry_blocked = true; return; } - if interrupted || self.desktop_away() { + let away = self.desktop_away(); + if interrupted || away { tracing::info!( reason, + away, "The second screen failed while Windows withheld the desktop or the display may \ have been asleep — not remembering it beyond this app run" ); self.memory.excuse(); + if away { + self.retry_when_back = Some(now); + } return; } if let Some(hold) = self.memory.note_failure(reason, now) { @@ -529,7 +569,15 @@ pub(crate) fn note_connected(shared: &SharedControl, now: Instant) { } return; } + let retrying = *status == VddStatus::Failed && state.retry_when_back.is_some(); drop((state, status)); + if retrying { + tracing::info!( + "Client connected — the second screen failed while the desktop was away, and is \ + tried again once the desktop is back" + ); + return; + } if let Some(reason) = remembered_failure(shared) { tracing::info!( second_screen_fallback = reason, @@ -2292,6 +2340,65 @@ mod tests { ); } + /// The PC was locked, or its displays went off, while the second screen + /// proved itself. The viewer stayed on the mirror for the rest of the app + /// run, even after the user came back. + #[test] + fn a_second_screen_that_failed_while_the_desktop_was_away_is_tried_again_once_it_is_back() { + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + let shared = second_screen_selected(); + let away = Arc::new(AtomicBool::new(false)); + shared.second_screen.lock().fake_desktop_away = Some(away.clone()); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + let start = Instant::now(); + assert_eq!(next_step(&shared, start), Step::Switch); + start_proving(&shared, 7, start); + away.store(true, Ordering::Relaxed); + assert_eq!( + next_step(&shared, start + PROVE_TIMEOUT), + Step::Fallback("no_frames") + ); + assert!(!shared.second_screen.lock().memory.saved()); + // Still away: the mirror stays, and the driver is left alone. + let mut elapsed = PROVE_TIMEOUT; + while elapsed < secs(60.0) { + elapsed += secs(0.25); + assert_eq!(next_step(&shared, start + elapsed), Step::Nothing); + } + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + away.store(false, Ordering::Relaxed); + let back = start + elapsed + DESKTOP_CHECK; + assert_eq!(next_step(&shared, back), Step::Attach); + assert_eq!(remembered_failure(&shared), None); + // Once only: the next failure decides afresh. + assert!(shared.second_screen.lock().retry_when_back.is_none()); + } + + /// A proof that lost the desktop while the user was there is excused + /// too, but tried again only at Try again: retrying at once could turn + /// a display that keeps dropping out on and off forever. + #[test] + fn an_interrupted_proof_with_the_desktop_there_is_not_tried_again_by_itself() { + let shared = second_screen_selected(); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + let start = Instant::now(); + assert_eq!(next_step(&shared, start), Step::Switch); + start_proving(&shared, 7, start); + note_interruption_while(&shared, 7, false); + assert_eq!( + next_step(&shared, start + PROVE_TIMEOUT), + Step::Fallback("no_frames") + ); + for second in 1..60 { + let now = start + PROVE_TIMEOUT + Duration::from_secs(second); + assert_eq!(next_step(&shared, now), Step::Nothing); + } + assert_eq!(remembered_failure(&shared), Some("no_frames")); + } + #[test] fn a_duplication_error_on_the_second_screen_goes_back_to_the_mirror_at_once() { let shared = second_screen_selected(); From 8ebc2549ced50bfda4c1ef964f6279d1dbf13613 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 13:23:26 -0500 Subject: [PATCH 030/226] Check that a moved PC is the saved one at the address it was found - The PC's identity answer now carries the port and the home network addresses it receives the stream on, and the proof covers them. The iPad reconnects only when the address it asked at is one of them. - The PC answers at most 8 identity checks per address and 32 in all every 10 s, over the network only, before any proof is made. - The iPad looks for a moved PC only over a direct Wi-Fi or Ethernet path, with the PC in that network's subnet. - The moved-PC e2e picks free ports, or checks the ones it is given. --- host/src/network.rs | 147 +++++++++ host/src/pairing.rs | 45 +-- host/src/transport/audio.rs | 1 + host/src/transport/mod.rs | 28 ++ host/src/transport/session.rs | 284 ++++++++++++++++-- .../App/ConnectionManager.swift | 16 +- ios/EternalMonitor/App/MovedPC.swift | 114 ++++++- ios/EternalMonitor/Network/WireProtocol.swift | 45 ++- ios/EternalMonitorTests/FakeUDPHost.swift | 36 ++- ios/EternalMonitorTests/MovedPCTests.swift | 88 +++++- .../WireProtocolGoldenTests.swift | 14 +- proto/src/v2/control.rs | 76 ++++- proto/testdata/v2_vectors.txt | 4 +- proto/tests/golden.rs | 35 ++- scripts/e2e_moved_pc.sh | 27 +- 15 files changed, 859 insertions(+), 101 deletions(-) diff --git a/host/src/network.rs b/host/src/network.rs index bf01b624..38fd856d 100644 --- a/host/src/network.rs +++ b/host/src/network.rs @@ -368,6 +368,107 @@ impl Drop for Listing { } } +/// The home network addresses among `adapters`: IPv4 on Wi-Fi and Ethernet +/// adapters that are up, in a private range or link-local, link-local last. +/// An iPad checking that it found this PC at an address gets these signed. +pub fn home_addresses_of(adapters: &[Adapter]) -> Vec { + let mut addresses: Vec = adapters + .iter() + .filter(|adapter| adapter.up && matches!(adapter.kind, Kind::Wifi | Kind::Ethernet)) + .flat_map(|adapter| adapter.ipv4.iter().copied()) + .filter(|ip| ip.is_private() || ip.is_link_local()) + .collect(); + addresses.sort_by_key(|ip| ip.is_link_local()); + let mut seen = Vec::with_capacity(addresses.len()); + addresses.retain(|ip| { + !seen.contains(ip) && { + seen.push(*ip); + true + } + }); + addresses +} + +/// How long one listing of the home network addresses answers. +const HOME_TTL: Duration = Duration::from_secs(2); + +/// This PC's home network addresses now (see [`home_addresses_of`]), listed +/// at most every 2 s. +pub fn home_addresses() -> Vec { + static LISTED: Mutex)>> = Mutex::new(None); + let mut listed_at = LISTED.lock(); + if let Some((_, addresses)) = listed_at.as_ref().filter(|(at, _)| at.elapsed() < HOME_TTL) { + return addresses.clone(); + } + let addresses = listed() + .or_else(interfaces) + .map(|adapters| home_addresses_of(&adapters)) + .unwrap_or_default(); + *listed_at = Some((Instant::now(), addresses.clone())); + addresses +} + +/// What kind of interface a Unix name and flags say: network cards are +/// named en*, eth* or wl*; point-to-point interfaces are tunnels. +#[cfg(unix)] +fn interface_kind(name: &str, flags: libc::c_int) -> Kind { + if flags & libc::IFF_LOOPBACK != 0 { + Kind::Virtual + } else if flags & libc::IFF_POINTOPOINT != 0 { + Kind::Vpn + } else if ["en", "eth", "wl"] + .iter() + .any(|prefix| name.starts_with(prefix)) + { + Kind::Ethernet + } else { + Kind::Other + } +} + +/// Unix interfaces with their IPv4 addresses, from `getifaddrs`. +#[cfg(unix)] +fn interfaces() -> Option> { + let mut list: *mut libc::ifaddrs = std::ptr::null_mut(); + if unsafe { libc::getifaddrs(&mut list) } != 0 { + return None; + } + let mut out: Vec = Vec::new(); + let mut next = list; + while !next.is_null() { + let entry = unsafe { &*next }; + next = entry.ifa_next; + if entry.ifa_addr.is_null() + || i32::from(unsafe { (*entry.ifa_addr).sa_family }) != libc::AF_INET + { + continue; + } + let sin = unsafe { &*entry.ifa_addr.cast::() }; + let ip = Ipv4Addr::from(u32::from_be(sin.sin_addr.s_addr)); + let name = unsafe { std::ffi::CStr::from_ptr(entry.ifa_name) } + .to_string_lossy() + .into_owned(); + let flags = entry.ifa_flags as libc::c_int; + match out.iter_mut().find(|adapter| adapter.name == name) { + Some(adapter) => adapter.ipv4.push(ip), + None => out.push(Adapter { + kind: interface_kind(&name, flags), + name, + up: flags & libc::IFF_UP != 0 && flags & libc::IFF_RUNNING != 0, + gateway: false, + ipv4: vec![ip], + }), + } + } + unsafe { libc::freeifaddrs(list) }; + Some(out) +} + +#[cfg(not(unix))] +fn interfaces() -> Option> { + None +} + /// The local address the OS would use toward a public address. pub fn default_route() -> Option { std::net::UdpSocket::bind("0.0.0.0:0") @@ -939,6 +1040,52 @@ mod tests { /// Windows says the list grew every time: three bigger buffers, then no /// list, so no hint. + /// What an identity proof signs: home network addresses on network + /// cards that are up, never a VPN's or a virtual switch's, even in a + /// private range. + #[test] + fn home_addresses_are_on_network_cards_that_are_up() { + let mut down = adapter(Kind::Ethernet, true, [192, 168, 7, 2]); + down.up = false; + let adapters = [ + adapter(Kind::Vpn, false, [10, 8, 0, 2]), + adapter(Kind::Virtual, false, [172, 20, 0, 1]), + adapter(Kind::Wifi, true, [169, 254, 3, 4]), + down, + adapter(Kind::Other, true, [192, 168, 9, 9]), + Adapter { + ipv4: vec![ + ip([192, 168, 1, 20]), + ip([203, 0, 113, 5]), + ip([192, 168, 1, 20]), + ], + ..adapter(Kind::Ethernet, true, [0, 0, 0, 0]) + }, + adapter(Kind::Wifi, true, [10, 0, 0, 31]), + ]; + assert_eq!( + home_addresses_of(&adapters), + [ + ip([192, 168, 1, 20]), + ip([10, 0, 0, 31]), + ip([169, 254, 3, 4]) + ] + ); + assert!(home_addresses_of(&[]).is_empty()); + } + + #[cfg(unix)] + #[test] + fn unix_interfaces_are_told_apart_by_flags_then_name() { + assert_eq!(interface_kind("en0", libc::IFF_UP), Kind::Ethernet); + assert_eq!(interface_kind("wlan0", libc::IFF_UP), Kind::Ethernet); + assert_eq!(interface_kind("utun3", libc::IFF_POINTOPOINT), Kind::Vpn); + assert_eq!(interface_kind("en5", libc::IFF_POINTOPOINT), Kind::Vpn); + assert_eq!(interface_kind("lo0", libc::IFF_LOOPBACK), Kind::Virtual); + assert_eq!(interface_kind("bridge100", libc::IFF_UP), Kind::Other); + assert_eq!(interface_kind("docker0", libc::IFF_UP), Kind::Other); + } + #[test] fn listing_gives_up_after_three_bigger_buffers() { let warned = AtomicBool::new(false); diff --git a/host/src/pairing.rs b/host/src/pairing.rs index 757b870f..85110ec3 100644 --- a/host/src/pairing.rs +++ b/host/src/pairing.rs @@ -41,13 +41,9 @@ pub fn parse_token(text: &str) -> Option<[u8; 16]> { (token != [0; 16]).then_some(token) } -fn identity_proof(token: &[u8; 16], nonce: &[u8; 16]) -> [u8; 32] { +fn identity_proof(token: &[u8; 16], message: &[u8]) -> [u8; 32] { let key = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, token); - let mut context = ring::hmac::Context::with_key(&key); - context.update(nonce); - context.update(eternal_wire::v2::control::IDENTIFY_CONTEXT); - context - .sign() + ring::hmac::sign(&key, message) .as_ref() .try_into() .expect("SHA-256 is 32 bytes") @@ -153,11 +149,16 @@ impl Pairing { Ok(()) } - /// The answer to an iPad checking that this is the PC it paired with: - /// HMAC-SHA256 keyed with the pairing token over `nonce ‖ context`. Only - /// a PC holding the token can make it, and the token never leaves. - pub fn identity_proof(&self, nonce: &[u8; 16]) -> [u8; 32] { - identity_proof(&self.token, nonce) + /// The answer to an iPad checking that this is the PC it paired with, at + /// the address it asked: HMAC-SHA256 keyed with the pairing token over + /// the nonce, the port and the addresses this PC receives the stream on + /// (`identity_message`). Only a PC holding the token can make it, and the + /// token never leaves. + pub fn identity_proof(&self, nonce: &[u8; 16], port: u16, addresses: &[[u8; 4]]) -> [u8; 32] { + identity_proof( + &self.token, + &eternal_wire::v2::control::identity_message(nonce, port, addresses), + ) } pub fn regenerate_token(&mut self) -> Result<(), getrandom::Error> { @@ -380,19 +381,29 @@ mod tests { let token = std::array::from_fn(|index| index as u8 + 1); let nonce = std::array::from_fn(|index| 0xA0 + index as u8); let pairing = Pairing::new(true, token).unwrap(); - let hex: String = pairing - .identity_proof(&nonce) + let addresses = [[192, 0, 2, 10], [198, 51, 100, 10]]; + let proof = |nonce: &[u8; 16], port, addresses: &[[u8; 4]]| { + pairing.identity_proof(nonce, port, addresses) + }; + let hex: String = proof(&nonce, 9876, &addresses) .iter() .map(|byte| format!("{byte:02x}")) .collect(); assert_eq!( hex, - "126dc93467af3b0621c4f9b420e0d323143259d743f85703b9fec83989eba353" + "18fb81d76bbb96e91101119775ad9766b0c332b2c49441e6f07ec33be1c6d94e" ); - // Another token, or another nonce, gives another proof. + // Another token, nonce, port or address gives another proof. let other = Pairing::new(true, [9; 16]).unwrap(); - assert_ne!(other.identity_proof(&nonce), pairing.identity_proof(&nonce)); - assert_ne!(pairing.identity_proof(&[0; 16]), pairing.identity_proof(&nonce)); + let signed = proof(&nonce, 9876, &addresses); + assert_ne!(other.identity_proof(&nonce, 9876, &addresses), signed); + assert_ne!(proof(&[0; 16], 9876, &addresses), signed); + assert_ne!(proof(&nonce, 9877, &addresses), signed); + assert_ne!(proof(&nonce, 9876, &addresses[..1]), signed); + assert_ne!( + proof(&nonce, 9876, &[[192, 0, 2, 10], [198, 51, 100, 11]]), + signed + ); } fn guess(nonce: u32) -> FailedRequest { diff --git a/host/src/transport/audio.rs b/host/src/transport/audio.rs index f765f853..b3d39a7e 100644 --- a/host/src/transport/audio.rs +++ b/host/src/transport/audio.rs @@ -136,6 +136,7 @@ mod tests { shared: &shared, stream_epoch: 7, pen_available: false, + listen: "127.0.0.1:0".parse().unwrap(), }; let peer = PeerId::udp("127.0.0.1:19891".parse().unwrap()); shared.session.lock().handle_control( diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 70cfb197..58331b8a 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -83,6 +83,8 @@ struct SharedConfigSource<'a> { shared: &'a SharedControl, stream_epoch: u32, pen_available: bool, + /// Where the socket receives the stream. + listen: SocketAddr, } impl ConfigSource for SharedConfigSource<'_> { @@ -115,6 +117,23 @@ impl ConfigSource for SharedConfigSource<'_> { } } + /// A socket on one address signs that one; one on every address signs + /// the PC's home network addresses. + fn identity_endpoints(&self) -> Option<(u16, Vec<[u8; 4]>)> { + let std::net::IpAddr::V4(ip) = self.listen.ip() else { + return None; + }; + let addresses = if ip.is_unspecified() { + crate::network::home_addresses() + } else { + vec![ip] + }; + Some(( + self.listen.port(), + addresses.iter().map(|ip| ip.octets()).collect(), + )) + } + fn host_caps(&self) -> u16 { // `start_sender` numbers sent frames contiguously per epoch. let numbering = eternal_wire::v2::control::HOSTCAP_CONTIGUOUS_FRAME_SEQ; @@ -225,6 +244,7 @@ pub async fn start_sender( shared: &shared, stream_epoch, pen_available: input_relay.pen_available(), + listen: local_addr, }; let abr_enabled = !std::env::var("ETERNAL_ABR").is_ok_and(|v| v.trim() == "0"); @@ -952,6 +972,7 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let peer = PeerId::udp("127.0.0.1:19893".parse().unwrap()); let hello = ControlMessage::Hello2(Hello2 { @@ -1016,6 +1037,7 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let hello = ControlMessage::Hello2(Hello2 { proto_min: 2, @@ -1066,6 +1088,7 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); @@ -1161,6 +1184,7 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); @@ -1267,6 +1291,7 @@ mod tests { shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let hello = ControlMessage::Hello2(Hello2 { proto_min: 2, @@ -1342,6 +1367,7 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); @@ -1405,6 +1431,7 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let held = shared .session @@ -1442,6 +1469,7 @@ mod tests { shared: &shared, stream_epoch: 1, pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), }; let expired = shared .session diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 7704e6a8..f3a99ce7 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -53,6 +53,50 @@ const REFUSED_FOR: Duration = Duration::from_secs(60); const REFUSED_SUMMARIES: u32 = 20; const REFUSAL_MINUTE: Duration = Duration::from_secs(60); +/// IDENTIFY answers per window: for one asking address, and in all. An +/// iPad's check sends at most 8 and stops at the first answer. +const IDENTIFY_WINDOW: Duration = Duration::from_secs(10); +const IDENTIFY_PER_SOURCE: u32 = 8; +const IDENTIFY_OVERALL: u32 = 32; +/// Asking addresses tracked in a window; more get no answer until it ends. +const IDENTIFY_SOURCES: usize = 16; + +/// Bounds what IDENTIFY costs: each answer is an HMAC, an address listing +/// and a datagram, and anyone on the network may ask. +#[derive(Debug, Default)] +struct IdentifyLimiter { + overall: Option<(Instant, u32)>, + sources: Vec<(std::net::IpAddr, Instant, u32)>, +} + +impl IdentifyLimiter { + /// Whether `ip` may be answered now, counting the answer if so. + fn allow(&mut self, ip: std::net::IpAddr, now: Instant) -> bool { + let over = |since: Instant| now.saturating_duration_since(since) >= IDENTIFY_WINDOW; + self.overall.take_if(|(since, _)| over(*since)); + self.sources.retain(|(_, since, _)| !over(*since)); + let overall = self.overall.get_or_insert((now, 0)); + if overall.1 >= IDENTIFY_OVERALL { + return false; + } + let index = match self.sources.iter().position(|(source, ..)| *source == ip) { + Some(index) => index, + None if self.sources.len() < IDENTIFY_SOURCES => { + self.sources.push((ip, now, 0)); + self.sources.len() - 1 + } + None => return false, + }; + let source = &mut self.sources[index]; + if source.2 >= IDENTIFY_PER_SOURCE { + return false; + } + source.2 += 1; + overall.1 += 1; + true + } +} + /// The refused attempts of the current minute. #[derive(Debug, Clone, Copy)] struct RefusalMinute { @@ -288,6 +332,11 @@ pub trait ConfigSource { fn host_caps(&self) -> u16 { 0 } + /// The port this host receives the stream on and its home network + /// addresses, which an identity proof signs. None answers no IDENTIFY. + fn identity_endpoints(&self) -> Option<(u16, Vec<[u8; 4]>)> { + None + } } pub struct Session { @@ -307,6 +356,7 @@ pub struct Session { /// Recently refused HELLO2 nonces: a retransmit is the same attempt. refused: std::collections::VecDeque<(u32, Instant)>, refusal_minute: Option, + identify_limiter: IdentifyLimiter, /// Injected so tests control randomness; production passes a seeded value /// derived from process entropy. next_session_id: u32, @@ -324,6 +374,7 @@ impl Session { ended_for_liveness: None, refused: std::collections::VecDeque::new(), refusal_minute: None, + identify_limiter: IdentifyLimiter::default(), next_session_id: session_id_seed.max(1), legacy_client_warned: false, } @@ -532,7 +583,10 @@ impl Session { // INPUT_EVENT moved the mouse and clicked, a stray RECEIVER_REPORT // steered the bitrate, and a late BYE from a superseded session tore // down the one that replaced it. - if !matches!(message, ControlMessage::Hello2(_) | ControlMessage::Identify(_)) { + if !matches!( + message, + ControlMessage::Hello2(_) | ControlMessage::Identify(_) + ) { match self.active.as_ref() { Some(session) if session.session_id == header_session_id && !session.awaiting_takeover => {} @@ -542,7 +596,7 @@ impl Session { } match message { ControlMessage::Hello2(hello) => self.handle_hello(source, hello, config, now), - ControlMessage::Identify(request) => Self::answer_identify(source, request, config), + ControlMessage::Identify(request) => self.answer_identify(source, request, config, now), ControlMessage::Bye(reason) => self.handle_bye(source, reason, now), ControlMessage::KeyframeRequest(request) => self.handle_keyframe(source, request, now), ControlMessage::ReceiverReport(report) => self.handle_report(source, report, now), @@ -852,25 +906,44 @@ impl Session { actions } - /// An iPad checking that this is the PC it paired with, before it - /// reconnects at a new address. Stateless: it starts no session and - /// works while another device streams. + /// An iPad checking that this is the PC it paired with, at the address + /// it asked, before it reconnects there. It starts no session and works + /// while another device streams. Only Wi-Fi asks, within + /// [`IdentifyLimiter`]'s bounds. fn answer_identify( + &mut self, source: PeerId, request: eternal_wire::v2::control::Identify, config: &impl ConfigSource, + now: Instant, ) -> Actions { let mut actions = Actions::default(); + let (LinkId::Udp, Some(asker)) = (source.link, source.addr) else { + return actions; + }; let Some(pairing) = config.pairing() else { return actions; }; - if request.version != eternal_wire::v2::control::IDENTIFY_VERSION { + if request.version != eternal_wire::v2::control::IDENTIFY_VERSION + || !self.identify_limiter.allow(asker.ip(), now) + { return actions; } + let Some((port, addresses)) = config.identity_endpoints() else { + return actions; + }; + let addresses: Vec<[u8; 4]> = addresses + .into_iter() + .take(eternal_wire::v2::control::IDENTITY_MAX_ADDRESSES) + .collect(); let identity = ControlMessage::Identity(eternal_wire::v2::control::Identity { version: eternal_wire::v2::control::IDENTIFY_VERSION, nonce: request.nonce, - proof: pairing.lock().identity_proof(&request.nonce), + port, + proof: pairing + .lock() + .identity_proof(&request.nonce, port, &addresses), + addresses, }); actions.replies.push(( source, @@ -2296,11 +2369,47 @@ mod tests { ControlMessage::Identify(eternal_wire::v2::control::Identify { version, nonce }) } - /// The PC proves it holds its pairing token, with pairing on or off, - /// without starting a session, and while another iPad streams. + /// A paired host on port 9876 with the home network addresses given. + struct IdentifyConfig(PairConfig, Vec<[u8; 4]>); + impl ConfigSource for IdentifyConfig { + fn stream_config(&self) -> StreamConfig { + TestConfig.stream_config() + } + fn host_name(&self) -> String { + TestConfig.host_name() + } + fn pairing(&self) -> Option<&parking_lot::Mutex> { + self.0.pairing() + } + fn identity_endpoints(&self) -> Option<(u16, Vec<[u8; 4]>)> { + Some((9876, self.1.clone())) + } + } + + fn identify_config(required: bool) -> IdentifyConfig { + IdentifyConfig( + pair_config(required), + vec![[192, 0, 2, 1], [198, 51, 100, 1]], + ) + } + + fn answered( + session: &mut Session, + asker: PeerId, + config: &impl ConfigSource, + now: Instant, + ) -> bool { + !session + .handle_control(asker, 0, identify(1, [3; 16]), config, now) + .replies + .is_empty() + } + + /// The PC proves it holds its pairing token at its own port and + /// addresses, with pairing on or off, without starting a session. #[test] fn identify_is_answered_with_the_token_proof_and_starts_nothing() { - let config = pair_config(true); + let config = identify_config(true); let mut session = Session::new(100); let now = Instant::now(); let asker = addr([192, 0, 2, 9], 50100); @@ -2308,32 +2417,159 @@ mod tests { let identity = identity_reply(&actions); assert_eq!(actions.replies[0].0, asker); assert_eq!(identity.nonce, [3; 16]); - assert_eq!(identity.proof, config.0.lock().identity_proof(&[3; 16])); - assert_ne!(identity.proof, crate::pairing::Pairing::new(true, [8; 16]).unwrap().identity_proof(&[3; 16])); + assert_eq!(identity.port, 9876); + assert_eq!(identity.addresses, config.1); + let proof = + |pairing: &crate::pairing::Pairing| pairing.identity_proof(&[3; 16], 9876, &config.1); + assert_eq!(identity.proof, proof(&config.0 .0.lock())); + assert_ne!( + identity.proof, + proof(&crate::pairing::Pairing::new(true, [8; 16]).unwrap()) + ); assert!(session.session_id().is_none()); assert!(actions.new_target.is_none()); - let streaming = addr([192, 0, 2, 5], 50000); - session.handle_control(streaming, 0, ControlMessage::Hello2(pair_hello(1)), &pair_config(false), now); - let live = session.session_id(); - assert!(live.is_some()); - let actions = session.handle_control(asker, 0, identify(1, [4; 16]), &pair_config(false), now); - assert_eq!(identity_reply(&actions).proof, config.0.lock().identity_proof(&[4; 16])); - assert_eq!(session.session_id(), live, "the stream goes on"); - assert!(actions.new_target.is_none() && !actions.client_lost); + // At most four addresses go out, and the proof signs those four. + let many = IdentifyConfig( + pair_config(false), + (1..=6).map(|last| [192, 0, 2, last]).collect(), + ); + let identity = + identity_reply(&session.handle_control(asker, 0, identify(1, [3; 16]), &many, now)); + assert_eq!(identity.addresses, many.1[..4]); + assert_eq!( + identity.proof, + many.0 + .0 + .lock() + .identity_proof(&[3; 16], 9876, &many.1[..4]) + ); } /// A layout this PC doesn't know gets no answer, as from a PC before - /// 0.3.5; so does a host without pairing state. + /// 0.3.5; so does a host without pairing state or addresses to sign, + /// and a request over a cable. #[test] fn identify_of_another_version_is_not_answered() { let mut session = Session::new(100); let now = Instant::now(); let asker = addr([192, 0, 2, 9], 50100); - let actions = session.handle_control(asker, 0, identify(2, [3; 16]), &pair_config(true), now); - assert!(actions.replies.is_empty()); - let actions = session.handle_control(asker, 0, identify(1, [3; 16]), &TestConfig, now); + let actions = + session.handle_control(asker, 0, identify(2, [3; 16]), &identify_config(true), now); assert!(actions.replies.is_empty()); + assert!(!answered(&mut session, asker, &TestConfig, now)); + assert!(!answered(&mut session, asker, &pair_config(true), now)); + assert!(!answered( + &mut session, + PeerId::usb(1), + &identify_config(true), + now + )); + assert!(answered(&mut session, asker, &identify_config(true), now)); + } + + /// A burst from one address gets 8 answers in 10 s, and the PC 32 in + /// all from at most 16 addresses; each count starts over when its + /// window ends. + #[test] + fn identify_answers_are_bounded_per_asker_and_in_all() { + let config = identify_config(true); + let mut session = Session::new(100); + let start = Instant::now(); + let asker = addr([192, 0, 2, 9], 50100); + let burst = (0..20) + .filter(|_| answered(&mut session, asker, &config, start)) + .count(); + assert_eq!(burst, IDENTIFY_PER_SOURCE as usize); + // Another port on the same address shares its count. + assert!(!answered( + &mut session, + addr([192, 0, 2, 9], 50101), + &config, + start + )); + // Others still get theirs, up to the PC's 32. + let others: usize = (10..13) + .map(|last| { + (0..20) + .filter(|_| { + answered(&mut session, addr([192, 0, 2, last], 50100), &config, start) + }) + .count() + }) + .sum(); + assert_eq!(others, (IDENTIFY_OVERALL - IDENTIFY_PER_SOURCE) as usize); + assert!(!answered( + &mut session, + addr([192, 0, 2, 40], 50100), + &config, + start + )); + + let later = start + IDENTIFY_WINDOW - Duration::from_millis(1); + assert!(!answered(&mut session, asker, &config, later)); + let next = start + IDENTIFY_WINDOW; + assert!(answered(&mut session, asker, &config, next)); + assert!(answered( + &mut session, + addr([192, 0, 2, 40], 50100), + &config, + next + )); + + // Past 16 addresses in a window, a new one waits for the next. + let mut session = Session::new(100); + for last in 0..IDENTIFY_SOURCES as u8 { + assert!(answered( + &mut session, + addr([198, 51, 100, last], 50100), + &config, + start + )); + } + assert!(!answered(&mut session, asker, &config, start)); + assert!(answered(&mut session, asker, &config, next)); + } + + /// While another iPad streams, IDENTIFY is answered within the same + /// bounds, and the stream goes on whatever arrives. + #[test] + fn identify_while_streaming_is_bounded_and_leaves_the_stream() { + let mut session = Session::new(100); + let now = Instant::now(); + let streaming = addr([192, 0, 2, 5], 50000); + session.handle_control( + streaming, + 0, + ControlMessage::Hello2(pair_hello(1)), + &pair_config(false), + now, + ); + let live = session.session_id(); + assert!(live.is_some()); + + let config = identify_config(false); + let asker = addr([192, 0, 2, 9], 50100); + let mut answers = 0; + for index in 0..40u8 { + let actions = session.handle_control(asker, 0, identify(1, [index; 16]), &config, now); + assert!( + actions.new_target.is_none() && !actions.client_lost && actions.goodbye.is_none() + ); + if !actions.replies.is_empty() { + let identity = identity_reply(&actions); + assert_eq!(identity.nonce, [index; 16]); + assert_eq!(actions.replies[0].0, asker, "never to the streaming iPad"); + answers += 1; + } + } + assert_eq!(answers, IDENTIFY_PER_SOURCE); + // The streaming iPad's own IDENTIFY counts on its own address. + assert!(answered(&mut session, streaming, &config, now)); + assert_eq!(session.session_id(), live, "the stream goes on"); + let actions = session.tick(&config, true, now + Duration::from_millis(100)); + assert!(!actions.client_lost); + assert_eq!(session.session_id(), live); } fn pair_config(required: bool) -> PairConfig { diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 2c677f84..42c8a35b 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -224,6 +224,8 @@ final class ConnectionManager: ObservableObject { /// Which addresses a saved PC may move from and to. Tests on loopback /// widen it. var homeAddress: (String) -> Bool = MovedPC.isHomeAddress + /// How this iPad reaches an address. Tests replace it. + var localRoute: (String, UInt16) async -> MovedPC.Route? = { await MovedPC.route(to: $0, port: $1) } /// "host:port" of each address picked from Bonjour's list in this /// launch. Saved entries keep the mark once they show a picture. private var foundAddresses = Set() @@ -1056,8 +1058,9 @@ final class ConnectionManager: ObservableObject { /// The attempt's HELLO2 got no answer at `host:port`. A paired PC that /// this iPad found with Bonjour is looked for by name. Where Bonjour has - /// it at another home network address, the iPad checks that it is the - /// same PC, and only then moves the attempt there with its pairing. + /// it at another home network address that this iPad reaches directly, + /// the iPad checks that it is the same PC, at that address, and only + /// then moves the attempt there with its pairing. /// Otherwise the attempt goes on as before. private func lookForMovedPC(_ connectionID: UUID, host: String, port: UInt16) { let address = PairingStore.address(host: host, port: port) @@ -1084,8 +1087,15 @@ final class ConnectionManager: ObservableObject { self.record(.info, "connection", "The saved PC wasn't found at another address") return } + let route = await self.localRoute(moved.host, moved.port) + guard current() else { return } + guard let route, MovedPC.isLocal(moved.host, over: route) else { + self.record(.info, "connection", "The saved PC was found, but not on this iPad's local network: staying at the saved address") + return + } self.record(.info, "connection", "Checking that the PC at \(moved.host):\(moved.port) is the saved one") - let same = await MovedPC.confirm(host: moved.host, token: token, link: self.makeUDPLink(moved.port)) + let same = await MovedPC.confirm(host: moved.host, port: moved.port, token: token, + link: self.makeUDPLink(moved.port)) guard current() else { return } guard same else { self.record(.info, "connection", "The PC there didn't show it is the saved one: staying at the saved address") diff --git a/ios/EternalMonitor/App/MovedPC.swift b/ios/EternalMonitor/App/MovedPC.swift index cdc80ef9..eaa02a28 100644 --- a/ios/EternalMonitor/App/MovedPC.swift +++ b/ios/EternalMonitor/App/MovedPC.swift @@ -5,8 +5,9 @@ import os /// Finding a saved PC again after its address changed: the router gave it a /// new one, or it moved between a cable and Wi-Fi. Only a paired PC that this -/// iPad found with Bonjour, on a home network address, moves by itself; the -/// iPad checks that it is the same PC before reconnecting. +/// iPad found with Bonjour, on a home network address that the iPad reaches +/// directly over Wi-Fi or Ethernet, moves by itself; the iPad checks that it +/// is the same PC, at that address, before reconnecting. enum MovedPC { /// The Bonjour service a PC named `pcName` advertises. static func serviceName(pcName: String) -> String { "EternalMonitor (\(pcName))" } @@ -35,7 +36,8 @@ enum MovedPC { } /// A home network address, which the router hands out and may hand to - /// another device in time. VPN and Tailscale addresses never move. + /// another device in time. VPN and Tailscale addresses never move; a VPN + /// in these ranges is caught by `isLocal`. static func isHomeAddress(_ host: String) -> Bool { guard host.split(separator: ".").count == 4, let bytes = IPv4Address(host)?.rawValue, bytes.count == 4 else { return false } return bytes[0] == 10 || (bytes[0] == 172 && (16...31).contains(bytes[1])) || (bytes[0] == 192 && bytes[1] == 168) @@ -62,21 +64,104 @@ enum MovedPC { return claims } - /// What the PC holding `token` answers to `nonce`. - static func proof(token: Data, nonce: Data) -> Data { - Data(HMAC.authenticationCode(for: nonce + WireIdentify.context, using: SymmetricKey(data: token))) + /// How the iPad reaches an address: the interface it sends from, whether + /// the path is Wi-Fi or wired Ethernet and nothing else, and that + /// interface's IPv4 address and netmask. + struct Route: Equatable { + var interface: String + var direct: Bool + var address: String + var netmask: String + } + + /// Interfaces that are tunnels, whatever the path calls them. + private static let tunnels = ["utun", "ipsec", "ppp", "tun", "tap", "wg"] + + /// Whether `host` is on the local network `route` sends from: a direct + /// Wi-Fi or Ethernet path, not a tunnel, with `host` in its subnet. A + /// VPN that uses home network ranges fails here. + static func isLocal(_ host: String, over route: Route) -> Bool { + let ipv4 = { (text: String) -> UInt32? in + guard text.split(separator: ".").count == 4, let bytes = IPv4Address(text)?.rawValue, bytes.count == 4 else { return nil } + return bytes.reduce(0) { $0 << 8 | UInt32($1) } + } + guard route.direct, !tunnels.contains(where: route.interface.hasPrefix), + let address = ipv4(route.address), let mask = ipv4(route.netmask), let candidate = ipv4(host), + mask != 0, mask != .max else { return false } + return address & mask == candidate & mask + } + + /// How this iPad reaches `host:port` now, from a UDP path to it that + /// sends nothing; nil if no path is ready within 2 s. + static func route(to host: String, port: UInt16) async -> Route? { + guard let remotePort = NWEndpoint.Port(rawValue: port) else { return nil } + let connection = NWConnection(to: .hostPort(host: NWEndpoint.Host(host), port: remotePort), using: .udp) + let queue = DispatchQueue(label: "com.eternal.moved-pc.route") + let path: NWPath? = await withCheckedContinuation { continuation in + let done = OSAllocatedUnfairLock(initialState: false) + let finish: @Sendable (NWPath?) -> Void = { path in + guard done.withLock({ finished in defer { finished = true }; return !finished }) else { return } + connection.stateUpdateHandler = nil + connection.cancel() + continuation.resume(returning: path) + } + connection.stateUpdateHandler = { [weak connection] state in + switch state { + case .ready: finish(connection?.currentPath) + case .failed, .cancelled: finish(nil) + default: break + } + } + queue.asyncAfter(deadline: .now() + 2) { finish(nil) } + connection.start(queue: queue) + } + guard let path, case .hostPort(.ipv4(let local), _)? = path.localEndpoint, + let interface = path.availableInterfaces.first?.name else { return nil } + let direct = (path.usesInterfaceType(.wifi) || path.usesInterfaceType(.wiredEthernet)) + && !path.usesInterfaceType(.other) && !path.usesInterfaceType(.cellular) + let address = local.rawValue.map { String($0) }.joined(separator: ".") + guard let netmask = netmask(interface: interface, address: address) else { return nil } + return Route(interface: interface, direct: direct, address: address, netmask: netmask) + } + + /// The netmask of `address` on `interface`, from the interface list. + private static func netmask(interface: String, address: String) -> String? { + var list: UnsafeMutablePointer? + guard getifaddrs(&list) == 0, let first = list else { return nil } + defer { freeifaddrs(first) } + let text = { (pointer: UnsafeMutablePointer) -> String in + var host = [CChar](repeating: 0, count: Int(NI_MAXHOST)) + getnameinfo(pointer, socklen_t(pointer.pointee.sa_len), &host, socklen_t(host.count), nil, 0, NI_NUMERICHOST) + return String(cString: host) + } + for entry in sequence(first: first, next: { $0.pointee.ifa_next }) { + guard String(cString: entry.pointee.ifa_name) == interface, let addr = entry.pointee.ifa_addr, + addr.pointee.sa_family == sa_family_t(AF_INET), text(addr) == address, + let mask = entry.pointee.ifa_netmask else { continue } + return text(mask) + } + return nil + } + + /// What the PC holding `token` answers to `nonce`, receiving at `port` + /// on `addresses` (4 bytes each). + static func proof(token: Data, nonce: Data, port: UInt16, addresses: [Data]) -> Data { + Data(HMAC.authenticationCode(for: WireIdentity.message(nonce: nonce, port: port, addresses: addresses), + using: SymmetricKey(data: token))) } /// Asks the PC at `host:port`, over `link` and nothing else, to show it - /// is the PC that `token` paired with. True once its answer checks out; - /// false after `attempts` requests go unanswered, as from a PC before - /// 0.3.5, or a wrong answer. The token itself is never sent. - static func confirm(host: String, token: Data, link: any MediaLink, attempts: Int = 8, + /// is the PC that `token` paired with, receiving at `host:port`. True + /// once its answer checks out and names that address; false after + /// `attempts` requests go without one, as from a PC before 0.3.5, or a + /// wrong proof. The token itself is never sent. + static func confirm(host: String, port: UInt16, token: Data, link: any MediaLink, attempts: Int = 8, interval: Double = 0.25) async -> Bool { var generator = SystemRandomNumberGenerator() let nonce = Data((0..<16).map { _ in UInt8.random(in: .min ... .max, using: &generator) }) let request = Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identify(WireIdentify(nonce: nonce))) let key = SymmetricKey(data: token) + let asked = IPv4Address(host)?.rawValue let box = LinkBox(link) return await withCheckedContinuation { continuation in let done = OSAllocatedUnfairLock(initialState: false) @@ -93,8 +178,13 @@ enum MovedPC { link.onControlDatagram = { data in guard case .identity(let identity)? = Wire.parseControl(data)?.message, identity.version == WireIdentify.version, identity.nonce == nonce else { return } - finish(HMAC.isValidAuthenticationCode(identity.proof, authenticating: nonce + WireIdentify.context, - using: key)) + let signed = WireIdentity.message(nonce: nonce, port: identity.port, addresses: identity.addresses) + guard HMAC.isValidAuthenticationCode(identity.proof, authenticating: signed, using: key) else { + return finish(false) + } + // The PC signs where it receives the stream. An answer for + // another address isn't the saved PC at this one. + if identity.port == port, let asked, identity.addresses.contains(asked) { finish(true) } } link.onError = { _ in finish(false) } link.onListenerReady = { _ in diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index 28a57d98..5b54864f 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -377,22 +377,39 @@ struct WirePong: Equatable { } /// iPad → PC, session 0: before reconnecting to a PC it found at a new -/// address, the iPad checks that it is the same PC. The PC answers with a -/// `WireIdentity`. 32 reserved bytes keep the request as long as the answer. +/// address, the iPad checks that it is the same PC, at that address. The PC +/// answers with a `WireIdentity`. Reserved bytes keep the request as long as +/// the answer. struct WireIdentify: Equatable { static let version: UInt8 = 1 /// What the proof signs after the nonce. static let context = Data("EternalMonitor identify v1".utf8) + /// Addresses an answer carries at most. + static let maxAddresses = 4 + /// Both bodies: version, nonce, port, count, address slots and proof. + static let bodySize = 1 + 16 + 2 + 1 + 4 * maxAddresses + 32 var version = WireIdentify.version var nonce: Data } -/// PC → iPad, session 0: HMAC-SHA256 keyed with the pairing token over -/// `nonce ‖ WireIdentify.context`. +/// PC → iPad, session 0: the port the PC receives the stream on, its home +/// network IPv4 addresses (4 bytes each), and HMAC-SHA256 keyed with the +/// pairing token over `message`. struct WireIdentity: Equatable { var version = WireIdentify.version var nonce: Data + var port: UInt16 + var addresses: [Data] var proof: Data + + /// What the proof signs: `nonce ‖ context ‖ port (LE) ‖ count ‖ addresses`. + static func message(nonce: Data, port: UInt16, addresses: [Data]) -> Data { + var message = nonce + WireIdentify.context + message.appendLE(port) + message.append(UInt8(addresses.count)) + for address in addresses { message.append(address) } + return message + } } struct WireInputEvent: Equatable { @@ -539,11 +556,17 @@ extension Wire { precondition(i.nonce.count == 16) body.append(i.version) body.append(i.nonce) - body.append(Data(repeating: 0, count: 32)) + body.append(Data(repeating: 0, count: WireIdentify.bodySize - 17)) case .identity(let i): - precondition(i.nonce.count == 16 && i.proof.count == 32) + precondition(i.nonce.count == 16 && i.proof.count == 32 && i.addresses.count <= WireIdentify.maxAddresses + && i.addresses.allSatisfy { $0.count == 4 }) body.append(i.version) body.append(i.nonce) + body.appendLE(i.port) + body.append(UInt8(i.addresses.count)) + for slot in 0.. Bool { for _ in 0..<100 { guard let (data, from, length) = receive() else { continue } @@ -100,13 +101,44 @@ final class FakeUDPHost: @unchecked Sendable { if case .hello2? = message { helloBeforeIdentify = true } guard case .identify(let request)? = message else { continue } client = (from, length) + let addresses = [Data([127, 0, 0, 1])] send(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identity(WireIdentity( - nonce: request.nonce, proof: proof ?? MovedPC.proof(token: token, nonce: request.nonce))))) + nonce: request.nonce, port: port, addresses: addresses, + proof: proof ?? MovedPC.proof(token: token, nonce: request.nonce, port: port, addresses: addresses))))) return true } return false } + /// Hands the next identify request to `pc` and gives the iPad whatever + /// `pc` answers, from this fake's port. False after thirty quiet seconds. + func relayIdentify(to pc: FakeUDPHost) -> Bool { + var target = sockaddr_in() + target.sin_len = UInt8(MemoryLayout.size) + target.sin_family = sa_family_t(AF_INET) + target.sin_addr.s_addr = inet_addr("127.0.0.1") + target.sin_port = pc.port.bigEndian + for _ in 0..<100 { + guard let (data, from, length) = receive() else { continue } + let message = Wire.parseControl(data)?.message + if case .hello2? = message { helloBeforeIdentify = true } + if case .identity? = message, client != nil { + send(data) + return true + } + guard case .identify? = message else { continue } + client = (from, length) + _ = data.withUnsafeBytes { bytes in + withUnsafePointer(to: &target) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { + sendto(fd, bytes.baseAddress, bytes.count, 0, $0, socklen_t(MemoryLayout.size)) + } + } + } + } + return false + } + /// The control messages received within `seconds`, by type. func received(within seconds: Double) -> [Wire.PacketType] { var types: [Wire.PacketType] = [] diff --git a/ios/EternalMonitorTests/MovedPCTests.swift b/ios/EternalMonitorTests/MovedPCTests.swift index 011e36de..6eab707b 100644 --- a/ios/EternalMonitorTests/MovedPCTests.swift +++ b/ios/EternalMonitorTests/MovedPCTests.swift @@ -132,12 +132,44 @@ final class MovedPCTests: XCTestCase { } /// The same known answer as the host's pairing tests and the wire's - /// golden `identity` vector. + /// golden `identity` vector. The proof signs the PC's port and addresses. func testTheIdentityProofIsTheSharedKnownAnswer() { - let proof = MovedPC.proof(token: Data(1...16), nonce: Data(0xA0...0xAF)) - XCTAssertEqual(proof.map { String(format: "%02x", $0) }.joined(), - "126dc93467af3b0621c4f9b420e0d323143259d743f85703b9fec83989eba353") - XCTAssertNotEqual(MovedPC.proof(token: Data(2...17), nonce: Data(0xA0...0xAF)), proof) + let addresses = [Data([192, 0, 2, 10]), Data([198, 51, 100, 10])] + let proof = { (token: Data, port: UInt16, addresses: [Data]) in + MovedPC.proof(token: token, nonce: Data(0xA0...0xAF), port: port, addresses: addresses) + } + let known = proof(Data(1...16), 9876, addresses) + XCTAssertEqual(known.map { String(format: "%02x", $0) }.joined(), + "18fb81d76bbb96e91101119775ad9766b0c332b2c49441e6f07ec33be1c6d94e") + XCTAssertNotEqual(proof(Data(2...17), 9876, addresses), known) + XCTAssertNotEqual(proof(Data(1...16), 9877, addresses), known) + XCTAssertNotEqual(proof(Data(1...16), 9876, [addresses[0]]), known) + } + + /// Only an address this iPad reaches directly, over Wi-Fi or Ethernet + /// and in that network's subnet, may be a moved PC. A VPN in a home + /// network range is not. + func testOnlyAnAddressOnTheLocalNetworkIsLocal() { + let wifi = MovedPC.Route(interface: "en0", direct: true, address: "192.168.1.23", netmask: "255.255.255.0") + XCTAssertTrue(MovedPC.isLocal("192.168.1.40", over: wifi)) + XCTAssertFalse(MovedPC.isLocal("192.168.2.40", over: wifi), "another subnet") + XCTAssertFalse(MovedPC.isLocal("10.0.0.40", over: wifi)) + XCTAssertFalse(MovedPC.isLocal("studio-pc.local", over: wifi)) + let wide = MovedPC.Route(interface: "en0", direct: true, address: "10.20.0.5", netmask: "255.255.0.0") + XCTAssertTrue(MovedPC.isLocal("10.20.200.9", over: wide)) + + // A VPN in a home network range: the path isn't Wi-Fi or Ethernet, + // or the interface is a tunnel, or it is a point-to-point address. + let vpn = MovedPC.Route(interface: "utun4", direct: false, address: "10.8.0.6", netmask: "255.255.255.0") + XCTAssertFalse(MovedPC.isLocal("10.8.0.1", over: vpn)) + XCTAssertFalse(MovedPC.isLocal("10.8.0.1", over: MovedPC.Route(interface: "utun4", direct: true, address: "10.8.0.6", + netmask: "255.255.255.0"))) + XCTAssertFalse(MovedPC.isLocal("192.168.1.40", over: MovedPC.Route(interface: "ipsec0", direct: true, + address: "192.168.1.23", netmask: "255.255.255.0"))) + XCTAssertFalse(MovedPC.isLocal("192.168.1.23", over: MovedPC.Route(interface: "en0", direct: true, + address: "192.168.1.23", netmask: "255.255.255.255"))) + XCTAssertFalse(MovedPC.isLocal("192.168.1.40", over: MovedPC.Route(interface: "en0", direct: false, + address: "192.168.1.23", netmask: "255.255.255.0"))) } // MARK: - Connecting @@ -153,6 +185,8 @@ final class MovedPCTests: XCTestCase { /// otherwise, paired with `token`; Bonjour has it at `new`. Loopback /// counts as a home address here. private func setUp(old: FakeUDPHost, new: FakeUDPHost, token: Data?, found: Bool? = true, + route: MovedPC.Route? = MovedPC.Route(interface: "en0", direct: true, address: "127.0.0.1", + netmask: "255.0.0.0"), connectTimeout: Double) throws -> Setup { RecentConnectionStore.shared.connections = [ RecentConnection(host: "127.0.0.1", port: old.port, lastUsed: .now, isUSB: false, name: "Test PC", found: found), @@ -163,6 +197,7 @@ final class MovedPCTests: XCTestCase { let manager = ConnectionManager(timing: .init(connectTimeoutSeconds: connectTimeout, reconnectGraceSeconds: 0.1, reconnectBackoffSeconds: { _ in 0.05 }), pairingStore: pairings) manager.homeAddress = { _ in true } + manager.localRoute = { _, _ in route } var asked: [String] = [] manager.findPCs = { name in asked.append(name) @@ -240,6 +275,49 @@ final class MovedPCTests: XCTestCase { } } + /// The answer at the address Bonjour gave comes from the saved PC at + /// another address, and says so: it isn't the saved PC at this address. + /// No HELLO2, no token. + func testAnAnswerFromAnotherAddressIsNotTheSamePC() async throws { + let old = FakeUDPHost(), relay = FakeUDPHost(), pc = FakeUDPHost() + let setup = try setUp(old: old, new: relay, token: Self.token, connectTimeout: 7) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + async let answered = Task.detached { pc.answerIdentify(token: Self.token) }.value + let types = await Task.detached { () -> [Wire.PacketType] in + guard relay.relayIdentify(to: pc) else { return [] } + return relay.received(within: 4) + }.value + let pcAnswered = await answered + XCTAssertTrue(pcAnswered, "the request reached the saved PC") + XCTAssertFalse(types.isEmpty, "the iPad asked the relay") + XCTAssertFalse(types.contains(.hello2), "\(types)") + XCTAssertFalse(relay.helloBeforeIdentify) + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: relay.port)) + } + + /// Bonjour has the PC at a home network address that this iPad reaches + /// over a VPN, or by no ready path: it isn't asked anything. + func testAPCReachedOverAVPNIsNeverAsked() async throws { + let vpn = MovedPC.Route(interface: "utun4", direct: false, address: "127.0.0.2", netmask: "255.0.0.0") + for route: MovedPC.Route? in [vpn, nil] { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, route: route, connectTimeout: 6) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let types = await Task.detached { new.received(within: 5) }.value + XCTAssertEqual(types, [], "\(String(describing: route))") + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.asked().count, 1) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + XCTAssertTrue(setup.manager.diagnostics.contains { $0.message.contains("not on this iPad's local network") }) + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token) + } + } + /// Unpaired, nothing proves which PC is which: with the saved PC gone, a /// same-named PC with pairing off isn't even looked for. func testAnUnpairedSavedPCIsNeverMoved() async throws { diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index 24f2a7fd..7c891611 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -40,8 +40,9 @@ final class WireProtocolGoldenTests: XCTestCase { ) } - /// The identify request and its answer: nonce A0..AF, and the proof of - /// pairing token 01..10, which the iPad checks as the PC's own. + /// The identify request and its answer: nonce A0..AF, port 9876, two + /// addresses, and the proof of pairing token 01..10 over them, which the + /// iPad checks as the PC's own. func testIdentifyAndIdentityVectors() throws { let nonce = Data(0xA0...0xAF) let request = try vector("identify") @@ -55,7 +56,14 @@ final class WireProtocolGoldenTests: XCTestCase { let (_, message) = try XCTUnwrap(Wire.parseControl(answer)) guard case .identity(let identity) = message else { return XCTFail("wrong type") } XCTAssertEqual(identity.nonce, nonce) - XCTAssertEqual(identity.proof, MovedPC.proof(token: Data(1...16), nonce: nonce)) + XCTAssertEqual(identity.port, 9876) + XCTAssertEqual(identity.addresses, [Data([192, 0, 2, 10]), Data([198, 51, 100, 10])]) + XCTAssertEqual(identity.proof, MovedPC.proof(token: Data(1...16), nonce: nonce, port: 9876, + addresses: identity.addresses)) + // More addresses than fit are refused, never read past. + var five = answer + five[16 + 1 + 16 + 2] = 5 + XCTAssertNil(Wire.parseControl(five)) XCTAssertEqual(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: message), answer) } diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index 684fa306..72a19744 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -320,27 +320,52 @@ pub const IDENTIFY_VERSION: u8 = 1; /// What an [`Identity`] proof signs after the nonce. pub const IDENTIFY_CONTEXT: &[u8] = b"EternalMonitor identify v1"; +/// At most this many addresses in an [`Identity`]. +pub const IDENTITY_MAX_ADDRESSES: usize = 4; + /// Client → host, session_id 0: before reconnecting to a PC it found at a new -/// address, the iPad checks that it is the same PC. The host answers with an -/// [`Identity`] whose proof only the PC holding the pairing token can make. -/// 32 reserved bytes keep the request as long as the answer. Hosts before -/// 0.3.5 don't know the type and drop it. +/// address, the iPad checks that it is the same PC, at that address. The host +/// answers with an [`Identity`] whose proof only the PC holding the pairing +/// token can make. Reserved bytes keep the request as long as the answer. +/// Hosts before 0.3.5 don't know the type and drop it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Identify { pub version: u8, pub nonce: [u8; 16], } -/// Host → client, session_id 0: `proof` is HMAC-SHA256 keyed with the -/// pairing token over `nonce ‖ IDENTIFY_CONTEXT`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Host → client, session_id 0: the port the host receives the stream on and +/// its home network IPv4 addresses (at most [`IDENTITY_MAX_ADDRESSES`]), and +/// `proof`, HMAC-SHA256 keyed with the pairing token over +/// [`identity_message`]. The iPad goes on only when the address it asked at +/// is one of them. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Identity { pub version: u8, pub nonce: [u8; 16], + pub port: u16, + pub addresses: Vec<[u8; 4]>, pub proof: [u8; 32], } -pub const IDENTIFY_BODY_SIZE: usize = 49; +/// Both bodies: version, nonce, port, address count, the address slots +/// (unused ones zero) and the proof. +pub const IDENTIFY_BODY_SIZE: usize = 1 + 16 + 2 + 1 + 4 * IDENTITY_MAX_ADDRESSES + 32; + +/// What an [`Identity`] proof signs: `nonce ‖ IDENTIFY_CONTEXT ‖ port (LE) ‖ +/// count ‖ addresses`. Only the first [`IDENTITY_MAX_ADDRESSES`] count. +pub fn identity_message(nonce: &[u8; 16], port: u16, addresses: &[[u8; 4]]) -> Vec { + let addresses = &addresses[..addresses.len().min(IDENTITY_MAX_ADDRESSES)]; + let mut message = Vec::with_capacity(16 + IDENTIFY_CONTEXT.len() + 3 + 4 * addresses.len()); + message.extend_from_slice(nonce); + message.extend_from_slice(IDENTIFY_CONTEXT); + message.extend_from_slice(&port.to_le_bytes()); + message.push(addresses.len() as u8); + for address in addresses { + message.extend_from_slice(address); + } + message +} pub const INPUT_EVENT_SIZE: usize = 30; pub const PEN_INPUT_EVENT_SIZE: usize = INPUT_EVENT_SIZE + 4; @@ -491,11 +516,17 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - ControlMessage::Identify(i) => { body.push(i.version); body.extend_from_slice(&i.nonce); - body.extend_from_slice(&[0; 32]); + body.extend_from_slice(&[0; IDENTIFY_BODY_SIZE - 17]); } ControlMessage::Identity(i) => { + let count = i.addresses.len().min(IDENTITY_MAX_ADDRESSES); body.push(i.version); body.extend_from_slice(&i.nonce); + body.extend_from_slice(&i.port.to_le_bytes()); + body.push(count as u8); + for slot in 0..IDENTITY_MAX_ADDRESSES { + body.extend_from_slice(i.addresses.get(slot).unwrap_or(&[0; 4])); + } body.extend_from_slice(&i.proof); } ControlMessage::InputEvent(e) => { @@ -721,14 +752,27 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), PacketType::Identify => { let version = r.u8()?; let nonce = r.take(16)?.try_into().unwrap(); - r.take(32)?; + r.take(IDENTIFY_BODY_SIZE - 17)?; ControlMessage::Identify(Identify { version, nonce }) } - PacketType::Identity => ControlMessage::Identity(Identity { - version: r.u8()?, - nonce: r.take(16)?.try_into().unwrap(), - proof: r.take(32)?.try_into().unwrap(), - }), + PacketType::Identity => { + let version = r.u8()?; + let nonce = r.take(16)?.try_into().unwrap(); + let port = r.u16()?; + let count = usize::from(r.u8()?); + if count > IDENTITY_MAX_ADDRESSES { + return Err(WireError::InvalidField("identity_addresses")); + } + let slots = r.take(4 * IDENTITY_MAX_ADDRESSES)?; + let addresses = slots.as_chunks::<4>().0[..count].to_vec(); + ControlMessage::Identity(Identity { + version, + nonce, + port, + addresses, + proof: r.take(32)?.try_into().unwrap(), + }) + } PacketType::Media | PacketType::MediaFec | PacketType::Audio | PacketType::Error => { return Err(WireError::InvalidField("packet_type")); } @@ -996,6 +1040,8 @@ mod tests { ControlMessage::Identity(Identity { version: IDENTIFY_VERSION, nonce: [7; 16], + port: 9876, + addresses: vec![[192, 0, 2, 7], [198, 51, 100, 7]], proof: [9; 32], }), ControlMessage::InputEvent(InputEvent { diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index cda185d1..d651c882 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -32,5 +32,5 @@ receiver_report_v030 454d021500003200785634123200000003000000881300007e1300000a0 hello_ack_paired 454d02110000390000000000030000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f100100 hello_ack_contiguous_seq 454d02110000390000000000050000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f102100 hello2_v030 454d02100000410000000000040000000202efbeadde94260300010074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e20100 -identify 454d021a00003100000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf0000000000000000000000000000000000000000000000000000000000000000 -identity 454d021b00003100000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf126dc93467af3b0621c4f9b420e0d323143259d743f85703b9fec83989eba353 +identify 454d021a00004400000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +identity 454d021b00004400000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf942602c000020ac633640a000000000000000018fb81d76bbb96e91101119775ad9766b0c332b2c49441e6f07ec33be1c6d94e diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index bdf567a4..a4402cc8 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -416,8 +416,9 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { }), ), )); - // The proof for pairing token 01..10: the same known answer the host's - // pairing tests and the iPad's identity tests check. + // The proof for pairing token 01..10, port 9876 and two addresses: the + // same known answer the host's pairing tests and the iPad's identity + // tests check. vectors.push(( "identity", encode_control( @@ -426,6 +427,8 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { &ControlMessage::Identity(Identity { version: IDENTIFY_VERSION, nonce, + port: 9876, + addresses: IDENTITY_ADDRESSES.to_vec(), proof: IDENTITY_PROOF, }), ), @@ -433,9 +436,10 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { vectors } +const IDENTITY_ADDRESSES: [[u8; 4]; 2] = [[192, 0, 2, 10], [198, 51, 100, 10]]; const IDENTITY_PROOF: [u8; 32] = [ - 0x12, 0x6d, 0xc9, 0x34, 0x67, 0xaf, 0x3b, 0x06, 0x21, 0xc4, 0xf9, 0xb4, 0x20, 0xe0, 0xd3, 0x23, - 0x14, 0x32, 0x59, 0xd7, 0x43, 0xf8, 0x57, 0x03, 0xb9, 0xfe, 0xc8, 0x39, 0x89, 0xeb, 0xa3, 0x53, + 0x18, 0xfb, 0x81, 0xd7, 0x6b, 0xbb, 0x96, 0xe9, 0x11, 0x01, 0x11, 0x97, 0x75, 0xad, 0x97, 0x66, + 0xb0, 0xc3, 0x32, 0xb2, 0xc4, 0x94, 0x41, 0xe6, 0xf0, 0x7e, 0xc3, 0x3b, 0xe1, 0xc6, 0xd9, 0x4e, ]; fn render(vectors: &[(&'static str, Vec)]) -> String { @@ -581,9 +585,30 @@ fn identify_request_is_as_long_as_its_answer() { assert_eq!(vectors["identify"].len(), vectors["identity"].len()); assert_eq!(vectors["identify"].len(), 16 + IDENTIFY_BODY_SIZE); match parse_control(&vectors["identity"]).unwrap().1 { - ControlMessage::Identity(identity) => assert_eq!(identity.proof, IDENTITY_PROOF), + ControlMessage::Identity(identity) => { + assert_eq!(identity.proof, IDENTITY_PROOF); + assert_eq!(identity.port, 9876); + assert_eq!(identity.addresses, IDENTITY_ADDRESSES); + } other => panic!("{other:?}"), } + // Five addresses don't fit: the answer is refused, never longer. + let mut five = vectors["identity"].clone(); + five[16 + 1 + 16 + 2] = 5; + assert!(parse_control(&five).is_err()); + // Only the addresses counted are read and signed. + let mut one = vectors["identity"].clone(); + one[16 + 1 + 16 + 2] = 1; + match parse_control(&one).unwrap().1 { + ControlMessage::Identity(identity) => { + assert_eq!(identity.addresses, [IDENTITY_ADDRESSES[0]]) + } + other => panic!("{other:?}"), + } + assert_ne!( + identity_message(&[0xA0; 16], 9876, &IDENTITY_ADDRESSES), + identity_message(&[0xA0; 16], 9876, &IDENTITY_ADDRESSES[..1]) + ); } /// A host that numbers frames contiguously says so in HELLO_ACK; one that diff --git a/scripts/e2e_moved_pc.sh b/scripts/e2e_moved_pc.sh index fc2ab65f..61311d70 100755 --- a/scripts/e2e_moved_pc.sh +++ b/scripts/e2e_moved_pc.sh @@ -10,8 +10,10 @@ # EM_PAIRING=off the PC has pairing off, so nothing proves it # EM_MOVED_HOST_BIN= the PC comes back as a build before 0.3.5 # -# On a shared Mac this is an iOS e2e run: hold the machine's e2e lock, and use -# your own simulator (EM_SIM_UDID) and a private port (EM_PORT, and the next). +# On a shared Mac this is an iOS e2e run: hold the machine's e2e lock and use +# your own simulator (EM_SIM_UDID). The PC's two ports are picked free when the +# run starts; EM_PORT and EM_NEW_PORT choose them instead, and the run stops +# before anything starts if either is taken. set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" @@ -20,8 +22,6 @@ export PATH="$HOME/.cargo/bin:$PATH" export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:-/opt/homebrew/opt/ffmpeg@7/lib/pkgconfig}" UDID="${EM_SIM_UDID:?set EM_SIM_UDID to your own simulator}" -PORT="${EM_PORT:-19893}" -NEW_PORT=$((PORT + 1)) EXPECT="${EM_EXPECT:-move}" PAIRING="${EM_PAIRING:-on}" TIMEOUT_SECS="${EM_TIMEOUT:-90}" @@ -49,6 +49,25 @@ a=ipaddress.ip_address(sys.argv[1]) sys.exit(0 if a.version==4 and a.is_private and not a.is_loopback and a not in ipaddress.ip_network('100.64.0.0/10') else 1) PY +# The PC's port before and after the move: each free for UDP and TCP on every +# address, checked together so the two differ. +PORTS=$(python3 - "${EM_PORT:-0}" "${EM_NEW_PORT:-0}" <<'PY' +import socket,sys +held=[] +for wanted in map(int, sys.argv[1:3]): + try: + udp=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); udp.bind(('0.0.0.0',wanted)); held.append(udp) + port=udp.getsockname()[1] + tcp=socket.socket(socket.AF_INET,socket.SOCK_STREAM); tcp.bind(('0.0.0.0',port)); held.append(tcp) + except OSError: + print(f"port {wanted} is in use"); sys.exit(1) + print(port) +PY +) || { echo "FAIL: $PORTS" >&2; exit 1; } +PORT=$(echo "$PORTS" | sed -n 1p) +NEW_PORT=$(echo "$PORTS" | sed -n 2p) +[ "$PORT" != "$NEW_PORT" ] || { echo "FAIL: EM_PORT and EM_NEW_PORT must differ" >&2; exit 1; } + HOST_PID="" SILENT_PID="" cleanup() { From cfb6783d25c5fc49893839febf8120d2073ca89b Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 13:40:24 -0500 Subject: [PATCH 031/226] Let the moved-PC e2e stand the Mac's interface in for the iPad's Wi-Fi The simulator reaches a PC on its own Mac over loopback, which no iPad does. Simulator builds take EM_ROUTE_INTERFACE as the interface for such a path; device builds don't have it. --- ios/EternalMonitor/App/MovedPC.swift | 8 ++++++-- ios/EternalMonitor/Core/E2ESignpost.swift | 11 +++++++++++ scripts/e2e_moved_pc.sh | 3 +++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/ios/EternalMonitor/App/MovedPC.swift b/ios/EternalMonitor/App/MovedPC.swift index eaa02a28..6e53aa92 100644 --- a/ios/EternalMonitor/App/MovedPC.swift +++ b/ios/EternalMonitor/App/MovedPC.swift @@ -116,9 +116,13 @@ enum MovedPC { connection.start(queue: queue) } guard let path, case .hostPort(.ipv4(let local), _)? = path.localEndpoint, - let interface = path.availableInterfaces.first?.name else { return nil } - let direct = (path.usesInterfaceType(.wifi) || path.usesInterfaceType(.wiredEthernet)) + var interface = path.availableInterfaces.first?.name else { return nil } + var direct = (path.usesInterfaceType(.wifi) || path.usesInterfaceType(.wiredEthernet)) && !path.usesInterfaceType(.other) && !path.usesInterfaceType(.cellular) + if path.usesInterfaceType(.loopback), let stand = E2E.loopbackRouteInterface { + interface = stand + direct = true + } let address = local.rawValue.map { String($0) }.joined(separator: ".") guard let netmask = netmask(interface: interface, address: address) else { return nil } return Route(interface: interface, direct: direct, address: address, netmask: netmask) diff --git a/ios/EternalMonitor/Core/E2ESignpost.swift b/ios/EternalMonitor/Core/E2ESignpost.swift index e2520c77..8fd93d18 100644 --- a/ios/EternalMonitor/Core/E2ESignpost.swift +++ b/ios/EternalMonitor/Core/E2ESignpost.swift @@ -62,6 +62,17 @@ enum E2E { return 9877 } + /// Simulator builds: `EM_ROUTE_INTERFACE=en0` stands that interface in + /// for the loopback path to a PC on the simulator's own Mac, which no + /// iPad has (see `MovedPC.route`). + static var loopbackRouteInterface: String? { + #if targetEnvironment(simulator) + return ProcessInfo.processInfo.environment["EM_ROUTE_INTERFACE"].flatMap { $0.isEmpty ? nil : $0 } + #else + return nil + #endif + } + static func firstFrame(width: Int, height: Int, link: String = "udp") { guard enabled else { return } let monotonicMs = DispatchTime.now().uptimeNanoseconds / 1_000_000 diff --git a/scripts/e2e_moved_pc.sh b/scripts/e2e_moved_pc.sh index 61311d70..94441a9d 100755 --- a/scripts/e2e_moved_pc.sh +++ b/scripts/e2e_moved_pc.sh @@ -156,7 +156,10 @@ start_host "$HOST_BIN" "$PORT" echo "==> Launching app: pairs and streams from the home network address, port $PORT" LINK="$LAN:$PORT" [ "$PAIRING" = on ] && LINK="eternaldisplay://$LINK?t=$TOKEN" +# The simulator reaches this Mac's own address over loopback; the Mac's +# network interface stands in for the iPad's Wi-Fi. SIMCTL_CHILD_EM_AUTOCONNECT="$LINK" SIMCTL_CHILD_EM_AUTOCONNECT_FOUND=1 SIMCTL_CHILD_EM_E2E_LOG=1 SIMCTL_CHILD_EM_USB_PORT=0 \ + SIMCTL_CHILD_EM_ROUTE_INTERFACE="$IFACE" \ xcrun simctl launch "$UDID" com.eternal.monitor -didSeeOnboarding YES -allowUSB NO -autoReconnect YES -playPCaudio NO >/dev/null wait_for "no first stream" "$TIMEOUT_SECS" frames_at_least 1 From fac2c2190a768450915684150de55f13e0c3876a Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 13:29:21 -0500 Subject: [PATCH 032/226] Bring an AMF session's first keyframe out on a still desktop FFmpeg 7.1's AMF wrapper asks AMF for output only when it submits the next frame. On a still desktop capture resends the picture once for a new viewer and then waits for the 750 ms keepalive, so the viewer's first keyframe sat inside AMF that long. AMD PCs were slow on 39-56% of reconnects; in field logs the keyframe left AMF 670-680 ms after it went in, and later packets followed each keepalive. While a fresh AMF session's first keyframe is still inside the encoder, the encode loop now waits 20 ms for a new picture and otherwise gives AMF the last one again every 20 ms, for up to 500 ms. NVENC (delay=0), QSV (async_depth=1) and libx264 return each packet with its own frame and are unchanged. ETERNAL_SYNTH_STILL=1 makes the synthetic source a desktop that never changes, resent as DXGI capture resends one, so headless hosts can time a new viewer's first frame there. The periodic AMF IDR request, which FFmpeg 7.1's AMF ignores, now logs at debug. --- host/src/capture/mod.rs | 67 +++++++++++++++++- host/src/capture/synthetic.rs | 37 +++++++++- host/src/encoder/mod.rs | 94 ++++++++++++++++++++++++- host/tests/amf_encoder.rs | 125 ++++++++++++++++++++++++++++++++++ 4 files changed, 315 insertions(+), 8 deletions(-) diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 1473f9a9..ee831d4f 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -36,7 +36,6 @@ pub(crate) const FPS_WINDOW: usize = 60; /// DXGI delivers no new frames. If a client is connected and we haven't sent anything for this /// long, resend the last frame so the encoder emits the pending IDR and the iPad gets a sync /// sample instead of timing out on a black screen. -#[cfg(any(windows, test))] pub(crate) const IDLE_KEEPALIVE: Duration = Duration::from_millis(750); /// With nobody connected the cached picture is refreshed this often, so a new @@ -200,6 +199,41 @@ impl FrameSlot { self.inner.condvar.wait(&mut state); } } + + /// As [`Self::blocking_take`], but gives up after `timeout`. + pub fn take_within(&self, timeout: Duration) -> Taken { + let deadline = Instant::now() + timeout; + let mut state = self.inner.state.lock(); + loop { + if let Some(frame) = state.frame.take() { + return Taken::Frame(frame); + } + if state.closed { + return Taken::Closed; + } + if self + .inner + .condvar + .wait_until(&mut state, deadline) + .timed_out() + { + return match state.frame.take() { + Some(frame) => Taken::Frame(frame), + None if state.closed => Taken::Closed, + None => Taken::Nothing, + }; + } + } + } +} + +/// What [`FrameSlot::take_within`] found. +#[derive(Debug)] +pub enum Taken { + Frame(RawFrame), + /// No frame arrived in time. + Nothing, + Closed, } /// Stamp a watchdog heartbeat (ms on the process clock). @@ -798,7 +832,6 @@ pub(crate) fn keepalive_allowed(seen_image: bool, second_screen: bool) -> bool { /// How long a keyframe request waits for the encoder to take the resent frame /// before the picture is resent again. -#[cfg(any(windows, test))] const KEYFRAME_RESEND: Duration = Duration::from_millis(100); /// Whether an idle display's last picture is due to be resent to the viewer. @@ -806,7 +839,6 @@ const KEYFRAME_RESEND: Duration = Duration::from_millis(100); /// at once when there is a real picture to send: a viewer back within the /// second screen's grace period otherwise waited for the next keepalive, up /// to 750 ms, before the encoder could start its session. -#[cfg(any(windows, test))] pub(crate) fn keepalive_due( since_send: Duration, keyframe_requested: bool, @@ -970,6 +1002,35 @@ fn pick_new_virtual_output<'a>(before: &[String], now: &'a [OutputInfo]) -> Opti #[cfg(test)] mod tests { + #[test] + fn a_timed_take_returns_a_frame_nothing_or_the_close() { + let (producer, consumer) = super::frame_slot(); + let frame = |frame_number| super::RawFrame { + frame_number, + timestamp: Instant::now(), + data: Arc::new(Vec::new()), + width: 2, + height: 2, + }; + let short = Duration::from_millis(5); + assert!(matches!(consumer.take_within(short), super::Taken::Nothing)); + producer.publish(frame(1)); + assert!(matches!( + consumer.take_within(short), + super::Taken::Frame(taken) if taken.frame_number == 1 + )); + let waiting = std::thread::spawn(move || consumer.take_within(Duration::from_secs(10))); + std::thread::sleep(Duration::from_millis(20)); + producer.publish(frame(2)); + assert!(matches!( + waiting.join().unwrap(), + super::Taken::Frame(taken) if taken.frame_number == 2 + )); + let (producer, consumer) = super::frame_slot(); + producer.close(); + assert!(matches!(consumer.take_within(short), super::Taken::Closed)); + } + #[test] fn pointer_only_frames_are_not_read_back_before_the_first_image() { assert!(!frame_has_image(0, false), "blank first frame"); diff --git a/host/src/capture/synthetic.rs b/host/src/capture/synthetic.rs index fadc35bb..563135b9 100644 --- a/host/src/capture/synthetic.rs +++ b/host/src/capture/synthetic.rs @@ -16,7 +16,7 @@ use std::time::Instant; use tracing::info; -use super::{frame_budget_for, heartbeat, FrameSlot, RawFrame}; +use super::{frame_budget_for, heartbeat, keepalive_due, FrameSlot, RawFrame}; use crate::control::SharedControl; use crate::stats::PIPELINE_STATS; @@ -44,6 +44,14 @@ fn synthetic_size() -> (u32, u32) { (DEFAULT_WIDTH, DEFAULT_HEIGHT) } +/// `ETERNAL_SYNTH_STILL=1`: a desktop that never changes. After its first +/// picture the source resends it only as DXGI capture does on a still +/// desktop, for the idle keepalive or a keyframe request (see +/// [`keepalive_due`]), so a test can time a new viewer's first frame there. +fn still_desktop() -> bool { + std::env::var("ETERNAL_SYNTH_STILL").is_ok_and(|v| v.trim() == "1") +} + /// Produce frames until `shared.running` clears or the channel closes. /// Mirrors the DXGI loop's contract: paced by `target_fps`, records capture /// stats, ships BGRA `RawFrame`s. @@ -78,6 +86,11 @@ pub fn run_capture_loop( // avoids normal sleep coalescing without burning an entire CPU core. let mut timer = super::timing::FrameTimer::new(); let mut next_deadline = Instant::now(); + // `ETERNAL_SYNTH_STILL=1`: the first picture, then only what DXGI sends + // for a desktop that never changes. + let mut still: Option>> = None; + let mut last_send = Instant::now(); + let still_mode = still_desktop(); loop { if !shared.running.load(Ordering::SeqCst) @@ -92,8 +105,28 @@ pub fn run_capture_loop( heartbeat(&shared.hb_capture_loop_ms); + let data = match still.as_ref() { + Some(picture) + if shared.client_connected() + && keepalive_due( + frame_start.duration_since(last_send), + shared.force_next_idr.load(Ordering::SeqCst), + true, + ) => + { + Arc::clone(picture) + } + Some(_) => { + timer.wait_until(frame_start + frame_budget); + continue; + } + None => render_next_frame(&mut buffers, width, height, *frame_number + 1), + }; *frame_number += 1; - let data = render_next_frame(&mut buffers, width, height, *frame_number); + last_send = frame_start; + if still_mode && still.is_none() { + still = Some(Arc::clone(&data)); + } PIPELINE_STATS.lock().record_capture(width, height); heartbeat(&shared.hb_capture_frame_ms); diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index a865edf5..53c3522e 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -14,7 +14,7 @@ use eternal_wire::h264::{ normalize_h264_payload_with_info, parameter_sets_from_extradata, H264BitstreamState, }; -use crate::capture::FrameSlot; +use crate::capture::{FrameSlot, RawFrame, Taken}; use crate::control::SharedControl; use crate::gpu::{GpuInfo, GpuVendor}; use crate::stats::PIPELINE_STATS; @@ -167,7 +167,29 @@ fn run_encode_loop( } }; - while let Some(raw_frame) = frames.blocking_take() { + let mut owed_keyframe: Option = None; + loop { + let raw_frame = match owed_keyframe.as_mut() { + None => match frames.blocking_take() { + Some(frame) => frame, + None => break, + }, + Some(owed) => match frames.take_within(KEYFRAME_FLUSH_WAIT) { + Taken::Frame(frame) => frame, + Taken::Closed => break, + Taken::Nothing => match owed.again() { + Some(frame) => frame, + None => { + info!( + given = owed.given, + "The encoder still holds the session's first keyframe; waiting for the next capture" + ); + owed_keyframe = None; + continue; + } + }, + }, + }; if !shared.running.load(Ordering::SeqCst) || !crate::supervisor::generation_is_current(generation) { @@ -491,6 +513,11 @@ fn run_encode_loop( let convert_start = Instant::now(); encoder.convert(&raw_frame.data, (raw_frame.width * 4) as usize)?; + // The picture is kept only while a session's first keyframe may stay + // inside the encoder, to be given to it again. + let picture = (holds_last_frame(&encoder.opened_encoder) + && (encoder.frames_submitted == 0 || owed_keyframe.is_some())) + .then(|| raw_frame.clone()); // Conversion finished synchronously. Recycle capture's buffer before // the codec or sender can block, including on the direct macOS path. drop(raw_frame.data); @@ -516,6 +543,7 @@ fn run_encode_loop( let encode_start = Instant::now(); encoder.send_frame(pts, raw_frame.timestamp, raw_frame.frame_number)?; + let mut keyframe_out = false; loop { match encoder.encoder.receive_packet(&mut encoder.packet) { Ok(()) => { @@ -560,6 +588,7 @@ fn run_encode_loop( au_info.is_keyframe(packet_is_key, &encoder.opened_encoder); (nal_data, is_keyframe) }; + keyframe_out |= is_keyframe; if !opened_is_hevc { encoder.observe_amf_diagnostics( &nal_data, @@ -625,6 +654,13 @@ fn run_encode_loop( Err(e) => return Err(e.into()), } } + owed_keyframe = match picture { + Some(picture) if !keyframe_out => Some(OwedKeyframe { + picture, + given: owed_keyframe.map_or(0, |owed| owed.given), + }), + _ => None, + }; } info!("Capture channel closed, encoder shutting down"); @@ -1639,6 +1675,48 @@ pub fn ignores_forced_keyframes(encoder_name: &str) -> bool { matches!(encoder_name, "h264_amf" | "hevc_amf") } +/// True for encoders that keep each frame until they are given the next. +/// FFmpeg 7.1's AMF wrapper asks AMF for output only when it submits a frame +/// (8.0 can wait for it, with `query_timeout`). On a still desktop a new +/// viewer's opening keyframe sat inside AMF until capture's next picture, +/// the 750 ms keepalive: AMD PCs were slow on 39-56% of reconnects. +pub fn holds_last_frame(encoder_name: &str) -> bool { + matches!(encoder_name, "h264_amf" | "hevc_amf") +} + +/// How long the encode loop waits for capture's next picture before it gives +/// an encoder that [holds its last frame](holds_last_frame) the last picture +/// again, to bring out a session's first keyframe. It is about one frame at +/// 60 fps, and longer than AMF takes to encode one. +const KEYFRAME_FLUSH_WAIT: Duration = Duration::from_millis(20); +/// How many times the last picture is given again before the keyframe is +/// left to capture's next picture: 500 ms, as an AMD driver can take a few +/// hundred to encode a fresh session's first frame, and capture's keepalive +/// follows at 750 ms. +const KEYFRAME_FLUSHES: u32 = 25; + +/// A session's first keyframe still inside an encoder that holds its last +/// frame, and the picture to give it again. +struct OwedKeyframe { + picture: RawFrame, + given: u32, +} + +impl OwedKeyframe { + fn again(&mut self) -> Option { + if self.given >= KEYFRAME_FLUSHES { + return None; + } + self.given += 1; + debug!( + frame = self.picture.frame_number, + given = self.given, + "Giving the encoder its last picture again to bring out the keyframe" + ); + Some(self.picture.clone()) + } +} + /// Whether a pending keyframe request needs a session reopen: the encoder /// ignores per-frame requests, the session has already sent its opening IDR, /// and the last such reopen is older than the client's own request limit. @@ -1830,7 +1908,17 @@ fn prepare_frame_for_encode( } else { "period" }; - if is_amf { + // The period runs all stream long, and FFmpeg 7.1's AMF ignores it + // (see `ignores_forced_keyframes`): debug only. + if !force_hit { + debug!( + encoder = encoder_name, + frame = frame_number, + period = AMF_FORCED_INTRA_PERIOD, + reason, + "[AMF] Forced IDR" + ); + } else if is_amf { info!( encoder = encoder_name, frame = frame_number, diff --git a/host/tests/amf_encoder.rs b/host/tests/amf_encoder.rs index 6a35ea53..61a578dc 100644 --- a/host/tests/amf_encoder.rs +++ b/host/tests/amf_encoder.rs @@ -491,6 +491,131 @@ fn amf_hevc_honours_keyframe_requests() { assert_keyframe_on_request(&run, Id::HEVC); } +/// How long the desktop stays still after a keyframe request. Capture resends +/// the last picture once for the request, and then nothing until the 750 ms +/// keepalive, so an encoder that holds a frame until the next one arrives +/// kept a new viewer waiting that long. Nothing is published in this window: +/// a keyframe that leaves at all left without a next frame. How fast it +/// leaves is printed, not checked, because an AMD driver's own reopen took +/// from 50 to over 400 ms in a fresh test process. +const STILL_WINDOW: Duration = Duration::from_millis(1500); + +/// Streams a few frames, goes still, then requests a keyframe with one +/// resent picture and nothing after it. Returns how long that picture's +/// keyframe took to leave the stage, or `None` if it never did. +fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { + ffmpeg_next::init().unwrap(); + let (producer, consumer) = frame_slot(); + let (tx, mut rx) = tokio::sync::mpsc::channel(NAL_CHANNEL_CAPACITY); + let shared = SharedControl::new(0, 8_000_000); + shared.running.store(true, Ordering::SeqCst); + let gpu = GpuInfo { + encoder_name: encoder.to_string(), + codec_display_name: encoder.to_string(), + ..GpuInfo::software_fallback() + }; + let generation = eternal_host::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let stage_shared = shared.clone(); + let stage = std::thread::spawn(move || { + run_encode_stage(consumer, tx, stage_shared, gpu, generation).map_err(|e| e.to_string()) + }); + let collector = std::thread::spawn(move || { + let mut packets = Vec::new(); + while let Some(nal) = rx.blocking_recv() { + packets.push((Instant::now(), nal)); + } + packets + }); + let publish = |number: u64| { + let (width, height) = LARGE; + let mut data = frame_buffer((width * height * 4) as usize); + render_synthetic_frame(&mut data, width, height, number); + producer.publish(RawFrame { + frame_number: number, + timestamp: Instant::now(), + data: Arc::new(data), + width, + height, + }); + }; + + for number in 0..10 { + publish(number); + std::thread::sleep(FRAME_INTERVAL); + } + // The desktop goes still, and a viewer connects. + std::thread::sleep(Duration::from_millis(500)); + const RESENT: u64 = 100; + let requested = Instant::now(); + shared.force_next_idr.store(true, Ordering::SeqCst); + publish(RESENT); + std::thread::sleep(STILL_WINDOW); + producer.close(); + + match stage.join() { + Ok(Ok(())) => {} + Ok(Err(error)) => panic!("{encoder} encode stage failed: {error}"), + Err(_) => panic!("{encoder} encode stage panicked"), + } + let software_fallback = PIPELINE_STATS.lock().using_software_fallback; + let packets = collector.join().unwrap(); + let keyframe = packets + .iter() + .find(|(_, nal)| nal.is_keyframe && nal.sequence == RESENT) + .map(|(at, _)| at.duration_since(requested)); + eprintln!( + "{encoder}: still-desktop keyframe after {keyframe:?}; packets {:?}", + packets + .iter() + .map(|(at, nal)| ( + nal.sequence, + nal.is_keyframe, + at.saturating_duration_since(requested).as_millis() + )) + .collect::>() + ); + (keyframe, software_fallback) +} + +fn assert_still_keyframe(encoder: &str) { + let (keyframe, software_fallback) = keyframe_on_a_still_desktop(encoder); + assert!( + encoder == "libx264" || !software_fallback, + "{encoder} fell back to libx264" + ); + assert!( + keyframe.is_some(), + "{encoder} held the requested keyframe until the next captured frame" + ); +} + +#[test] +fn software_encoder_sends_a_requested_keyframe_on_a_still_desktop() { + let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + assert_still_keyframe("libx264"); +} + +#[test] +fn amf_sends_a_requested_keyframe_on_a_still_desktop() { + for name in ["h264_amf", "hevc_amf"] { + if !amf_available(name) { + continue; + } + let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + assert_still_keyframe(name); + } +} + +#[cfg(windows)] +#[test] +#[ignore = "native: an NVIDIA card"] +fn nvenc_sends_a_requested_keyframe_on_a_still_desktop() { + let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + for name in ["h264_nvenc", "hevc_nvenc"] { + assert_still_keyframe(name); + } +} + #[test] fn amf_fails_cleanly_without_an_amd_driver() { if amf_expectation() != Some(AmfExpectation::Fallback) { From 7f0df487044ffb770a1ce3f12558b20585df427c Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 13:48:46 -0500 Subject: [PATCH 033/226] Replace a failing window device too, and let go of the lost one On DX12 the replacement device could not draw: eframe kept the lost device alive, so D3D12 handed out no new one on that GPU and the window's old swapchain stayed. The window now drops eframe's copy. A wgpu error on the new device replaces it again, at most every 500 ms, instead of panicking. --- host/src/gui/mod.rs | 2 +- host/src/gui/renderer.rs | 19 +++++++++++++------ host/src/startup/mod.rs | 11 ++++++++--- vendor/README.md | 7 +++++-- vendor/egui-wgpu/src/winit.rs | 28 ++++++++++++++++++++-------- 5 files changed, 47 insertions(+), 20 deletions(-) diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 53b1a294..051e6e33 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -1078,7 +1078,7 @@ impl eframe::App for AnalyzerApp { fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { renderer::frame_started(ui.ctx()); - renderer::simulate_device_loss(frame); + renderer::release_render_state(frame); crate::startup::simulate_window_failure(); let ctx = ui.ctx().clone(); if let Some(language) = self.pending_language.take() { diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 15275474..f5d76675 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -456,16 +456,23 @@ pub(super) fn frame_started(ctx: &egui::Context) { } } -/// Lose the window's graphics device on purpose when -/// [`crate::startup::SIMULATE_ENV`] asks: DX12 removes the device, as a -/// driver reset does; other backends destroy it. -pub(super) fn simulate_device_loss(frame: &eframe::Frame) { - if !crate::startup::simulate_device_loss_now() { +/// Drop eframe's copy of the window's first device, which is kept for paint +/// callbacks the window has none of. A lost device is only replaced once +/// nothing holds it: D3D12 keeps one device per GPU in a process, and the +/// window's old swapchain stays with it. When [`crate::startup::SIMULATE_ENV`] +/// asks for a device loss, the copy is kept until then, and the device is lost +/// on purpose: DX12 removes it, as a driver reset does; other backends destroy +/// it. +pub(super) fn release_render_state(frame: &mut eframe::Frame) { + if crate::startup::device_loss_requested() && !crate::startup::simulate_device_loss_now() { return; } - let Some(render_state) = frame.wgpu_render_state() else { + let Some(render_state) = frame.wgpu_render_state.take() else { return; }; + if !crate::startup::device_loss_requested() { + return; + } warn!("Simulated loss of the window's graphics device"); #[cfg(windows)] if remove_dx12_device(&render_state.device) { diff --git a/host/src/startup/mod.rs b/host/src/startup/mod.rs index 13ef0209..a4977226 100644 --- a/host/src/startup/mod.rs +++ b/host/src/startup/mod.rs @@ -585,11 +585,16 @@ pub(crate) fn simulate_window_failure() { } } -/// True once, 15 s after the window works, when [`SIMULATE_ENV`] asks for -/// `device-lost-later`: the window's graphics device is to be lost then. +/// [`SIMULATE_ENV`] asks for `device-lost-later`. +pub(crate) fn device_loss_requested() -> bool { + simulated() == "device-lost-later" +} + +/// True once, 15 s after the window works, when [`device_loss_requested`]: +/// the window's graphics device is to be lost then. pub(crate) fn simulate_device_loss_now() -> bool { static DONE: AtomicBool = AtomicBool::new(false); - simulated() == "device-lost-later" + device_loss_requested() && SHOWN_AT .get() .is_some_and(|shown| shown.elapsed() > Duration::from_secs(15)) diff --git a/vendor/README.md b/vendor/README.md index f3338770..5a1a2f3b 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -18,14 +18,17 @@ panic ends the app and the stream with it. 0.37.0 still does. The patch: - `Renderer::update_buffers` skips a frame whose staging buffer can't be made, and `Renderer::render` then draws nothing. -- The winit `Painter` watches each device's lost callback, and keeps wgpu's - default error handler from panicking on what a lost device reports. +- The winit `Painter` watches each device's lost callback. A wgpu error + outside an error scope counts as a loss too, where wgpu's default handler + panics: a window whose device or surface fails is drawn on a new one. - When the device is lost, or a frame's upload fails, the `Painter` drops the device, its surfaces and egui's renderer. eframe calls `set_window` before each paint, which makes new ones; until that works the window skips frames and tries again every 500 ms. The new device's first frame uploads the font atlas again. Other textures are the app's to load again (`host/src/gui/renderer.rs` counts devices for that). +- A D3D12 device is only replaced once nothing holds the lost one, so the host + drops the copy eframe keeps in `Frame` (`release_render_state`). Drop this copy when an egui-wgpu release survives device loss, or carry the patch to the version eframe moves to. diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs index 55a922a3..6ee4d76c 100644 --- a/vendor/egui-wgpu/src/winit.rs +++ b/vendor/egui-wgpu/src/winit.rs @@ -15,7 +15,7 @@ use egui::{Context, Event, UserData, ViewportId, ViewportIdMap, ViewportIdSet}; use std::sync::Arc; /// EternalMonitor patch: how often a lost device's replacement is tried again -/// while it can't be created, as during a driver reset. +/// while it can't be created or doesn't work, as during a driver reset. const REBUILD_RETRY: Duration = Duration::from_millis(500); struct SurfaceState { @@ -59,6 +59,8 @@ pub struct Painter { rebuilding: bool, /// The new device's first paint uploads the font atlas again. restore_fonts: bool, + /// When the last device was dropped, so a replacement that fails at once waits. + dropped_at: Option, } impl Painter { @@ -103,6 +105,7 @@ impl Painter { device_lost: Arc::new(AtomicBool::new(false)), rebuilding: false, restore_fonts: false, + dropped_at: None, } } @@ -288,8 +291,9 @@ impl Painter { Ok(()) } - /// EternalMonitor patch: flag the loss of `device`, and keep wgpu's default handler - /// from panicking on the errors a lost device then reports. + /// EternalMonitor patch: flag the loss of `device`. An error wgpu reports outside an + /// error scope flags it too, instead of panicking as wgpu's default handler does: a + /// window whose device or surface fails is drawn again on a new one. fn watch_device(&mut self, device: &wgpu::Device) { let lost = Arc::new(AtomicBool::new(false)); let flag = Arc::clone(&lost); @@ -299,11 +303,10 @@ impl Painter { }); let flag = Arc::clone(&lost); device.on_uncaptured_error(Arc::new(move |error| { - if flag.load(Ordering::SeqCst) { + if flag.swap(true, Ordering::SeqCst) { log::debug!("wgpu error on the lost device: {error}"); } else { - // As wgpu's default handler. - panic!("wgpu error: {error}\n"); + log::error!("wgpu error; replacing the window's device: {error}"); } })); self.device_lost = lost; @@ -320,6 +323,7 @@ impl Painter { self.screen_capture_state = None; self.rebuilding = true; self.restore_fonts = false; + self.dropped_at = Some(web_time::Instant::now()); self.context.request_repaint(); } @@ -582,9 +586,17 @@ impl Painter { let capture = !capture_data.is_empty(); let mut vsync_sec = 0.0; - // EternalMonitor patch: a lost device draws nothing again; replace it. + // EternalMonitor patch: a lost device draws nothing again; replace it, at most + // every REBUILD_RETRY. if self.device_lost.load(Ordering::SeqCst) { - self.drop_lost_device(); + if self + .dropped_at + .is_some_and(|dropped| dropped.elapsed() < REBUILD_RETRY) + { + self.context.request_repaint_after(REBUILD_RETRY); + } else { + self.drop_lost_device(); + } return vsync_sec; } From d39ce5b3d63010adca78ee1c9e6a94e9fa8e58ad Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:00:58 -0500 Subject: [PATCH 034/226] Test that an answer signing other addresses, or none, isn't the saved PC The fake PC can sign any addresses. At the asked port, an answer signing another address or no address leads to no HELLO2 and keeps the pairing. The forwarding helper gets a plainer name. --- ios/EternalMonitorTests/FakeUDPHost.swift | 9 +++--- ios/EternalMonitorTests/MovedPCTests.swift | 37 ++++++++++++++++++---- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index 204b81fa..f3d6228e 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -92,16 +92,15 @@ final class FakeUDPHost: @unchecked Sendable { } /// Answers the next identify request as the PC holding `token` would, - /// receiving at this fake's port on loopback, or with `proof` instead. - /// False after thirty quiet seconds. - func answerIdentify(token: Data, proof: Data? = nil) -> Bool { + /// receiving at this fake's port on `addresses` (loopback unless given), + /// or with `proof` instead. False after thirty quiet seconds. + func answerIdentify(token: Data, proof: Data? = nil, addresses: [Data] = [Data([127, 0, 0, 1])]) -> Bool { for _ in 0..<100 { guard let (data, from, length) = receive() else { continue } let message = Wire.parseControl(data)?.message if case .hello2? = message { helloBeforeIdentify = true } guard case .identify(let request)? = message else { continue } client = (from, length) - let addresses = [Data([127, 0, 0, 1])] send(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identity(WireIdentity( nonce: request.nonce, port: port, addresses: addresses, proof: proof ?? MovedPC.proof(token: token, nonce: request.nonce, port: port, addresses: addresses))))) @@ -112,7 +111,7 @@ final class FakeUDPHost: @unchecked Sendable { /// Hands the next identify request to `pc` and gives the iPad whatever /// `pc` answers, from this fake's port. False after thirty quiet seconds. - func relayIdentify(to pc: FakeUDPHost) -> Bool { + func forwardIdentify(to pc: FakeUDPHost) -> Bool { var target = sockaddr_in() target.sin_len = UInt8(MemoryLayout.size) target.sin_family = sa_family_t(AF_INET) diff --git a/ios/EternalMonitorTests/MovedPCTests.swift b/ios/EternalMonitorTests/MovedPCTests.swift index 6eab707b..4a8d95c3 100644 --- a/ios/EternalMonitorTests/MovedPCTests.swift +++ b/ios/EternalMonitorTests/MovedPCTests.swift @@ -279,24 +279,47 @@ final class MovedPCTests: XCTestCase { /// another address, and says so: it isn't the saved PC at this address. /// No HELLO2, no token. func testAnAnswerFromAnotherAddressIsNotTheSamePC() async throws { - let old = FakeUDPHost(), relay = FakeUDPHost(), pc = FakeUDPHost() - let setup = try setUp(old: old, new: relay, token: Self.token, connectTimeout: 7) + let old = FakeUDPHost(), forwarder = FakeUDPHost(), pc = FakeUDPHost() + let setup = try setUp(old: old, new: forwarder, token: Self.token, connectTimeout: 7) defer { setup.manager.cancel() } setup.manager.connect(host: "127.0.0.1", port: old.port) async let answered = Task.detached { pc.answerIdentify(token: Self.token) }.value let types = await Task.detached { () -> [Wire.PacketType] in - guard relay.relayIdentify(to: pc) else { return [] } - return relay.received(within: 4) + guard forwarder.forwardIdentify(to: pc) else { return [] } + return forwarder.received(within: 4) }.value let pcAnswered = await answered XCTAssertTrue(pcAnswered, "the request reached the saved PC") - XCTAssertFalse(types.isEmpty, "the iPad asked the relay") + XCTAssertFalse(types.isEmpty, "the iPad asked the forwarder") XCTAssertFalse(types.contains(.hello2), "\(types)") - XCTAssertFalse(relay.helloBeforeIdentify) + XCTAssertFalse(forwarder.helloBeforeIdentify) try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token) - XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: relay.port)) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: forwarder.port)) + } + + /// The PC at the address and port asked proves it holds the pairing but + /// signs other addresses, or none: it isn't the saved PC at this + /// address. No HELLO2, no token, and the pairing stays. + func testAnAnswerSigningOtherAddressesIsNotTheSamePC() async throws { + for signed in [[Data([192, 0, 2, 7])], []] { + let old = FakeUDPHost(), new = FakeUDPHost() + let setup = try setUp(old: old, new: new, token: Self.token, connectTimeout: 7) + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: old.port) + let types = await Task.detached { () -> [Wire.PacketType] in + guard new.answerIdentify(token: Self.token, addresses: signed) else { return [] } + return new.received(within: 4) + }.value + XCTAssertFalse(types.isEmpty, "\(signed): the iPad kept asking") + XCTAssertFalse(types.contains(.hello2), "\(signed): \(types)") + XCTAssertFalse(new.helloBeforeIdentify, "\(signed)") + try await until("timed out", seconds: 15) { setup.manager.state == .disconnected } + XCTAssertEqual(setup.manager.problem, .noReply(usb: false), "\(signed)") + XCTAssertEqual(try setup.pairings.token(address: address(old)), Self.token, "\(signed)") + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: new.port), "\(signed)") + } } /// Bonjour has the PC at a home network address that this iPad reaches From 303254fd380bb5f5036cb462a08b62457e850976 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:01:44 -0500 Subject: [PATCH 035/226] Keep the idle picture going while only the pointer moves With the iPad drawing the pointer, pointer-only updates are released without a frame. A moving pointer keeps Desktop Duplication from timing out, so the idle keepalive and keyframe answers, which ran only on a timeout, stopped: on the reference PC the watchdog restarted the stream after 5 s of pointer motion. Pointer-only updates now run the same keepalive check as a timeout. --- host/src/capture/dxgi.rs | 41 ++++++++++++++++------- host/src/capture/mod.rs | 70 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 11 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 0bad0f4d..07cb4bc1 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -16,10 +16,10 @@ use super::recovery::{ LossStep, OpenStep, LOSS_GIVE_UP, }; use super::{ - client_connected, frame_budget_for, frame_buffer, frame_has_image, heartbeat, idle_heartbeat, + acquired, client_connected, frame_budget_for, frame_buffer, heartbeat, idle_heartbeat, keepalive_allowed, keepalive_due, read_back_image, reconcile_virtual_display, recovery_poll, - resolve_target, FrameSlot, FrameUse, IdleGate, OutputInfo, RawFrame, ACQUIRE_TIMEOUT_MS, - FPS_WINDOW, FRAME_PADDING, + resolve_target, Acquired, FrameSlot, FrameUse, IdleGate, OutputInfo, RawFrame, + ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, }; use crate::control::{CaptureTarget, SharedControl}; use crate::gpu_preference::Outcome; @@ -498,15 +498,34 @@ pub fn run_capture_loop( // --- Copy desktop texture to staging and read pixels into a // recycled buffer (the readback is the ONLY full-frame copy) --- - // A pointer-only update needs no frame when the iPad draws - // the pointer: the transport sends the position. - if !frame_has_image(frame_info.LastPresentTime, seen_image) - || (draws_pointer && frame_info.LastPresentTime == 0) - { - unsafe { - dupl.ReleaseFrame()?; + // A pointer-only update needs no new frame when the iPad + // draws the pointer: the transport sends the position. + match acquired(frame_info.LastPresentTime, seen_image, draws_pointer) { + Acquired::ReadBack => {} + skip => { + unsafe { + dupl.ReleaseFrame()?; + } + if skip == Acquired::PointerOnly + && client_connected(&shared) + && keepalive_due( + last_send.elapsed(), + shared.force_next_idr.load(Ordering::SeqCst), + seen_image, + ) + && keepalive_allowed(seen_image, second_screen) + { + resend_last_frame( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + (tex_width, tex_height), + ); + } + continue; } - continue; } let frame_data = read_back_image(&shared, generation, &mut seen_image, || { windows::core::Result::Ok(if let Some(ref resource) = desktop_resource { diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index e83117b1..77cec87d 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -108,6 +108,34 @@ pub(crate) fn frame_has_image(last_present_time: i64, seen_image: bool) -> bool last_present_time != 0 || seen_image } +/// What the capture loop does with an acquired frame. +#[cfg(any(windows, test))] +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Acquired { + /// Read the image back for the encoder. + ReadBack, + /// Release it: there is no image yet ([`frame_has_image`]). + Skip, + /// Release it: only the pointer moved, and the iPad draws the pointer. + /// The last picture still goes out as on a still desktop + /// ([`keepalive_due`]): a moving pointer keeps the duplication from ever + /// timing out, and without it the watchdog took the silence for a stuck + /// capture and restarted the stream, and keyframe requests went + /// unanswered. + PointerOnly, +} + +#[cfg(any(windows, test))] +pub(crate) fn acquired(last_present_time: i64, seen_image: bool, draws_pointer: bool) -> Acquired { + if !frame_has_image(last_present_time, seen_image) { + Acquired::Skip + } else if draws_pointer && last_present_time == 0 { + Acquired::PointerOnly + } else { + Acquired::ReadBack + } +} + pub(crate) fn frame_budget_for(target_fps: u32) -> Duration { let fps = target_fps.max(1) as u64; Duration::from_micros(1_000_000 / fps) @@ -978,6 +1006,48 @@ mod tests { assert!(frame_has_image(0, true)); } + /// While only the pointer moves and the iPad draws it, the duplication + /// returns pointer updates (here 600 a second) and never times out. The + /// last picture must still go out every keepalive, and a keyframe request + /// be answered within its resend time, as on a still desktop. + #[test] + fn pointer_only_updates_keep_the_picture_alive_for_an_ipad_that_draws_it() { + assert_eq!(acquired(0, false, true), Acquired::Skip, "no image yet"); + assert_eq!(acquired(0, true, false), Acquired::ReadBack, "older iPad"); + assert_eq!( + acquired(42, true, true), + Acquired::ReadBack, + "the desktop changed" + ); + let mut since_send = Duration::ZERO; + let mut resends = Vec::new(); + let step = Duration::from_micros(1_600); + let mut now = Duration::ZERO; + let (mut asked, mut pending) = (false, false); + while now < Duration::from_secs(3) { + now += step; + since_send += step; + if !asked && now >= Duration::from_millis(2_000) { + (asked, pending) = (true, true); + } + assert_eq!(acquired(0, true, true), Acquired::PointerOnly); + if keepalive_due(since_send, pending, true) { + resends.push(now); + since_send = Duration::ZERO; + pending = false; + } + } + assert!(resends.len() >= 4, "{resends:?}"); + for pair in resends.windows(2) { + assert!(pair[1] - pair[0] <= IDLE_KEEPALIVE + step, "{resends:?}"); + } + assert!( + resends.iter().any(|at| *at >= Duration::from_millis(2_000) + && *at <= Duration::from_millis(2_000) + KEYFRAME_RESEND + step), + "the keyframe request waited: {resends:?}" + ); + } + pub(super) fn connect_viewer(shared: &SharedControl) { use crate::transport::link::PeerId; use crate::transport::session::ConfigSource; From 14385618e26ba69df5804748c0ed9152aea1715f Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:07:29 -0500 Subject: [PATCH 036/226] Sign the home network address first, answer nothing without one, and list off the session lock - Addresses on an adapter with a default gateway come first, so the four signed always include the home network's. - With no address to sign, the PC sends no identity answer. - The home network addresses are listed on a thread of their own at most every 2 s, and first when the stream starts. An identity check reads the last listing and never waits for one. --- host/src/network.rs | 177 ++++++++++++++++++++++++++++++++------ host/src/transport/mod.rs | 63 +++++++++++--- 2 files changed, 202 insertions(+), 38 deletions(-) diff --git a/host/src/network.rs b/host/src/network.rs index 38fd856d..bb92e4e8 100644 --- a/host/src/network.rs +++ b/host/src/network.rs @@ -369,43 +369,107 @@ impl Drop for Listing { } /// The home network addresses among `adapters`: IPv4 on Wi-Fi and Ethernet -/// adapters that are up, in a private range or link-local, link-local last. +/// adapters that are up, in a private range or link-local. Those on an +/// adapter with a default gateway come first and link-local ones last, so a +/// cap on how many are signed keeps the home network's. /// An iPad checking that it found this PC at an address gets these signed. pub fn home_addresses_of(adapters: &[Adapter]) -> Vec { - let mut addresses: Vec = adapters + let mut addresses: Vec<(Ipv4Addr, bool)> = adapters .iter() .filter(|adapter| adapter.up && matches!(adapter.kind, Kind::Wifi | Kind::Ethernet)) - .flat_map(|adapter| adapter.ipv4.iter().copied()) - .filter(|ip| ip.is_private() || ip.is_link_local()) + .flat_map(|adapter| adapter.ipv4.iter().map(|&ip| (ip, adapter.gateway))) + .filter(|(ip, _)| ip.is_private() || ip.is_link_local()) .collect(); - addresses.sort_by_key(|ip| ip.is_link_local()); + addresses.sort_by_key(|&(ip, gateway)| (ip.is_link_local(), !gateway)); let mut seen = Vec::with_capacity(addresses.len()); - addresses.retain(|ip| { - !seen.contains(ip) && { - seen.push(*ip); - true + for (ip, _) in addresses { + if !seen.contains(&ip) { + seen.push(ip); } - }); - addresses + } + seen } /// How long one listing of the home network addresses answers. const HOME_TTL: Duration = Duration::from_secs(2); -/// This PC's home network addresses now (see [`home_addresses_of`]), listed -/// at most every 2 s. -pub fn home_addresses() -> Vec { - static LISTED: Mutex)>> = Mutex::new(None); - let mut listed_at = LISTED.lock(); - if let Some((_, addresses)) = listed_at.as_ref().filter(|(at, _)| at.elapsed() < HOME_TTL) { - return addresses.clone(); +/// The home network addresses from the last listing, listed on a thread of +/// its own at most every 2 s. Asking never waits: an identity check asks +/// under the session lock, and no listing runs while it is held. +pub struct HomeCache { + list: fn() -> Option>, + ttl: Duration, + state: Arc>, +} + +#[derive(Default)] +struct HomeState { + listed: Option<(Instant, Vec)>, + listing: bool, +} + +impl HomeCache { + fn with(list: fn() -> Option>, ttl: Duration) -> Self { + Self { + list, + ttl, + state: Arc::default(), + } + } + + /// The last listing's addresses. A missing or stale one starts a listing + /// and reads as before (or none) until it is done. + pub fn get(&self) -> Vec { + let (addresses, start) = { + let mut state = self.state.lock(); + let stale = state + .listed + .as_ref() + .is_none_or(|(at, _)| at.elapsed() >= self.ttl); + let start = stale && !state.listing; + state.listing |= start; + let addresses = state + .listed + .as_ref() + .map(|(_, addresses)| addresses.clone()); + (addresses.unwrap_or_default(), start) + }; + if start { + let (list, listing) = (self.list, HomeListing(self.state.clone())); + // A thread that can't start drops the closure, and with it the guard. + let _ = std::thread::Builder::new() + .name("home-addresses".into()) + .spawn(move || { + let addresses = list() + .map(|adapters| home_addresses_of(&adapters)) + .unwrap_or_default(); + listing.0.lock().listed = Some((Instant::now(), addresses)); + }); + } + addresses } - let addresses = listed() - .or_else(interfaces) - .map(|adapters| home_addresses_of(&adapters)) - .unwrap_or_default(); - *listed_at = Some((Instant::now(), addresses.clone())); - addresses +} + +/// Ends a listing when dropped, so a worker that panics loses only that +/// refresh. +struct HomeListing(Arc>); + +impl Drop for HomeListing { + fn drop(&mut self) { + self.0.lock().listing = false; + } +} + +fn listed_home() -> Option> { + listed().or_else(interfaces) +} + +/// This PC's home network addresses (see [`home_addresses_of`] and +/// [`HomeCache`]). +pub fn home_addresses() -> Vec { + static HOME: std::sync::LazyLock = + std::sync::LazyLock::new(|| HomeCache::with(listed_home, HOME_TTL)); + HOME.get() } /// What kind of interface a Unix name and flags say: network cards are @@ -1074,6 +1138,71 @@ mod tests { assert!(home_addresses_of(&[]).is_empty()); } + /// The home network's address goes first, so the 4 signed include it + /// however many other addresses the PC has. + #[test] + fn home_addresses_with_a_gateway_come_first() { + let direct = Adapter { + ipv4: (1..=4).map(|last| ip([10, 0, 9, last])).collect(), + ..adapter(Kind::Ethernet, false, [0, 0, 0, 0]) + }; + let adapters = [ + adapter(Kind::Wifi, true, [169, 254, 3, 4]), + direct, + adapter(Kind::Wifi, true, [192, 168, 1, 20]), + ]; + let addresses = home_addresses_of(&adapters); + assert_eq!(addresses[0], ip([192, 168, 1, 20])); + assert_eq!( + addresses[1..5], + (1..=4).map(|last| ip([10, 0, 9, last])).collect::>()[..] + ); + assert_eq!(addresses.last(), Some(&ip([169, 254, 3, 4]))); + } + + static HOME_GATE: Mutex<()> = Mutex::new(()); + static HOME_CALLS: AtomicUsize = AtomicUsize::new(0); + static HOME_THREAD: Mutex> = Mutex::new(None); + + fn gated_home() -> Option> { + *HOME_THREAD.lock() = std::thread::current().name().map(str::to_string); + HOME_CALLS.fetch_add(1, Ordering::SeqCst); + let _open = HOME_GATE.lock(); + tailscale_and_wifi() + } + + /// An identity check asks under the session lock: while the adapters are + /// still being listed it gets none at once, from one listing on a thread + /// of its own, and the addresses once it is done. + #[test] + fn home_addresses_never_wait_for_the_listing() { + let cache = Arc::new(HomeCache::with(gated_home, HOME_TTL)); + let closed = HOME_GATE.lock(); + for _ in 0..50 { + let (answer, asked) = std::sync::mpsc::channel(); + let asking = cache.clone(); + std::thread::spawn(move || answer.send(asking.get())); + let addresses = asked + .recv_timeout(Duration::from_secs(1)) + .expect("asking waited for the listing"); + assert!(addresses.is_empty()); + } + let deadline = Instant::now() + Duration::from_secs(10); + while HOME_CALLS.load(Ordering::SeqCst) == 0 { + assert!(Instant::now() < deadline, "no listing started"); + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(HOME_THREAD.lock().as_deref(), Some("home-addresses")); + drop(closed); + let deadline = Instant::now() + Duration::from_secs(10); + while cache.get().is_empty() { + assert!(Instant::now() < deadline, "the addresses never came"); + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(cache.get(), [ip([192, 168, 1, 20])]); + assert_eq!(HOME_CALLS.load(Ordering::SeqCst), 1); + } + #[cfg(unix)] #[test] fn unix_interfaces_are_told_apart_by_flags_then_name() { diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 8f86f936..51457df2 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -78,6 +78,29 @@ pub mod probe { } } +/// What an identity proof signs for a socket on `listen`: that address, or +/// for one on every address, `home()`. None, and no answer, without an +/// address to sign. +fn identity_endpoints( + listen: SocketAddr, + home: impl FnOnce() -> Vec, +) -> Option<(u16, Vec<[u8; 4]>)> { + let std::net::IpAddr::V4(ip) = listen.ip() else { + return None; + }; + let addresses = if ip.is_unspecified() { + home() + } else { + vec![ip] + }; + (!addresses.is_empty()).then(|| { + ( + listen.port(), + addresses.iter().map(|ip| ip.octets()).collect(), + ) + }) +} + /// Stream parameters advertised in HELLO_ACK and heartbeats. struct SharedConfigSource<'a> { shared: &'a SharedControl, @@ -117,21 +140,8 @@ impl ConfigSource for SharedConfigSource<'_> { } } - /// A socket on one address signs that one; one on every address signs - /// the PC's home network addresses. fn identity_endpoints(&self) -> Option<(u16, Vec<[u8; 4]>)> { - let std::net::IpAddr::V4(ip) = self.listen.ip() else { - return None; - }; - let addresses = if ip.is_unspecified() { - crate::network::home_addresses() - } else { - vec![ip] - }; - Some(( - self.listen.port(), - addresses.iter().map(|ip| ip.octets()).collect(), - )) + identity_endpoints(self.listen, crate::network::home_addresses) } fn notice(&self) -> u8 { @@ -233,6 +243,10 @@ pub async fn start_sender( .max(1); let local_addr = socket.local_addr()?; + if bind_ip.is_unspecified() { + // Lists the home network addresses before anyone asks for them. + crate::network::home_addresses(); + } info!(%local_addr, stream_epoch, "UDP transport ready — waiting for a v2 client HELLO"); // The Address row and QR code follow every bind, so a chosen network // that came back shows its address. Resolved before locking: the stats @@ -1029,6 +1043,27 @@ mod tests { crate::diagnostics::tests::assert_contract(&summaries[0]); } + /// A socket on one address signs that address; one on every address + /// signs the home network addresses. With none to sign, or IPv6, the PC + /// answers nothing rather than an empty list. + #[test] + fn identity_endpoints_need_an_address_to_sign() { + let all = SocketAddr::from(([0, 0, 0, 0], 9876)); + let home = || vec![std::net::Ipv4Addr::new(192, 0, 2, 4)]; + assert_eq!( + identity_endpoints(all, home), + Some((9876, vec![[192, 0, 2, 4]])) + ); + assert_eq!(identity_endpoints(all, Vec::new), None); + let one = SocketAddr::from(([198, 51, 100, 3], 9876)); + assert_eq!( + identity_endpoints(one, || panic!("lists nothing")), + Some((9876, vec![[198, 51, 100, 3]])) + ); + let v6 = SocketAddr::from((std::net::Ipv6Addr::LOCALHOST, 9876)); + assert_eq!(identity_endpoints(v6, home), None); + } + #[tokio::test] async fn repairs_current_and_previous_frames_before_the_pacing_wait_ends() { let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); From fd6c1bc032061e5c024585715401ce781da4bfd6 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:28:58 -0500 Subject: [PATCH 037/226] Send vdd_only and panel_lost as their own summary fallbacks The backend's summary check lists both since its last migration, so the summary no longer folds them into other. The contract test follows it, and a new test carries both through a session's summary to the upload. --- host/src/diagnostics.rs | 54 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index c29b2c10..57bb1dbc 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -286,8 +286,8 @@ fn take_fallback(session_id: u32) -> Option<&'static str> { /// A `second_screen_fallback` reason as the summary names it. `remembered` /// is a failure saved by an earlier app run. The backend's check of the -/// summary lists each token, so the newer reasons, `vdd_only` and -/// `panel_lost`, go as `other` until it lists them too. +/// summary lists each token (`copy_session_summaries`), so a new reason goes +/// as `other` until it lists that one too. fn fallback_token(reason: &str) -> &'static str { match reason { "attach_timeout" => "attach_timeout", @@ -296,6 +296,8 @@ fn fallback_token(reason: &str) -> &'static str { "no_frames" => "no_frames", "duplication_failed" => "duplication_failed", "remembered" => "remembered", + "vdd_only" => "vdd_only", + "panel_lost" => "panel_lost", _ => "other", } } @@ -592,7 +594,7 @@ pub(crate) mod tests { r" stage=(started|hello_sent|accepted|video_sent|first_frame|steady)", r" outcome=(streamed|no_reply|no_picture|pairing|user_ended|app_ended|error)", r" failure=(none|no_reply|no_video|no_frame|vdd_enable_failed|vdd_attach_timeout|vdd_not_attached|capture_failed|encoder_failed|usb_lost|signal_lost|other)", - r" fallback=(none|attach_timeout|enable_failed|no_driver|no_frames|duplication_failed|remembered|other)", + r" fallback=(none|attach_timeout|enable_failed|no_driver|no_frames|duplication_failed|remembered|vdd_only|panel_lost|other)", r" end=(goodbye|liveness|superseded|shutdown|timeout|user|background|error)", r" ttff_ms=(0|[1-9][0-9]{0,5}|-) duration_ms=(0|[1-9][0-9]{0,9})", r" os=(win:[0-9]{1,6}\.[0-9]{1,6}|ipados:[0-9]{1,3}\.[0-9]{1,3}(\.[0-9]{1,3})?) hw=(pc|iPad[0-9]{1,2},[0-9]{1,2})", @@ -885,8 +887,8 @@ pub(crate) mod tests { ("no_frames", "no_frames"), ("duplication_failed", "duplication_failed"), ("remembered", "remembered"), - ("vdd_only", "other"), - ("panel_lost", "other"), + ("vdd_only", "vdd_only"), + ("panel_lost", "panel_lost"), ("C:\\Users\\someone", "other"), ] { assert_eq!(fallback_token(reason), token); @@ -994,6 +996,48 @@ pub(crate) mod tests { ); } + /// The hybrid laptop's two fallbacks along the whole path: the session's + /// summary names each (the backend's check lists both), passes that + /// check, and uploads as written, in its own category. + #[test] + fn a_lone_display_or_lost_panel_reaches_the_uploaded_summary() { + use tracing_subscriber::prelude::*; + let _slot = FALLBACK_SLOT.lock(); + let summaries = std::sync::Arc::new(parking_lot::Mutex::new(Vec::::new())); + let subscriber = tracing_subscriber::registry() + .with(FallbackWatch) + .with(SummaryCapture(summaries.clone())); + let shared = SharedControl::new(9876, 15_000_000); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + tracing::subscriber::with_default(subscriber, || { + for (id, reason) in [(1, "vdd_only"), (2, "panel_lost")] { + note_session_start(id, &shared); + tracing::warn!(second_screen_fallback = reason, "fell back"); + let ended = SessionEnd { + session_id: Some(id), + ..end( + EndReason::Goodbye(ByeReason::UserDisconnect), + Some(400), + 60_000, + ) + }; + log_session_end(&ended, &shared); + } + }); + let lines = summaries.lock().clone(); + assert_eq!(lines.len(), 2, "{lines:?}"); + for (line, token) in lines.iter().zip(["vdd_only", "panel_lost"]) { + assert_contract(line); + assert!(line.contains(&format!(" fallback={token} ")), "{line}"); + let logged = format!("2026-11-07T01:02:03.123456Z INFO {line}"); + assert_eq!( + crate::telemetry::session_summary(&logged), + Some(line.as_str()) + ); + assert_eq!(crate::telemetry::uploaded(&logged), *line); + } + } + #[test] fn windows_11_is_named_from_its_build_not_its_registry_label() { // The customer's PC: Windows 11 26H2 with the legacy label. From 2284eac45b0b573d172826c5a9a4a4e275c473f4 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:30:32 -0500 Subject: [PATCH 038/226] Don't count a closed lid as a lost panel A lid the user closes withdraws the built-in panel's target just as the hybrid laptop's lost panel does. The guard now reads the lid from Windows' GUID_LIDSWITCH_STATE_CHANGE notifications and only counts a withdrawn panel as lost while the lid is open, so closing the lid while the virtual display turns on, with or without an external monitor, goes as on any PC: no panel_lost, no restart notice, nothing remembered. --- host/src/display_guard.rs | 159 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 152 insertions(+), 7 deletions(-) diff --git a/host/src/display_guard.rs b/host/src/display_guard.rs index adc03793..8cdc64f0 100644 --- a/host/src/display_guard.rs +++ b/host/src/display_guard.rs @@ -33,8 +33,9 @@ //! Windows lists the panel's target as unavailable, every layout that names //! it is refused, and only a restart gives it back. The guard tells that //! case apart, on a PC with more than one graphics card, by the panel's -//! target: the virtual display is on, the panel is not, and no built-in -//! target is offered any more. It then makes no `SetDisplayConfig` call and +//! target: the virtual display is on, the panel is not, no built-in target +//! is offered any more, and Windows doesn't say the lid is closed (a closed +//! lid withdraws the panel on purpose). It then makes no `SetDisplayConfig` call and //! never turns the display off: the desktop stays on it, which the iPad //! still shows. Only when Windows ends the session does the turn off run, //! so the restart that brings the panel back starts without the display. @@ -158,6 +159,9 @@ pub trait Displays { fn built_in(&mut self) -> Option; /// The PC has more than one graphics card. fn hybrid(&self) -> bool; + /// Windows says the laptop's lid is closed. False when it hasn't said, + /// as on a PC without a lid. + fn lid_closed(&self) -> bool; /// Windows is ending the session: shutting down, restarting or signing out. fn session_ending(&self) -> bool; fn sleep(&mut self, duration: Duration); @@ -189,7 +193,8 @@ pub enum BuiltIn { /// At least one is available for a path. Offered, /// Listed, and none available: a closed lid, or the hybrid laptop's - /// panel after the virtual display took it. + /// panel after the virtual display took it ([`lost_panel`] tells them + /// apart). Withdrawn, } @@ -756,14 +761,16 @@ fn alone(screens: &[Screen]) -> bool { /// The hybrid laptop's black panel: on a PC with more than one graphics /// card, the virtual display is on in `screens`, the built-in panel is off, -/// and Windows no longer offers any built-in target. After a turn on this -/// is read once the screens settle ([`SETTLE`] after the Enable, while no -/// real screen came back). A closed lid reads the same; holding the display -/// on then costs nothing. +/// and Windows no longer offers any built-in target, with the lid open. +/// After a turn on this is read once the screens settle ([`SETTLE`] after +/// the Enable, while no real screen came back). A lid the user closed +/// withdraws the panel too, and is no failure: the panel comes back when it +/// opens. fn lost_panel(displays: &mut D, screens: &[Screen]) -> bool { has_vdd(screens) && !screens.iter().any(|screen| screen.internal) && displays.hybrid() + && !displays.lid_closed() && displays.built_in() == Some(BuiltIn::Withdrawn) } @@ -863,6 +870,7 @@ mod windows_impl { impl Windows { pub fn new(owner: crate::startup::ProcessId, log: fn(bool, &str)) -> Self { + lid::watch(); Self { log, // Without its start time a process id could be a later process's. @@ -944,6 +952,89 @@ mod windows_impl { } } + /// Whether Windows says the lid is closed, from its + /// GUID_LIDSWITCH_STATE_CHANGE notifications: it sends the current state + /// once the callback is registered, and each change after it. + mod lid { + use std::sync::atomic::{AtomicU8, Ordering}; + use std::sync::Once; + use std::time::Duration; + + use windows::core::GUID; + use windows::Win32::Foundation::{ERROR_SUCCESS, HANDLE}; + use windows::Win32::System::Power::{ + PowerSettingRegisterNotification, DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS, + POWERBROADCAST_SETTING, + }; + use windows::Win32::UI::WindowsAndMessaging::{ + DEVICE_NOTIFY_CALLBACK, PBT_POWERSETTINGCHANGE, + }; + + const LIDSWITCH_STATE_CHANGE: GUID = + GUID::from_u128(0xba3e0f4d_b817_4094_a2d1_d56379e6a0f3); + const UNKNOWN: u8 = u8::MAX; + /// 0 closed, 1 open. + static STATE: AtomicU8 = AtomicU8::new(UNKNOWN); + + unsafe extern "system" fn changed( + _context: *const core::ffi::c_void, + kind: u32, + setting: *const core::ffi::c_void, + ) -> u32 { + if kind == PBT_POWERSETTINGCHANGE && !setting.is_null() { + let setting = setting.cast::(); + let header = unsafe { std::ptr::read_unaligned(setting) }; + if header.PowerSetting == LIDSWITCH_STATE_CHANGE && header.DataLength >= 4 { + let data = unsafe { std::ptr::addr_of!((*setting).Data) }.cast::(); + let value = unsafe { std::ptr::read_unaligned(data) }; + STATE.store(value.min(1) as u8, Ordering::Relaxed); + } + } + 0 + } + + /// Start listening. Later calls do nothing. + pub(super) fn watch() { + static WATCH: Once = Once::new(); + WATCH.call_once(|| { + // Windows keeps both for as long as the process runs. + let parameters = Box::leak(Box::new(DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS { + Callback: Some(changed), + Context: std::ptr::null_mut(), + })); + let mut registration = std::ptr::null_mut(); + let result = unsafe { + PowerSettingRegisterNotification( + &LIDSWITCH_STATE_CHANGE, + DEVICE_NOTIFY_CALLBACK, + HANDLE((parameters as *mut DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS).cast()), + &mut registration, + ) + }; + if result != ERROR_SUCCESS { + tracing::warn!( + error = result.0, + "Could not watch the lid; a built-in screen Windows withdraws counts as \ + lost" + ); + } + }); + } + + /// Closed, as Windows last said. A process that just started waits + /// briefly for the first notification. + pub(super) fn closed() -> bool { + watch(); + for _ in 0..20 { + match STATE.load(Ordering::Relaxed) { + UNKNOWN => std::thread::sleep(Duration::from_millis(10)), + state => return state == 0, + } + } + false + } + } + impl super::Displays for Windows { type Saved = Saved; @@ -1006,6 +1097,10 @@ mod windows_impl { crate::diagnostics::host_facts().hybrid == Some(true) } + fn lid_closed(&self) -> bool { + lid::closed() + } + fn session_ending(&self) -> bool { unsafe { GetSystemMetrics(SM_SHUTTINGDOWN) != 0 } } @@ -1123,6 +1218,7 @@ mod tests { /// What `QDC_ALL_PATHS` says of the panel; `None` (unread) by default. built_in: Option, hybrid: bool, + lid_closed: bool, session_ending: bool, slept: Duration, logs: Vec<(bool, String)>, @@ -1221,6 +1317,10 @@ mod tests { self.hybrid } + fn lid_closed(&self) -> bool { + self.lid_closed + } + fn session_ending(&self) -> bool { self.session_ending } @@ -1616,6 +1716,48 @@ mod tests { assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); } + /// A lid the user closes while the virtual display turns on withdraws + /// the panel too, and is no failure. With an external monitor on, the + /// turn on goes as on any PC; with the panel alone before, as #185's + /// steps go. Neither is a lost panel. + #[test] + fn a_lid_closed_during_the_turn_on_is_not_a_lost_panel() { + let mut fake = hybrid_laptop(); + fake.lid_closed = true; + fake.reads = VecDeque::from([ + Some(vec![laptop_panel(), monitor()]), + Some(vec![monitor(), lone_vdd()]), + ]); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Left); + assert!(!guard.panel_lost() && !guard.alone()); + assert_eq!(fake.set_display_config_calls(), 0); + assert!(!fake.text().contains("panel_lost"), "{}", fake.text()); + + let mut fake = hybrid_laptop(); + fake.lid_closed = true; + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); + assert!(!guard.panel_lost()); + assert!(!fake.text().contains("panel_lost"), "{}", fake.text()); + } + + /// The start-up turn off of a closed laptop whose virtual display is + /// still on from an earlier run, with no lost panel ever: it goes as + /// before, and the display is turned off. + #[test] + fn a_closed_lid_start_up_turn_off_runs_as_before() { + let mut fake = hybrid_laptop(); + fake.lid_closed = true; + fake.reads = VecDeque::from([Some(vec![lone_vdd()])]); + let mut guard = Guard::new(); + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.extend_calls, 1); + assert!(!fake.text().contains("panel_lost"), "{}", fake.text()); + } + #[test] fn a_healthy_pc_sees_no_set_display_config_call() { // Panel and monitor stay through the session; the monitor as the @@ -1921,6 +2063,9 @@ mod tests { fn hybrid(&self) -> bool { self.0.hybrid() } + fn lid_closed(&self) -> bool { + self.0.lid_closed() + } fn session_ending(&self) -> bool { self.0.session_ending() } From 981456f90c3e461eb5722d5bf6a48c46c93f4ff7 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:31:50 -0500 Subject: [PATCH 039/226] Sign no home network address once its listing has expired The addresses are given out only while the listing that found them started less than 2 s ago. An expired listing reads as none, so the PC sends no identity answer, and a new listing starts on its own thread. A listing that is already expired when it ends is dropped. --- host/src/network.rs | 50 +++++++++++++++++------ host/src/transport/session.rs | 74 +++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 13 deletions(-) diff --git a/host/src/network.rs b/host/src/network.rs index bb92e4e8..be458d75 100644 --- a/host/src/network.rs +++ b/host/src/network.rs @@ -395,7 +395,8 @@ const HOME_TTL: Duration = Duration::from_secs(2); /// The home network addresses from the last listing, listed on a thread of /// its own at most every 2 s. Asking never waits: an identity check asks -/// under the session lock, and no listing runs while it is held. +/// under the session lock, and no listing runs while it is held. Addresses +/// older than that are never given out: the PC may have lost them. pub struct HomeCache { list: fn() -> Option>, ttl: Duration, @@ -409,7 +410,7 @@ struct HomeState { } impl HomeCache { - fn with(list: fn() -> Option>, ttl: Duration) -> Self { + pub(crate) fn with(list: fn() -> Option>, ttl: Duration) -> Self { Self { list, ttl, @@ -417,33 +418,35 @@ impl HomeCache { } } - /// The last listing's addresses. A missing or stale one starts a listing - /// and reads as before (or none) until it is done. + /// The last listing's addresses while it is fresh: it started less than + /// `ttl` ago. A missing or expired one reads as none, so nothing is + /// signed, and starts a listing on its own thread. A listing that is + /// already expired when it ends is dropped. pub fn get(&self) -> Vec { let (addresses, start) = { let mut state = self.state.lock(); - let stale = state + let fresh = state .listed .as_ref() - .is_none_or(|(at, _)| at.elapsed() >= self.ttl); - let start = stale && !state.listing; + .filter(|(started, _)| started.elapsed() < self.ttl); + let addresses = fresh.map(|(_, addresses)| addresses.clone()); + let start = addresses.is_none() && !state.listing; state.listing |= start; - let addresses = state - .listed - .as_ref() - .map(|(_, addresses)| addresses.clone()); (addresses.unwrap_or_default(), start) }; if start { - let (list, listing) = (self.list, HomeListing(self.state.clone())); + let (list, ttl, listing) = (self.list, self.ttl, HomeListing(self.state.clone())); // A thread that can't start drops the closure, and with it the guard. let _ = std::thread::Builder::new() .name("home-addresses".into()) .spawn(move || { + let started = Instant::now(); let addresses = list() .map(|adapters| home_addresses_of(&adapters)) .unwrap_or_default(); - listing.0.lock().listed = Some((Instant::now(), addresses)); + if started.elapsed() < ttl { + listing.0.lock().listed = Some((started, addresses)); + } }); } addresses @@ -1160,6 +1163,27 @@ mod tests { assert_eq!(addresses.last(), Some(&ip([169, 254, 3, 4]))); } + static SLOW_CALLS: AtomicUsize = AtomicUsize::new(0); + + fn slow_home() -> Option> { + SLOW_CALLS.fetch_add(1, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(150)); + tailscale_and_wifi() + } + + /// A listing that took longer than its lifetime is already stale when + /// it arrives: it is dropped, and the addresses stay unknown. + #[test] + fn a_listing_older_than_its_lifetime_is_dropped() { + let cache = HomeCache::with(slow_home, Duration::from_millis(50)); + let deadline = Instant::now() + Duration::from_secs(10); + while SLOW_CALLS.load(Ordering::SeqCst) < 3 { + assert!(cache.get().is_empty(), "a stale listing was signed"); + assert!(Instant::now() < deadline, "no listing finished"); + std::thread::sleep(Duration::from_millis(5)); + } + } + static HOME_GATE: Mutex<()> = Mutex::new(()); static HOME_CALLS: AtomicUsize = AtomicUsize::new(0); static HOME_THREAD: Mutex> = Mutex::new(None); diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 4939ac10..3a3b7b1b 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -2501,6 +2501,80 @@ mod tests { } } + /// A paired host on every address whose home network addresses come + /// from `HomeCache`, listed by `held_home`. + struct ListedConfig(PairConfig, crate::network::HomeCache); + impl ConfigSource for ListedConfig { + fn stream_config(&self) -> StreamConfig { + TestConfig.stream_config() + } + fn host_name(&self) -> String { + TestConfig.host_name() + } + fn pairing(&self) -> Option<&parking_lot::Mutex> { + self.0.pairing() + } + fn identity_endpoints(&self) -> Option<(u16, Vec<[u8; 4]>)> { + super::super::identity_endpoints(SocketAddr::from(([0, 0, 0, 0], 9876)), || { + self.1.get() + }) + } + } + + /// Held while the test keeps the PC's adapters from being listed. + static HOME_HELD: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); + + fn held_home() -> Option> { + let _open = HOME_HELD.lock(); + Some(vec![crate::network::Adapter { + name: String::new(), + kind: crate::network::Kind::Wifi, + up: true, + gateway: true, + ipv4: vec![std::net::Ipv4Addr::new(192, 168, 1, 20)], + }]) + } + + /// Once the last listing of the PC's addresses has expired, IDENTIFY gets + /// no answer until a new listing comes, however long that takes: the PC + /// never signs an address it may no longer have. + #[test] + fn identify_signs_no_expired_listing() { + let lifetime = Duration::from_secs(1); + let config = ListedConfig( + pair_config(true), + crate::network::HomeCache::with(held_home, lifetime), + ); + let mut session = Session::new(100); + let start = Instant::now(); + // A new asker and a new limiter window each time, so only the + // listing decides. + let mut asked = 0u32; + let mut ask = |session: &mut Session| { + asked += 1; + let asker = addr([192, 0, 2, (asked % 250) as u8 + 1], 50100); + answered(session, asker, &config, start + IDENTIFY_WINDOW * asked) + }; + let deadline = Instant::now() + Duration::from_secs(10); + while !ask(&mut session) { + assert!(Instant::now() < deadline, "the addresses were never listed"); + std::thread::sleep(Duration::from_millis(5)); + } + + let held = HOME_HELD.lock(); + std::thread::sleep(lifetime + Duration::from_millis(100)); + for _ in 0..30 { + assert!(!ask(&mut session), "signed an expired listing"); + std::thread::sleep(Duration::from_millis(50)); + } + drop(held); + let deadline = Instant::now() + Duration::from_secs(10); + while !ask(&mut session) { + assert!(Instant::now() < deadline, "a fresh listing never came"); + std::thread::sleep(Duration::from_millis(5)); + } + } + fn identify_config(required: bool) -> IdentifyConfig { IdentifyConfig( pair_config(required), From 64be3d641526f26a83b23c75fc6f907c10cc8a97 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:32:10 -0500 Subject: [PATCH 040/226] Turn a lost panel's display off at quit, start-up and in the reporter The hold kept the virtual display on through the quit and the failure reporter, and only a turn off during Windows' shutdown let it go. A user who quit EternalMonitor before restarting left it on for good: its enabled state outlasts the restart, so the PC started with it again. Once the host quits nothing streams the display, as the iPad's stream ends with it, so holding it gains nothing. The turn offs that are cleanups (at start-up, at quit, and in the failure reporter after the host ended) now run while the panel is lost, still with no SetDisplayConfig call, using the installed Disable task. The hold stays for turn offs while the host runs. --- ARCHITECTURE.md | 6 ++- host/src/display_guard.rs | 94 ++++++++++++++++++++++++++++++++++----- host/src/vdd.rs | 4 ++ 3 files changed, 90 insertions(+), 14 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 28800203..2b663b49 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -201,8 +201,10 @@ acts while the virtual display is still on: a turn on that leaves it alone switches Windows to "Extend" (else the saved layout or "PC screen only"), and a turn off never runs while it is the only screen, until Windows ends the session. On a hybrid laptop whose panel the virtual display takes away until a restart -(Windows withdraws its target), it tries nothing and keeps the display on, and -Second screen mirrors there until the user chooses it again. It also ships a UDP +(Windows withdraws its target while the lid is open), it tries nothing and +keeps the display on while the host runs; the turn offs at start-up, quit and +in the failure reporter run, so the restart starts without it. Second screen +mirrors there until the user chooses it again. It also ships a UDP firewall rule. Runtime upgrade, task-ACL and uninstall checks remain hardware gates. diff --git a/host/src/display_guard.rs b/host/src/display_guard.rs index 8cdc64f0..29632185 100644 --- a/host/src/display_guard.rs +++ b/host/src/display_guard.rs @@ -35,10 +35,13 @@ //! case apart, on a PC with more than one graphics card, by the panel's //! target: the virtual display is on, the panel is not, no built-in target //! is offered any more, and Windows doesn't say the lid is closed (a closed -//! lid withdraws the panel on purpose). It then makes no `SetDisplayConfig` call and -//! never turns the display off: the desktop stays on it, which the iPad -//! still shows. Only when Windows ends the session does the turn off run, -//! so the restart that brings the panel back starts without the display. +//! lid withdraws the panel on purpose). It then makes no `SetDisplayConfig` +//! call, and while the host runs it doesn't turn the display off: the +//! desktop stays on it, which the iPad still shows. A turn off that nothing +//! streams after, at start-up, at quit, in the failure reporter or once +//! Windows ends the session, runs, so the restart that brings the panel +//! back starts without the display: its enabled state would outlast the +//! restart. //! //! The layout from before a turn on is also kept on disk, so the failure //! reporter, a new process, compares with it when the host that turned @@ -164,6 +167,9 @@ pub trait Displays { fn lid_closed(&self) -> bool; /// Windows is ending the session: shutting down, restarting or signing out. fn session_ending(&self) -> bool; + /// This turn off is a cleanup, after which nothing streams the display: + /// at start-up, at quit, or in the failure reporter once the host ended. + fn cleanup(&self) -> bool; fn sleep(&mut self, duration: Duration); fn log(&mut self, warn: bool, text: &str); /// The host whose layout from before its turn on is kept on disk: this @@ -467,6 +473,16 @@ impl Guard { return true; } if lost_panel(displays, screens) { + if displays.cleanup() { + let text = format!( + "The built-in screen is still unavailable, and nothing streams the virtual \ + display after this — turning it off, so the restart that brings the screen \ + back starts without it outcome=released reason=panel_lost {}", + line(Some(screens)) + ); + displays.log(true, &text); + return true; + } let text = format!( "Leaving the virtual display on: the built-in screen is still unavailable, and \ turning the display off would leave the PC with no screen outcome=kept_on \ @@ -806,7 +822,8 @@ static GUARD: parking_lot::Mutex> = /// Run a driver change for `owner`'s display with the guard around it. /// `log` writes one line; `display_off` says the driver tasks know the -/// display is off; `done` tells a change that succeeded. A turn off that +/// display is off; `cleanup`, that nothing streams the display after this +/// turn off; `done` tells a change that succeeded. A turn off that /// would leave the PC with no screen does not run: `kept_on` is its result. #[cfg(windows)] pub(crate) fn around( @@ -814,11 +831,12 @@ pub(crate) fn around( owner: crate::startup::ProcessId, log: fn(bool, &str), display_off: bool, + cleanup: bool, change: impl FnOnce() -> T, done: impl FnOnce(&T) -> bool, kept_on: impl FnOnce() -> T, ) -> T { - let mut displays = windows_impl::Windows::new(owner, log); + let mut displays = windows_impl::Windows::new(owner, log, cleanup); if !GUARD.lock().before(enabled, display_off, &mut displays) { return kept_on(); } @@ -864,15 +882,17 @@ mod windows_impl { pub struct Windows { log: fn(bool, &str), + cleanup: bool, owner: Option, file: Option, } impl Windows { - pub fn new(owner: crate::startup::ProcessId, log: fn(bool, &str)) -> Self { + pub fn new(owner: crate::startup::ProcessId, log: fn(bool, &str), cleanup: bool) -> Self { lid::watch(); Self { log, + cleanup, // Without its start time a process id could be a later process's. owner: (owner.start != 0).then(|| owner.to_string()), file: crate::settings::app_data_dir().map(|directory| directory.join(RECORD_FILE)), @@ -1105,6 +1125,10 @@ mod windows_impl { unsafe { GetSystemMetrics(SM_SHUTTINGDOWN) != 0 } } + fn cleanup(&self) -> bool { + self.cleanup + } + fn sleep(&mut self, duration: Duration) { std::thread::sleep(duration); } @@ -1220,6 +1244,7 @@ mod tests { hybrid: bool, lid_closed: bool, session_ending: bool, + cleanup: bool, slept: Duration, logs: Vec<(bool, String)>, owner: Option, @@ -1325,6 +1350,10 @@ mod tests { self.session_ending } + fn cleanup(&self) -> bool { + self.cleanup + } + fn sleep(&mut self, duration: Duration) { self.slept += duration; } @@ -1659,16 +1688,54 @@ mod tests { assert_eq!(fake.reapply_calls, [1]); } - /// The host that starts after a restart with the display still on - /// (Windows ended before the turn off ran) has no layout from before. - /// Its start-up turn off must not leave the PC with no screen either. + /// Quitting before the restart. Once the host quits, nothing streams the + /// display: the iPad's stream ends with it. The quit's turn off, a + /// cleanup, runs then, with no `SetDisplayConfig` call, so the restart + /// starts without the display, whose enabled state would outlast it. So + /// does the failure reporter's after a host that ended on its own, + /// comparing with the layout that host kept. #[test] - fn a_start_up_turn_off_with_the_panel_withdrawn_leaves_the_display_on() { + fn quitting_with_the_panel_lost_turns_the_display_off_for_the_restart() { let mut fake = hybrid_laptop(); - fake.reads = VecDeque::from([Some(vec![lone_vdd()])]); let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::PanelLost); + // While the host runs, held. assert!(!turn_off_allowed(&mut guard, &mut fake)); + // The quit. + fake.cleanup = true; + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.set_display_config_calls(), 0); + let (warn, text) = fake.logs.last().unwrap(); + assert!(*warn); + assert_eq!( + text, + "The built-in screen is still unavailable, and nothing streams the virtual display \ + after this — turning it off, so the restart that brings the screen back starts \ + without it outcome=released reason=panel_lost paths=0 [] vdd=1" + ); + // The failure reporter, a new process, after the host ended. + let mut reporter = Guard::new(); + assert!(fake.kept.is_some()); + assert!(turn_off_allowed(&mut reporter, &mut fake)); + assert!(fake.text().contains( + "Comparing with the screens kept from before the virtual display turned on: paths=1" + )); + assert_eq!(fake.set_display_config_calls(), 0); + } + + /// The host that starts after a restart with the display still on (a + /// power cut ended the last one) has no layout from before. Its + /// start-up turn off is a cleanup too: it runs, with no call. + #[test] + fn a_start_up_turn_off_with_the_panel_withdrawn_turns_it_off() { + let mut fake = hybrid_laptop(); + fake.cleanup = true; + fake.reads = VecDeque::from([Some(vec![lone_vdd()])]); + let mut guard = Guard::new(); + assert!(turn_off_allowed(&mut guard, &mut fake)); assert_eq!(fake.set_display_config_calls(), 0); + assert!(fake.text().contains("outcome=released reason=panel_lost")); // On a PC with one graphics card, it goes as before. fake.hybrid = false; assert!(turn_off_allowed(&mut guard, &mut fake)); @@ -2069,6 +2136,9 @@ mod tests { fn session_ending(&self) -> bool { self.0.session_ending() } + fn cleanup(&self) -> bool { + self.0.cleanup() + } fn sleep(&mut self, duration: Duration) { self.0.sleep(duration) } diff --git a/host/src/vdd.rs b/host/src/vdd.rs index ffd1ed03..7b5e2f0f 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -470,6 +470,8 @@ fn request_on_turn( crate::startup::ProcessId::current(), log_screens, display_off, + // Start-up, quit and panics: nothing streams it after. + !enabled && purpose == Purpose::Cleanup, || run_task(if enabled { TASK_ENABLE } else { TASK_DISABLE }, progress), |outcome| *outcome == TaskOutcome::Done, // Left on: the next turn off tries again. @@ -539,6 +541,8 @@ pub(crate) fn disable_for_ended( crate::startup::reporter_log, // Only for a turn on. false, + // The host ended. + true, || run_task(TASK_DISABLE, &mut || {}), |outcome| *outcome == TaskOutcome::Done, || TaskOutcome::Failed(None), From c39e5c92993caf4e14365eda885ea37d6a9cc8ba Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:35:55 -0500 Subject: [PATCH 041/226] Ask for a restart only while the lost panel is still off The Stream page showed the restart notice for a lost panel saved before the last restart too, when the panel was back. The memory now knows when Windows last started, and the notice asks for a restart only for a lost panel from this app run or saved since that start. After the restart it says calmly that Second screen is paused on this laptop until Try again. --- host/src/capture/second_screen.rs | 21 +++++++ host/src/capture/second_screen/memory.rs | 74 +++++++++++++++++++++++- host/src/gui/page_stream.rs | 8 ++- host/src/gui/tests.rs | 23 ++++++++ 4 files changed, 122 insertions(+), 4 deletions(-) diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 11ddffc4..536e8a49 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -41,6 +41,10 @@ use tracing::warn; use crate::control::{CaptureTarget, SharedControl, VddStatus}; use crate::gpu_preference::AdapterId; pub(crate) use memory::PANEL_LOST; + +/// The Stream page's reason for a lost panel from before the last restart: +/// the panel is back, and Second screen only waits to be chosen again. +pub(crate) const PANEL_LOST_BEFORE: &str = "panel_lost_before"; use memory::{Hold, Memory}; #[cfg(any(windows, test, debug_assertions))] @@ -172,6 +176,8 @@ pub(crate) struct State { failure: Option<&'static str>, /// The failure saved for later app runs. memory: Memory, + /// This app run's Enable took the built-in panel away. + panel_lost_this_run: bool, /// Windows refused the mirror's screen from the card EternalMonitor /// captures on, which the display configuration then wired to another /// card: see [`note_mirror_refused`]. @@ -501,6 +507,8 @@ pub fn load_memory(shared: &SharedControl) { Instant::now(), ); memory.set_rule(rule, dir.join(memory::RULE_FILE_NAME)); + let uptime = unsafe { windows::Win32::System::SystemInformation::GetTickCount64() } / 1000; + memory.set_boot(wall.saturating_sub(uptime)); shared.second_screen.lock().memory = memory; #[cfg(not(test))] display_power::watch(); @@ -701,6 +709,18 @@ pub(crate) fn remembered_failure(shared: &SharedControl) -> Option<&'static str> } } +/// [`remembered_failure`] for the Stream page's notice, where a lost panel +/// from before the last restart is [`PANEL_LOST_BEFORE`]: only one that is +/// still off asks for a restart. +pub(crate) fn notice_failure(shared: &SharedControl) -> Option<&'static str> { + let failure = remembered_failure(shared); + let state = shared.second_screen.lock(); + if failure == Some(PANEL_LOST) && !state.panel_lost_this_run && !state.memory.panel_lost_now() { + return Some(PANEL_LOST_BEFORE); + } + failure +} + /// The user asked to try again: a failed start, or one whose thread is gone, /// is tried once more at the next connection, and a saved failure is /// forgotten. @@ -1524,6 +1544,7 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura *status = VddStatus::Failed; let mut state = shared.second_screen.lock(); state.failure = Some(PANEL_LOST); + state.panel_lost_this_run = true; state.remember_panel_lost(now); } AttachResult::TimedOut => { diff --git a/host/src/capture/second_screen/memory.rs b/host/src/capture/second_screen/memory.rs index fd07daca..2567cdff 100644 --- a/host/src/capture/second_screen/memory.rs +++ b/host/src/capture/second_screen/memory.rs @@ -223,6 +223,8 @@ struct Failed { reason: &'static str, failures: u32, failed_at: u64, + /// Noted by this app run, not read from the file. + this_run: bool, } /// Where the failure is saved and what it is for. Without one, as in unit @@ -269,6 +271,8 @@ pub(crate) struct Memory { /// failure and its count stay for the next app run. excused: bool, rule: Option, + /// When Windows last started, in seconds since 1970, once known. + booted_at: Option, } impl Memory { @@ -313,6 +317,7 @@ impl Memory { reason, failures: record.failures, failed_at: record.failed_at, + this_run: false, }), }; let saver = Arc::new(Saver::new(spawn)); @@ -331,6 +336,7 @@ impl Memory { failed, excused: false, rule: None, + booted_at: None, }; if let Some(failed) = memory.failed { let hold = memory.hold(now); @@ -447,6 +453,25 @@ impl Memory { }) } + /// When Windows last started, in seconds since 1970: a lost panel saved + /// before it was mended by that restart. + pub(crate) fn set_boot(&mut self, booted_at: u64) { + self.booted_at = Some(booted_at); + } + + /// Whether the saved failure is a lost panel that is still off: noted by + /// this app run, or saved since Windows last started. Not one from before + /// a restart, nor any while the start is unknown. + pub(crate) fn panel_lost_now(&self) -> bool { + self.failed.is_some_and(|failed| { + failed.reason == PANEL_LOST + && (failed.this_run + || self + .booted_at + .is_some_and(|booted_at| failed.failed_at >= booted_at)) + }) + } + /// Whether a failure is saved, held or due, and not /// [`Memory::excused`]. pub(crate) fn saved(&self) -> bool { @@ -482,6 +507,7 @@ impl Memory { reason, failures, failed_at, + this_run: true, }); self.hold(now) } @@ -931,9 +957,9 @@ fn overwrite_file(path: &Path, data: &[u8]) -> std::io::Result<()> { #[cfg(test)] mod tests { use super::super::{ - attach, next_step, note_attached, note_connected, note_image, remembered_failure, - retries_later, set_facts, start_proving, Driver, Failure, Step, TurnedOn, - BLOCKED_EXTEND_RETRY, PROVE_TIMEOUT, + attach, next_step, note_attached, note_connected, note_image, notice_failure, + remembered_failure, retries_later, set_facts, start_proving, Driver, Failure, Step, + TurnedOn, BLOCKED_EXTEND_RETRY, PANEL_LOST_BEFORE, PROVE_TIMEOUT, }; use super::*; use crate::capture::tests::{connect_viewer, hybrid_laptop, refuse_moved_mirror, NVIDIA}; @@ -1181,6 +1207,48 @@ mod tests { remove_folder(&path); } + /// The Stream page asks for a restart only while the panel is off: in + /// the app run that lost it, and in a later one before the PC restarts. + /// After the restart the panel is back, and the notice only says that + /// Second screen waits to be chosen again. + #[test] + fn a_lost_panel_asks_for_a_restart_only_until_the_pc_restarted() { + let path = temp_file("panel-lost-notice"); + let (memory, start) = app_run(&path, Duration::ZERO); + let shared = second_screen_with(memory); + connect_viewer(&shared); + assert_eq!(next_step(&shared, start), Step::Attach); + let driver = NeverAttaches { + turned_on: TurnedOn::PanelLost, + ..NeverAttaches::new(start) + }; + attach(&shared, &driver, ATTACH_TIMEOUT); + assert_eq!(notice_failure(&shared), Some(PANEL_LOST)); + // The user quits and starts the app again, with no restart between: + // Windows started an hour before the failure. + for (booted_at, notice) in [ + (T0 - HOUR, PANEL_LOST), + // The restart, ten minutes after it. + (T0 + 600, PANEL_LOST_BEFORE), + ] { + let (mut memory, later) = app_run(&path, Duration::from_secs(1200)); + memory.set_boot(booted_at); + let shared = second_screen_with(memory); + connect_viewer(&shared); + note_connected(&shared, later); + assert_eq!(next_step(&shared, later), Step::Nothing, "held"); + assert_eq!(remembered_failure(&shared), Some(PANEL_LOST)); + assert_eq!(notice_failure(&shared), Some(notice), "{booted_at}"); + } + // Without the start known, a saved one counts as from before. + let (memory, later) = app_run(&path, Duration::from_secs(1200)); + let shared = second_screen_with(memory); + connect_viewer(&shared); + assert_eq!(next_step(&shared, later), Step::Nothing); + assert_eq!(notice_failure(&shared), Some(PANEL_LOST_BEFORE)); + remove_folder(&path); + } + /// The field case: the display never attaches. The next app run mirrors /// at once, without the driver or the 30 s wait, says why once per /// connection, and tries again after the wait. diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index 107b704d..b18a44b3 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -127,7 +127,7 @@ impl AnalyzerApp { } // The user asked for the extended (virtual) display but it could not be brought up, // so capture silently fell back to the primary monitor. Say so. - let vdd_failure = crate::capture::second_screen::remembered_failure(&self.control.shared); + let vdd_failure = crate::capture::second_screen::notice_failure(&self.control.shared); if let Some(mut notice) = second_screen_notice(vdd_status, vdd_health, vdd_problem, vdd_failure) { @@ -923,6 +923,12 @@ pub(super) fn second_screen_notice( true, false, ), + // The same, after the restart that brought the panel back. + (Failed, _, _) if failure == Some("panel_lost_before") => notice( + "Second screen is paused on this laptop because it turned off the built-in screen last time. Click Try again to use it.", + true, + false, + ), (Failed | TaskFailed, VddTaskHealth::NotResponding, _) => notice( "Windows' display driver stopped responding. Restart the PC to use the second screen.", false, diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index 337c4ade..bf894812 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -709,6 +709,29 @@ fn a_lost_built_in_screen_says_to_restart_not_to_extend() { } } +/// After the restart that brought the panel back, the notice is calm: no +/// restart, only that Second screen is paused until it is chosen again. +#[test] +fn a_lost_built_in_screen_from_before_the_restart_only_says_it_is_paused() { + use crate::control::VddStatus; + use crate::vdd::VddTaskHealth; + use crate::vdd_device::VddProblem; + for problem in [None, Some(VddProblem::NotExtended)] { + let paused = second_screen_notice( + VddStatus::Failed, + VddTaskHealth::Ok, + problem, + Some("panel_lost_before"), + ) + .unwrap(); + assert_eq!( + paused.body, + "Second screen is paused on this laptop because it turned off the built-in screen last time. Click Try again to use it." + ); + assert!(paused.retry && !paused.installer); + } +} + /// A second screen failure remembered for this PC is tried again later by /// itself; the notice says so beside Try again. #[test] From cafdf427f567f68d5b180cfc83a590a98f643588 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:36:03 -0500 Subject: [PATCH 042/226] Let go of the kept picture when a newer capture arrives While an AMF session's first keyframe is owed, the encode loop keeps the last picture to give it again. It kept that picture until the newer capture had been encoded, so with another frame in the mailbox all three of the synthetic source's buffers could be in use and the next render found none free. The retry state now lives in OwedKeyframe::next, which drops the kept picture as soon as a newer capture is taken, before it is converted. The retries also stop at a 500 ms wall-clock deadline from when the keyframe went in, rather than after a count of waits. --- host/src/encoder/mod.rs | 152 +++++++++++++++++++++++++++++++--------- 1 file changed, 118 insertions(+), 34 deletions(-) diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index 53c3522e..4af8d288 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -174,20 +174,17 @@ fn run_encode_loop( Some(frame) => frame, None => break, }, - Some(owed) => match frames.take_within(KEYFRAME_FLUSH_WAIT) { - Taken::Frame(frame) => frame, - Taken::Closed => break, - Taken::Nothing => match owed.again() { - Some(frame) => frame, - None => { - info!( - given = owed.given, - "The encoder still holds the session's first keyframe; waiting for the next capture" - ); - owed_keyframe = None; - continue; - } - }, + Some(owed) => match owed.next(&frames) { + Next::Frame(frame) => frame, + Next::Closed => break, + Next::GaveUp => { + info!( + given = owed.given, + "The encoder still holds the session's first keyframe; waiting for the next capture" + ); + owed_keyframe = None; + continue; + } }, }; if !shared.running.load(Ordering::SeqCst) @@ -655,10 +652,13 @@ fn run_encode_loop( } } owed_keyframe = match picture { - Some(picture) if !keyframe_out => Some(OwedKeyframe { - picture, - given: owed_keyframe.map_or(0, |owed| owed.given), - }), + Some(picture) if !keyframe_out => { + let mut owed = owed_keyframe + .take() + .unwrap_or_else(|| OwedKeyframe::until(Instant::now() + KEYFRAME_FLUSH_FOR)); + owed.picture = Some(picture); + Some(owed) + } _ => None, }; } @@ -1689,31 +1689,64 @@ pub fn holds_last_frame(encoder_name: &str) -> bool { /// again, to bring out a session's first keyframe. It is about one frame at /// 60 fps, and longer than AMF takes to encode one. const KEYFRAME_FLUSH_WAIT: Duration = Duration::from_millis(20); -/// How many times the last picture is given again before the keyframe is -/// left to capture's next picture: 500 ms, as an AMD driver can take a few -/// hundred to encode a fresh session's first frame, and capture's keepalive -/// follows at 750 ms. -const KEYFRAME_FLUSHES: u32 = 25; +/// How long after the keyframe went in the last picture is given again, +/// before the keyframe is left to capture's next picture: an AMD driver can +/// take a few hundred ms to encode a fresh session's first frame, and +/// capture's keepalive follows at 750 ms. +const KEYFRAME_FLUSH_FOR: Duration = Duration::from_millis(500); /// A session's first keyframe still inside an encoder that holds its last /// frame, and the picture to give it again. struct OwedKeyframe { - picture: RawFrame, + /// The last picture encoded. It pins one of capture's few buffers. + picture: Option, + /// When the picture stops being given again. + until: Instant, given: u32, } +/// What the encode loop encodes next while a keyframe is owed. +enum Next { + Frame(RawFrame), + Closed, + /// The deadline passed: capture's next picture brings the keyframe out. + GaveUp, +} + impl OwedKeyframe { - fn again(&mut self) -> Option { - if self.given >= KEYFRAME_FLUSHES { - return None; + fn until(until: Instant) -> Self { + Self { + picture: None, + until, + given: 0, + } + } + + /// Capture's next picture if one arrives within [`KEYFRAME_FLUSH_WAIT`], + /// else the last picture again, until the deadline. + fn next(&mut self, frames: &FrameSlot) -> Next { + let wait = KEYFRAME_FLUSH_WAIT.min(self.until.saturating_duration_since(Instant::now())); + match frames.take_within(wait) { + // The newer picture replaces the kept one, which is let go before + // the newer one is converted: capture has no buffer to spare. + Taken::Frame(frame) => { + self.picture = None; + Next::Frame(frame) + } + Taken::Closed => Next::Closed, + Taken::Nothing => match self.picture.take() { + Some(picture) if Instant::now() < self.until => { + self.given += 1; + debug!( + frame = picture.frame_number, + given = self.given, + "Giving the encoder its last picture again to bring out the keyframe" + ); + Next::Frame(picture) + } + _ => Next::GaveUp, + }, } - self.given += 1; - debug!( - frame = self.picture.frame_number, - given = self.given, - "Giving the encoder its last picture again to bring out the keyframe" - ); - Some(self.picture.clone()) } } @@ -4668,6 +4701,57 @@ mod tests { assert!(!pending); } + fn raw(frame_number: u64, data: &std::sync::Arc>) -> RawFrame { + RawFrame { + frame_number, + timestamp: Instant::now(), + data: std::sync::Arc::clone(data), + width: 2, + height: 2, + } + } + + /// Capture recycles three buffers, and the mailbox may hold one while the + /// encoder works on another. A kept picture that outlived a newer capture + /// took the third: the synthetic source found no free buffer and panicked. + #[test] + fn a_newer_capture_releases_the_kept_picture_before_it_is_encoded() { + let (producer, consumer) = crate::capture::frame_slot(); + let kept = std::sync::Arc::new(vec![0u8; 16]); + let newer = std::sync::Arc::new(vec![0u8; 16]); + let mut owed = OwedKeyframe::until(Instant::now() + Duration::from_secs(5)); + owed.picture = Some(raw(1, &kept)); + producer.publish(raw(2, &newer)); + let Next::Frame(frame) = owed.next(&consumer) else { + panic!("no frame"); + }; + assert_eq!(frame.frame_number, 2); + assert_eq!( + std::sync::Arc::strong_count(&kept), + 1, + "the kept picture still pins a capture buffer while the newer one is encoded" + ); + } + + #[test] + fn the_kept_picture_is_given_again_only_until_the_deadline() { + let (_producer, consumer) = crate::capture::frame_slot(); + let kept = std::sync::Arc::new(vec![0u8; 16]); + let mut owed = OwedKeyframe::until(Instant::now() + Duration::from_millis(60)); + owed.picture = Some(raw(1, &kept)); + let started = Instant::now(); + let Next::Frame(frame) = owed.next(&consumer) else { + panic!("the picture was not given again"); + }; + assert_eq!(frame.frame_number, 1); + assert!(started.elapsed() >= KEYFRAME_FLUSH_WAIT); + // The loop keeps the picture again after encoding it. + owed.picture = Some(frame); + std::thread::sleep(Duration::from_millis(80)); + assert!(matches!(owed.next(&consumer), Next::GaveUp)); + assert_eq!(owed.given, 1); + } + #[test] fn amf_options_keep_the_settings_the_ipad_decoder_needs() { let h264 = encoder_options("h264_amf"); From 4a67b21d93c354558c7d595ffb32caf096ea07ea Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:37:06 -0500 Subject: [PATCH 043/226] Wait for the pointer's next frame budget on a high-resolution timer On the reference PC the last position of each burst of pointer updates went 12 to 25 ms after the burst, not within the 8 ms budget: Tokio's timer wakes on Windows' 15.6 ms tick. The budget's end now waits on the transport's high-resolution deadline thread, as pen spacing does. The 100 ms repeat stays on Tokio's timer. --- host/src/transport/mod.rs | 19 +++++++++---- host/src/transport/pointer.rs | 52 ++++++++++++++++++++++++++++------- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 375d22b5..83e15871 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -292,6 +292,7 @@ pub async fn start_sender( let mut pen_tick = tokio::time::interval(Duration::from_millis(50)); pen_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let mut pen_spacing = Deadline::new("pen-spacing")?; + let mut pointer_budget = Deadline::new("pointer-budget")?; loop { // A pen backlog sends one bounded chunk at the top of every pass, @@ -307,12 +308,17 @@ pub async fn start_sender( .keep_pen_alive(*shared.capture_geometry.lock()); } pen_spacing.arm(state.input_relay.pen_spaced_until()); - let pointer_due = { + // Tokio's timer can wake 15 ms late on Windows, longer than a frame + // budget: the budget's end waits on a high-resolution timer. + let pointer_repeat = { let (shape, position) = shared.pointer.latest(); let budget = crate::capture::frame_budget_for(shared.effective_fps()); - state - .pointer - .next_poll(budget, position, shape.map(|shape| shape.id)) + pointer_budget.arm(state.pointer.budget_end( + budget, + position, + shape.map(|shape| shape.id), + )); + state.pointer.repeat_due(budget) }; tokio::select! { // Service timers and queued control before another paced frame. @@ -400,7 +406,10 @@ pub async fn start_sender( _ = shared.pointer.changed() => { send_pointer(&mut state.pointer, &links, &shared).await; } - _ = tokio::time::sleep_until(pointer_due.unwrap_or_else(Instant::now).into()), if pointer_due.is_some() => { + _ = pointer_budget.wait(), if pointer_budget.armed() => { + send_pointer(&mut state.pointer, &links, &shared).await; + } + _ = tokio::time::sleep_until(pointer_repeat.unwrap_or_else(Instant::now).into()), if pointer_repeat.is_some() => { send_pointer(&mut state.pointer, &links, &shared).await; } Some(packet) = audio_rx.recv() => { diff --git a/host/src/transport/pointer.rs b/host/src/transport/pointer.rs index 248150d1..f2623fd8 100644 --- a/host/src/transport/pointer.rs +++ b/host/src/transport/pointer.rs @@ -135,16 +135,33 @@ impl PointerSender { } /// When [`Self::poll`] next has something to send without new input: - /// the end of the budget a newer position waits for, or a repeat. + /// the start of the next budget, for a newer position that waits for + /// it, or a repeat. pub fn next_poll( &self, budget: Duration, latest: Option, shape_id: Option, ) -> Option { - self.session_id?; - let (sent, at) = self.sent?; - let epoch = self.epoch?; + [ + self.budget_end(budget, latest, shape_id), + self.repeat_due(budget), + ] + .into_iter() + .flatten() + .min() + } + + /// The start of the next budget, while a newer position waits for it. + /// At most one budget away, and later than the one before: the loop + /// waits for it on a high-resolution timer. + pub fn budget_end( + &self, + budget: Duration, + latest: Option, + shape_id: Option, + ) -> Option { + let (sent, _) = self.sent?; let waiting = latest.is_some_and(|position| { ( position.visible, @@ -158,14 +175,21 @@ impl PointerSender { sent.shape_id, ) }); + waiting.then(|| self.next_slot(budget)).flatten() + } + + /// When the last position goes again, no earlier than the next budget. + pub fn repeat_due(&self, budget: Duration) -> Option { + Some(self.repeat_at?.max(self.next_slot(budget)?)) + } + + fn next_slot(&self, budget: Duration) -> Option { + self.session_id?; + let (_, at) = self.sent?; + let epoch = self.epoch?; let budget = budget.max(Duration::from_nanos(1)); let slot = at.saturating_duration_since(epoch).as_nanos() / budget.as_nanos(); - let next_slot = epoch + budget * (slot as u32 + 1); - let budget_end = waiting.then_some(next_slot); - [budget_end, self.repeat_at.map(|due| due.max(next_slot))] - .into_iter() - .flatten() - .min() + Some(epoch + budget * (slot as u32 + 1)) } } @@ -230,6 +254,7 @@ mod tests { // wakeups would. let mut sent: Vec<(u64, i32)> = Vec::new(); let mut latest = None; + let mut last_end: Option = None; let mut next = updates.iter().peekable(); for ms in 0..=t + 200 { while next.peek().is_some_and(|(at_ms, _)| *at_ms == ms) { @@ -239,6 +264,13 @@ mod tests { for x in states(&sender.poll(now, BUDGET, (Some(shape(1)), latest))) { sent.push((ms, x)); } + // What the loop's high-resolution timer waits for: within one + // budget, and never earlier than the end it waited for before. + if let Some(end) = sender.budget_end(BUDGET, latest, Some(1)) { + assert!(end > now && end <= now + BUDGET, "{ms} ms"); + assert!(last_end.is_none_or(|last| end >= last), "{ms} ms"); + last_end = Some(end); + } } let slot = |ms: u64| Duration::from_millis(ms).as_nanos() / BUDGET.as_nanos(); for pair in sent.windows(2) { From 0d411b2da9d268fb65408a18352e45d4c9793b50 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:37:16 -0500 Subject: [PATCH 044/226] Give a picture given again its own PTS under ETERNAL_LEGACY_PTS The encode loop gives an AMF session its last picture again with that picture's capture number and time. In legacy mode next_pts returned the capture number unchanged, so those separately encoded frames shared one PTS. Legacy PTS now stay strictly increasing too, as real-time PTS already did. --- host/src/encoder/mod.rs | 45 ++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index 4af8d288..d3f56416 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -890,13 +890,15 @@ impl EncoderState { } fn next_pts(&mut self, captured: Instant, frame_number: u64) -> i64 { - if self.legacy_pts { - return frame_number as i64; - } - let epoch = *self.pts_epoch.get_or_insert(captured); - let real = captured.saturating_duration_since(epoch).as_micros() as i64; - // Strict monotonicity (keepalive resends and clock quirks). - let pts = real.max(self.last_pts + 1); + let pts = if self.legacy_pts { + frame_number as i64 + } else { + let epoch = *self.pts_epoch.get_or_insert(captured); + captured.saturating_duration_since(epoch).as_micros() as i64 + }; + // Strict monotonicity (keepalive resends, a picture given again with + // its capture number and time, and clock quirks). + let pts = pts.max(self.last_pts + 1); self.last_pts = pts; pts } @@ -4711,6 +4713,35 @@ mod tests { } } + /// A picture given again to bring out a keyframe keeps its capture time + /// and number, but is a separate frame to the encoder: it must not share + /// the PTS of the frame before it, under ETERNAL_LEGACY_PTS too. + #[test] + fn a_picture_given_again_gets_its_own_pts_in_both_pts_modes() { + ffmpeg_next::init().unwrap(); + for legacy in [false, true] { + let mut state = libx264_at(640, 360); + state.legacy_pts = legacy; + let captured = Instant::now(); + // Frame 7, given again twice, then capture's next frame. + let frames = [(captured, 7), (captured, 7), (captured, 7)] + .into_iter() + .chain([(captured + Duration::from_millis(16), 8)]); + let mut pts = Vec::new(); + for (at, n) in frames { + let next = state.next_pts(at, n); + state.frame.set_pts(Some(next)); + state.send_frame(next, at, n).unwrap(); + while state.encoder.receive_packet(&mut state.packet).is_ok() {} + pts.push(next); + } + assert!( + pts.windows(2).all(|pair| pair[0] < pair[1]), + "legacy={legacy}: PTS {pts:?}" + ); + } + } + /// Capture recycles three buffers, and the mailbox may hold one while the /// encoder works on another. A kept picture that outlived a newer capture /// took the third: the synthetic source found no free buffer and panicked. From 68f0055095cfa493d7e50c66a7d3df35f2bce96b Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:38:14 -0500 Subject: [PATCH 045/226] Wait for the encoder's warm-up in the still-desktop keyframe test The test slept for a fixed time before requesting the keyframe, so an encoder that opened slowly could still have the warm-up frames to encode, and the request did not reliably take the reopen a reconnect takes. It now waits, for up to 10 s, until the warm-up output has come and gone quiet, then requests the keyframe and waits for it for up to 1.5 s. --- host/tests/amf_encoder.rs | 82 +++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 24 deletions(-) diff --git a/host/tests/amf_encoder.rs b/host/tests/amf_encoder.rs index 61a578dc..cdd2cf9e 100644 --- a/host/tests/amf_encoder.rs +++ b/host/tests/amf_encoder.rs @@ -491,18 +491,23 @@ fn amf_hevc_honours_keyframe_requests() { assert_keyframe_on_request(&run, Id::HEVC); } -/// How long the desktop stays still after a keyframe request. Capture resends -/// the last picture once for the request, and then nothing until the 750 ms -/// keepalive, so an encoder that holds a frame until the next one arrives -/// kept a new viewer waiting that long. Nothing is published in this window: -/// a keyframe that leaves at all left without a next frame. How fast it -/// leaves is printed, not checked, because an AMD driver's own reopen took +/// How long a keyframe requested on a still desktop may take to come out. +/// Capture resends the last picture once for the request, and then nothing +/// until the 750 ms keepalive, so an encoder that holds a frame until the +/// next one arrives kept a new viewer waiting that long. Nothing is published +/// meanwhile: a keyframe that comes out at all came out without a next frame. +/// How fast is printed, not checked, because an AMD driver's own reopen took /// from 50 to over 400 ms in a fresh test process. const STILL_WINDOW: Duration = Duration::from_millis(1500); - -/// Streams a few frames, goes still, then requests a keyframe with one -/// resent picture and nothing after it. Returns how long that picture's -/// keyframe took to leave the stage, or `None` if it never did. +/// How long the encoder may take to open and encode the warm-up frames. +const WARM_UP_LIMIT: Duration = Duration::from_secs(10); +/// The warm-up is over once no packet has come for this long. +const WARM_UP_QUIET: Duration = Duration::from_millis(300); + +/// Streams a few frames and waits for the encoder to encode them, goes still, +/// then requests a keyframe with one resent picture and nothing after it, as +/// for a reconnecting viewer. Returns how long that picture's keyframe took +/// to leave the stage, or `None` if it did not within [`STILL_WINDOW`]. fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { ffmpeg_next::init().unwrap(); let (producer, consumer) = frame_slot(); @@ -519,12 +524,13 @@ fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { let stage = std::thread::spawn(move || { run_encode_stage(consumer, tx, stage_shared, gpu, generation).map_err(|e| e.to_string()) }); + let (packets_tx, packets) = std::sync::mpsc::channel(); let collector = std::thread::spawn(move || { - let mut packets = Vec::new(); while let Some(nal) = rx.blocking_recv() { - packets.push((Instant::now(), nal)); + if packets_tx.send((Instant::now(), nal)).is_err() { + break; + } } - packets }); let publish = |number: u64| { let (width, height) = LARGE; @@ -543,13 +549,45 @@ fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { publish(number); std::thread::sleep(FRAME_INTERVAL); } - // The desktop goes still, and a viewer connects. - std::thread::sleep(Duration::from_millis(500)); + // The request must find a session that has encoded frames, so that it + // takes the reopen a reconnect takes, whatever the encoder's open cost. + let warm_up_until = Instant::now() + WARM_UP_LIMIT; + let mut seen = Vec::new(); + loop { + match packets.recv_timeout(WARM_UP_QUIET) { + Ok(packet) => seen.push(packet), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) if !seen.is_empty() => break, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => assert!( + Instant::now() < warm_up_until, + "{encoder} encoded nothing within {WARM_UP_LIMIT:?}" + ), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + panic!( + "{encoder} stage ended during the warm-up: {:?}", + stage.join() + ) + } + } + } + + // A viewer connects to the still desktop. const RESENT: u64 = 100; let requested = Instant::now(); shared.force_next_idr.store(true, Ordering::SeqCst); publish(RESENT); - std::thread::sleep(STILL_WINDOW); + let mut keyframe = None; + while keyframe.is_none() { + let left = STILL_WINDOW.saturating_sub(requested.elapsed()); + match packets.recv_timeout(left) { + Ok((at, nal)) => { + if nal.is_keyframe && nal.sequence == RESENT { + keyframe = Some(at.duration_since(requested)); + } + seen.push((at, nal)); + } + Err(_) => break, + } + } producer.close(); match stage.join() { @@ -557,20 +595,16 @@ fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { Ok(Err(error)) => panic!("{encoder} encode stage failed: {error}"), Err(_) => panic!("{encoder} encode stage panicked"), } + collector.join().unwrap(); let software_fallback = PIPELINE_STATS.lock().using_software_fallback; - let packets = collector.join().unwrap(); - let keyframe = packets - .iter() - .find(|(_, nal)| nal.is_keyframe && nal.sequence == RESENT) - .map(|(at, _)| at.duration_since(requested)); eprintln!( "{encoder}: still-desktop keyframe after {keyframe:?}; packets {:?}", - packets - .iter() + seen.iter() .map(|(at, nal)| ( nal.sequence, nal.is_keyframe, - at.saturating_duration_since(requested).as_millis() + at.saturating_duration_since(requested).as_millis() as i64 + - requested.saturating_duration_since(*at).as_millis() as i64 )) .collect::>() ); From ae1b7a502fa2fb2c653f4c976d86a736130aed0d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:42:41 -0500 Subject: [PATCH 046/226] Allow the boot time setter's dead code off Windows Only the Windows memory loader reads when Windows started, so clippy on other platforms flagged the setter as unused. --- host/src/capture/second_screen/memory.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/host/src/capture/second_screen/memory.rs b/host/src/capture/second_screen/memory.rs index 2567cdff..9ed2916b 100644 --- a/host/src/capture/second_screen/memory.rs +++ b/host/src/capture/second_screen/memory.rs @@ -455,6 +455,7 @@ impl Memory { /// When Windows last started, in seconds since 1970: a lost panel saved /// before it was mended by that restart. + #[cfg_attr(not(any(windows, test)), allow(dead_code))] // Windows reads the start. pub(crate) fn set_boot(&mut self, booted_at: u64) { self.booted_at = Some(booted_at); } From e7078824311daad7285a4414cc8f350a40a37ee2 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:52:17 -0500 Subject: [PATCH 047/226] Take a panel for lost only once Windows says the lid is open The lid watch read an unknown lid as open after a 200 ms wait, and a failed registration the same way, so a lid closed on purpose could still be recorded as a lost panel. Windows sends the lid's first state once it knows the lid device, with no bound on when. The lid is now open, closed or unknown, and a lost panel is recorded, and the display held while the host runs, only when Windows has said it is open. A withdrawn panel with the lid unknown goes through #185's steps. The cleanup at quit, start-up and in the reporter still turns the display off unless the lid is known closed, so the restart starts without it either way. --- host/src/display_guard.rs | 224 ++++++++++++++++++++++++++++---------- 1 file changed, 164 insertions(+), 60 deletions(-) diff --git a/host/src/display_guard.rs b/host/src/display_guard.rs index 29632185..1d9f77e3 100644 --- a/host/src/display_guard.rs +++ b/host/src/display_guard.rs @@ -34,8 +34,8 @@ //! it is refused, and only a restart gives it back. The guard tells that //! case apart, on a PC with more than one graphics card, by the panel's //! target: the virtual display is on, the panel is not, no built-in target -//! is offered any more, and Windows doesn't say the lid is closed (a closed -//! lid withdraws the panel on purpose). It then makes no `SetDisplayConfig` +//! is offered any more, and Windows has said the lid is open (a closed lid +//! withdraws the panel on purpose). It then makes no `SetDisplayConfig` //! call, and while the host runs it doesn't turn the display off: the //! desktop stays on it, which the iPad still shows. A turn off that nothing //! streams after, at start-up, at quit, in the failure reporter or once @@ -162,9 +162,8 @@ pub trait Displays { fn built_in(&mut self) -> Option; /// The PC has more than one graphics card. fn hybrid(&self) -> bool; - /// Windows says the laptop's lid is closed. False when it hasn't said, - /// as on a PC without a lid. - fn lid_closed(&self) -> bool; + /// The laptop's lid, as Windows last said. + fn lid(&self) -> Lid; /// Windows is ending the session: shutting down, restarting or signing out. fn session_ending(&self) -> bool; /// This turn off is a cleanup, after which nothing streams the display: @@ -191,6 +190,51 @@ pub trait Saved: Sized { fn decode(text: &str) -> Option; } +/// The laptop's lid, from Windows' GUID_LIDSWITCH_STATE_CHANGE +/// notifications. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Lid { + /// Windows hasn't said: before its first notification, which comes once + /// it knows the lid device and its state with no bound on when, after a + /// failed registration, and on a PC without a lid. + #[default] + Unknown, + Open, + Closed, +} + +/// What the lid notifications said so far: [`Lid::Unknown`] until the +/// first one, and for good when there is none. +pub struct LidWatch(std::sync::atomic::AtomicU8); + +impl LidWatch { + const UNKNOWN: u8 = u8::MAX; + + pub const fn new() -> Self { + Self(std::sync::atomic::AtomicU8::new(Self::UNKNOWN)) + } + + /// One notification's value: 0 closed, 1 open. + pub fn note(&self, value: u32) { + let state = u8::from(value != 0); + self.0.store(state, std::sync::atomic::Ordering::Relaxed); + } + + pub fn lid(&self) -> Lid { + match self.0.load(std::sync::atomic::Ordering::Relaxed) { + Self::UNKNOWN => Lid::Unknown, + 0 => Lid::Closed, + _ => Lid::Open, + } + } +} + +impl Default for LidWatch { + fn default() -> Self { + Self::new() + } +} + /// The built-in panel's targets, as `QDC_ALL_PATHS` lists them. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BuiltIn { @@ -472,17 +516,24 @@ impl Guard { ); return true; } - if lost_panel(displays, screens) { - if displays.cleanup() { - let text = format!( - "The built-in screen is still unavailable, and nothing streams the virtual \ - display after this — turning it off, so the restart that brings the screen \ - back starts without it outcome=released reason=panel_lost {}", - line(Some(screens)) - ); - displays.log(true, &text); - return true; - } + let withdrawn = withdrawn_panel(displays, screens); + let lid = displays.lid(); + // Also with the lid unknown: a lid in fact closed loses nothing. + if withdrawn && lid != Lid::Closed && displays.cleanup() { + let reason = match lid { + Lid::Open => "reason=panel_lost", + _ => "reason=panel_withdrawn lid=unknown", + }; + let text = format!( + "The built-in screen is still unavailable, and nothing streams the virtual \ + display after this — turning it off, so the restart that brings the screen \ + back starts without it outcome=released {reason} {}", + line(Some(screens)) + ); + displays.log(true, &text); + return true; + } + if withdrawn && lid == Lid::Open { let text = format!( "Leaving the virtual display on: the built-in screen is still unavailable, and \ turning the display off would leave the PC with no screen outcome=kept_on \ @@ -775,21 +826,25 @@ fn alone(screens: &[Screen]) -> bool { has_vdd(screens) && real(screens).next().is_none() } -/// The hybrid laptop's black panel: on a PC with more than one graphics -/// card, the virtual display is on in `screens`, the built-in panel is off, -/// and Windows no longer offers any built-in target, with the lid open. -/// After a turn on this is read once the screens settle ([`SETTLE`] after -/// the Enable, while no real screen came back). A lid the user closed -/// withdraws the panel too, and is no failure: the panel comes back when it -/// opens. -fn lost_panel(displays: &mut D, screens: &[Screen]) -> bool { +/// On a PC with more than one graphics card, the virtual display is on in +/// `screens`, the built-in panel is off, and Windows no longer offers any +/// built-in target: the hybrid laptop's black panel, or a closed lid. +fn withdrawn_panel(displays: &mut D, screens: &[Screen]) -> bool { has_vdd(screens) && !screens.iter().any(|screen| screen.internal) && displays.hybrid() - && !displays.lid_closed() && displays.built_in() == Some(BuiltIn::Withdrawn) } +/// The hybrid laptop's black panel: [`withdrawn_panel`] with the lid known +/// to be open. After a turn on this is read once the screens settle +/// ([`SETTLE`] after the Enable, while no real screen came back). A lid the +/// user closed withdraws the panel too, and is no failure: the panel comes +/// back when it opens. Nor is a lid Windows hasn't reported yet. +fn lost_panel(displays: &mut D, screens: &[Screen]) -> bool { + displays.lid() == Lid::Open && withdrawn_panel(displays, screens) +} + /// The screens once a real one is back, or as they are after [`SETTLE`]. A /// PC that has one returns at once; only one that lost them all waits. fn settle(displays: &mut D, before: &[Screen]) -> Option> { @@ -972,13 +1027,11 @@ mod windows_impl { } } - /// Whether Windows says the lid is closed, from its - /// GUID_LIDSWITCH_STATE_CHANGE notifications: it sends the current state - /// once the callback is registered, and each change after it. + /// The lid from Windows' GUID_LIDSWITCH_STATE_CHANGE notifications: the + /// current state comes once Windows knows the lid device and its state, + /// then each change. mod lid { - use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::Once; - use std::time::Duration; use windows::core::GUID; use windows::Win32::Foundation::{ERROR_SUCCESS, HANDLE}; @@ -992,9 +1045,7 @@ mod windows_impl { const LIDSWITCH_STATE_CHANGE: GUID = GUID::from_u128(0xba3e0f4d_b817_4094_a2d1_d56379e6a0f3); - const UNKNOWN: u8 = u8::MAX; - /// 0 closed, 1 open. - static STATE: AtomicU8 = AtomicU8::new(UNKNOWN); + static WATCH: super::super::LidWatch = super::super::LidWatch::new(); unsafe extern "system" fn changed( _context: *const core::ffi::c_void, @@ -1006,8 +1057,7 @@ mod windows_impl { let header = unsafe { std::ptr::read_unaligned(setting) }; if header.PowerSetting == LIDSWITCH_STATE_CHANGE && header.DataLength >= 4 { let data = unsafe { std::ptr::addr_of!((*setting).Data) }.cast::(); - let value = unsafe { std::ptr::read_unaligned(data) }; - STATE.store(value.min(1) as u8, Ordering::Relaxed); + WATCH.note(unsafe { std::ptr::read_unaligned(data) }); } } 0 @@ -1015,8 +1065,8 @@ mod windows_impl { /// Start listening. Later calls do nothing. pub(super) fn watch() { - static WATCH: Once = Once::new(); - WATCH.call_once(|| { + static REGISTER: Once = Once::new(); + REGISTER.call_once(|| { // Windows keeps both for as long as the process runs. let parameters = Box::leak(Box::new(DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS { Callback: Some(changed), @@ -1034,24 +1084,16 @@ mod windows_impl { if result != ERROR_SUCCESS { tracing::warn!( error = result.0, - "Could not watch the lid; a built-in screen Windows withdraws counts as \ - lost" + "Could not watch the lid; a built-in screen Windows withdraws is not \ + taken for lost" ); } }); } - /// Closed, as Windows last said. A process that just started waits - /// briefly for the first notification. - pub(super) fn closed() -> bool { + pub(super) fn lid() -> super::super::Lid { watch(); - for _ in 0..20 { - match STATE.load(Ordering::Relaxed) { - UNKNOWN => std::thread::sleep(Duration::from_millis(10)), - state => return state == 0, - } - } - false + WATCH.lid() } } @@ -1117,8 +1159,8 @@ mod windows_impl { crate::diagnostics::host_facts().hybrid == Some(true) } - fn lid_closed(&self) -> bool { - lid::closed() + fn lid(&self) -> super::Lid { + lid::lid() } fn session_ending(&self) -> bool { @@ -1242,7 +1284,7 @@ mod tests { /// What `QDC_ALL_PATHS` says of the panel; `None` (unread) by default. built_in: Option, hybrid: bool, - lid_closed: bool, + lid: Lid, session_ending: bool, cleanup: bool, slept: Duration, @@ -1342,8 +1384,8 @@ mod tests { self.hybrid } - fn lid_closed(&self) -> bool { - self.lid_closed + fn lid(&self) -> Lid { + self.lid } fn session_ending(&self) -> bool { @@ -1632,7 +1674,7 @@ mod tests { fn hybrid_laptop() -> Fake { let mut fake = Fake::reading([vec![laptop_panel()], vec![lone_vdd()]]); fake.owner = Some(HOST.into()); - (fake.hybrid, fake.built_in) = (true, Some(BuiltIn::Withdrawn)); + (fake.hybrid, fake.built_in, fake.lid) = (true, Some(BuiltIn::Withdrawn), Lid::Open); (fake.refuse_extend, fake.refuse_reapply) = (true, true); fake } @@ -1790,7 +1832,7 @@ mod tests { #[test] fn a_lid_closed_during_the_turn_on_is_not_a_lost_panel() { let mut fake = hybrid_laptop(); - fake.lid_closed = true; + fake.lid = Lid::Closed; fake.reads = VecDeque::from([ Some(vec![laptop_panel(), monitor()]), Some(vec![monitor(), lone_vdd()]), @@ -1803,12 +1845,74 @@ mod tests { assert!(!fake.text().contains("panel_lost"), "{}", fake.text()); let mut fake = hybrid_laptop(); - fake.lid_closed = true; + fake.lid = Lid::Closed; + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); + assert!(!guard.panel_lost()); + assert!(!fake.text().contains("panel_lost"), "{}", fake.text()); + } + + /// Windows sends the lid's first state once it knows the lid device, + /// with no bound on when. Until then the lid is unknown, and a + /// withdrawn panel is not taken for lost: it goes through #185's steps, + /// and nothing records a lost panel. Once Windows says the lid is open, + /// the next turn on is. + #[test] + fn a_lid_state_windows_has_not_sent_yet_is_no_lost_panel() { + let watch = LidWatch::new(); + let mut fake = hybrid_laptop(); + fake.lid = watch.lid(); + assert_eq!(fake.lid, Lid::Unknown); let mut guard = Guard::new(); guard.before(true, false, &mut fake); assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); assert!(!guard.panel_lost()); assert!(!fake.text().contains("panel_lost"), "{}", fake.text()); + // The first notification, late: open. + watch.note(1); + let mut fake = hybrid_laptop(); + fake.lid = watch.lid(); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::PanelLost); + watch.note(0); + assert_eq!(watch.lid(), Lid::Closed); + } + + /// With the lid watch's registration failed, the lid stays unknown for + /// good. No lost panel is ever recorded, so a lid closed on purpose + /// can't become one; while the host runs, a turn off goes as #185's do. + /// The cleanup at quit still turns the display off, so the restart + /// starts without it whatever the lid was. + #[test] + fn a_failed_lid_registration_records_no_lost_panel_yet_quitting_turns_it_off() { + let mut fake = hybrid_laptop(); + fake.lid = LidWatch::new().lid(); + let mut guard = Guard::new(); + guard.before(true, false, &mut fake); + assert_eq!(guard.after(true, true, &mut fake), Outcome::Failed); + assert!(!guard.panel_lost()); + let calls = fake.set_display_config_calls(); + // While the host runs: #185's steps, then held. + assert!(!turn_off_allowed(&mut guard, &mut fake)); + assert!(fake.set_display_config_calls() > calls); + assert!( + !fake.text().contains("reason=panel_lost"), + "{}", + fake.text() + ); + // The quit. + let calls = fake.set_display_config_calls(); + fake.cleanup = true; + assert!(turn_off_allowed(&mut guard, &mut fake)); + assert_eq!(fake.set_display_config_calls(), calls); + assert!(fake + .logs + .last() + .unwrap() + .1 + .contains("outcome=released reason=panel_withdrawn lid=unknown")); } /// The start-up turn off of a closed laptop whose virtual display is @@ -1817,7 +1921,7 @@ mod tests { #[test] fn a_closed_lid_start_up_turn_off_runs_as_before() { let mut fake = hybrid_laptop(); - fake.lid_closed = true; + fake.lid = Lid::Closed; fake.reads = VecDeque::from([Some(vec![lone_vdd()])]); let mut guard = Guard::new(); assert!(turn_off_allowed(&mut guard, &mut fake)); @@ -2130,8 +2234,8 @@ mod tests { fn hybrid(&self) -> bool { self.0.hybrid() } - fn lid_closed(&self) -> bool { - self.0.lid_closed() + fn lid(&self) -> Lid { + self.0.lid() } fn session_ending(&self) -> bool { self.0.session_ending() From 074e7badf46dd6d62da44f4817fe562a875ec8f5 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:52:58 -0500 Subject: [PATCH 048/226] Pass the guard one driver change, not eight arguments Adding the cleanup flag gave around eight arguments, which Windows clippy refuses (too_many_arguments). What the change is now travels as one Change, and around keeps the three closures. --- host/src/display_guard.rs | 40 +++++++++++++++++++++++++++------------ host/src/vdd.rs | 26 +++++++++++++++---------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/host/src/display_guard.rs b/host/src/display_guard.rs index 1d9f77e3..fe5b839d 100644 --- a/host/src/display_guard.rs +++ b/host/src/display_guard.rs @@ -875,27 +875,43 @@ fn wait_for_real(displays: &mut D) -> Option> { static GUARD: parking_lot::Mutex> = parking_lot::Mutex::new(Guard::new()); -/// Run a driver change for `owner`'s display with the guard around it. -/// `log` writes one line; `display_off` says the driver tasks know the -/// display is off; `cleanup`, that nothing streams the display after this -/// turn off; `done` tells a change that succeeded. A turn off that -/// would leave the PC with no screen does not run: `kept_on` is its result. +/// One driver change, for [`around`]. +#[cfg(windows)] +pub(crate) struct Change { + /// A turn on, else a turn off. + pub enabled: bool, + /// The host whose display it is. + pub owner: crate::startup::ProcessId, + /// Writes one line. + pub log: fn(bool, &str), + /// The driver tasks know the display is off. + pub display_off: bool, + /// Nothing streams the display after this turn off. + pub cleanup: bool, +} + +/// Run `change` with the guard around it: `run` makes it, `done` tells one +/// that succeeded. A turn off that would leave the PC with no screen does +/// not run: `kept_on` is its result. #[cfg(windows)] pub(crate) fn around( - enabled: bool, - owner: crate::startup::ProcessId, - log: fn(bool, &str), - display_off: bool, - cleanup: bool, - change: impl FnOnce() -> T, + change: Change, + run: impl FnOnce() -> T, done: impl FnOnce(&T) -> bool, kept_on: impl FnOnce() -> T, ) -> T { + let Change { + enabled, + owner, + log, + display_off, + cleanup, + } = change; let mut displays = windows_impl::Windows::new(owner, log, cleanup); if !GUARD.lock().before(enabled, display_off, &mut displays) { return kept_on(); } - let result = change(); + let result = run(); let done = done(&result); GUARD.lock().after(enabled, done, &mut displays); result diff --git a/host/src/vdd.rs b/host/src/vdd.rs index 7b5e2f0f..dbde864d 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -465,13 +465,16 @@ fn request_on_turn( // Read here, after the gate decided and before the task runs. let display_off = TASKS.gate.lock().known_enabled == Some(false); crate::startup::display::change(enabled, progress, |progress| { - outcome = crate::display_guard::around( + let change = crate::display_guard::Change { enabled, - crate::startup::ProcessId::current(), - log_screens, + owner: crate::startup::ProcessId::current(), + log: log_screens, display_off, // Start-up, quit and panics: nothing streams it after. - !enabled && purpose == Purpose::Cleanup, + cleanup: !enabled && purpose == Purpose::Cleanup, + }; + outcome = crate::display_guard::around( + change, || run_task(if enabled { TASK_ENABLE } else { TASK_DISABLE }, progress), |outcome| *outcome == TaskOutcome::Done, // Left on: the next turn off tries again. @@ -535,14 +538,17 @@ pub(crate) fn disable_for_ended( || crate::startup::display::other_host(host, reporters), || { // The host's layout from before its turn on, which it kept on disk. - crate::display_guard::around( - false, - host, - crate::startup::reporter_log, + let change = crate::display_guard::Change { + enabled: false, + owner: host, + log: crate::startup::reporter_log, // Only for a turn on. - false, + display_off: false, // The host ended. - true, + cleanup: true, + }; + crate::display_guard::around( + change, || run_task(TASK_DISABLE, &mut || {}), |outcome| *outcome == TaskOutcome::Done, || TaskOutcome::Failed(None), From 42ee21fc323037b09f0829b7f68d1e91b87b84bf Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 15:00:53 -0500 Subject: [PATCH 049/226] Say where input_soak's log must not go --- host/examples/input_soak.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/host/examples/input_soak.rs b/host/examples/input_soak.rs index 0c393924..6d0141e1 100644 --- a/host/examples/input_soak.rs +++ b/host/examples/input_soak.rs @@ -9,6 +9,10 @@ //! (no clicks, no ink) and taps of Right Ctrl alone. //! //! `input_soak ` +//! +//! Keep the log, and stdout if redirected, on a disk nothing else is busy +//! writing: a log write that waits seconds stops the input with it, and the +//! host then rightly ends the session for silence. use std::io::Write; use std::net::{SocketAddr, UdpSocket}; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; From 82c0c7396316ba34f43213821424a1b968f48ee9 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 15:09:31 -0500 Subject: [PATCH 050/226] Stream on libx264 when a hardware encoder open hangs, remember failed encoders, and change Quick Sync's bitrate in place A hardware open was waited for up to its 15 s limit, past the iPad's 10 s first-frame timeout. The pipeline's first open in the FFmpeg lane is now waited for 3 s; libx264 then streams while the open goes on in the background, and the stream switches to it if it opens within its limit. Recovery tries from libx264 open in the background too. An NVENC, AMF or Quick Sync open that fails for its own reason is remembered at its size for the launch, so its FFmpeg errors are logged once; Restart forgets it. AMD and Intel cards whose encoder failed get a driver-update note in the fallback notice. FFmpeg 7.1.1's Quick Sync encoder resets its rate control when bit_rate changes, so h264_qsv and hevc_qsv take bitrate changes in place. A session that refuses one is reopened, and Quick Sync bitrate reopens then wait 5 s. --- host/src/encoder/mod.rs | 1482 +++++++++++++++++++++++++++++++---- host/src/gui/mod.rs | 4 + host/src/gui/page_stream.rs | 27 +- host/src/gui/tests.rs | 51 +- host/src/stall.rs | 444 +++++++++-- host/src/stats.rs | 5 + 6 files changed, 1792 insertions(+), 221 deletions(-) diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index a865edf5..8028a11f 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -107,13 +107,20 @@ fn run_encode_loop( } info!(encoder = %encoder_name, "Found encoder codec"); { + let (nvenc_failed, vendors) = { + let failures = LAUNCH_FAILURES.lock(); + (failures.any("_nvenc"), failures.vendors.clone()) + }; let mut stats = PIPELINE_STATS.lock(); stats.set_codec_name(&codec_display_name); // Fresh pipeline start: clear any stale software-fallback flag. The encode loop // re-sets it below only if a hardware encoder actually fails to open. A deliberate // libx264 override is not a fallback, so it intentionally leaves this false. stats.set_software_fallback(false); - stats.nvidia_driver_outdated = false; + // A driver note stays while the failure that gave it is remembered, + // as that encoder is not tried again to give it anew. + stats.nvidia_driver_outdated &= nvenc_failed; + stats.driver_update_vendors = vendors; } let mut encoder_state: Option = None; @@ -144,6 +151,11 @@ fn run_encode_loop( let mut hevc_unavailable_warned = false; let mut hevc_open_failed = false; let mut stall_recovery = StallRecovery::default(); + // An open in the lane going on in the background while libx264 streams. + let mut background: Option = None; + // When a picture is due while no session is open: FIRST_PICTURE_WAIT + // after the first try of an encoder. + let mut first_picture_by: Option = None; let mut last_keyframe_reopen: Option = None; let mut pending_keyframe = false; let mut bitrate_pacer = BitratePacer::default(); @@ -223,11 +235,17 @@ fn run_encode_loop( } else { encoder_name.clone() }; - // No open given up on still holds FFmpeg's lock. Until then, an open - // in that lane would only be refused. + // No open given up on, or going on in the background, still holds + // FFmpeg's lock. Until then, an open in that lane would only be + // refused. let lane_clear = crate::stall::abandoned(crate::stall::Call::EncoderOpen) == 0; let desired_encoder = stall_recovery.encoder_for_frame(&preferred, Instant::now(), lane_clear); + // While an encoder opens in the background, libx264 streams on. + let desired_encoder = match (&background, &encoder_state) { + (Some(_), Some(state)) => state.opened_encoder.clone(), + _ => desired_encoder, + }; forget_failed_reopen( &mut failed_reopen, &mut negotiated, @@ -267,6 +285,14 @@ fn run_encode_loop( let size_changed = encoder_state .as_ref() .is_some_and(|state| !state.accepts_frame(raw_frame.width, raw_frame.height)); + if size_changed { + if let Some(behind) = background.take() { + info!( + encoder = behind.open.encoder(), + "The capture size changed while an encoder opened in the background; that session will not be used" + ); + } + } // Latch requests before deciding whether to reopen. Requests received // during the rate limit must survive until AMF can emit an IDR. pending_keyframe |= shared.force_next_idr.swap(false, Ordering::SeqCst); @@ -280,18 +306,23 @@ fn run_encode_loop( }); if encoder_state.is_none() { - match EncoderState::new( + let picture_by = + *first_picture_by.get_or_insert_with(|| Instant::now() + FIRST_PICTURE_WAIT); + let wait = || picture_by.saturating_duration_since(Instant::now()); + match open_session( + &LAUNCH_FAILURES, &desired_encoder, raw_frame.width, raw_frame.height, desired_bitrate, target_fps, &mut input_mode, - first_open(&desired_encoder), + first_open(&desired_encoder, wait()), &mut opening, ) { Ok(state) => { encoder_state = Some(state); + first_picture_by = None; bitrate_pacer.opened(Instant::now()); if stall_recovery.opened(&desired_encoder, &preferred) && desired_encoder != "libx265" @@ -300,6 +331,36 @@ fn run_encode_loop( } note_encoder_opened(&shared, &desired_encoder); } + // No picture yet, and the iPad would give up waiting for + // it: libx264 streams while the open goes on. + Err(e) if is_still_opening(e.as_ref()) => { + let open = take_still_opening(e.as_ref()).expect("the open going on"); + warn!( + encoder = %desired_encoder, + waited_ms = open.pending.running_for().as_millis() as u64, + "The encoder has not opened yet; streaming on libx264 while it opens in the background" + ); + let state = open_session( + &LAUNCH_FAILURES, + "libx264", + raw_frame.width, + raw_frame.height, + desired_bitrate, + target_fps, + &mut input_mode, + Opening::First, + &mut opening, + )?; + stall_recovery.outwaited(&desired_encoder, "libx264", Instant::now()); + background = Some(Behind::first(open, desired_bitrate)); + encoder_state = Some(state); + first_picture_by = None; + bitrate_pacer.opened(Instant::now()); + PIPELINE_STATS + .lock() + .set_software_fallback(encoder_name != "libx264"); + note_encoder_opened(&shared, "libx264"); + } Err(e) if is_hevc_encoder(&desired_encoder) => { warn!( encoder = %desired_encoder, @@ -320,29 +381,53 @@ fn run_encode_loop( "Preferred encoder failed to open; trying remaining encoders" ); let stalled = is_open_stall(e.as_ref()); + let mut still_opening = None; let (name, state) = open_remaining_encoders( (&desired_encoder, &e.to_string()), crate::gpu::vendors_present, |name| { - EncoderState::new( + let opened = open_session( + &LAUNCH_FAILURES, name, raw_frame.width, raw_frame.height, desired_bitrate, target_fps, &mut input_mode, - first_open(name), + first_open(name, wait()), &mut opening, - ) + ); + if let Err(error) = &opened { + if let Some(open) = take_still_opening(error.as_ref()) { + still_opening = Some(open); + } + } + opened }, )?; - // After a stall the preferred encoder stays preferred, - // to be tried again; a real error leaves it for good. - if !stall_recovery.failed(stalled, &preferred, name, Instant::now()) { - encoder_name = name.to_string(); + if let Some(open) = still_opening { + // The preferred encoder failed for a reason of its + // own (a stalled one leaves no lane for another), + // and libx264 streams while the next one opens. + warn!( + encoder = open.encoder(), + waited_ms = open.pending.running_for().as_millis() as u64, + "The encoder has not opened yet; streaming on libx264 while it opens in the background" + ); + encoder_name = open.encoder().to_string(); + stall_recovery.opened(name, &preferred); + stall_recovery.outwaited(open.encoder(), name, Instant::now()); + background = Some(Behind::first(open, desired_bitrate)); + } else { + // After a stall the preferred encoder stays preferred, + // to be tried again; a real error leaves it for good. + if !stall_recovery.failed(stalled, &preferred, name, Instant::now()) { + encoder_name = name.to_string(); + } + stall_recovery.opened(name, &preferred); } - stall_recovery.opened(name, &preferred); encoder_state = Some(state); + first_picture_by = None; bitrate_pacer.opened(Instant::now()); PIPELINE_STATS .lock() @@ -351,74 +436,133 @@ fn run_encode_loop( } Err(e) => return Err(e), } - } else if size_changed - || encoder_state.as_ref().is_some_and(|state| { - reopen_needed( - ( - &state.opened_encoder, - state.opened_bitrate, - state.opened_fps, - ), - (&desired_encoder, wanted_bitrate, target_fps), - keyframe_reopen, - failed_reopen.as_ref(), - lane_clear, - ) - }) - { - // Bitrate changed (ABR rung or the user's slider), the - // negotiated codec changed (HEVC toggle / client capability), the - // capture size changed (display mode change), or AMF owes a - // keyframe: hardware encoders ignore parameter pokes on an open - // context, so each means a session reopen (~50-200ms hiccup, no - // transport teardown, same stream epoch) followed by a forced IDR. - // So does a try of the preferred encoder after a stalled open. - // Every reopen takes the newest bitrate target, paced or not. - if keyframe_reopen { - last_keyframe_reopen = Some(Instant::now()); - } - let (previous, current_encoder, reason) = encoder_state - .as_ref() - .map(|s| { - let reason = reopen_reason( - (&s.opened_encoder, s.opened_fps), - (&desired_encoder, target_fps), - size_changed, - keyframe_reopen, - stall_recovery - .owed - .as_ref() - .is_some_and(|owed| owed.preferred == desired_encoder), + } else { + // A reopen this frame: the open that went on in the background, + // once it returned, or a new one. + let reopened = if let Some(result) = background + .as_mut() + .and_then(|behind| behind.open.poll()) + { + let behind = background.take().expect("it was just polled"); + if let Err(error) = &result { + note_open_failure( + &LAUNCH_FAILURES, + behind.open.encoder(), + behind.open.args.size, + error.as_ref(), ); - (s.opened_bitrate, s.opened_encoder.clone(), reason) - }) - .unwrap_or_default(); - stall_recovery.starting(&desired_encoder); - match EncoderState::new( - &desired_encoder, - raw_frame.width, - raw_frame.height, - desired_bitrate, - target_fps, - &mut input_mode, + } + Some((result, behind.reopen)) + } else if background.is_none() + && (size_changed + || encoder_state.as_ref().is_some_and(|state| { + reopen_needed( + ( + &state.opened_encoder, + state.opened_bitrate, + state.opened_fps, + ), + (&desired_encoder, wanted_bitrate, target_fps), + keyframe_reopen, + failed_reopen.as_ref(), + lane_clear, + ) + })) + { + // Bitrate changed (ABR rung or the user's slider), the + // negotiated codec changed (HEVC toggle / client capability), the + // capture size changed (display mode change), or AMF owes a + // keyframe: hardware encoders ignore parameter pokes on an open + // context, so each means a session reopen (~50-200ms hiccup, no + // transport teardown, same stream epoch) followed by a forced IDR. + // So does a try of the preferred encoder after a stalled open. + // Every reopen takes the newest bitrate target, paced or not. + if keyframe_reopen { + last_keyframe_reopen = Some(Instant::now()); + } + let (previous, current_encoder, reason) = encoder_state + .as_ref() + .map(|s| { + let reason = reopen_reason( + (&s.opened_encoder, s.opened_fps), + (&desired_encoder, target_fps), + size_changed, + keyframe_reopen, + stall_recovery + .owed + .as_ref() + .is_some_and(|owed| owed.preferred == desired_encoder), + ); + (s.opened_bitrate, s.opened_encoder.clone(), reason) + }) + .unwrap_or_default(); + stall_recovery.starting(&desired_encoder); // The old session cannot take a frame of the new size. - if size_changed { + let how = if size_changed { Opening::First + } else if opens_behind(¤t_encoder, &desired_encoder) { + Opening::Behind } else { Opening::Reopen - }, - &mut opening, - ) { - Ok(state) => { + }; + let reopen = Reopening { + encoder: desired_encoder.clone(), + from: previous, + current: current_encoder, + reason, + size_changed, + keyframe: keyframe_reopen, + bitrate: desired_bitrate, + fps: target_fps, + first: false, + in_background: false, + }; + match open_session( + &LAUNCH_FAILURES, + &desired_encoder, + raw_frame.width, + raw_frame.height, + desired_bitrate, + target_fps, + &mut input_mode, + how, + &mut opening, + ) { + Err(e) if is_still_opening(e.as_ref()) => { + let open = take_still_opening(e.as_ref()).expect("the open going on"); + info!( + from = %reopen.current, + encoder = %desired_encoder, + reason, + "Opening the encoder in the background; the stream goes on meanwhile" + ); + background = Some(Behind { + open, + reopen: Reopening { + in_background: true, + ..reopen + }, + }); + None + } + result => Some((result, reopen)), + } + } else { + None + }; + match reopened { + None => {} + Some((Ok(state), reopen)) => { info!( - from = previous, - to = desired_bitrate, - encoder = %desired_encoder, + from = reopen.from, + to = state.opened_bitrate, + encoder = %reopen.encoder, width = state.frame.width(), height = state.frame.height(), - size_changed, - keyframe = keyframe_reopen, - reason, + size_changed = reopen.size_changed, + keyframe = reopen.keyframe, + reason = reopen.reason, + in_background = reopen.in_background, "Reopened encoder session" ); encoder_state = Some(state); @@ -427,57 +571,80 @@ fn run_encode_loop( pending_keyframe = true; // Back from libx264 to hardware (libx265 only replaces a // libx264 that was already preferred). - if stall_recovery.opened(&desired_encoder, &preferred) - && desired_encoder != "libx265" + if stall_recovery.opened(&reopen.encoder, &preferred) + && reopen.encoder != "libx265" { PIPELINE_STATS.lock().set_software_fallback(false); } - note_encoder_opened(&shared, &desired_encoder); + note_encoder_opened(&shared, &reopen.encoder); } // The old session cannot take a frame of the new size. A // pipeline restart opens a fresh encoder, with the software // fallback if the hardware one rejects this size. - Err(e) if size_changed => { + Some((Err(e), reopen)) if reopen.size_changed => { return Err(format!( - "{desired_encoder} could not reopen at {}x{}: {e}", - raw_frame.width, raw_frame.height + "{} could not reopen at {}x{}: {e}", + reopen.encoder, raw_frame.width, raw_frame.height ) .into()); } - Err(e) => { + // The pipeline's first open, which went on in the background. + Some((Err(e), reopen)) if reopen.first => { + if is_open_stall(e.as_ref()) { + // Tried again once the lane is clear, as after any + // stalled open (see `StallRecovery::outwaited`). + warn!( + error = %e, + encoder = %reopen.encoder, + "The encoder did not open in the background; streaming on libx264 until it is tried again" + ); + } else { + warn!( + error = %e, + encoder = %reopen.encoder, + "The encoder failed to open in the background; staying on libx264 for this pipeline" + ); + stall_recovery.owed = None; + if is_hevc_encoder(&reopen.encoder) { + hevc_open_failed = true; + } else { + encoder_name = "libx264".to_string(); + } + } + } + Some((Err(e), reopen)) => { let stalled = is_open_stall(e.as_ref()); if stalled { warn!( error = %e, - encoder = %desired_encoder, + encoder = %reopen.encoder, "Encoder reopen stalled — keeping the current session" ); - } else if is_hevc_encoder(&desired_encoder) { + } else if is_hevc_encoder(&reopen.encoder) { warn!( error = %e, - encoder = %desired_encoder, + encoder = %reopen.encoder, "HEVC reopen failed — keeping the current session" ); } else { warn!( error = %e, - bitrate = desired_bitrate, + bitrate = reopen.bitrate, "Encoder reopen failed — keeping the current session" ); } // A stall is tried again once the lane is clear, while // tries are left. - if !stall_recovery.failed(stalled, &preferred, ¤t_encoder, Instant::now()) + if !stall_recovery.failed(stalled, &preferred, &reopen.current, Instant::now()) { give_up_reopen( - &desired_encoder, - ¤t_encoder, + &reopen.encoder, + &reopen.current, stall_recovery.owed.is_some(), &mut encoder_name, &mut hevc_open_failed, ); - failed_reopen = - Some((desired_encoder.clone(), desired_bitrate, target_fps)); + failed_reopen = Some((reopen.encoder.clone(), reopen.bitrate, reopen.fps)); } } } @@ -514,9 +681,17 @@ fn run_encode_loop( ); let encode_start = Instant::now(); - encoder.send_frame(pts, raw_frame.timestamp, raw_frame.frame_number)?; + // A bitrate Quick Sync took in place applies with this frame: an + // error now is its refusal. + let mut reset_refused = None; + if let Err(error) = encoder.send_frame(pts, raw_frame.timestamp, raw_frame.frame_number) { + if !encoder.bitrate_reset_unconfirmed { + return Err(error.into()); + } + reset_refused = Some(error); + } - loop { + while reset_refused.is_none() { match encoder.encoder.receive_packet(&mut encoder.packet) { Ok(()) => { let encode_us = encode_start.elapsed().as_micros(); @@ -622,9 +797,23 @@ fn run_encode_loop( } } Err(e) if is_eagain(&e) => break, + Err(e) if encoder.bitrate_reset_unconfirmed => reset_refused = Some(e), Err(e) => return Err(e.into()), } } + if let Some(error) = reset_refused { + QSV_RESET_REFUSED.store(true, Ordering::Relaxed); + warn!( + %error, + encoder = %encoder.opened_encoder, + bitrate = encoder.opened_bitrate, + "Quick Sync did not take the new bitrate on the open session; reopening it, as for its bitrate changes from now on" + ); + encoder_state = None; + pending_keyframe = true; + continue; + } + encoder.bitrate_reset_unconfirmed = false; } info!("Capture channel closed, encoder shutting down"); @@ -662,6 +851,27 @@ struct EncoderState { /// time_base 1/60 (escape hatch until the real-PTS path is verified on /// NVENC/AMF hardware). legacy_pts: bool, + /// A Quick Sync bitrate change was made in place, and no frame has gone + /// through the session since. Quick Sync applies it with the next frame, + /// so an error there means it refused the change. + bitrate_reset_unconfirmed: bool, +} + +/// Set once a Quick Sync session refused a bitrate change in place: from +/// then on, this launch, its bitrate changes reopen it, as before. +static QSV_RESET_REFUSED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Encoders that take a new bitrate on the open session, without the reopen +/// that costs a stutter and an IDR. FFmpeg 7.1's libx264 wrapper applies +/// bitrate and VBV changes through x264_encoder_reconfig. Its Quick Sync +/// encoders compare `bit_rate` with the value they opened at before each +/// frame, and reset the session's rate control to it (qsvenc.c, +/// `update_bitrate` and `MFXVideoENCODE_Reset`), with no new sequence. A +/// Quick Sync open takes 630 ms at the median in the field, against 144 ms +/// for NVENC and 43 ms for AMF, so each bitrate reopen froze the picture +/// that long. Unless `qsv_refused`: a session refused the change once. +fn takes_bitrate_in_place(encoder_name: &str, qsv_refused: bool) -> bool { + encoder_name == "libx264" || (encoder_name.ends_with("_qsv") && !qsv_refused) } impl EncoderState { @@ -671,30 +881,43 @@ impl EncoderState { bitrate_bps: u32, target_fps: u32, ) -> bool { - if self.opened_encoder != "libx264" - || desired_encoder != self.opened_encoder + if !takes_bitrate_in_place( + &self.opened_encoder, + QSV_RESET_REFUSED.load(Ordering::Relaxed), + ) || desired_encoder != self.opened_encoder || target_fps != self.opened_fps || bitrate_bps == self.opened_bitrate { return false; } - // FFmpeg 7.1's libx264 wrapper applies bitrate and VBV changes through - // x264_encoder_reconfig before the next frame. Preserve its reference - // frames, PTS epoch and buffers instead of allocating a second encoder - // for every ABR step. Other encoders retain the existing reopen path. + // Preserve the encoder's reference frames, PTS epoch and buffers + // instead of allocating a second encoder for every ABR step. Other + // encoders retain the existing reopen path. self.encoder.set_bit_rate(bitrate_bps.max(500_000) as usize); - self.encoder - .set_max_bit_rate(bitrate_bps.max(500_000) as usize); - // A half-second VBV is enabled at open, as required for x264's live - // bitrate reconfiguration. The context belongs to this encode thread. - unsafe { - (*self.encoder.as_mut_ptr()).rc_buffer_size = (bitrate_bps.max(500_000) / 2) as i32 - }; - info!( - from = self.opened_bitrate, - to = bitrate_bps, - "Updated software encoder bitrate" - ); + if self.opened_encoder == "libx264" { + self.encoder + .set_max_bit_rate(bitrate_bps.max(500_000) as usize); + // A half-second VBV is enabled at open, as required for x264's live + // bitrate reconfiguration. The context belongs to this encode thread. + unsafe { + (*self.encoder.as_mut_ptr()).rc_buffer_size = (bitrate_bps.max(500_000) / 2) as i32 + }; + info!( + from = self.opened_bitrate, + to = bitrate_bps, + "Updated software encoder bitrate" + ); + } else { + // Quick Sync opened in VBR with no maximum rate or buffer size, + // so only the target changes. + self.bitrate_reset_unconfirmed = true; + info!( + from = self.opened_bitrate, + to = bitrate_bps, + encoder = %self.opened_encoder, + "Updated hardware encoder bitrate without reopening" + ); + } self.opened_bitrate = bitrate_bps; true } @@ -790,7 +1013,38 @@ impl EncoderState { encoder.set_gop(30); configure_encoder_flags(&mut encoder, encoder_name); - let encoder = match open_encoder(encoder, encoder_name, opening, progress) { + let args = SessionArgs { + encoder_name: encoder_name.to_string(), + size: (width, height), + bitrate_bps, + target_fps, + pixel_format, + legacy_pts, + }; + match open_encoder(encoder, encoder_name, opening, progress) { + Opened::Now(result) => Self::opened(result, &args), + Opened::Behind(pending) => Err(Box::new(StillOpening( + encoder_name.to_string(), + parking_lot::Mutex::new(Some(BackgroundOpen { pending, args })), + ))), + } + } + + /// The session from an open that returned with `result`, or its error. + fn opened( + result: OpenResult, + args: &SessionArgs, + ) -> Result> { + let SessionArgs { + ref encoder_name, + size: (width, height), + bitrate_bps, + target_fps, + pixel_format, + legacy_pts, + } = *args; + let encoder_name = encoder_name.as_str(); + let encoder = match result { Ok(Ok(encoder)) => encoder, Ok(Err(error)) => { note_nvidia_driver(encoder_name, error, crate::gpu::nvidia_driver_versions); @@ -850,6 +1104,7 @@ impl EncoderState { last_pts: -1, in_flight: InFlightFrames::default(), legacy_pts, + bitrate_reset_unconfirmed: false, }) } @@ -977,21 +1232,50 @@ type OpenResult = Result< crate::stall::Stall, >; -/// What streams while an encoder session opens, which sets its time limit. +/// What streams while an encoder session opens, which sets its time limit +/// and how long it is waited for. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Opening { - /// Nothing: the pipeline's first open and its fallbacks, or a reopen at - /// a new capture size, which the old session cannot encode. + /// Nothing: the pipeline's last resort, libx264, or a reopen at a new + /// capture size, which the old session cannot encode. First, /// As `First`, with another encoder to try if this one fails: every /// first open but libx264's. FFmpeg's errors while it opens are logged - /// as warnings (see [`crate::diagnostics::while_probing`]). - Probe, + /// as warnings (see [`crate::diagnostics::while_probing`]). One in the + /// lane is waited for only until the pipeline's first picture is due, + /// the time given (see [`FIRST_PICTURE_WAIT`]): then it goes on opening + /// in the background while libx264 streams. + Probe(Duration), /// The live session, which goes on if the reopen is given up on. The /// picture freezes until then, so the limit is shorter. Reopen, + /// A reopen in the lane while a session that needs none streams + /// (libx264): it opens in the background, and the stream goes on + /// meanwhile, so it has a first open's limits (see [`opens_behind`]). + Behind, +} + +/// How long a pipeline may go without a picture, from its first try of an +/// encoder in the lane. The iPad gives up on a session after 10 s without +/// one. A hardware open that hung was waited for up to its 15 s limit, so +/// libx264 took over too late, and the iPad showed no picture (field +/// triage, Nov 2026). Past this, libx264 streams while the open goes on in +/// the background, and the stream switches to it if it opens within its +/// limit. Above the slowest NVENC and AMF opens in the field (2.6 s and +/// 1.2 s) and Quick Sync's p99 (3.2 s): the few slower Quick Sync opens +/// start on libx264 for a moment. +const FIRST_PICTURE_WAIT: Duration = Duration::from_secs(3); + +/// An encoder open that returned, or one that goes on in the background. +enum Opened { + Now(OpenResult), + Behind(PendingOpen), } +type PendingOpen = crate::stall::Pending< + Result, +>; + /// Open `encoder` with its options. An encoder that opens under FFmpeg's /// process-wide lock opens under its time limit (see [`crate::stall`]); one /// that returns after it is closed on the call's thread. @@ -1000,7 +1284,7 @@ fn open_encoder( encoder_name: &str, opening: Opening, progress: &mut dyn FnMut(), -) -> OpenResult { +) -> Opened { open_encoder_on( crate::stall::lanes(), encoder, @@ -1015,47 +1299,140 @@ fn open_encoder_on( encoder: ffmpeg_next::codec::encoder::video::Video, encoder_name: &str, opening: Opening, - progress: &mut dyn FnMut(), -) -> OpenResult { + mut progress: &mut dyn FnMut(), +) -> Opened { let name = encoder_name.to_string(); + let probing = matches!(opening, Opening::Probe(_)); // On the thread that makes the call, where FFmpeg logs. let open = move || { let open = || encoder.open_with(encoder_options(&name)); - if opening == Opening::Probe { + if probing { crate::diagnostics::while_probing(open) } else { open() } }; if !takes_codec_lock(encoder_name) { - return Ok(open()); + return Opened::Now(Ok(open())); + } + let limits = open_limits(encoder_name, opening); + match crate::stall::start_on(lanes, crate::stall::Call::EncoderOpen, limits, open) { + Ok(pending) => match wait_or_park(pending, opening, &mut progress) { + Ok(result) => Opened::Now(result), + Err(pending) => Opened::Behind(pending), + }, + Err(stall) => Opened::Now(Err(stall)), } - crate::stall::run_on( - lanes, - crate::stall::Call::EncoderOpen, - open_limits(encoder_name, opening), - open, - progress, - ) } -/// A pipeline's first open of `encoder_name`: anything but libx264 has -/// another encoder after it. -fn first_open(encoder_name: &str) -> Opening { +/// Waits for an open in the lane as `opening` says: to its limit, until the +/// first picture is due, or not at all. Err: it goes on, parked. +fn wait_or_park Instant>( + mut pending: crate::stall::Pending, + opening: Opening, + progress: &mut impl FnMut(), +) -> Result, crate::stall::Pending> { + let wait = match opening { + Opening::Probe(wait) => wait, + Opening::Behind => Duration::ZERO, + Opening::First | Opening::Reopen => Duration::MAX, + }; + if !wait.is_zero() { + if let Some(result) = pending.wait_up_to(wait, progress) { + return Ok(result); + } + } + match pending.park() { + Some(result) => Ok(result), + None => Err(pending), + } +} + +/// A pipeline's first open of `encoder_name`, `wait` before its first +/// picture is due: anything but libx264 has another encoder after it. +fn first_open(encoder_name: &str, wait: Duration) -> Opening { if encoder_name == "libx264" { Opening::First } else { - Opening::Probe + Opening::Probe(wait) } } fn open_limits(encoder_name: &str, opening: Opening) -> crate::stall::Limits { match opening { - Opening::First | Opening::Probe => crate::stall::encoder_open_limits(encoder_name), + Opening::First | Opening::Probe(_) | Opening::Behind => { + crate::stall::encoder_open_limits(encoder_name) + } Opening::Reopen => crate::stall::encoder_reopen_limits(encoder_name), } } +/// Whether a reopen from the `current` session to `wanted` opens in the +/// background: libx264 needs no lane, so it streams on while an encoder in +/// the lane opens, instead of freezing for as long as the open takes. +fn opens_behind(current: &str, wanted: &str) -> bool { + !takes_codec_lock(current) && takes_codec_lock(wanted) +} + +/// What a session opens with, to finish an open that returns later. +struct SessionArgs { + encoder_name: String, + size: (u32, u32), + bitrate_bps: u32, + target_fps: u32, + pixel_format: ffmpeg_next::format::Pixel, + legacy_pts: bool, +} + +/// An encoder session that opens in the background while libx264 streams. +struct BackgroundOpen { + pending: PendingOpen, + args: SessionArgs, +} + +impl BackgroundOpen { + fn encoder(&self) -> &str { + &self.args.encoder_name + } + + /// The session once its open returned, or the open's error; past its + /// limit, its stall. None while it opens within its limit. + fn poll(&mut self) -> Option>> { + let result = self.pending.poll()?; + Some(EncoderState::opened(result, &self.args)) + } +} + +/// An open in the lane that had not returned when its wait ended and goes +/// on in the background, carried out of [`EncoderState::new`] to the encode +/// loop, which streams on libx264 meanwhile. +struct StillOpening(String, parking_lot::Mutex>); + +impl std::fmt::Debug for StillOpening { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("StillOpening").field(&self.0).finish() + } +} + +impl std::fmt::Display for StillOpening { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} has not opened yet", self.0) + } +} + +impl std::error::Error for StillOpening {} + +fn is_still_opening(error: &(dyn std::error::Error + Send + Sync + 'static)) -> bool { + error.is::() +} + +/// The open that goes on in the background, from the error that carries it. +fn take_still_opening( + error: &(dyn std::error::Error + Send + Sync + 'static), +) -> Option { + error.downcast_ref::()?.1.lock().take() +} + /// Encoders FFmpeg 7.1 opens under its one process-wide lock /// (FF_CODEC_CAP_NOT_INIT_THREADSAFE): the NVENC, AMF and QSV hardware /// encoders, which wait on a graphics driver, and libx265. While one of them @@ -1128,6 +1505,14 @@ fn open_remaining_encoders( } match open(name) { Ok(state) => return Ok((name, state)), + // Every other open in the lane would wait behind it. + Err(error) if is_still_opening(error.as_ref()) => { + info!( + encoder = name, + "Hardware encoder is still opening; libx264 streams while it opens in the background" + ); + break; + } Err(error) => { warn!(encoder = name, %error, "Hardware encoder did not open"); note(&mut reasons, name, error.to_string()); @@ -1202,6 +1587,154 @@ fn note_nvidia_driver( PIPELINE_STATS.lock().nvidia_driver_outdated = true; } +/// Hardware encoders that failed to open this launch, each at the encoded +/// size it failed at. Each is not tried again at that size, so its FFmpeg +/// errors are logged once instead of at every pipeline start and reopen +/// (field triage, Nov 2026: 13-44 lines per PC, over and over, for an NVENC +/// API too old, an AMF component or a Quick Sync session that could not be +/// created). Forgotten when the user clicks Restart, as the fallback notice +/// asks after a driver update. A stall is not a failure: the driver did not +/// answer in time, which says nothing about the encoder. +#[derive(Default)] +pub(crate) struct LaunchFailures { + opens: Vec<(String, (u32, u32))>, + /// AMD and Intel cards whose encoder failed to open: the fallback + /// notice asks for their driver update. + vendors: Vec, +} + +static LAUNCH_FAILURES: parking_lot::Mutex = + parking_lot::Mutex::new(LaunchFailures { + opens: Vec::new(), + vendors: Vec::new(), + }); + +impl LaunchFailures { + /// Why `encoder` is not opened at `size`: it failed there this launch. + fn earlier(&self, encoder: &str, size: (u32, u32)) -> Option { + self.opens + .iter() + .any(|(name, at)| name == encoder && *at == size) + .then(|| { + format!( + "{encoder} failed to open at {}x{} earlier in this launch; not tried again until Restart", + size.0, size.1 + ) + }) + } + + /// Whether an encoder named with `suffix` failed this launch. + fn any(&self, suffix: &str) -> bool { + self.opens.iter().any(|(name, _)| name.ends_with(suffix)) + } + + /// `encoder` failed to open at `size`, for a reason of its own. When it + /// is AMD's or Intel's, and `vendors_present` (read only then) lists + /// that vendor's card, that card's driver could not start it. + fn failed( + &mut self, + encoder: &str, + size: (u32, u32), + vendors_present: impl FnOnce() -> Option>, + ) { + if !is_hardware_encoder(encoder) || self.earlier(encoder, size).is_some() { + return; + } + self.opens.push((encoder.to_string(), size)); + let vendor = if encoder.ends_with("_amf") { + GpuVendor::Amd + } else if encoder.ends_with("_qsv") { + GpuVendor::Intel + } else { + return; + }; + if self.vendors.contains(&vendor) + || !vendors_present().is_some_and(|present| present.contains(&vendor)) + { + return; + } + warn!( + %vendor, + encoder, + "The graphics driver could not start video encoding on this card; video is encoded on the processor unless another graphics card can, and updating the driver may enable hardware encoding" + ); + self.vendors.push(vendor); + } +} + +fn is_hardware_encoder(encoder_name: &str) -> bool { + ["_nvenc", "_amf", "_qsv"] + .iter() + .any(|suffix| encoder_name.ends_with(suffix)) +} + +/// Forget this launch's failed opens and their driver notes, so the next +/// pipeline tries every encoder again: the user clicked Restart, maybe +/// after a driver update. +pub fn forget_launch_failures() { + let forgotten = std::mem::take(&mut *LAUNCH_FAILURES.lock()); + if !forgotten.opens.is_empty() { + info!( + encoders = forgotten.opens.len(), + "Trying the encoders that failed to open again" + ); + } +} + +/// Opens a session as [`EncoderState::new`] does, unless `encoder_name` +/// failed to open at this size earlier in this launch, and remembers a +/// hardware encoder's own failure in `failures`. +#[allow(clippy::too_many_arguments)] +fn open_session( + failures: &parking_lot::Mutex, + encoder_name: &str, + capture_width: u32, + capture_height: u32, + bitrate_bps: u32, + target_fps: u32, + input_mode: &mut EncoderInput, + opening: Opening, + progress: &mut dyn FnMut(), +) -> Result> { + let size = encoded_size(capture_width, capture_height); + if let Some(why) = failures.lock().earlier(encoder_name, size) { + return Err(why.into()); + } + let opened = EncoderState::new( + encoder_name, + capture_width, + capture_height, + bitrate_bps, + target_fps, + input_mode, + opening, + progress, + ); + if let Err(error) = &opened { + note_open_failure(failures, encoder_name, size, error.as_ref()); + } + opened +} + +/// Remembers a hardware encoder's own failure to open at `size`, and shows +/// its driver note on the Stream page. +fn note_open_failure( + failures: &parking_lot::Mutex, + encoder_name: &str, + size: (u32, u32), + error: &(dyn std::error::Error + Send + Sync + 'static), +) { + if is_open_stall(error) { + return; + } + let vendors = { + let mut failures = failures.lock(); + failures.failed(encoder_name, size, crate::gpu::vendors_present); + failures.vendors.clone() + }; + PIPELINE_STATS.lock().driver_update_vendors = vendors; +} + /// An encoder open that its time limit gave up on, or refused while an /// earlier one was stuck. The driver did not answer in time, which says /// nothing about whether the encoder works. @@ -1216,8 +1749,10 @@ impl std::fmt::Display for OpenStalled { impl std::error::Error for OpenStalled {} +/// An open given up on or refused, or one still going on in the background: +/// neither says the encoder does not work. fn is_open_stall(error: &(dyn std::error::Error + Send + Sync + 'static)) -> bool { - error.downcast_ref::().is_some() + error.downcast_ref::().is_some() || is_still_opening(error) } /// Opens at the pixel format `input` picks from `supported`. When nothing @@ -1250,6 +1785,51 @@ fn open_at_input( } } +/// A reopen, for its log line and for what follows if it fails. +struct Reopening { + encoder: String, + /// The bitrate of the session it replaces. + from: u32, + /// The session that streams until it opens. + current: String, + reason: &'static str, + size_changed: bool, + keyframe: bool, + bitrate: u32, + fps: u32, + /// The pipeline's first open, which went on past its first picture's + /// wait (see [`FIRST_PICTURE_WAIT`]). + first: bool, + in_background: bool, +} + +/// An open going on in the background, and the reopen it makes once it +/// returns. +struct Behind { + open: BackgroundOpen, + reopen: Reopening, +} + +impl Behind { + /// The pipeline's first open of `open`'s encoder, while libx264 streams + /// at `bitrate`. + fn first(open: BackgroundOpen, bitrate: u32) -> Self { + let reopen = Reopening { + encoder: open.encoder().to_string(), + from: bitrate, + current: "libx264".to_string(), + reason: "recovery", + size_changed: false, + keyframe: false, + bitrate: open.args.bitrate_bps, + fps: open.args.target_fps, + first: true, + in_background: true, + }; + Self { open, reopen } + } +} + /// How long after a stalled open each new try of the preferred encoder /// waits, at the earliest: at most three tries a pipeline generation. const STALL_RETRY_WAITS: [Duration; 3] = [ @@ -1394,6 +1974,20 @@ impl StallRecovery { true } + /// The pipeline's first open of `preferred` had not returned when its + /// first picture was due, and goes on in the background while `using` + /// streams. If it opens, [`Self::opened`] ends the recovery. If it is + /// given up on at its limit, the first try is due once the lane is + /// clear, as after an open that stalled while the stream waited. + fn outwaited(&mut self, preferred: &str, using: &str, now: Instant) { + self.trying = false; + self.owed = Some(Owed { + preferred: preferred.to_string(), + using: using.to_string(), + next_try: now + STALL_RETRY_WAITS[0], + }); + } + /// No more tries this generation. fn end(&mut self) { self.owed = None; @@ -1429,14 +2023,21 @@ fn reopen_needed( /// ABR step just after an AMF keyframe reopen) reopens once. const BITRATE_DECREASE_GAP: Duration = Duration::from_secs(1); -/// Paces reopens for a bitrate decrease alone. Hardware encoders (and +/// How long after a Quick Sync session opened a bitrate change alone may +/// wait, up or down, once Quick Sync refused to take one in place (see +/// [`takes_bitrate_in_place`]): each of its reopens freezes the picture +/// for about 0.6 s, and one PC in the field reopened 148 times. +const QSV_BITRATE_REOPEN_GAP: Duration = Duration::from_secs(5); + +/// Paces reopens for a bitrate change alone. Hardware encoders (and /// libx265) take a new bitrate only by reopening, and each reopen costs an /// IDR and a visible stutter. A decrease takes effect at once, or 1 s after /// the session opened if that is later, so the stream never overruns a /// congested link more than 1 s longer than before, and a decrease right /// after another reopen does not make a second one at once. An increase /// takes effect at once, as do codec, frame rate, size, keyframe and -/// recovery reopens, and every reopen opens at the newest target. +/// recovery reopens, and every reopen opens at the newest target. Quick +/// Sync, whose reopens are slowest, waits 5 s for either. #[derive(Default)] struct BitratePacer { /// When the open session opened, for any reason. @@ -1446,14 +2047,21 @@ struct BitratePacer { impl BitratePacer { /// The bitrate this frame wants. `opened` and `wanted` are encoder, /// bitrate and frame rate. The newest target, unless only the bitrate - /// differs, downwards, within 1 s of the session's open: then the - /// session's own. + /// differs, downwards, within 1 s of the session's open, or either way + /// within 5 s of a Quick Sync session's: then the session's own. fn wanted(&self, opened: (&str, u32, u32), wanted: (&str, u32, u32), now: Instant) -> u32 { + let gap = if opened.0.ends_with("_qsv") { + QSV_BITRATE_REOPEN_GAP + } else if wanted.1 < opened.1 { + BITRATE_DECREASE_GAP + } else { + Duration::ZERO + }; let waits = (opened.0, opened.2) == (wanted.0, wanted.2) - && wanted.1 < opened.1 + && wanted.1 != opened.1 && self .opened_at - .is_some_and(|at| now.duration_since(at) < BITRATE_DECREASE_GAP); + .is_some_and(|at| now.duration_since(at) < gap); if waits { opened.1 } else { @@ -2840,6 +3448,15 @@ mod tests { (result, lines.lines().map(str::to_string).collect()) } + /// An open that returned, as every open but a wait-limited one in the + /// lane does. + fn now(opened: Opened) -> OpenResult { + match opened { + Opened::Now(result) => result, + Opened::Behind(_) => panic!("the open went on in the background"), + } + } + /// A real FFmpeg open that fails with an FFmpeg error line: libx264, /// which opens on the calling thread, with no picture size. fn failing_open(opening: Opening) -> Vec { @@ -2853,7 +3470,13 @@ mod tests { video.set_format(ffmpeg_next::format::Pixel::YUV420P); video.set_time_base(ffmpeg_next::Rational(1, 60)); let (opened, lines) = logged(|| { - open_encoder_on(crate::stall::lanes(), video, "libx264", opening, &mut || {}) + now(open_encoder_on( + crate::stall::lanes(), + video, + "libx264", + opening, + &mut || {}, + )) }); assert!(matches!(opened, Ok(Err(_))), "the open must fail"); lines @@ -2864,7 +3487,7 @@ mod tests { #[test] fn ffmpeg_errors_are_warnings_only_while_an_encoder_is_one_of_several() { - let probed = failing_open(Opening::Probe); + let probed = failing_open(Opening::Probe(FIRST_PICTURE_WAIT)); assert!(!probed.is_empty()); assert!( probed @@ -2887,14 +3510,15 @@ mod tests { ); } // The pipeline's first open probes, except the last resort. - assert_eq!(first_open("h264_nvenc"), Opening::Probe); - assert_eq!(first_open("hevc_amf"), Opening::Probe); - assert_eq!(first_open("libx265"), Opening::Probe); - assert_eq!(first_open("h264_videotoolbox"), Opening::Probe); - assert_eq!(first_open("libx264"), Opening::First); + let wait = FIRST_PICTURE_WAIT; + assert_eq!(first_open("h264_nvenc", wait), Opening::Probe(wait)); + assert_eq!(first_open("hevc_amf", wait), Opening::Probe(wait)); + assert_eq!(first_open("libx265", wait), Opening::Probe(wait)); + assert_eq!(first_open("h264_videotoolbox", wait), Opening::Probe(wait)); + assert_eq!(first_open("libx264", wait), Opening::First); // A probe has the first open's limits. assert_eq!( - open_limits("h264_qsv", Opening::Probe), + open_limits("h264_qsv", Opening::Probe(FIRST_PICTURE_WAIT)), open_limits("h264_qsv", Opening::First) ); } @@ -3193,8 +3817,11 @@ mod tests { /// Encoder, bitrate and frame rate of the open session. opened: (String, u32, u32), pacer: BitratePacer, - /// Bitrate changes the open libx264 session took in place. + /// Bitrate changes the open libx264 or Quick Sync session took in + /// place. in_place: usize, + /// A Quick Sync session refused a bitrate change in place. + qsv_refused: bool, /// The user's preference. hevc_wanted: bool, bitrate: u32, @@ -3221,6 +3848,7 @@ mod tests { opened: (opened.to_string(), 15_000_000, 60), pacer: BitratePacer::default(), in_place: 0, + qsv_refused: false, hevc_wanted: is_hevc_encoder(opened), bitrate: 15_000_000, fps: 60, @@ -3256,8 +3884,8 @@ mod tests { self.undecodable_retry .frame(undecodable, &mut self.failed_reopen, self.now); // As `update_bitrate_without_reopening`. - if (self.opened.0.as_str(), self.opened.2) == ("libx264", self.fps) - && desired == "libx264" + if takes_bitrate_in_place(&self.opened.0, self.qsv_refused) + && (self.opened.0.as_str(), self.opened.2) == (desired.as_str(), self.fps) && self.opened.1 != self.bitrate { self.opened.1 = self.bitrate; @@ -3796,10 +4424,12 @@ mod tests { #[test] fn a_reopen_goes_ahead_as_soon_as_the_lane_clears_whenever_a_try_is_due() { - // QSV streams. ABR steps stall its reopen, twice: the next tries + // QSV streams, and reopens for a bitrate change since it refused + // one in place. ABR steps stall its reopen, twice: the next tries // are due 2 s and then 8 s later. Each time the lane clears 1 s // after the stall, and a bitrate change reopens on the next frame. let mut p = Pipeline::new("h264_qsv", "h264_qsv", Instant::now()); + p.qsv_refused = true; for (bitrate, stalls) in [(10_000_000, true), (12_000_000, true), (14_000_000, false)] { p.bitrate = bitrate; let reopen = if stalls { @@ -3910,13 +4540,9 @@ mod tests { #[test] fn a_drop_reopens_at_once_or_within_a_second_of_the_last_open() { - for name in [ - "h264_nvenc", - "h264_amf", - "h264_qsv", - "hevc_nvenc", - "libx265", - ] { + // Quick Sync takes a new bitrate in place; see + // `quick_sync_takes_every_bitrate_change_in_place_until_it_refuses_one`. + for name in ["h264_nvenc", "h264_amf", "hevc_nvenc", "libx265"] { let h264 = match name { "hevc_nvenc" => "h264_nvenc", "libx265" => "libx264", @@ -4100,6 +4726,435 @@ mod tests { assert_eq!(play(&mut p, &[], 60_000), []); } + #[test] + fn quick_sync_takes_every_bitrate_change_in_place_until_it_refuses_one() { + for name in ["libx264", "h264_qsv", "hevc_qsv"] { + assert!(takes_bitrate_in_place(name, false), "{name}"); + } + for name in ["h264_nvenc", "hevc_nvenc", "h264_amf", "hevc_amf", "libx265"] { + assert!(!takes_bitrate_in_place(name, false), "{name}"); + } + assert!(!takes_bitrate_in_place("h264_qsv", true)); + assert!(takes_bitrate_in_place("libx264", true)); + // As libx264: rises and drops, some within a second of each other, + // each on the frame it arrives, and no reopen. The field's worst PC + // reopened Quick Sync 148 times, about 0.6 s of frozen picture each. + let t0 = Instant::now(); + let mut p = Pipeline::new("h264_qsv", "h264_qsv", t0); + for (at, bitrate) in [ + (0, 12_000_000), + (100, 10_000_000), + (5_000, 15_000_000), + (5_100, 12_000_000), + ] { + p.now = t0 + Duration::from_millis(at); + p.bitrate = bitrate; + assert_eq!(p.frame(Reopen::Opens), None, "at {at} ms"); + assert_eq!(p.opened.1, bitrate, "at {at} ms"); + } + assert_eq!(p.in_place, 4); + // A frame rate change still reopens. + p.fps = 30; + assert_eq!(p.frame(Reopen::Opens).as_deref(), Some("h264_qsv")); + // Once a session refused a change, a change reopens, at most every + // 5 s, rises too: here 5 s after that frame rate reopen, and 5 s + // after the previous bitrate reopen. + p.qsv_refused = true; + let reopens = play( + &mut p, + &[ + (0, Some(10_000_000)), + (1_000, Some(14_000_000)), + (6_000, Some(8_000_000)), + (6_500, Some(9_000_000)), + ], + 15_000, + ); + assert_eq!(reopens, [(5_008, 14_000_000), (10_016, 9_000_000)]); + assert_eq!(p.in_place, 4); + } + + #[test] + fn a_quick_sync_bitrate_reopen_waits_5_s_for_either_direction() { + let t0 = Instant::now(); + let at = |ms: u64| t0 + Duration::from_millis(ms); + let mut pacer = BitratePacer::default(); + pacer.opened(t0); + let qsv = ("h264_qsv", 10_000_000, 60); + assert_eq!(pacer.wanted(qsv, ("h264_qsv", 12_000_000, 60), at(1_000)), 10_000_000); + assert_eq!(pacer.wanted(qsv, ("h264_qsv", 8_000_000, 60), at(4_999)), 10_000_000); + assert_eq!(pacer.wanted(qsv, ("h264_qsv", 8_000_000, 60), at(5_000)), 8_000_000); + assert_eq!(pacer.wanted(qsv, ("h264_qsv", 12_000_000, 60), at(5_000)), 12_000_000); + // A frame rate or codec change goes ahead at the newest target. + assert_eq!(pacer.wanted(qsv, ("h264_qsv", 12_000_000, 30), at(10)), 12_000_000); + assert_eq!(pacer.wanted(qsv, ("hevc_qsv", 12_000_000, 60), at(10)), 12_000_000); + // Other encoders keep #165's pacing: a rise at once, a drop 1 s on. + let nvenc = ("h264_nvenc", 10_000_000, 60); + assert_eq!(pacer.wanted(nvenc, ("h264_nvenc", 12_000_000, 60), at(10)), 12_000_000); + assert_eq!(pacer.wanted(nvenc, ("h264_nvenc", 8_000_000, 60), at(999)), 10_000_000); + assert_eq!(pacer.wanted(nvenc, ("h264_nvenc", 8_000_000, 60), at(1_000)), 8_000_000); + } + + /// A clock that stands still until the test moves it: the ms since the + /// test began are in the returned atomic. + fn paused_clock() -> ( + std::sync::Arc, + impl Fn() -> Instant + Clone + Send + 'static, + ) { + let start = Instant::now(); + let ms = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let read = std::sync::Arc::clone(&ms); + (ms, move || { + start + Duration::from_millis(read.load(Ordering::SeqCst)) + }) + } + + /// A fake hardware open of `name` in `lanes`, on `clock`, that hangs + /// until the returned sender sends, then opens. + fn hung_open Instant>( + lanes: &'static crate::stall::Lanes, + name: &'static str, + opening: Opening, + clock: N, + ) -> ( + std::sync::mpsc::Sender<()>, + crate::stall::Pending<&'static str, N>, + ) { + let (release, hang) = stuck_open(); + let pending = crate::stall::start_in( + lanes, + crate::stall::Call::EncoderOpen, + crate::stall::Wait { + limits: open_limits(name, opening), + slice: Duration::from_millis(1), + now: clock, + }, + move || { + hang(); + name + }, + crate::stall::no_hooks(), + ) + .unwrap(); + (release, pending) + } + + /// Waits for the pipeline's first open of `name` as the encode loop + /// does, each 1 ms heartbeat moving the paused clock 100 ms on. + fn first_open_of Instant>( + pending: crate::stall::Pending<&'static str, N>, + ms: &std::sync::atomic::AtomicU64, + ) -> Result< + Result<&'static str, crate::stall::Stall>, + crate::stall::Pending<&'static str, N>, + > { + wait_or_park(pending, Opening::Probe(FIRST_PICTURE_WAIT), &mut || { + ms.fetch_add(100, Ordering::SeqCst); + }) + } + + #[test] + fn a_hung_first_open_leaves_the_picture_to_libx264_when_it_is_due() { + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let (ms, clock) = paused_clock(); + let ((parked, opened_late), lines) = logged(|| { + let opening = Opening::Probe(FIRST_PICTURE_WAIT); + let (release, pending) = hung_open(lanes, "h264_qsv", opening, clock.clone()); + let Err(mut parked) = first_open_of(pending, &ms) else { + panic!("the hung open returned") + }; + // Given up waiting for when the first picture was due, 7 s + // before the iPad's timeout; not given up on. + let parked_at = parked.running_for(); + // libx264 needs no lane and opens at once. Every hardware open + // is refused at once instead of waiting on FFmpeg's lock. + assert_eq!( + open_in(lanes, "libx264", Opening::First, || {}).unwrap(), + "libx264" + ); + for name in ["h264_nvenc", "h264_amf", "hevc_qsv", "libx265"] { + let error = open_in(lanes, name, Opening::First, || {}).unwrap_err(); + assert!(is_open_stall(error.as_ref()), "{name}: {error}"); + assert!(error.to_string().contains("not started"), "{error}"); + } + // The stream: libx264 while QSV opens. + let t0 = Instant::now(); + let at = |ms: u64| t0 + Duration::from_millis(ms); + let mut recovery = StallRecovery::default(); + recovery.outwaited("h264_qsv", "libx264", at(3_000)); + for ms in [3_000, 4_000, 5_000, 14_000] { + assert_eq!( + recovery.encoder_for_frame("h264_qsv", at(ms), lane_clear(lanes)), + "libx264", + "at {ms} ms" + ); + } + // QSV opens at 4.4 s, the slowest open in the field, within + // its 15 s limit: the stream takes that very session. + ms.store(4_400, Ordering::SeqCst); + assert!(parked.poll().is_none()); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + let opened = parked.poll(); + assert!(recovery.opened("h264_qsv", "h264_qsv")); + assert!(recovery.owed.is_none()); + assert_eq!(recovery.tries, 0); + (parked_at, opened) + }); + assert_eq!(parked, FIRST_PICTURE_WAIT); + assert_eq!(opened_late, Some(Ok("h264_qsv"))); + assert!( + !lines.iter().any(|line| line.contains("stall=encoder_open")), + "{lines:?}" + ); + } + + #[test] + fn an_open_that_returns_before_the_first_picture_is_due_is_not_parked() { + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let (ms, clock) = paused_clock(); + let opening = Opening::Probe(FIRST_PICTURE_WAIT); + let (release, pending) = hung_open(lanes, "h264_nvenc", opening, clock); + // NVENC's slowest open in the field: 2.6 s. The clock stops there, + // however long the hand-over takes on a loaded machine. + let mut beats = 0; + let mut released = false; + let opened = wait_or_park(pending, opening, &mut || { + beats += 1; + if !released && ms.fetch_add(100, Ordering::SeqCst) + 100 >= 2_600 { + release.send(()).unwrap(); + released = true; + } + }); + assert!(matches!(opened, Ok(Ok("h264_nvenc")))); + assert!(lane_clear(lanes)); + // The watchdog's heartbeat ran meanwhile. + assert!(beats >= 26, "{beats}"); + // A deadline already past (an earlier encoder of the fallback used + // the wait up) parks at once; so does a background reopen. + for opening in [Opening::Probe(Duration::ZERO), Opening::Behind] { + let (_ms, clock) = paused_clock(); + let (release, pending) = hung_open(lanes, "h264_amf", opening, clock); + let Err(parked) = wait_or_park(pending, opening, &mut || panic!("it waited")) else { + panic!("returned") + }; + assert_eq!(parked.running_for(), Duration::ZERO); + release.send(()).unwrap(); + drop(parked); + wait_for_clear_lane(lanes); + } + // A reopen of a live session still waits, to its 9 s limit. + let (ms, clock) = paused_clock(); + let (release, pending) = hung_open(lanes, "h264_qsv", Opening::Reopen, clock); + let gave_up = wait_or_park(pending, Opening::Reopen, &mut || { + ms.fetch_add(100, Ordering::SeqCst); + }); + let Ok(Err(crate::stall::Stall::TimedOut { after, .. })) = gave_up else { + panic!("not given up on at the reopen limit") + }; + assert_eq!(after, Duration::from_secs(9)); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + } + + #[test] + fn a_first_open_hung_past_its_limit_is_tried_again_in_the_background() { + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let (ms, clock) = paused_clock(); + let t0 = Instant::now(); + let at = |ms: u64| t0 + Duration::from_millis(ms); + let mut recovery = StallRecovery::default(); + let opening = Opening::Probe(FIRST_PICTURE_WAIT); + let (release, pending) = hung_open(lanes, "h264_nvenc", opening, clock.clone()); + let Err(mut parked) = first_open_of(pending, &ms) else { + panic!("the hung open returned") + }; + recovery.outwaited("h264_nvenc", "libx264", at(3_000)); + // Given up on at NVENC's 15 s limit, with the one stall line; the + // lane stays taken until the driver returns. + ms.store(15_000, Ordering::SeqCst); + let (result, lines) = logged(|| parked.poll()); + assert!(matches!( + result, + Some(Err(crate::stall::Stall::TimedOut { .. })) + )); + assert!( + lines + .iter() + .any(|line| line.contains("stall=encoder_open after_ms=15000")), + "{lines:?}" + ); + assert!(!lane_clear(lanes)); + assert_eq!( + recovery.encoder_for_frame("h264_nvenc", at(20_000), false), + "libx264" + ); + // The driver returns at 30 s. The try is due: it opens in the + // background, so libx264 streams on instead of freezing for it. + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + let desired = recovery.encoder_for_frame("h264_nvenc", at(30_000), true); + assert_eq!(desired, "h264_nvenc"); + assert!(opens_behind("libx264", &desired)); + assert_eq!( + open_limits("h264_nvenc", Opening::Behind), + crate::stall::encoder_open_limits("h264_nvenc") + ); + recovery.starting(&desired); + ms.store(30_000, Ordering::SeqCst); + let (release, pending) = hung_open(lanes, "h264_nvenc", Opening::Behind, clock); + let Err(mut trying) = + wait_or_park(pending, Opening::Behind, &mut || panic!("a background open waited")) + else { + panic!("returned") + }; + assert_eq!( + recovery.encoder_for_frame("h264_nvenc", at(30_016), lane_clear(lanes)), + "libx264" + ); + ms.store(30_200, Ordering::SeqCst); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + assert_eq!(trying.poll(), Some(Ok("h264_nvenc"))); + assert!(recovery.opened("h264_nvenc", "h264_nvenc")); + assert!(recovery.owed.is_none()); + } + + #[test] + fn only_a_switch_from_libx264_into_the_lane_opens_in_the_background() { + for (current, wanted, behind) in [ + ("libx264", "h264_nvenc", true), + ("libx264", "hevc_amf", true), + ("libx264", "h264_qsv", true), + ("libx264", "libx265", true), + ("libx264", "libx264", false), + ("h264_nvenc", "hevc_nvenc", false), + ("h264_qsv", "h264_qsv", false), + ("hevc_amf", "libx264", false), + ("h264_videotoolbox", "hevc_videotoolbox", false), + ] { + assert_eq!(opens_behind(current, wanted), behind, "{current} -> {wanted}"); + } + } + + #[test] + fn a_hardware_encoder_that_failed_is_remembered_at_its_size_for_the_launch() { + use GpuVendor::{Amd, Intel, Nvidia}; + let hd = (1920, 1080); + let mut failures = LaunchFailures::default(); + failures.failed("h264_nvenc", hd, || panic!("NVENC never reads the cards")); + assert!(failures.earlier("h264_nvenc", hd).is_some()); + // Another size, another encoder: tried. + assert!(failures.earlier("h264_nvenc", (2732, 2048)).is_none()); + assert!(failures.earlier("hevc_nvenc", hd).is_none()); + assert!(failures.any("_nvenc")); + assert!(!failures.any("_amf")); + // Software encoders are never left out. + failures.failed("libx264", hd, || panic!("read")); + failures.failed("libx265", hd, || panic!("read")); + assert!(failures.earlier("libx264", hd).is_none()); + assert!(failures.earlier("libx265", hd).is_none()); + // AMF on a PC with an AMD card: its driver gets the note. + failures.failed("h264_amf", hd, || Some(vec![Nvidia, Amd])); + assert_eq!(failures.vendors, [Amd]); + // Quick Sync on a PC without an Intel card: no note, as that + // failure is expected there. Still remembered. + failures.failed("h264_qsv", hd, || Some(vec![Nvidia, Amd])); + assert_eq!(failures.vendors, [Amd]); + assert!(failures.earlier("h264_qsv", hd).is_some()); + // Unknown cards (DXGI listed some, not all): no note. + failures.failed("hevc_qsv", hd, || None); + assert_eq!(failures.vendors, [Amd]); + failures.failed("h264_qsv", (1280, 720), || Some(vec![Intel])); + assert_eq!(failures.vendors, [Amd, Intel]); + // Each failure is kept once, and the cards are read once a vendor. + failures.failed("hevc_amf", hd, || panic!("read again")); + failures.failed("h264_nvenc", hd, || panic!("read")); + assert_eq!( + failures + .opens + .iter() + .filter(|(name, size)| name == "h264_nvenc" && *size == hd) + .count(), + 1 + ); + let why = failures.earlier("h264_amf", hd).unwrap(); + assert!( + why.contains("h264_amf failed to open at 1920x1080 earlier in this launch"), + "{why}" + ); + assert!(why.contains("Restart"), "{why}"); + } + + #[test] + fn an_encoder_that_failed_to_open_is_not_opened_again_or_logged_again() { + ffmpeg_next::init().unwrap(); + crate::diagnostics::install_ffmpeg_logging(); + let failures = parking_lot::Mutex::new(LaunchFailures::default()); + // A real open that fails: Quick Sync on a PC without an Intel card + // ("Error creating a MFX session"), or not in this FFmpeg build + // (macOS). Both are the encoder's own failure. + let open = |failures: &parking_lot::Mutex| { + logged(|| { + open_session( + failures, + "h264_qsv", + 1280, + 720, + 8_000_000, + 60, + &mut EncoderInput::Yuv420, + Opening::First, + &mut || {}, + ) + .err() + .map(|error| error.to_string()) + }) + }; + let (Some(first), first_lines) = open(&failures) else { + eprintln!("Quick Sync opens on this PC; nothing to remember"); + return; + }; + eprintln!("first open: {first}"); + assert!(failures.lock().earlier("h264_qsv", (1280, 720)).is_some()); + // Every pipeline after it: no open, no FFmpeg line, and the reason. + for _ in 0..3 { + let (again, lines) = open(&failures); + let again = again.expect("opened"); + assert!(again.contains("earlier in this launch"), "{again}"); + assert!( + !lines + .iter() + .any(|line| line.contains("ffmpeg") || line.contains("Opening encoder session")), + "{lines:?}" + ); + } + eprintln!( + "first open logged {} FFmpeg lines", + first_lines + .iter() + .filter(|line| line.contains("ffmpeg")) + .count() + ); + // libx264 is never held back, and a Restart tries Quick Sync again. + assert!(open_session( + &failures, + "libx264", + 1280, + 720, + 8_000_000, + 60, + &mut EncoderInput::Yuv420, + Opening::First, + &mut || {}, + ) + .is_ok()); + *failures.lock() = LaunchFailures::default(); + let (retried, _) = open(&failures); + let retried = retried.expect("opened"); + assert!(!retried.contains("earlier in this launch"), "{retried}"); + } + #[test] fn the_reopen_line_says_why() { let nvenc = ("h264_nvenc", 60); @@ -4239,7 +5294,13 @@ mod tests { // stays on H.264. if let Some(video) = context("libx265") { let started = Instant::now(); - let opened = open_encoder_on(lanes, video, "libx265", Opening::First, &mut || {}); + let opened = now(open_encoder_on( + lanes, + video, + "libx265", + Opening::First, + &mut || {}, + )); assert!( matches!( opened, @@ -4255,7 +5316,13 @@ mod tests { // libx264 opens meanwhile. let video = context("libx264").expect("libx264 is in every build"); assert!(matches!( - open_encoder_on(lanes, video, "libx264", Opening::First, &mut || {}), + now(open_encoder_on( + lanes, + video, + "libx264", + Opening::First, + &mut || {} + )), Ok(Ok(_)) )); let _ = release.send(()); @@ -4378,10 +5445,17 @@ mod tests { // FPS and codec changes must still use the existing reopen path. assert!(!state.update_bitrate_without_reopening("libx264", 4_000_000, 120)); assert!(!state.update_bitrate_without_reopening("libx265", 4_000_000, 60)); - for name in ["h264_nvenc", "h264_amf", "h264_qsv", "libx265"] { + for name in ["h264_nvenc", "h264_amf", "libx265"] { state.opened_encoder = name.to_string(); assert!(!state.update_bitrate_without_reopening(name, 4_000_000, 60)); } + // Quick Sync takes it in place, to be confirmed by its next frame. + state.opened_encoder = "h264_qsv".to_string(); + assert!(!state.bitrate_reset_unconfirmed); + assert!(state.update_bitrate_without_reopening("h264_qsv", 4_000_000, 60)); + assert_eq!(state.opened_bitrate, 4_000_000); + assert!(state.bitrate_reset_unconfirmed); + assert!(!state.update_bitrate_without_reopening("h264_qsv", 4_000_000, 60)); } #[test] @@ -4750,9 +5824,10 @@ mod tests { } /// Native, on a PC with NVENC: the encode stage's first NVENC open is - /// held for twice its limit (`ETERNAL_SIMULATE_STALL`). The stage falls - /// through to software at once and streams, the late NVENC session is - /// closed on its own thread, and the stage then goes back to NVENC. + /// held for twice its limit (`ETERNAL_SIMULATE_STALL`). The stage streams + /// on software once the first picture is due, gives the open up at its + /// limit, closes the late NVENC session on its own thread, and then goes + /// back to NVENC through an open in the background. /// `cargo test -p eternal-host --lib a_real_nvenc_open -- --ignored --nocapture` #[cfg(windows)] #[test] @@ -4764,8 +5839,8 @@ mod tests { let started = stage.started; // When each step's line first shows, to within a frame. let steps = [ + "streaming on libx264 while it opens in the background", "stall=encoder_open", - "streaming on the fallback", "stall_ended=encoder_open", "Back on the preferred encoder", ]; @@ -4787,9 +5862,10 @@ mod tests { for _ in 0..30 { stage.publish(); } - let [stalled, _, ended, back] = seen.map(Option::unwrap); + let [outwaited, stalled, ended, back] = seen.map(Option::unwrap); + let first_packet = stage.packets.lock().first().copied().expect("no packet"); // A picture throughout: x264 while the open was stuck, NVENC after. - let on_x264 = stage.packets_between(stalled + Duration::from_secs(1), ended); + let on_x264 = stage.packets_between(outwaited + Duration::from_secs(1), ended); let on_nvenc = stage.packets_between(back, Duration::MAX); let text = stage.finish(); let at = |needle: &str| { @@ -4800,7 +5876,21 @@ mod tests { assert!(order.windows(2).all(|pair| pair[0] < pair[1]), "{order:?}"); let back_line = text[order[3]..].lines().next().unwrap(); assert!(back_line.contains("h264_nvenc"), "{back_line}"); - eprintln!("stalled at {stalled:?}, stall ended at {ended:?}, back on NVENC at {back:?}"); + assert!( + text.contains("Opening the encoder in the background"), + "the try froze the stream:\n{text}" + ); + eprintln!( + "first picture at {first_packet:?}, x264 from {outwaited:?}, stalled at {stalled:?}, \ + stall ended at {ended:?}, back on NVENC at {back:?}" + ); + // The first picture well before the iPad's 10 s timeout: 3 s, then + // libx264's open and first frame. + assert!( + outwaited >= Duration::from_secs(3) && outwaited < Duration::from_secs(5), + "{outwaited:?}" + ); + assert!(first_packet < Duration::from_secs(5), "{first_packet:?}"); // Given up on at NVENC's 15 s limit; the held open returns at 30 s. assert!( stalled >= Duration::from_secs(15) && stalled < Duration::from_secs(18), @@ -4816,6 +5906,62 @@ mod tests { assert!(!stats.using_software_fallback); } + /// Native, on a PC with NVENC: the first NVENC open takes 6 s, past the + /// first picture's 3 s but within its 15 s limit. The stage streams on + /// software from 3 s and switches to that same NVENC session when it + /// opens, with no stall and no second open. + /// `cargo test -p eternal-host --lib a_slow_real_nvenc_open -- --ignored --nocapture` + #[cfg(windows)] + #[test] + #[ignore = "native: needs an NVIDIA GPU"] + fn a_slow_real_nvenc_open_is_taken_once_it_opens() { + use std::time::Duration; + std::env::set_var("ETERNAL_SIMULATE_STALL", "encoder_open"); + std::env::set_var("ETERNAL_SIMULATE_STALL_MS", "6000"); + let mut stage = NativeStage::start(); + let (outwaited, from) = stage.publish_until( + "streaming on libx264 while it opens in the background", + 0, + Duration::from_secs(20), + ); + let (switched, _) = + stage.publish_until("in_background=true", from, Duration::from_secs(20)); + for _ in 0..30 { + stage.publish(); + } + let first_packet = stage.packets.lock().first().copied().expect("no packet"); + let on_x264 = stage.packets_between(outwaited + Duration::from_millis(500), switched); + let after = stage.packets_between(switched, Duration::MAX); + let text = stage.finish(); + eprintln!( + "first picture at {first_packet:?}, x264 from {outwaited:?}, NVENC from {switched:?}; \ + {on_x264} packets on x264, {after} after" + ); + assert!(first_packet < Duration::from_secs(5), "{first_packet:?}"); + assert!( + switched >= Duration::from_secs(6) && switched < Duration::from_secs(8), + "{switched:?}" + ); + assert!(on_x264 >= 30, "{on_x264} packets on x264"); + assert!(after >= 20, "{after} packets on NVENC"); + let line = text + .lines() + .find(|line| line.contains("in_background=true")) + .unwrap(); + assert!(line.contains("Reopened encoder session"), "{line}"); + assert!(line.contains("h264_nvenc"), "{line}"); + assert!(!text.contains("stall=encoder_open"), "{text}"); + // NVENC opened once: the session that went on in the background. + assert_eq!( + text.matches("Encoder opened").count(), + 2, + "x264 and NVENC, once each:\n{text}" + ); + let stats = PIPELINE_STATS.lock(); + assert_eq!(stats.codec_name, "H.264 (NVENC)"); + assert!(!stats.using_software_fallback); + } + /// Native, on a PC with NVENC: an ABR step's NVENC reopen /// is held for 10 s, so it is given up on at the 9 s reopen limit and /// the old session streams on. The lane clears 1 s later, before the diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 53d7cf8e..7191faf4 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -81,6 +81,7 @@ struct StatsSnapshot { codec_name: String, using_software_fallback: bool, nvidia_driver_outdated: bool, + driver_update_vendors: Vec, gpu_name: String, capture_display: String, transport_fps: f64, @@ -119,6 +120,7 @@ impl StatsSnapshot { codec_name: s.codec_name.clone(), using_software_fallback: s.using_software_fallback, nvidia_driver_outdated: s.nvidia_driver_outdated, + driver_update_vendors: s.driver_update_vendors.clone(), gpu_name: s.gpu_name.clone(), capture_display: s.capture_display.clone(), transport_fps: s.transport_fps, @@ -820,6 +822,8 @@ impl AnalyzerApp { fn restart_stream(&mut self) { self.restart_pending = false; + // The user may have updated a driver, as the fallback notice asks. + crate::encoder::forget_launch_failures(); self.control.request_restart(); } diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index 405e3d7f..2403019b 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -85,7 +85,11 @@ impl AnalyzerApp { WARNING, Icon::Warning, SOFTWARE_FALLBACK_TITLE, - &software_fallback_body(customized, snap.nvidia_driver_outdated), + &software_fallback_body( + customized, + snap.nvidia_driver_outdated, + &snap.driver_update_vendors, + ), if customized { 230.0 } else { 110.0 }, |ui| { if customized { @@ -831,9 +835,17 @@ pub(super) const SOFTWARE_FALLBACK_TITLE: &str = "Video is using the processor instead of the graphics card"; /// What the CPU fallback notice says: undo custom settings first, say the -/// NVIDIA driver is too old when that is what the graphics card refused, or -/// suggest a driver update. -pub(super) fn software_fallback_body(customized: bool, nvidia_driver_outdated: bool) -> String { +/// NVIDIA driver is too old when that is what the graphics card refused, +/// name the AMD or Intel driver whose card could not start encoding +/// (`driver_vendors`), or suggest a driver update. +pub(super) fn software_fallback_body( + customized: bool, + nvidia_driver_outdated: bool, + driver_vendors: &[crate::gpu::GpuVendor], +) -> String { + use crate::gpu::GpuVendor::{Amd, Intel}; + let amd = driver_vendors.contains(&Amd); + let intel = driver_vendors.contains(&Intel); if customized { "The graphics card could not encode video with the options chosen in Settings, so the picture may lag and the PC may run hot. Go back to the recommended options first.".to_string() } else if nvidia_driver_outdated { @@ -841,6 +853,13 @@ pub(super) fn software_fallback_body(customized: bool, nvidia_driver_outdated: b "The NVIDIA graphics driver on this PC is too old for the graphics card to encode video, so the picture may lag and the PC may run hot. Update the NVIDIA driver to version {} or newer, if your graphics card supports it, then click Restart. Older cards keep using the processor.", crate::encoder::NVENC_MIN_DRIVER ) + } else if amd && intel { + "The AMD and Intel graphics drivers on this PC could not start video encoding on the graphics cards, so the picture may lag and the PC may run hot. Update the AMD and Intel graphics drivers, then click Restart.".to_string() + } else if amd || intel { + let vendor = if amd { "AMD" } else { "Intel" }; + format!( + "The {vendor} graphics driver on this PC could not start video encoding on the graphics card, so the picture may lag and the PC may run hot. Update the {vendor} graphics driver, then click Restart." + ) } else { "The picture may lag and the PC may run hot. Update your graphics driver from NVIDIA, AMD or Intel, then click Restart.".to_string() } diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index e74b54a8..4bd0c58c 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -424,7 +424,7 @@ fn display_choices_use_the_names_shared_with_the_ipad() { #[test] fn an_outdated_nvidia_driver_is_named_with_the_version_to_install() { - let body = software_fallback_body(false, true); + let body = software_fallback_body(false, true, &[]); assert!( body.contains("NVIDIA driver to version 570 or newer"), "{body}" @@ -441,10 +441,53 @@ fn an_outdated_nvidia_driver_is_named_with_the_version_to_install() { } // Custom settings are still the first thing to undo. assert_eq!( - software_fallback_body(true, true), - software_fallback_body(true, false) + software_fallback_body(true, true, &[]), + software_fallback_body(true, false, &[]) ); - assert!(!software_fallback_body(false, false).contains("570")); + assert!(!software_fallback_body(false, false, &[]).contains("570")); +} + +#[test] +fn an_amd_or_intel_card_that_could_not_encode_is_named_for_its_driver_update() { + use crate::gpu::GpuVendor::{Amd, Intel, Nvidia}; + let amd = software_fallback_body(false, false, &[Amd]); + assert!( + amd.contains("The AMD graphics driver on this PC could not start video encoding"), + "{amd}" + ); + assert!(amd.contains("Update the AMD graphics driver, then click Restart"), "{amd}"); + let intel = software_fallback_body(false, false, &[Intel]); + assert!( + intel.contains("Update the Intel graphics driver, then click Restart"), + "{intel}" + ); + assert!(!intel.contains("AMD"), "{intel}"); + let both = software_fallback_body(false, false, &[Intel, Amd]); + assert!( + both.contains("Update the AMD and Intel graphics drivers, then click Restart"), + "{both}" + ); + for body in [&amd, &intel, &both] { + for jargon in ["AMF", "Quick Sync", "QSV", "MFX", "libx264", "encoder", "http"] { + assert!(!body.contains(jargon), "{body}"); + } + } + // An outdated NVIDIA driver names the version to install, and custom + // settings are still the first thing to undo. + assert_eq!( + software_fallback_body(false, true, &[Amd]), + software_fallback_body(false, true, &[]) + ); + assert_eq!( + software_fallback_body(true, false, &[Amd]), + software_fallback_body(true, false, &[]) + ); + // No vendor named: the general advice, as before. + assert_eq!( + software_fallback_body(false, false, &[Nvidia]), + software_fallback_body(false, false, &[]) + ); + assert!(software_fallback_body(false, false, &[]).contains("NVIDIA, AMD or Intel")); } #[test] diff --git a/host/src/stall.rs b/host/src/stall.rs index 9d8b5a20..a4fba44e 100644 --- a/host/src/stall.rs +++ b/host/src/stall.rs @@ -205,7 +205,8 @@ impl std::fmt::Display for Stall { impl std::error::Error for Stall {} -/// Calls of each kind that were abandoned and still run. +/// Calls of each kind that still run with no caller waiting for them: +/// abandoned, or parked (see [`Pending::park`]). pub(crate) struct Lanes([AtomicUsize; CALLS]); impl Lanes { @@ -225,8 +226,8 @@ pub(crate) fn lanes() -> &'static Lanes { &LANES } -/// Abandoned calls of this kind still running, process-wide: while one -/// runs, the next of its kind is refused. +/// Abandoned or parked calls of this kind still running, process-wide: +/// while one runs, the next of its kind is refused. pub fn abandoned(call: Call) -> usize { LANES.abandoned(call) } @@ -255,8 +256,26 @@ pub(crate) fn run_on( call: Call, limits: Limits, f: F, - progress: impl FnMut(), + mut progress: impl FnMut(), ) -> Result +where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, +{ + start_on(lanes, call, limits, f)? + .wait_up_to(limits.limit, &mut progress) + .expect("the limit ends the wait") +} + +/// Start `f` on its own thread under `limits`, to wait for it with +/// [`Pending::wait_up_to`]. Refused, as by [`run`], while an earlier call of +/// its kind is stuck. +pub(crate) fn start_on( + lanes: &'static Lanes, + call: Call, + limits: Limits, + f: F, +) -> Result, Stall> where T: Send + 'static, F: FnOnce() -> T + Send + 'static, @@ -264,7 +283,7 @@ where let wait = Wait { limits, slice: WAIT_SLICE, - now: Instant::now, + now: Instant::now as fn() -> Instant, }; let delay = simulated_delay(call, limits); let hooks = Hooks { @@ -275,7 +294,7 @@ where }, handoff: || {}, }; - run_in(lanes, call, wait, f, progress, hooks) + start_in(lanes, call, wait, f, hooks) } /// Points on the call's thread: `before` the call (a simulated stall), and @@ -298,6 +317,7 @@ pub(crate) fn no_hooks() -> Hooks { type Outcome = std::thread::Result; /// [`run`] with its lanes, clock and hooks given. +#[cfg(test)] pub(crate) fn run_in( lanes: &'static Lanes, call: Call, @@ -310,7 +330,57 @@ where T: Send + 'static, F: FnOnce() -> T + Send + 'static, { - use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; + let limit = wait.limits.limit; + start_in(lanes, call, wait, f, hooks)? + .wait_up_to(limit, &mut progress) + .expect("the limit ends the wait") +} + +/// Who takes a call's result. Decided under one lock that the call's thread +/// holds from reading it until its result is in the channel, and that the +/// caller holds while it looks in the channel before changing it, so exactly +/// one of them owns the result, and the lane is freed whenever it was taken. +/// Never held across the call itself. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Owner { + /// The caller waits for it. + Waiting, + /// The caller stopped waiting before the limit and takes the result + /// later (see [`Pending::park`]). The lane counts the call meanwhile. + Parked, + /// The caller gave up on it: its thread releases the result. + Nobody, +} + +/// A time-limited call that runs on its own thread, from [`start_in`]. +pub(crate) struct Pending Instant> { + call: Call, + limits: Limits, + slice: Duration, + now: N, + lanes: &'static Lanes, + owner: Arc>, + rx: mpsc::Receiver>, + started: Instant, + /// Its result was returned, or it was given up on. + done: bool, +} + +/// Start `f` on its own thread under `wait`'s limits, or refuse it while an +/// earlier call of its kind still runs with nobody waiting. +pub(crate) fn start_in( + lanes: &'static Lanes, + call: Call, + wait: Wait, + f: F, + hooks: Hooks, +) -> Result, Stall> +where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, + N: Fn() -> Instant, +{ + use std::panic::{catch_unwind, AssertUnwindSafe}; let lane = &lanes.0[call as usize]; let abandoned = lane.load(SeqCst); if abandoned >= MAX_ABANDONED { @@ -321,14 +391,9 @@ where ); return Err(Stall::Refused { call }); } - // Set by the caller when it gives up. The call's thread holds the same - // lock from reading it until its result is in the channel, and the - // caller looks in the channel under it before giving up, so exactly one - // of them owns the result, and the lane is freed whenever it was taken. - // Never held across the call itself. - let gave_up = Arc::new(Mutex::new(false)); + let owner = Arc::new(Mutex::new(Owner::Waiting)); let (tx, rx) = mpsc::channel::>(); - let thread_gave_up = Arc::clone(&gave_up); + let thread_owner = Arc::clone(&owner); let real_start = Instant::now(); let Hooks { before, handoff } = hooks; let spawned = std::thread::Builder::new() @@ -340,13 +405,23 @@ where before(); f() })); - let given_up = thread_gave_up.lock(); + let owner = thread_owner.lock(); handoff(); - if !*given_up { - let _ = tx.send(outcome); - return; + match *owner { + Owner::Waiting => { + let _ = tx.send(outcome); + return; + } + // The caller takes it from the channel. The call no longer + // holds anything up. + Owner::Parked => { + let _ = tx.send(outcome); + lanes.0[call as usize].fetch_sub(1, SeqCst); + return; + } + Owner::Nobody => {} } - drop(given_up); + drop(owner); let panicked = outcome.is_err(); // Released here, before the lane lets a new call start. A drop // that panics must not keep the lane taken either. @@ -364,38 +439,140 @@ where return Err(Stall::Refused { call }); } let started = (wait.now)(); - let finish = |outcome: Outcome, took: Duration| match outcome { - Ok(value) => { - note_finished(call, wait.limits, took); - Ok(value) + Ok(Pending { + call, + limits: wait.limits, + slice: wait.slice, + now: wait.now, + lanes, + owner, + rx, + started, + done: false, + }) +} + +impl Instant> Pending { + /// Wait until the call returns, until `up_to` after it started, or until + /// its limit, calling `progress` every slice meanwhile. Past its limit it + /// is given up on. None: it still runs, after `up_to`. + pub(crate) fn wait_up_to( + &mut self, + up_to: Duration, + progress: &mut impl FnMut(), + ) -> Option> { + loop { + match self.rx.recv_timeout(self.slice) { + Ok(outcome) => { + let took = (self.now)().saturating_duration_since(self.started); + return Some(self.finish(outcome, took)); + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + panic!("{} ended without a result", self.call.name()) + } + } + let after = (self.now)().saturating_duration_since(self.started); + if after >= self.limits.limit { + return Some(self.give_up(after)); + } + if after >= up_to { + return None; + } + progress(); } - Err(panic) => resume_unwind(panic), - }; - loop { - match rx.recv_timeout(wait.slice) { - Ok(outcome) => { - return finish(outcome, (wait.now)().saturating_duration_since(started)); + } + + /// Stop waiting for a call that still runs, to take its result later + /// with [`Self::poll`]. Until it returns, the lane counts it as it counts + /// an abandoned call, so the next of its kind is refused instead of + /// waiting behind it. Its result, if it already returned. + pub(crate) fn park(&mut self) -> Option> { + let mut owner = self.owner.lock(); + if let Ok(outcome) = self.rx.try_recv() { + drop(owner); + let took = (self.now)().saturating_duration_since(self.started); + return Some(self.finish(outcome, took)); + } + if *owner == Owner::Waiting { + *owner = Owner::Parked; + self.lanes.0[self.call as usize].fetch_add(1, SeqCst); + } + None + } + + /// A parked call's result once it returned. Past its limit it is given + /// up on, as when waited for. None while it runs within its limit. + pub(crate) fn poll(&mut self) -> Option> { + let after = (self.now)().saturating_duration_since(self.started); + match self.rx.try_recv() { + Ok(outcome) => Some(self.finish(outcome, after)), + Err(mpsc::TryRecvError::Empty) if after >= self.limits.limit => { + Some(self.give_up(after)) } - Err(mpsc::RecvTimeoutError::Timeout) => {} - Err(mpsc::RecvTimeoutError::Disconnected) => { - panic!("{} ended without a result", call.name()) + Err(mpsc::TryRecvError::Empty) => None, + Err(mpsc::TryRecvError::Disconnected) => { + panic!("{} ended without a result", self.call.name()) } } - let after = (wait.now)().saturating_duration_since(started); - if after >= wait.limits.limit { - let mut gave_up = gave_up.lock(); - // It may have returned since the last look. - if let Ok(outcome) = rx.try_recv() { - drop(gave_up); - return finish(outcome, after); + } + + /// How long the call has run, by its caller's clock. + pub(crate) fn running_for(&self) -> Duration { + (self.now)().saturating_duration_since(self.started) + } + + fn finish(&mut self, outcome: Outcome, took: Duration) -> Result { + self.done = true; + match outcome { + Ok(value) => { + note_finished(self.call, self.limits, took); + Ok(value) } - *gave_up = true; - lane.fetch_add(1, SeqCst); - drop(gave_up); - log_timed_out(call, wait.limits, after); - return Err(Stall::TimedOut { call, after }); + Err(panic) => std::panic::resume_unwind(panic), + } + } + + fn give_up(&mut self, after: Duration) -> Result { + let mut owner = self.owner.lock(); + // It may have returned since the last look. + if let Ok(outcome) = self.rx.try_recv() { + drop(owner); + return self.finish(outcome, after); } - progress(); + // A parked call is counted already. + if *owner == Owner::Waiting { + self.lanes.0[self.call as usize].fetch_add(1, SeqCst); + } + *owner = Owner::Nobody; + drop(owner); + self.done = true; + log_timed_out(self.call, self.limits, after); + Err(Stall::TimedOut { + call: self.call, + after, + }) + } +} + +/// Dropped before its result was taken (a pipeline that ends while an +/// encoder opens in the background): the call's thread releases the result +/// when it returns, and the lane counts the call until then. +impl Drop for Pending { + fn drop(&mut self) { + if self.done { + return; + } + let mut owner = self.owner.lock(); + // Returned already: its result is released here, and a parked + // call's thread freed the lane when it handed it over. + if self.rx.try_recv().is_ok() { + return; + } + if *owner == Owner::Waiting { + self.lanes.0[self.call as usize].fetch_add(1, SeqCst); + } + *owner = Owner::Nobody; } } @@ -922,6 +1099,183 @@ mod tests { }); } + /// A clock that stands still until the test moves it: the time since + /// the test began, in ms, is the returned atomic. + fn paused_clock() -> (Arc, impl Fn() -> Instant + Clone) { + let start = Instant::now(); + let ms = Arc::new(AtomicU64::new(0)); + let read = Arc::clone(&ms); + (ms, move || start + Duration::from_millis(read.load(SeqCst))) + } + + #[test] + fn a_parked_call_holds_its_lane_and_is_taken_once_it_returns() { + let lanes = lanes(); + let (ms, now) = paused_clock(); + let dropped_on = Arc::new(Mutex::new(None)); + let (release, call) = hang(Device(Arc::clone(&dropped_on))); + let wait = || Wait { + limits: encoder_open_limits("h264_qsv"), + slice: Duration::from_millis(1), + now: now.clone(), + }; + let mut pending = start_in(lanes, Call::EncoderOpen, wait(), call, no_hooks()).unwrap(); + // Waited for until 3 s, then parked: no stall line, and the lane + // counts it. + ms.store(3_000, SeqCst); + assert!(pending.wait_up_to(Duration::from_secs(3), &mut || {}).is_none()); + assert!(pending.park().is_none()); + assert_eq!(lanes.abandoned(Call::EncoderOpen), 1); + assert_eq!(pending.running_for(), Duration::from_secs(3)); + // The next open in the lane is refused at once, as behind a stuck + // one, instead of waiting on FFmpeg's lock behind it. + let refused = run_in( + lanes, + Call::EncoderOpen, + wait(), + || (), + || panic!("a refused call waited"), + no_hooks(), + ); + assert_eq!( + refused, + Err(Stall::Refused { + call: Call::EncoderOpen + }) + ); + ms.store(14_000, SeqCst); + assert!(pending.poll().is_none()); + // It returns within its limit: the lane frees as it hands over, and + // the caller takes the result on its own thread. + release.send(()).unwrap(); + wait_until("the parked call's lane did not free", || { + lanes.abandoned(Call::EncoderOpen) == 0 + }); + let device = pending.poll().expect("returned").expect("not a stall"); + drop(device); + assert_eq!(*dropped_on.lock(), Some(std::thread::current().id())); + drop(pending); + assert_eq!(lanes.abandoned(Call::EncoderOpen), 0); + } + + #[test] + fn a_parked_call_past_its_limit_is_given_up_on_as_one_waited_for() { + let lanes = lanes(); + let (ms, now) = paused_clock(); + let dropped_on = Arc::new(Mutex::new(None)); + let (release, call) = hang(Device(Arc::clone(&dropped_on))); + let mut pending = start_in( + lanes, + Call::EncoderOpen, + Wait { + limits: encoder_open_limits("h264_nvenc"), + slice: Duration::from_millis(1), + now, + }, + call, + no_hooks(), + ) + .unwrap(); + assert!(pending.park().is_none()); + ms.store(14_999, SeqCst); + assert!(pending.poll().is_none()); + ms.store(15_000, SeqCst); + let Some(Err(Stall::TimedOut { call, after })) = pending.poll() else { + panic!("not given up on at the limit") + }; + assert_eq!( + (call, after), + (Call::EncoderOpen, Duration::from_secs(15)) + ); + // Counted once, from the park, until it returns; then released on + // its own thread. + assert_eq!(lanes.abandoned(Call::EncoderOpen), 1); + release.send(()).unwrap(); + wait_until("the given-up call's lane did not free", || { + lanes.abandoned(Call::EncoderOpen) == 0 + }); + wait_until("the late result was not released", || { + dropped_on.lock().is_some() + }); + assert_ne!(*dropped_on.lock(), Some(std::thread::current().id())); + } + + #[test] + fn a_result_in_before_the_park_or_the_look_is_taken() { + let lanes = lanes(); + let (ms, now) = paused_clock(); + let wait = || Wait { + limits: encoder_open_limits("h264_amf"), + slice: Duration::from_millis(1), + now: now.clone(), + }; + // It returned while the caller was about to park it: the park finds + // the result, once the call's thread has let go of the hand-over. + let (inside_tx, inside_rx) = mpsc::channel::<()>(); + let mut pending = start_in( + lanes, + Call::EncoderOpen, + wait(), + || 5, + Hooks { + before: || {}, + handoff: move || { + let _ = inside_tx.send(()); + }, + }, + ) + .unwrap(); + inside_rx.recv_timeout(Duration::from_secs(60)).unwrap(); + assert_eq!(pending.park(), Some(Ok(5))); + assert_eq!(lanes.abandoned(Call::EncoderOpen), 0); + // A parked call that returned before the limit is taken even when + // the caller looks only after the limit. + let (release, call) = hang(6); + let mut pending = start_in(lanes, Call::EncoderOpen, wait(), call, no_hooks()).unwrap(); + assert!(pending.park().is_none()); + release.send(()).unwrap(); + wait_until("the parked call's lane did not free", || { + lanes.abandoned(Call::EncoderOpen) == 0 + }); + ms.store(60_000, SeqCst); + assert_eq!(pending.poll(), Some(Ok(6))); + } + + #[test] + fn a_pending_call_dropped_before_its_result_frees_its_lane_when_it_returns() { + for park in [true, false] { + let lanes = lanes(); + let (_ms, now) = paused_clock(); + let dropped_on = Arc::new(Mutex::new(None)); + let (release, call) = hang(Device(Arc::clone(&dropped_on))); + let mut pending = start_in( + lanes, + Call::EncoderOpen, + Wait { + limits: encoder_open_limits("h264_nvenc"), + slice: Duration::from_millis(1), + now, + }, + call, + no_hooks(), + ) + .unwrap(); + if park { + assert!(pending.park().is_none()); + } + drop(pending); + assert_eq!(lanes.abandoned(Call::EncoderOpen), 1, "park={park}"); + release.send(()).unwrap(); + wait_until("a dropped call kept its lane", || { + lanes.abandoned(Call::EncoderOpen) == 0 + }); + wait_until("the dropped call's result was not released", || { + dropped_on.lock().is_some() + }); + assert_ne!(*dropped_on.lock(), Some(std::thread::current().id())); + } + } + #[test] fn the_stall_line_is_unquoted_for_counting() { let out = Arc::new(Mutex::new(Vec::::new())); diff --git a/host/src/stats.rs b/host/src/stats.rs index 98c9b58a..a7a36bd3 100644 --- a/host/src/stats.rs +++ b/host/src/stats.rs @@ -46,6 +46,9 @@ pub struct PipelineStats { /// linked FFmpeg's NVENC needs. The fallback notice then says to update /// it. pub nvidia_driver_outdated: bool, + /// AMD and Intel cards on the PC whose encoder failed to open: the + /// fallback notice says to update their driver. + pub driver_update_vendors: Vec, encode_timestamps: VecDeque, // Transport @@ -109,6 +112,7 @@ impl PipelineStats { codec_name: String::new(), using_software_fallback: false, nvidia_driver_outdated: false, + driver_update_vendors: Vec::new(), encode_timestamps: VecDeque::with_capacity(128), transport_fps: 0.0, @@ -150,6 +154,7 @@ impl PipelineStats { self.codec_name = previous.codec_name; self.using_software_fallback = previous.using_software_fallback; self.nvidia_driver_outdated = previous.nvidia_driver_outdated; + self.driver_update_vendors = previous.driver_update_vendors; self.mdns_active = previous.mdns_active; previous.audio } From 99b1102f8405aab2ebb37afa3bb5e60dfb461a61 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 15:35:00 -0500 Subject: [PATCH 051/226] Hand every stream's input to one thread, the old stream's releases first Each transport started an input thread of its own, and dropping its handle only closed that thread's queue. After a restart while Windows held the old thread, the new one injected at once. When Windows let the old one go, its cleanup released keys and buttons the new stream had pressed since, and every restart in a stuck stretch left one more thread waiting on Windows. - SharedControl keeps one input thread for the app, started with the first stream. Each stream generation is a handle on it. - A generation's start and end are queued in order. Its end, or the next start, releases what its input holds before the next generation's input lands. Input an ended generation queued late is dropped: its end released what it held. - A quit waits up to 500 ms for the last releases to reach Windows. The relay no longer goes with a thread that a process exit skips. - input::hold::hold_next holds only the next injection, as Windows does a thread it holds while others go on. input_restart.rs restarts the real loop on one port while Windows holds the old stream's input. The iPad presses A again on the new one. On the previous commit A ends up released: the new KeyDown lands, then the old KeyUp. --- host/src/control.rs | 4 + host/src/input/injector.rs | 476 +++++++++++++++++++++++++----------- host/src/input/mod.rs | 45 +++- host/src/main.rs | 7 + host/src/transport/mod.rs | 6 +- host/tests/input_restart.rs | 237 ++++++++++++++++++ host/tests/input_stall.rs | 8 +- 7 files changed, 620 insertions(+), 163 deletions(-) create mode 100644 host/tests/input_restart.rs diff --git a/host/src/control.rs b/host/src/control.rs index e7cd0ce8..1c331501 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -84,6 +84,9 @@ pub struct SharedControl { /// normalized touch coordinates onto this rect. `None` until the first /// capture stage comes up. pub capture_geometry: Arc>>, + /// The one thread that hands iPad input to Windows, for every stream + /// generation of this app. + pub input: Arc, /// The user's "prefer HEVC" setting. The encoder switches to an H.265 /// session only when this is on AND the connected client advertised HEVC /// decode in its HELLO2. @@ -196,6 +199,7 @@ impl SharedControl { hb_encode_frame_ms: Arc::new(AtomicU64::new(0)), transport_beat: Default::default(), capture_geometry: Arc::new(Mutex::new(None)), + input: Arc::default(), hevc_enabled: Arc::new(AtomicBool::new(false)), audio_enabled: Arc::new(AtomicBool::new(true)), active_codec: Arc::new(AtomicU8::new(eternal_wire::v2::control::CODEC_H264)), diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs index feb8572a..0d83e238 100644 --- a/host/src/input/injector.rs +++ b/host/src/input/injector.rs @@ -4,8 +4,14 @@ //! `GetSystemMetrics`, creating the pen device) can wait while Windows //! reconfigures its displays. They ran on the transport loop, which then //! answered nobody: no heartbeats, no HELLO answers, and a stopped stream -//! kept its port. Now the transport only queues input, and this thread -//! hands it to Windows in order. +//! kept its port. Now the transport only queues input, and one thread for +//! the whole app hands it to Windows in order. +//! +//! Each stream generation queues on the same thread, behind the one before +//! it. A generation's end releases what its input still holds before the +//! next one's input lands, however long Windows kept the thread waiting, +//! and a restart never starts a second thread that could inject alongside +//! a stuck one. //! //! The queue is bounded. While Windows takes nothing, a pointer or Pencil //! move replaces the move queued just before it, so a backlog never replays @@ -15,7 +21,8 @@ //! key releases and session resets are never dropped or merged. use std::collections::VecDeque; -use std::sync::Arc; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; use eternal_wire::v2::control::InputEvent; @@ -47,8 +54,8 @@ const SLOW_CALL: Duration = Duration::from_millis(250); /// milliseconds or worse on Windows. const PRECISE_WAIT: Duration = Duration::from_millis(2); -/// How long the transport waits at its start for the thread to say whether -/// Windows gave it a pen device. +/// How long the first stream waits for the thread to say whether Windows +/// gave it a pen device. Later streams know at once. const START_WAIT: Duration = Duration::from_secs(1); /// What the injection thread drives: the input relay, or a test's fake. @@ -86,8 +93,14 @@ impl Sink for InputRelay { } } -#[derive(Debug, Clone, Copy)] +type Geometry = Arc>>; + +#[derive(Debug, Clone)] enum Command { + /// A stream generation starts: what the one before it holds is + /// released first. `geometry` is the capture rect a still Pencil + /// follows. + Begin { geometry: Geometry }, Relay { session_id: u32, event: InputEvent, @@ -96,6 +109,8 @@ enum Command { }, /// Release what the PC holds: the session ended, expired or is on hold. Reset, + /// The generation's stream stopped: release what it holds. + End, } impl Command { @@ -134,10 +149,11 @@ impl Command { && event.buttons == earlier_event.buttons } - /// An end or reset: what keeps the PC from holding a key or button. + /// An end, reset or generation change: what keeps the PC from holding + /// a key or button. fn must_land(&self) -> bool { match self { - Command::Reset => true, + Command::Begin { .. } | Command::Reset | Command::End => true, Command::Relay { event, .. } => event.phase > PHASE_MOVED, } } @@ -145,6 +161,7 @@ impl Command { #[derive(Debug)] struct Queued { + generation: u64, command: Command, at: Instant, } @@ -152,27 +169,33 @@ struct Queued { #[derive(Debug, Default)] struct State { commands: VecDeque, - closed: bool, + /// The thread is handing a command to Windows. + busy: bool, merged: u64, dropped: u64, } +/// What the transport's handles and the thread share. #[derive(Debug)] struct Shared { state: Mutex, wake: Condvar, - capacity: usize, - merge_from: usize, - stale_after: Duration, + /// The queue is empty and the thread waits: for `InputThread::finish`. + idle: Condvar, + limits: Limits, + next_generation: AtomicU64, + /// Whether the relay has a native pen device: 0 not known yet, 1 no, + /// 2 yes. + pen: AtomicU8, } impl Shared { - fn push(&self, command: Command, now: Instant) { + fn push(&self, generation: u64, command: Command, now: Instant) { let mut state = self.state.lock(); let waiting = state.commands.len(); - if waiting >= self.merge_from { + if waiting >= self.limits.merge_from { if let Some(last) = state.commands.back_mut() { - if command.merges_with(&last.command) { + if last.generation == generation && command.merges_with(&last.command) { // The newer sample, in the older one's place: it has // waited as long. last.command = command; @@ -182,75 +205,141 @@ impl Shared { } } // Ends and resets may pass the bound, by as many as were pressed. - if waiting >= self.capacity && !(command.must_land() && waiting < 2 * self.capacity) { + let capacity = self.limits.capacity; + if waiting >= capacity && !(command.must_land() && waiting < 2 * capacity) { state.dropped += 1; return; } - state.commands.push_back(Queued { command, at: now }); + state.commands.push_back(Queued { + generation, + command, + at: now, + }); drop(state); self.wake.notify_one(); } } -/// The transport's handle on the injection thread. Dropping it lets the -/// thread hand over what is queued and release what the PC holds, without -/// waiting for Windows. +/// The app's one input thread, started with the first stream that takes +/// input. `SharedControl` keeps it across stream generations. +#[derive(Default)] +pub struct InputThread { + shared: OnceLock>, +} + +impl std::fmt::Debug for InputThread { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InputThread") + .field("started", &self.shared.get().is_some()) + .finish() + } +} + +impl InputThread { + /// A new stream generation's handle; starts the thread with the relay + /// the first time. + pub fn begin(&self, geometry: Geometry) -> Injector { + let shared = self.shared.get_or_init(|| { + #[cfg(all(test, not(windows)))] + let recorder = super::recorder::current(); + spawn(Limits::default(), move || { + #[cfg(all(test, not(windows)))] + super::recorder::adopt(recorder); + InputRelay::default() + }) + }); + Injector::begin(Arc::clone(shared), geometry) + } + + /// A thread driving a test's fake instead of Windows. + #[cfg(test)] + fn with_sink(limits: Limits, sink: impl FnOnce() -> S + Send + 'static) -> Self { + let thread = Self::default(); + let _ = thread.shared.set(spawn(limits, sink)); + thread + } + + /// At quit: wait up to `limit` for what is queued, the last stream's + /// releases among it, to reach Windows. Whether it did. + pub fn finish(&self, limit: Duration) -> bool { + let Some(shared) = self.shared.get() else { + return true; + }; + let deadline = Instant::now() + limit; + let mut state = shared.state.lock(); + while state.busy || !state.commands.is_empty() { + if shared.idle.wait_until(&mut state, deadline).timed_out() { + warn!( + waiting = state.commands.len(), + "Quitting before Windows took the last input" + ); + return false; + } + } + true + } +} + +fn spawn(limits: Limits, sink: impl FnOnce() -> S + Send + 'static) -> Arc { + let shared = Arc::new(Shared { + state: Mutex::new(State::default()), + wake: Condvar::new(), + idle: Condvar::new(), + limits, + next_generation: AtomicU64::new(1), + pen: AtomicU8::new(0), + }); + let worker = Arc::clone(&shared); + let spawned = std::thread::Builder::new() + .name("input-inject".into()) + .spawn(move || { + let mut sink = sink(); + note_pen(&worker, &sink); + run(&worker, &mut sink); + }); + if let Err(error) = spawned { + warn!(%error, "Could not start the input thread; input is ignored"); + shared.pen.store(1, Ordering::SeqCst); + } + shared +} + +fn note_pen(shared: &Shared, sink: &impl Sink) { + let pen = if sink.pen_available() { 2 } else { 1 }; + shared.pen.store(pen, Ordering::SeqCst); +} + +/// One stream generation's handle on the input thread. Dropping it +/// releases what the generation's input holds, in turn, without waiting +/// for Windows. #[derive(Debug)] pub struct Injector { shared: Arc, + generation: u64, pen_available: bool, } impl Injector { - /// Starts the thread with a new relay. `geometry` is the capture rect a - /// still Pencil follows. - pub fn start(geometry: Arc>>) -> Self { - #[cfg(all(test, not(windows)))] - let recorder = super::recorder::current(); - Self::spawn(geometry, Limits::default(), move || { - #[cfg(all(test, not(windows)))] - super::recorder::adopt(recorder); - InputRelay::default() - }) - } - - fn spawn( - geometry: Arc>>, - limits: Limits, - sink: impl FnOnce() -> S + Send + 'static, - ) -> Self { - let shared = Arc::new(Shared { - state: Mutex::new(State::default()), - wake: Condvar::new(), - capacity: limits.capacity, - merge_from: limits.merge_from, - stale_after: limits.stale_after, - }); - let (ready_tx, ready) = std::sync::mpsc::channel(); - let worker = Arc::clone(&shared); - let spawned = std::thread::Builder::new() - .name("input-inject".into()) - .spawn(move || { - let mut sink = sink(); - let _ = ready_tx.send(sink.pen_available()); - run(&worker, &mut sink, &geometry); - }); - let pen_available = match spawned { - Ok(_) => ready.recv_timeout(START_WAIT).unwrap_or_else(|_| { - warn!( - wait_ms = START_WAIT.as_millis() as u64, - "Windows did not set up input in time; the Pencil uses the mouse this stream" - ); - false - }), - Err(error) => { - warn!(%error, "Could not start the input thread; input is ignored this stream"); - shared.state.lock().closed = true; - false + fn begin(shared: Arc, geometry: Geometry) -> Self { + let generation = shared.next_generation.fetch_add(1, Ordering::SeqCst); + shared.push(generation, Command::Begin { geometry }, Instant::now()); + let started = Instant::now(); + let pen_available = loop { + match shared.pen.load(Ordering::SeqCst) { + 0 if started.elapsed() < START_WAIT => std::thread::sleep(Duration::from_millis(1)), + 0 => { + warn!( + wait_ms = START_WAIT.as_millis() as u64, + "Windows did not set up input in time; the Pencil uses the mouse this stream" + ); + break false; + } + pen => break pen == 2, } }; Self { shared, + generation, pen_available, } } @@ -262,27 +351,27 @@ impl Injector { /// Queue one session-validated event. Never waits for Windows. pub fn relay(&self, session_id: u32, event: InputEvent, output: CaptureGeometry, late: bool) { - self.shared.push( - Command::Relay { - session_id, - event, - output, - late, - }, - Instant::now(), - ); + self.push(Command::Relay { + session_id, + event, + output, + late, + }); } /// Queue a release of every key, button and pen contact the PC holds. pub fn reset(&self) { - self.shared.push(Command::Reset, Instant::now()); + self.push(Command::Reset); + } + + fn push(&self, command: Command) { + self.shared.push(self.generation, command, Instant::now()); } } impl Drop for Injector { fn drop(&mut self) { - self.shared.state.lock().closed = true; - self.shared.wake.notify_one(); + self.push(Command::End); } } @@ -307,7 +396,6 @@ enum Next { Command(Queued), /// A pen frame's spacing or the refresh tick is due. Due, - Closed, } /// Input that waited for Windows, counted until the queue empties. @@ -317,20 +405,29 @@ struct Backlog { longest: Duration, } -fn run(shared: &Shared, sink: &mut S, geometry: &Mutex>) { +/// The generation whose input the thread takes now. +struct Current { + generation: u64, + geometry: Geometry, + /// Its input reached the relay, so its end has something to release. + relayed: bool, +} + +fn run(shared: &Shared, sink: &mut S) { let mut timer = crate::capture::timing::FrameTimer::new(); let mut next_tick = Instant::now() + PEN_TICK; let mut backlog = Backlog::default(); + let mut current: Option = None; loop { let next = { let mut state = shared.state.lock(); + state.busy = false; loop { if let Some(queued) = state.commands.pop_front() { + state.busy = true; break Next::Command(queued); } - if state.closed { - break Next::Closed; - } + shared.idle.notify_all(); let (merged, dropped) = ( std::mem::take(&mut state.merged), std::mem::take(&mut state.dropped), @@ -346,8 +443,9 @@ fn run(shared: &Shared, sink: &mut S, geometry: &Mutex break Next::Due, Some(at) if spaced.is_some() && at - now <= PRECISE_WAIT => { @@ -367,32 +465,61 @@ fn run(shared: &Shared, sink: &mut S, geometry: &Mutex { - let started = Instant::now(); - let waited = started.saturating_duration_since(at); - backlog.longest = backlog.longest.max(waited); - match command { - Command::Relay { - session_id, - event, - output, - late, - } => { - let stale = waited > shared.stale_after; - backlog.stale += u64::from(stale && !late); - sink.relay(session_id, &event, output, late || stale); + if let Next::Command(Queued { + generation, + command, + at, + }) = next + { + let started = Instant::now(); + let waited = started.saturating_duration_since(at); + backlog.longest = backlog.longest.max(waited); + let ours = current + .as_ref() + .is_some_and(|current| current.generation == generation); + match command { + Command::Begin { geometry } => { + // A generation that never said it ended releases what + // it holds before the new one's input. + if current.take().is_some_and(|current| current.relayed) { + sink.reset(); } - Command::Reset => sink.reset(), + current = Some(Current { + generation, + geometry, + relayed: false, + }); } - note_slow(started, "input"); + Command::End if ours => { + if current.take().is_some_and(|current| current.relayed) { + sink.reset(); + } + } + Command::Reset if ours => sink.reset(), + Command::Relay { + session_id, + event, + output, + late, + } if ours => { + let stale = waited > shared.limits.stale_after; + backlog.stale += u64::from(stale && !late); + sink.relay(session_id, &event, output, late || stale); + if let Some(current) = current.as_mut() { + current.relayed = true; + } + } + // An ended generation's: its end released what it held. + Command::End | Command::Reset | Command::Relay { .. } => {} } - // The relay goes with the thread, releasing what the PC holds. - Next::Closed => return, - Next::Due => {} + note_slow(started, "input"); + note_pen(shared, sink); } // Pen frames held back for their spacing, and the refresh a still // contact needs, go between commands. + let Some(current) = current.as_ref() else { + continue; + }; let now = Instant::now(); let spaced_due = sink.spaced_until().is_some_and(|at| at <= now); let tick_due = sink.spaced_until().is_none() && sink.tick_needed() && next_tick <= now; @@ -401,7 +528,7 @@ fn run(shared: &Shared, sink: &mut S, geometry: &Mutex, calls: Arc>>, go: mpsc::Sender<()>, @@ -495,12 +623,14 @@ mod tests { let (go, hold) = mpsc::channel(); let (entered_tx, entered) = mpsc::channel(); let recorded = Arc::clone(&calls); - let injector = Injector::spawn(Arc::new(Mutex::new(Some(OUTPUT))), limits, move || Fake { + let thread = InputThread::with_sink(limits, move || Fake { calls: recorded, hold: Some(hold), entered: Some(entered_tx), }); + let injector = thread.begin(geometry()); Harness { + thread, injector: Some(injector), calls, go, @@ -508,6 +638,10 @@ mod tests { } } + fn geometry() -> Geometry { + Arc::new(Mutex::new(Some(OUTPUT))) + } + impl Harness { fn send(&self, kind: u8, phase: u8, id: u32) { self.injector @@ -536,6 +670,12 @@ mod tests { } self.calls.lock().clone() } + + /// Nothing more reaches Windows. + fn settled(&self, count: usize) { + assert!(self.thread.finish(Duration::from_secs(5))); + assert_eq!(self.calls.lock().len(), count, "{:?}", self.calls.lock()); + } } fn event(kind: u8, phase: u8, id: u32) -> InputEvent { @@ -645,8 +785,7 @@ mod tests { Call::Reset, ]; assert_eq!(harness.wait_for(expected.len()), expected); - std::thread::sleep(Duration::from_millis(20)); - assert_eq!(harness.calls.lock().len(), expected.len()); + harness.settled(expected.len()); } #[test] @@ -710,7 +849,7 @@ mod tests { } #[test] - fn a_dropped_handle_returns_at_once_and_its_input_still_lands() { + fn a_dropped_handle_returns_at_once_and_its_input_still_lands_then_is_released() { let mut harness = harness(Limits::default()); harness.stuck(); harness.send(KIND_KEY, PHASE_ENDED, 1); @@ -718,13 +857,55 @@ mod tests { drop(harness.injector.take()); assert!(started.elapsed() < Duration::from_millis(100)); harness.go.send(()).unwrap(); - assert_eq!( - harness.wait_for(2), - [ - relayed(KIND_KEY, PHASE_BEGAN, 0), - relayed(KIND_KEY, PHASE_ENDED, 1), - ] - ); + let expected = [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_KEY, PHASE_ENDED, 1), + Call::Reset, + ]; + assert_eq!(harness.wait_for(3), expected); + harness.settled(expected.len()); + } + + /// A stream restarts while Windows holds the old one's input. The new + /// generation's handle is ready at once, on the same thread, and its + /// input lands only after the old one's release: a key it presses + /// stays down. Restarts over and over start no thread of their own. + #[test] + fn a_restart_while_windows_is_stuck_releases_the_old_input_before_the_new() { + let mut harness = harness(Limits::default()); + harness.stuck(); + drop(harness.injector.take()); + let started = Instant::now(); + let mut generations: Vec<_> = (0..20).map(|_| harness.thread.begin(geometry())).collect(); + assert!(started.elapsed() < Duration::from_millis(100)); + let newest = generations.pop().unwrap(); + generations.clear(); + newest.relay(1, event(KIND_KEY, PHASE_BEGAN, 1), OUTPUT, false); + harness.go.send(()).unwrap(); + let expected = [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + Call::Reset, + relayed(KIND_KEY, PHASE_BEGAN, 1), + ]; + assert_eq!(harness.wait_for(3), expected); + // The old generations' ends, queued behind the new one's start, + // released nothing of it. + harness.settled(expected.len()); + drop(newest); + assert_eq!(harness.wait_for(4)[3], Call::Reset); + } + + #[test] + fn quitting_waits_for_queued_releases_but_not_past_its_limit() { + let harness = harness(Limits::default()); + harness.stuck(); + harness.send(KIND_KEY, PHASE_ENDED, 1); + let started = Instant::now(); + assert!(!harness.thread.finish(Duration::from_millis(50))); + assert!(started.elapsed() < Duration::from_secs(1)); + harness.go.send(()).unwrap(); + assert!(harness.thread.finish(Duration::from_secs(5))); + assert_eq!(harness.calls.lock().len(), 2); } /// The real relay, with a native pen device that takes every frame @@ -739,39 +920,36 @@ mod tests { let (taken_tx, taken_rx) = mpsc::channel(); let delivered = Arc::new(Mutex::new(None)); let at = Arc::clone(&delivered); - let injector = Injector::spawn( - Arc::new(Mutex::new(Some(OUTPUT))), - Limits::default(), - move || { - let taken = native_pen::install(); - *at.lock() = Some(Instant::now()); - native_pen::with(|device| { - for x in 0..SAMPLES as u16 { - let sample = PenSample { - phase: if x == 0 { - PenPhase::Down - } else { - PenPhase::Move - }, - x: i32::from(x), - y: 10, - x_norm: x, - y_norm: 10, - pressure: 512, - tilt_x: 0, - tilt_y: 0, - }; - device.deliver(sample, native_pen::SCREEN, Some(OUTPUT), Instant::now()); - } - }); - assert!( - taken.lock().len() < SAMPLES / 2, - "the spacing must hold the backlog back" - ); - taken_tx.send(taken).unwrap(); - InputRelay::default() - }, - ); + let thread = InputThread::with_sink(Limits::default(), move || { + let taken = native_pen::install(); + *at.lock() = Some(Instant::now()); + native_pen::with(|device| { + for x in 0..SAMPLES as u16 { + let sample = PenSample { + phase: if x == 0 { + PenPhase::Down + } else { + PenPhase::Move + }, + x: i32::from(x), + y: 10, + x_norm: x, + y_norm: 10, + pressure: 512, + tilt_x: 0, + tilt_y: 0, + }; + device.deliver(sample, native_pen::SCREEN, Some(OUTPUT), Instant::now()); + } + }); + assert!( + taken.lock().len() < SAMPLES / 2, + "the spacing must hold the backlog back" + ); + taken_tx.send(taken).unwrap(); + InputRelay::default() + }); + let injector = thread.begin(geometry()); let taken = taken_rx.recv().unwrap(); let started = Instant::now(); while taken.lock().len() < SAMPLES && started.elapsed() < Duration::from_secs(2) { diff --git a/host/src/input/mod.rs b/host/src/input/mod.rs index 85126401..ba288d20 100644 --- a/host/src/input/mod.rs +++ b/host/src/input/mod.rs @@ -11,7 +11,7 @@ mod hid; pub use hid::hid_to_scancode; mod injector; -pub use injector::Injector; +pub use injector::{Injector, InputThread}; #[cfg(windows)] pub mod windows_inject; @@ -509,34 +509,59 @@ pub(crate) mod native_pen { pub mod hold { use parking_lot::{Condvar, Mutex}; - /// Whether injections wait, and how many are waiting. - static HELD: Mutex<(bool, usize)> = Mutex::new((false, 0)); + #[derive(Default)] + struct Held { + on: bool, + /// Only the first injection waits; later ones pass, as on a thread + /// Windows is not holding. + first_only: bool, + waiting: usize, + } + + static HELD: Mutex = Mutex::new(Held { + on: false, + first_only: false, + waiting: 0, + }); static RELEASED: Condvar = Condvar::new(); + /// Every injection waits until [`release`]. pub fn hold() { - HELD.lock().0 = true; + *HELD.lock() = Held { + on: true, + ..Held::default() + }; + } + + /// The next injection waits until [`release`]; the rest pass. + pub fn hold_next() { + *HELD.lock() = Held { + on: true, + first_only: true, + waiting: 0, + }; } pub fn release() { - HELD.lock().0 = false; + HELD.lock().on = false; RELEASED.notify_all(); } /// Injections waiting now. pub fn waiting() -> usize { - HELD.lock().1 + HELD.lock().waiting } pub(super) fn pass() { let mut held = HELD.lock(); - if !held.0 { + if !held.on || (held.first_only && held.waiting > 0) { return; } - held.1 += 1; - while held.0 { + held.waiting += 1; + while held.on { RELEASED.wait(&mut held); } - held.1 -= 1; + held.waiting -= 1; } } diff --git a/host/src/main.rs b/host/src/main.rs index 9fae06d2..18e3bfee 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -367,6 +367,13 @@ fn main() -> Result<(), Box> { tracing::warn!("Supervisor did not shut down within 3s — forcing exit"); } + // The last stream's releases of held keys and buttons reach Windows, + // unless Windows is stuck. + gui_control + .shared + .input + .finish(std::time::Duration::from_millis(500)); + // The window is gone and the display is off, so a slow disk keeps no one // waiting on screen: give the second screen record up to 30 s to land. eternal_host::capture::second_screen::finish_saving(&gui_control.shared); diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 92622865..937935b5 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -226,7 +226,9 @@ pub async fn start_sender( .lock() .set_target_addr(shared.target_addr.lock().to_string()); - let input = crate::input::Injector::start(std::sync::Arc::clone(&shared.capture_geometry)); + let input = shared + .input + .begin(std::sync::Arc::clone(&shared.capture_geometry)); let config = SharedConfigSource { shared: &shared, stream_epoch, @@ -1159,7 +1161,7 @@ mod tests { LoopState { retransmit_ring: RetransmitRing::default(), fault: FaultInjector::new(0.0, 0.0, Duration::ZERO), - input: crate::input::Injector::start(Default::default()), + input: crate::input::InputThread::default().begin(Default::default()), abr: AbrController::new(15_000_000, true), } } diff --git a/host/tests/input_restart.rs b/host/tests/input_restart.rs new file mode 100644 index 00000000..95db5a51 --- /dev/null +++ b/host/tests/input_restart.rs @@ -0,0 +1,237 @@ +//! A stream restarts while Windows holds the old generation's input. The +//! iPad keeps its session and presses a key again on the new generation. +//! Each generation used to inject on a thread of its own: the new one's +//! press landed at once, and when Windows let the old thread go, its +//! cleanup released that key, which then read as up while the iPad held +//! it. Now the old generation's release lands first. +//! +//! The real transport loop, twice on one port, with the off-Windows input +//! recorder. Only the old generation's stuck injection is held. +#![cfg(not(windows))] + +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use eternal_host::control::SharedControl; +use eternal_host::input::{hold, recorder, CaptureGeometry, Injection, KIND_KEY, PHASE_BEGAN}; +use eternal_host::transport; +use eternal_wire::v2::control::{ + encode_control, parse_control, ControlMessage, Hello2, HelloStatus, InputEvent, + CAP_DECODE_H264, FEATURE_WANTS_INPUT, +}; +use tokio::sync::mpsc; + +const KEY_A: u16 = 0x04; +const KEY_B: u16 = 0x05; +/// The scan code Windows gets for A. +const SCAN_A: u16 = 0x1e; + +/// One stream generation: the real transport loop on its own thread. +struct Generation { + nal_tx: mpsc::Sender, + thread: std::thread::JoinHandle<()>, +} + +fn start(shared: &SharedControl, port: u16) -> Generation { + shared.running.store(true, Ordering::SeqCst); + let (nal_tx, nal_rx) = mpsc::channel(4); + let control = shared.clone(); + let thread = std::thread::spawn(move || { + let (_audio_tx, audio_rx) = mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(transport::start_sender( + nal_rx, + audio_rx, + port, + control, + supervisor_tx, + )) + .unwrap(); + }); + Generation { nal_tx, thread } +} + +fn stop(shared: &SharedControl, generation: Generation) { + shared.stop(); + drop(generation.nal_tx); + let stopping = Instant::now(); + while !generation.thread.is_finished() { + assert!( + stopping.elapsed() < Duration::from_secs(3), + "the stopped loop waits for Windows" + ); + std::thread::sleep(Duration::from_millis(10)); + } + generation.thread.join().unwrap(); +} + +struct Ipad { + socket: UdpSocket, + host: SocketAddr, + msg_seq: u32, + event_id: u32, + session_id: u32, +} + +impl Ipad { + fn send(&mut self, message: &ControlMessage) { + self.msg_seq += 1; + self.socket + .send_to( + &encode_control(self.session_id, self.msg_seq, message), + self.host, + ) + .unwrap(); + } + + fn connect(&mut self) { + let started = Instant::now(); + let mut bytes = [0u8; 2048]; + loop { + assert!( + started.elapsed() < Duration::from_secs(10), + "no HELLO answer" + ); + self.send(&ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 1, + listen_port: self.socket.local_addr().unwrap().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_WANTS_INPUT, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "Input restart iPad".into(), + device_id: 1, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + })); + let until = Instant::now() + Duration::from_millis(300); + while Instant::now() < until { + if let Ok((len, _)) = self.socket.recv_from(&mut bytes) { + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&bytes[..len]) { + if ack.status == HelloStatus::Ok { + self.session_id = ack.session_id; + return; + } + } + } + } + } + } + + fn press(&mut self, keycode: u16) { + self.event_id += 1; + let event = InputEvent { + input_ver: 1, + kind: KIND_KEY, + phase: PHASE_BEGAN, + buttons: 0, + event_id: self.event_id, + x_norm: 0, + y_norm: 0, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode, + modifiers: 0, + client_time_us: 0, + tilt_x: 0, + tilt_y: 0, + }; + self.send(&ControlMessage::InputEvent(event)); + } +} + +fn wait_until(what: &str, mut done: impl FnMut() -> bool) { + let started = Instant::now(); + while !done() { + assert!(started.elapsed() < Duration::from_secs(5), "{what}"); + std::thread::sleep(Duration::from_millis(5)); + } +} + +/// Whether Windows ends up holding A: its last press or release. +fn a_is_down(injected: &[Injection]) -> Option { + injected.iter().rev().find_map(|injection| match injection { + Injection::KeyDown { scan: SCAN_A, .. } => Some(true), + Injection::KeyUp { scan: SCAN_A, .. } => Some(false), + _ => None, + }) +} + +#[test] +fn a_restart_while_windows_holds_input_never_releases_the_new_generations_keys() { + eternal_host::clock::init(); + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + // Bound first, so its port is never the one the PC takes. + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let port = UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap(); + let shared = SharedControl::new(port, 10_000_000); + shared.pairing.lock().required = false; + *shared.capture_geometry.lock() = Some(CaptureGeometry { + left: 0, + top: 0, + width: 1280, + height: 720, + }); + recorder::take(); + let old = start(&shared, port); + let mut ipad = Ipad { + socket, + host: SocketAddr::from(([127, 0, 0, 1], port)), + msg_seq: 0, + event_id: 0, + session_id: 0, + }; + ipad.connect(); + ipad.press(KEY_A); + wait_until("A goes down", || a_is_down(&recorder::peek()) == Some(true)); + + // Windows stops taking the old generation's input, and the stream + // restarts on the same port. + hold::hold_next(); + ipad.press(KEY_B); + wait_until("B reaches Windows", || hold::waiting() == 1); + stop(&shared, old); + let new = start(&shared, port); + + // The iPad, still in its session, presses A again on the new stream. + let pressed_again = ipad.event_id + 1; + let started = Instant::now(); + while shared.transport_beat.read().is_none() || ipad.event_id < pressed_again { + assert!( + started.elapsed() < Duration::from_secs(5), + "the new loop starts" + ); + if shared.transport_beat.read().is_some() { + ipad.press(KEY_A); + } + std::thread::sleep(Duration::from_millis(20)); + } + std::thread::sleep(Duration::from_millis(300)); + + // Windows answers again. Once everything queued has landed, A is down. + hold::release(); + assert!(shared.input.finish(Duration::from_secs(5))); + std::thread::sleep(Duration::from_millis(200)); + let injected = recorder::take(); + assert_eq!(a_is_down(&injected), Some(true), "{injected:?}"); + + stop(&shared, new); +} diff --git a/host/tests/input_stall.rs b/host/tests/input_stall.rs index f1199476..b4b5f3b3 100644 --- a/host/tests/input_stall.rs +++ b/host/tests/input_stall.rs @@ -121,6 +121,10 @@ fn input_windows_holds_never_stops_the_transport_loop() { eternal_host::clock::init(); // Never reach a real iPad over USB. std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + // Bound first, so their ports are never the one the PC takes. + let unbound = SocketAddr::from(([127, 0, 0, 1], 0)); + let mut ipad = Ipad::new(unbound, 1); + let mut other = Ipad::new(unbound, 2); let port = UdpSocket::bind("127.0.0.1:0") .and_then(|s| s.local_addr()) .map(|a| a.port()) @@ -153,7 +157,8 @@ fn input_windows_holds_never_stops_the_transport_loop() { .unwrap(); }); let host = SocketAddr::from(([127, 0, 0, 1], port)); - let mut ipad = Ipad::new(host, 1); + ipad.host = host; + other.host = host; let session_id = ipad.accepted(); // Windows stops taking input, and the iPad presses a key. @@ -191,7 +196,6 @@ fn input_windows_holds_never_stops_the_transport_loop() { // iPad's HELLO is answered, and the watch sees the loop moving. let mut watch = Watch::default(); let mut verdicts = Vec::new(); - let mut other = Ipad::new(host, 2); other.hello(); let mut heartbeats = 0; let mut answered = false; From 7a04153779d23b7aeb6e55b3b7f5d7f67636b7db Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 15:36:38 -0500 Subject: [PATCH 052/226] Let go of the capture rect before Windows refreshes a still Pencil The input thread held the capture rect's lock through the pen refresh, Windows calls included: the guard lived to the end of the statement that passed the rect on. The transport takes the same lock for every input datagram. A refresh Windows held then stopped the loop at the iPad's next touch: no heartbeats, no HELLO answers, no goodbye. The rect is copied out first. transport::tests::a_pen_refresh_windows_holds_never_stops_the_loop runs the real loop with a fake Windows that holds the first refresh, while the iPad types. Before this change it heard no heartbeats in 3 s. --- host/src/input/injector.rs | 10 +- host/src/input/mod.rs | 2 + host/src/transport/mod.rs | 186 +++++++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 1 deletion(-) diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs index 0d83e238..8b2b9ce2 100644 --- a/host/src/input/injector.rs +++ b/host/src/input/injector.rs @@ -252,6 +252,11 @@ impl InputThread { } /// A thread driving a test's fake instead of Windows. + #[cfg(test)] + pub(crate) fn with_fake(sink: impl FnOnce() -> S + Send + 'static) -> Self { + Self::with_sink(Limits::default(), sink) + } + #[cfg(test)] fn with_sink(limits: Limits, sink: impl FnOnce() -> S + Send + 'static) -> Self { let thread = Self::default(); @@ -527,8 +532,11 @@ fn run(shared: &Shared, sink: &mut S) { if tick_due { next_tick = now + PEN_TICK; } + // Copied out first: the transport takes this lock for every + // input datagram, and Windows can hold the refresh. + let output = *current.geometry.lock(); let started = Instant::now(); - sink.keep_alive(*current.geometry.lock()); + sink.keep_alive(output); note_slow(started, "pen_refresh"); } } diff --git a/host/src/input/mod.rs b/host/src/input/mod.rs index ba288d20..1b8d790d 100644 --- a/host/src/input/mod.rs +++ b/host/src/input/mod.rs @@ -11,6 +11,8 @@ mod hid; pub use hid::hid_to_scancode; mod injector; +#[cfg(test)] +pub(crate) use injector::Sink; pub use injector::{Injector, InputThread}; #[cfg(windows)] diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 937935b5..bfc0885f 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -1558,6 +1558,192 @@ mod tests { ); } + /// Windows holds the input thread inside a still Pencil's refresh. The + /// loop takes the capture rect for every input datagram, so a refresh + /// that kept it while Windows held the call stopped the loop at the + /// iPad's next touch: no heartbeats, no HELLO answers, no goodbye. + #[test] + fn a_pen_refresh_windows_holds_never_stops_the_loop() { + use crate::input::{CaptureGeometry, InputThread, Sink, KIND_KEY, PHASE_BEGAN}; + use eternal_wire::v2::control::{HelloStatus, InputEvent, FEATURE_WANTS_INPUT}; + + /// Holds the first refresh until told to go on. + struct StuckRefresh { + entered: Option>, + hold: Option>, + } + impl Sink for StuckRefresh { + fn relay(&mut self, _: u32, _: &InputEvent, _: CaptureGeometry, _: bool) {} + fn reset(&mut self) {} + fn keep_alive(&mut self, _: Option) { + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(hold) = self.hold.take() { + let _ = hold.recv(); + } + } + fn pen_available(&self) -> bool { + true + } + fn spaced_until(&self) -> Option { + None + } + fn tick_needed(&self) -> bool { + true + } + } + + let ipad = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + ipad.set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let port = std::net::UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .unwrap() + .port(); + let host = SocketAddr::from(([127, 0, 0, 1], port)); + let (entered_tx, entered) = std_mpsc::channel(); + let (go, hold) = std_mpsc::channel(); + let mut shared = SharedControl::new(port, 15_000_000); + shared.input = std::sync::Arc::new(InputThread::with_fake(move || StuckRefresh { + entered: Some(entered_tx), + hold: Some(hold), + })); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + *shared.capture_geometry.lock() = Some(CaptureGeometry { + left: 0, + top: 0, + width: 1280, + height: 720, + }); + let (nal_tx, nal_rx) = mpsc::channel(4); + let control = shared.clone(); + let transport = std::thread::spawn(move || { + let (_audio_tx, audio_rx) = mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(start_sender(nal_rx, audio_rx, port, control, supervisor_tx)) + .unwrap(); + }); + // The refresh ticks from the stream's start; Windows holds it. + entered + .recv_timeout(Duration::from_secs(5)) + .expect("the input thread refreshes the pen"); + + let mut msg_seq = 0; + let mut send = |session_id, message: &ControlMessage| { + msg_seq += 1; + ipad.send_to(&encode_control(session_id, msg_seq, message), host) + .unwrap(); + }; + let heard = |until: Instant| { + let mut messages = Vec::new(); + let mut bytes = [0u8; 2048]; + while Instant::now() < until { + if let Ok((len, _)) = ipad.recv_from(&mut bytes) { + if let Ok((_, message)) = parse_control(&bytes[..len]) { + messages.push(message); + } + } + } + messages + }; + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 1, + listen_port: 0, + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_WANTS_INPUT, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "Pen refresh test".into(), + device_id: 1, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + let started = Instant::now(); + let session_id = loop { + assert!( + started.elapsed() < Duration::from_secs(5), + "no HELLO answer" + ); + send(0, &hello); + let accepted = heard(Instant::now() + Duration::from_millis(200)) + .into_iter() + .find_map(|message| match message { + ControlMessage::HelloAck(ack) if ack.status == HelloStatus::Ok => { + Some(ack.session_id) + } + _ => None, + }); + if let Some(id) = accepted { + break id; + } + }; + + // The iPad keeps typing while Windows holds the refresh; the loop + // keeps its heartbeats going. + let mut watch = beat::Watch::default(); + let mut verdicts = Vec::new(); + let mut heartbeats = 0; + let until = Instant::now() + Duration::from_secs(3); + let mut event_id = 0; + while Instant::now() < until { + event_id += 1; + send( + session_id, + &ControlMessage::InputEvent(InputEvent { + input_ver: 1, + kind: KIND_KEY, + phase: PHASE_BEGAN, + buttons: 0, + event_id, + x_norm: 0, + y_norm: 0, + pressure_x1000: 0, + scroll_dx: 0, + scroll_dy: 0, + keycode: 0x01, + modifiers: 0, + client_time_us: 0, + tilt_x: 0, + tilt_y: 0, + }), + ); + heartbeats += heard(Instant::now() + Duration::from_millis(100)) + .iter() + .filter(|message| matches!(message, ControlMessage::Heartbeat(_))) + .count(); + verdicts.extend(watch.check( + shared.transport_beat.read(), + crate::clock::host_now_us() / 1000, + )); + } + let stuck = heartbeats < 2 || !verdicts.is_empty(); + if stuck { + // Let a stuck loop go, so the test ends. + let _ = go.send(()); + } + assert!( + heartbeats >= 2, + "heartbeats while Windows held the refresh: {heartbeats}" + ); + assert_eq!(verdicts, []); + shared.stop(); + drop(nal_tx); + transport.join().unwrap(); + let _ = go.send(()); + } + #[test] fn a_failed_send_names_the_link_and_repeats_identically() { let out = std::sync::Arc::new(parking_lot::Mutex::new(Vec::::new())); From d6980b0dd4a3d7ed433aa2c99b12dd1d72c5f63b Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 15:38:18 -0500 Subject: [PATCH 053/226] Never lose a release at the input queue's bound Ends and resets could pass the bound, but only up to twice its size. A longer stretch with Windows stuck dropped every command after that: a later key-up or the disconnect's reset vanished, and keys or buttons stayed down. At the bound, a key-up, button release or reset now marks the last command queued. Once that command lands, everything its generation holds is released. The queue never grows past the bound. A stream generation's start and end always go in; there is one of each per stream. releases_past_the_hard_limit_still_release_everything fills the real 1024-command bound with presses, adds 1024 key-ups and the disconnect's reset, then lets Windows answer. Before this change the reset was lost and the last thing Windows got was a key-up. --- host/src/input/injector.rs | 121 +++++++++++++++++++++++++++++-------- 1 file changed, 95 insertions(+), 26 deletions(-) diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs index 8b2b9ce2..64162cb6 100644 --- a/host/src/input/injector.rs +++ b/host/src/input/injector.rs @@ -18,7 +18,9 @@ //! a path the iPad already left. Input that waited longer than //! [`INPUT_LATE`] goes to the relay as late, which lands only a release of //! what the PC holds, as for input that arrived late from the iPad. Ends, -//! key releases and session resets are never dropped or merged. +//! key releases and session resets are never merged, and never lost: at +//! the bound, one becomes a release of everything after the last command +//! queued. use std::collections::VecDeque; use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; @@ -34,9 +36,10 @@ use super::{ }; use crate::transport::session::INPUT_LATE; -/// Commands kept at most. A second of a Pencil at 240 Hz with every other -/// kind of input; while Windows is stuck, moves merge and the queue grows -/// only by presses and releases. +/// Commands kept at most, besides one start and one end per stream +/// generation. A second of a Pencil at 240 Hz with every other kind of +/// input; while Windows is stuck, moves merge and the queue grows only by +/// presses and releases. const CAPACITY: usize = 1024; /// From this many waiting commands, a move replaces the move before it. @@ -149,11 +152,11 @@ impl Command { && event.buttons == earlier_event.buttons } - /// An end, reset or generation change: what keeps the PC from holding - /// a key or button. - fn must_land(&self) -> bool { + /// An end or reset: what keeps the PC from holding a key or button. + fn releases(&self) -> bool { match self { - Command::Begin { .. } | Command::Reset | Command::End => true, + Command::Reset | Command::End => true, + Command::Begin { .. } => false, Command::Relay { event, .. } => event.phase > PHASE_MOVED, } } @@ -164,6 +167,9 @@ struct Queued { generation: u64, command: Command, at: Instant, + /// A release came while the queue was full: once this command has + /// landed, release everything its generation holds. + then_reset: bool, } #[derive(Debug, Default)] @@ -173,6 +179,9 @@ struct State { busy: bool, merged: u64, dropped: u64, + /// Releases that came while the queue was full, each now part of a + /// release of everything. + folded: u64, } /// What the transport's handles and the thread share. @@ -195,7 +204,10 @@ impl Shared { let waiting = state.commands.len(); if waiting >= self.limits.merge_from { if let Some(last) = state.commands.back_mut() { - if last.generation == generation && command.merges_with(&last.command) { + if last.generation == generation + && !last.then_reset + && command.merges_with(&last.command) + { // The newer sample, in the older one's place: it has // waited as long. last.command = command; @@ -204,16 +216,28 @@ impl Shared { } } } - // Ends and resets may pass the bound, by as many as were pressed. - let capacity = self.limits.capacity; - if waiting >= capacity && !(command.must_land() && waiting < 2 * capacity) { - state.dropped += 1; + // A generation's start and end always go in: there is one of each + // per stream. + let boundary = matches!(command, Command::Begin { .. } | Command::End); + if waiting >= self.limits.capacity && !boundary { + let barrier = state + .commands + .back_mut() + .filter(|last| command.releases() && last.generation == generation); + match barrier { + Some(last) => { + last.then_reset = true; + state.folded += 1; + } + None => state.dropped += 1, + } return; } state.commands.push_back(Queued { generation, command, at: now, + then_reset: false, }); drop(state); self.wake.notify_one(); @@ -433,17 +457,19 @@ fn run(shared: &Shared, sink: &mut S) { break Next::Command(queued); } shared.idle.notify_all(); - let (merged, dropped) = ( + let (merged, dropped, folded) = ( std::mem::take(&mut state.merged), std::mem::take(&mut state.dropped), + std::mem::take(&mut state.folded), ); - if backlog.stale > 0 || merged > 0 || dropped > 0 { + if backlog.stale > 0 || merged > 0 || dropped > 0 || folded > 0 { info!( stale = backlog.stale, merged, dropped, + folded, longest_ms = backlog.longest.as_millis() as u64, - "Input waited for Windows: moves merged, and only releases sent of what waited too long" + "Input waited for Windows: moves merged, only releases sent of what waited too long, and everything released for releases past the bound" ); backlog = Backlog::default(); } @@ -474,6 +500,7 @@ fn run(shared: &Shared, sink: &mut S) { generation, command, at, + then_reset, }) = next { let started = Instant::now(); @@ -517,6 +544,13 @@ fn run(shared: &Shared, sink: &mut S) { // An ended generation's: its end released what it held. Command::End | Command::Reset | Command::Relay { .. } => {} } + if then_reset + && current + .as_ref() + .is_some_and(|current| current.generation == generation) + { + sink.reset(); + } note_slow(started, "input"); note_pen(shared, sink); } @@ -828,7 +862,7 @@ mod tests { } #[test] - fn a_full_queue_drops_new_moves_and_presses_but_keeps_releases() { + fn a_full_queue_drops_new_moves_and_presses_and_turns_a_release_into_a_reset() { let harness = harness(Limits { capacity: 3, merge_from: 100, @@ -843,16 +877,51 @@ mod tests { harness.send(KIND_KEY, PHASE_ENDED, 6); harness.injector.as_ref().unwrap().reset(); harness.go.send(()).unwrap(); + // The key-up and the reset after it become one release of + // everything, after the last press queued. + let expected = [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_KEY, PHASE_BEGAN, 1), + relayed(KIND_KEY, PHASE_BEGAN, 2), + relayed(KIND_KEY, PHASE_BEGAN, 3), + Call::Reset, + ]; + assert_eq!(harness.wait_for(5), expected); + harness.settled(expected.len()); + } + + /// Windows stays stuck while the queue fills to its bound and then + /// releases keep coming: the key-ups, and the disconnect's reset after + /// them. However many pile up, what the PC holds is released once + /// Windows answers, and nothing pressed lands after that. + #[test] + fn releases_past_the_hard_limit_still_release_everything() { + let harness = harness(Limits::default()); + harness.stuck(); + for id in 1..=CAPACITY as u32 { + harness.send(KIND_KEY, PHASE_BEGAN, id); + } + for id in 1..=CAPACITY as u32 { + harness.send(KIND_KEY, PHASE_ENDED, 10_000 + id); + } + harness.send(KIND_KEY, PHASE_BEGAN, 20_000); + harness.injector.as_ref().unwrap().reset(); + harness.go.send(()).unwrap(); + assert!(harness.thread.finish(Duration::from_secs(5))); + let calls = harness.calls.lock().clone(); assert_eq!( - harness.wait_for(6), - [ - relayed(KIND_KEY, PHASE_BEGAN, 0), - relayed(KIND_KEY, PHASE_BEGAN, 1), - relayed(KIND_KEY, PHASE_BEGAN, 2), - relayed(KIND_KEY, PHASE_BEGAN, 3), - relayed(KIND_KEY, PHASE_ENDED, 6), - Call::Reset, - ] + calls.last(), + Some(&Call::Reset), + "{:?}", + &calls[calls.len() - 3..] + ); + assert!(calls.len() <= 2 * CAPACITY + 1, "{}", calls.len()); + let pressed: Vec<_> = calls[1..=CAPACITY].to_vec(); + assert_eq!( + pressed, + (1..=CAPACITY as u32) + .map(|id| relayed(KIND_KEY, PHASE_BEGAN, id)) + .collect::>() ); } From 11b59142385da660c9d83fd397bd80089c0af066 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 15:39:06 -0500 Subject: [PATCH 054/226] Measure the hop from the transport to the input thread the_hop_to_the_input_thread_is_short queues 2000 events at 1 kHz and prints p50, p99 and the worst time to the call that hands each to Windows. On macOS at load 23: p50 5 us, p99 26-32 us, worst under 60 us. --- host/src/input/injector.rs | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs index 64162cb6..6fe7bc45 100644 --- a/host/src/input/injector.rs +++ b/host/src/input/injector.rs @@ -985,6 +985,60 @@ mod tests { assert_eq!(harness.calls.lock().len(), 2); } + /// The hop the thread adds: from the transport queueing an event to the + /// call that hands it to Windows, at a steady 1 kHz. Prints p50, p99 and + /// the worst; the bound only catches a thread that waits for a timer. + #[test] + fn the_hop_to_the_input_thread_is_short() { + struct Timed(Arc>>); + impl Sink for Timed { + fn relay(&mut self, _: u32, event: &InputEvent, _: CaptureGeometry, _: bool) { + self.0.lock().push((event.event_id, Instant::now())); + } + fn reset(&mut self) {} + fn keep_alive(&mut self, _: Option) {} + fn pen_available(&self) -> bool { + false + } + fn spaced_until(&self) -> Option { + None + } + fn tick_needed(&self) -> bool { + false + } + } + const EVENTS: u32 = 2000; + let taken = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&taken); + let thread = InputThread::with_sink(Limits::default(), move || Timed(recorded)); + let injector = thread.begin(geometry()); + let mut queued = Vec::new(); + let mut next = Instant::now(); + for id in 0..EVENTS { + next += Duration::from_millis(1); + std::thread::sleep(next.saturating_duration_since(Instant::now())); + queued.push(Instant::now()); + injector.relay(1, event(KIND_HOVER, PHASE_MOVED, id), OUTPUT, false); + } + assert!(thread.finish(Duration::from_secs(5))); + let mut hops: Vec<_> = taken + .lock() + .iter() + .map(|(id, at)| *at - queued[*id as usize]) + .collect(); + assert_eq!(hops.len(), EVENTS as usize); + hops.sort(); + let at = |q: f64| hops[((hops.len() - 1) as f64 * q) as usize]; + let (p50, p99, worst) = (at(0.5), at(0.99), *hops.last().unwrap()); + eprintln!( + "input hop to the input thread: p50 {} us, p99 {} us, max {} us", + p50.as_micros(), + p99.as_micros(), + worst.as_micros() + ); + assert!(p99 < Duration::from_millis(50), "{p99:?}"); + } + /// The real relay, with a native pen device that takes every frame /// (Windows' own off Windows). A stall delivers 64 samples at once; the /// spacing holds back all but the first burst, and the thread sends the From a636f3077122a96598e713684f44347f12f864f3 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 15:57:48 -0500 Subject: [PATCH 055/226] Back off replacing a lost window device, and give up after ten failures The painter tried again on every repaint, each time listing adapters and asking for a device on the window's thread. Now a replacement waits as winit::Replacement says: at once, then 0.5 s doubling to 30 s, and after ten failures in a row (including new devices lost again before they drew for 10 s, as with a wgpu error on every frame) the window stays blank. While a replacement waits, painting only asks for a repaint. Also: configure_surface replaces the device instead of panicking when the adapter supports no surface; Renderer::upload_failed and SKIPPED_FRAMES are exported for the app; the test checks upload_failed before and after the loss. --- host/tests/window_device_loss.rs | 87 ++++++++++++- vendor/README.md | 21 +++- vendor/egui-wgpu/src/renderer.rs | 8 +- vendor/egui-wgpu/src/winit.rs | 203 ++++++++++++++++++++++++++----- 4 files changed, 280 insertions(+), 39 deletions(-) diff --git a/host/tests/window_device_loss.rs b/host/tests/window_device_loss.rs index 5741d8f3..67d7b84c 100644 --- a/host/tests/window_device_loss.rs +++ b/host/tests/window_device_loss.rs @@ -1,9 +1,13 @@ //! The window survives the loss of its graphics device, as in a driver //! reset. egui-wgpu 0.36 panicked on the next frame ("Failed to create //! staging buffer for index data"), which ended the app and the stream; the -//! patched copy in `vendor/egui-wgpu` skips the frame and replaces the device. +//! patched copy in `vendor/egui-wgpu` skips the frame and replaces the device +//! on a schedule (`Replacement`), giving up after repeated failures. + +use std::time::{Duration, Instant}; use eframe::egui; +use eframe::egui_wgpu::winit::Replacement; use eframe::egui_wgpu::{Renderer, RendererOptions, ScreenDescriptor}; use eframe::wgpu; @@ -56,6 +60,7 @@ fn a_lost_device_skips_the_frame_instead_of_panicking() { let mut encoder = device.create_command_encoder(&Default::default()); renderer.update_buffers(&device, &queue, &mut encoder, &primitives, &screen); queue.submit([encoder.finish()]); + assert!(!renderer.upload_failed()); // wgpu now refuses every staging buffer, as after a driver reset. device.destroy(); @@ -64,4 +69,84 @@ fn a_lost_device_skips_the_frame_instead_of_panicking() { let mut encoder = device.create_command_encoder(&Default::default()); let callbacks = renderer.update_buffers(&device, &queue, &mut encoder, &primitives, &screen); assert!(callbacks.is_empty()); + assert!(renderer.upload_failed()); +} + +const SECOND: Duration = Duration::from_secs(1); + +/// The waits between tries until the painter gives up, when every +/// replacement fails at once. +fn waits_until_given_up(replacement: &mut Replacement, mut now: Instant) -> Vec { + let mut waits = Vec::new(); + while let Some(wait) = replacement.wait(now) { + waits.push(wait); + now += wait; + assert!(waits.len() <= 20, "never gives up: {waits:?}"); + replacement.failed(now); + } + waits +} + +#[test] +fn a_lost_device_is_replaced_at_once_then_backs_off_then_gives_up() { + let start = Instant::now(); + let mut replacement = Replacement::default(); + assert_eq!(replacement.wait(start), Some(Duration::ZERO)); + replacement.drew(start); + assert!(!replacement.lost(start + SECOND)); + let ms = Duration::from_millis; + assert_eq!( + waits_until_given_up(&mut replacement, start + SECOND), + [ + ms(0), + ms(500), + ms(1000), + ms(2000), + ms(4000), + ms(8000), + ms(16000), + ms(30000), + ms(30000), + ms(30000), + ] + ); + assert_eq!(replacement.wait(start + Duration::from_secs(3600)), None); +} + +#[test] +fn a_new_device_that_fails_at_once_every_time_ends_in_giving_up() { + // As with a wgpu error on every frame: each new device is lost again + // right after it is made. + let mut now = Instant::now(); + let mut replacement = Replacement::default(); + let mut gave_up = 0; + for _ in 0..30 { + let Some(wait) = replacement.wait(now) else { + break; + }; + now += wait; + replacement.drew(now); + now += Duration::from_millis(20); + gave_up += usize::from(replacement.lost(now)); + } + assert_eq!(replacement.wait(now), None); + assert_eq!(gave_up, 1, "said once"); +} + +#[test] +fn a_device_that_drew_for_a_while_starts_the_count_again() { + let mut now = Instant::now(); + let mut replacement = Replacement::default(); + replacement.lost(now); + for _ in 0..(Replacement::MAX_FAILURES - 1) { + replacement.failed(now); + } + // One failure short of giving up, a new device works for a while. + now += replacement.wait(now).unwrap(); + replacement.drew(now); + now += Replacement::STABLE_AFTER; + assert!(!replacement.lost(now)); + assert_eq!(replacement.wait(now), Some(Duration::ZERO)); + replacement.failed(now); + assert_eq!(replacement.wait(now), Some(Duration::from_millis(500))); } diff --git a/vendor/README.md b/vendor/README.md index 5a1a2f3b..c01a3b66 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -23,12 +23,25 @@ The patch: panics: a window whose device or surface fails is drawn on a new one. - When the device is lost, or a frame's upload fails, the `Painter` drops the device, its surfaces and egui's renderer. eframe calls `set_window` before - each paint, which makes new ones; until that works the window skips frames - and tries again every 500 ms. The new device's first frame uploads the font - atlas again. Other textures are the app's to load again + each paint, which makes new ones when `winit::Replacement` says: at once, + then after 0.5 s, doubling up to 30 s. After 10 failures in a row (devices + that could not be made, or new ones lost again before they drew for 10 s) + it gives up, says so once, and the window stays blank. Streaming runs on + its own devices and is not affected. The new device's first frame uploads + the font atlas again. Other textures are the app's to load again (`host/src/gui/renderer.rs` counts devices for that). +- `configure_surface` replaces the device, instead of panicking, when the + adapter supports no surface any more. +- `Renderer::upload_failed` and `winit::SKIPPED_FRAMES` let the app see + skipped frames: the host counts them, so a backend that only skips frames + is never recorded as one that drew. - A D3D12 device is only replaced once nothing holds the lost one, so the host drops the copy eframe keeps in `Frame` (`release_render_state`). Drop this copy when an egui-wgpu release survives device loss, or carry the -patch to the version eframe moves to. +patch to the version eframe moves to. When eframe moves on (0.37 uses +egui-wgpu 0.37) this patch stops applying, because `[patch.crates-io]` only +replaces 0.36.2: Cargo warns that the patch was not used. Then +`host/tests/window_device_loss.rs` is the guard. It fails to build without +the patch's `upload_failed` and `Replacement`, and on an unpatched release it +fails with the old panic. diff --git a/vendor/egui-wgpu/src/renderer.rs b/vendor/egui-wgpu/src/renderer.rs index ff0a36f0..43940ed3 100644 --- a/vendor/egui-wgpu/src/renderer.rs +++ b/vendor/egui-wgpu/src/renderer.rs @@ -261,10 +261,16 @@ pub struct Renderer { /// EternalMonitor patch: the last [`Self::update_buffers`] could not upload its meshes, /// as happens once the device is lost, so [`Self::render`] draws nothing. - pub(crate) upload_failed: bool, + upload_failed: bool, } impl Renderer { + /// EternalMonitor patch: the last [`Self::update_buffers`] could not upload its meshes, + /// as happens once the device is lost; [`Self::render`] then draws nothing. + pub fn upload_failed(&self) -> bool { + self.upload_failed + } + /// Creates a renderer for a egui UI. /// /// `output_color_format` should preferably be [`wgpu::TextureFormat::Rgba8Unorm`] or diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs index 6ee4d76c..5bace98d 100644 --- a/vendor/egui-wgpu/src/winit.rs +++ b/vendor/egui-wgpu/src/winit.rs @@ -9,14 +9,85 @@ use crate::{ capture::{CaptureReceiver, CaptureSender, CaptureState, capture_channel}, }; use core::num::NonZeroU32; -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use core::time::Duration; use egui::{Context, Event, UserData, ViewportId, ViewportIdMap, ViewportIdSet}; use std::sync::Arc; +use web_time::Instant; -/// EternalMonitor patch: how often a lost device's replacement is tried again -/// while it can't be created or doesn't work, as during a driver reset. -const REBUILD_RETRY: Duration = Duration::from_millis(500); +/// EternalMonitor patch: frames the painter skipped because its device was lost or was +/// being replaced. An app that tells a working window by its frames counts these as +/// failures. +pub static SKIPPED_FRAMES: AtomicU64 = AtomicU64::new(0); + +/// EternalMonitor patch: when the painter tries to replace a lost device, and when it +/// stops trying. +/// +/// The first try is at once. Each failure, a device that could not be made or a new one +/// lost before it drew for [`Self::STABLE_AFTER`], waits twice as long as the last, from +/// 0.5 s up to 30 s. After [`Self::MAX_FAILURES`] the painter gives up and the window +/// stays blank. A device that drew for [`Self::STABLE_AFTER`] starts the count again. +#[derive(Debug, Default)] +pub struct Replacement { + failures: u32, + next_try: Option, + drawing_since: Option, + gave_up: bool, +} + +impl Replacement { + pub const MAX_FAILURES: u32 = 10; + pub const STABLE_AFTER: Duration = Duration::from_secs(10); + const FIRST_WAIT: Duration = Duration::from_millis(500); + const LONGEST_WAIT: Duration = Duration::from_secs(30); + + /// The device was lost at `now`. True when the painter just gave up. + pub fn lost(&mut self, now: Instant) -> bool { + let drew = self + .drawing_since + .is_some_and(|since| now.duration_since(since) >= Self::STABLE_AFTER); + self.drawing_since = None; + if drew { + self.failures = 0; + self.next_try = Some(now); + return false; + } + if self.next_try.is_none() { + // The device the window started with: its replacement is the first try. + self.next_try = Some(now); + return false; + } + self.failed(now) + } + + /// A replacement could not be made at `now`. True when the painter just gave up. + pub fn failed(&mut self, now: Instant) -> bool { + if self.gave_up { + return false; + } + self.failures += 1; + if self.failures >= Self::MAX_FAILURES { + self.gave_up = true; + return true; + } + let wait = Self::FIRST_WAIT.saturating_mul(1 << (self.failures - 1).min(16)); + self.next_try = Some(now + wait.min(Self::LONGEST_WAIT)); + false + } + + /// The device presented a frame at `now`. + pub fn drew(&mut self, now: Instant) { + self.drawing_since.get_or_insert(now); + } + + /// How long until the next try, or `None` once the painter gave up. + pub fn wait(&self, now: Instant) -> Option { + (!self.gave_up).then(|| { + self.next_try + .map_or(Duration::ZERO, |at| at.saturating_duration_since(now)) + }) + } +} struct SurfaceState { surface: wgpu::Surface<'static>, @@ -54,13 +125,13 @@ pub struct Painter { /// Set when the current device is lost: by its lost callback, or when a frame's upload /// fails. The next paint drops the device and everything made on it. device_lost: Arc, - /// The lost device was dropped; [`Self::set_window`] builds a new one, and failing - /// that is retried instead of returned. + /// The lost device was dropped; [`Self::set_window`] builds a new one when + /// [`Self::replacement`] says, and a failure is retried instead of returned. rebuilding: bool, /// The new device's first paint uploads the font atlas again. restore_fonts: bool, - /// When the last device was dropped, so a replacement that fails at once waits. - dropped_at: Option, + /// When to try replacing a lost device, and when to stop. + replacement: Replacement, } impl Painter { @@ -105,7 +176,7 @@ impl Painter { device_lost: Arc::new(AtomicBool::new(false)), rebuilding: false, restore_fonts: false, - dropped_at: None, + replacement: Replacement::default(), } } @@ -120,6 +191,7 @@ impl Painter { surface_state: &SurfaceState, render_state: &RenderState, config: &SurfaceConfig, + device_lost: &AtomicBool, ) { profiling::function_scope!(); @@ -140,16 +212,24 @@ impl Painter { let width = surface_state.width; let height = surface_state.height; + // EternalMonitor patch: a lost device's adapter may support no surface before its + // lost callback runs. Replace the device instead of panicking. + let Some(default_config) = + surface_state + .surface + .get_default_config(&render_state.adapter, width, height) + else { + log::warn!("The surface isn't supported by this adapter; replacing the device"); + device_lost.store(true, Ordering::SeqCst); + return; + }; let mut surf_config = wgpu::SurfaceConfiguration { usage: wgpu::TextureUsages::RENDER_ATTACHMENT, format: render_state.target_format, present_mode, alpha_mode: surface_state.alpha_mode, view_formats: vec![render_state.target_format], - ..surface_state - .surface - .get_default_config(&render_state.adapter, width, height) - .expect("The surface isn't supported by this adapter") + ..default_config }; if let Some(desired_maximum_frame_latency) = desired_maximum_frame_latency { @@ -219,6 +299,18 @@ impl Painter { if let Some(window) = window { let size = window.inner_size(); if !self.surfaces.contains_key(&viewport_id) { + // EternalMonitor patch: a lost device is replaced when `replacement` says, + // not on every repaint, and not at all once it gave up. + if self.rebuilding { + match self.replacement.wait(Instant::now()) { + None => return Ok(()), + Some(wait) if !wait.is_zero() => { + self.context.request_repaint_after(wait); + return Ok(()); + } + Some(_) => {} + } + } let added = match self.instance.create_surface(window) { Ok(surface) => self.add_surface(surface, viewport_id, size).await, Err(err) => Err(err.into()), @@ -227,8 +319,14 @@ impl Painter { // EternalMonitor patch: while a lost device is replaced, the window // skips frames until a new one can be made. Err(err) if self.rebuilding => { - log::warn!("Could not replace the lost wgpu device yet: {err}"); - self.context.request_repaint_after(REBUILD_RETRY); + if self.replacement.failed(Instant::now()) { + Self::log_gave_up(&err); + } else { + log::warn!("Could not replace the lost wgpu device yet: {err}"); + if let Some(wait) = self.replacement.wait(Instant::now()) { + self.context.request_repaint_after(wait); + } + } } added => added?, } @@ -315,7 +413,6 @@ impl Painter { /// EternalMonitor patch: forget a lost device and everything made on it. eframe calls /// [`Self::set_window`] before each paint, which builds a new device and surfaces. fn drop_lost_device(&mut self) { - log::warn!("Rebuilding the window renderer after its wgpu device was lost"); self.render_state = None; self.surfaces.clear(); self.depth_texture_view.clear(); @@ -323,8 +420,23 @@ impl Painter { self.screen_capture_state = None; self.rebuilding = true; self.restore_fonts = false; - self.dropped_at = Some(web_time::Instant::now()); - self.context.request_repaint(); + let now = Instant::now(); + if self.replacement.lost(now) { + Self::log_gave_up(&"the new device was lost too"); + } else { + log::warn!("Rebuilding the window renderer after its wgpu device was lost"); + if let Some(wait) = self.replacement.wait(now) { + self.context.request_repaint_after(wait); + } + } + } + + /// EternalMonitor patch: said once, when [`Replacement`] gives up. + fn log_gave_up(last: &dyn core::fmt::Display) { + log::error!( + "Gave up replacing the window's lost wgpu device after {} failures ({last}); the window stays blank", + Replacement::MAX_FAILURES + ); } /// Inserts a freshly created surface into [`Self::surfaces`] and configures it. @@ -415,7 +527,12 @@ impl Painter { surface_state.width = width; surface_state.height = height; - Self::configure_surface(surface_state, render_state, &self.config.surface); + Self::configure_surface( + surface_state, + render_state, + &self.config.surface, + &self.device_lost, + ); if let Some(depth_format) = self.options.depth_stencil_format { self.depth_texture_view.insert( @@ -515,7 +632,12 @@ impl Painter { .lock() .setPresentsWithTransaction(resizing); - Self::configure_surface(state, render_state, &self.config.surface); + Self::configure_surface( + state, + render_state, + &self.config.surface, + &self.device_lost, + ); } } } @@ -586,19 +708,21 @@ impl Painter { let capture = !capture_data.is_empty(); let mut vsync_sec = 0.0; - // EternalMonitor patch: a lost device draws nothing again; replace it, at most - // every REBUILD_RETRY. - if self.device_lost.load(Ordering::SeqCst) { - if self - .dropped_at - .is_some_and(|dropped| dropped.elapsed() < REBUILD_RETRY) - { - self.context.request_repaint_after(REBUILD_RETRY); - } else { - self.drop_lost_device(); + // EternalMonitor patch: while a lost device is replaced there is nothing to draw + // with; set_window makes the next one when `replacement` says. A device lost since + // the last frame draws nothing again, so it goes. + if self.rebuilding { + SKIPPED_FRAMES.fetch_add(1, Ordering::SeqCst); + if let Some(wait) = self.replacement.wait(Instant::now()) { + self.context.request_repaint_after(wait); } return vsync_sec; } + if self.device_lost.load(Ordering::SeqCst) { + SKIPPED_FRAMES.fetch_add(1, Ordering::SeqCst); + self.drop_lost_device(); + return vsync_sec; + } // If the previous frame produced `CurrentSurfaceTexture::Lost`, the action match // below set `needs_recreate`. Recreate the surface now, before re-borrowing @@ -689,14 +813,20 @@ impl Painter { // EternalMonitor patch: skip a frame whose upload failed; the next one replaces the // device. - if render_state.renderer.read().upload_failed || self.device_lost.load(Ordering::SeqCst) { + if render_state.renderer.read().upload_failed() || self.device_lost.load(Ordering::SeqCst) { + SKIPPED_FRAMES.fetch_add(1, Ordering::SeqCst); self.device_lost.store(true, Ordering::SeqCst); self.context.request_repaint_of(viewport_id); return vsync_sec; } if surface_state.needs_reconfigure { - Self::configure_surface(surface_state, render_state, &self.config.surface); + Self::configure_surface( + surface_state, + render_state, + &self.config.surface, + &self.device_lost, + ); surface_state.needs_reconfigure = false; } @@ -718,7 +848,12 @@ impl Painter { other => { match (*self.config.on_surface_status)(&other) { SurfaceErrorAction::Reconfigure => { - Self::configure_surface(surface_state, render_state, &self.config.surface); + Self::configure_surface( + surface_state, + render_state, + &self.config.surface, + &self.device_lost, + ); self.context.request_repaint_of(viewport_id); } SurfaceErrorAction::RecreateSurface => { @@ -876,6 +1011,8 @@ impl Painter { render_state.queue.present(output_frame); vsync_sec += start.elapsed().as_secs_f32(); } + // EternalMonitor patch: a device that keeps drawing starts the replacement count again. + self.replacement.drew(Instant::now()); vsync_sec } From d54c43ee785b50916d2929db1feca101ee030490 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 15:58:03 -0500 Subject: [PATCH 056/226] Count a device lost before the window works as that backend's failure The patched painter skips frames without a surface error, so the window could be confirmed working, and its backend recorded as drawing, while it only skipped frames. Frames the painter skipped and replacement devices now restart the wait like surface errors do. A device lost before the window is confirmed records a failure for the GPU and backend it started on, and ends the launch's marker so the replacement drawing later doesn't count for it. --- host/src/gui/renderer.rs | 86 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index f5d76675..27136d70 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -66,6 +66,9 @@ const RETRY_AFTER: u32 = 5; /// What ended a launch that crashed in no module the reporter could name, or /// that no reporter described. pub(crate) const UNKNOWN: &str = "unknown"; +/// What ended a launch whose graphics device was lost before the window +/// worked. +pub(crate) const DEVICE_LOST: &str = "device_lost"; /// What ended a launch that was ended without a crash, as by Task Manager. #[cfg(any(windows, test))] pub(crate) const TERMINATED: &str = "terminated"; @@ -441,7 +444,7 @@ pub(super) fn frame_started(ctx: &egui::Context) { }; let now_drawing = tracker.observe( ctx.cumulative_frame_nr(), - FAILED_ACQUIRES.load(Ordering::SeqCst), + failures(), visible, Instant::now(), ); @@ -456,6 +459,41 @@ pub(super) fn frame_started(ctx: &egui::Context) { } } +/// Surface errors, devices and frames the painter skipped while its device +/// was lost or being replaced: any of them starts the wait for a working +/// window again. egui-wgpu reports no surface error for a frame it skips. +fn failures() -> u64 { + FAILED_ACQUIRES.load(Ordering::SeqCst) + + devices() + + eframe::egui_wgpu::winit::SKIPPED_FRAMES.load(Ordering::SeqCst) +} + +/// The window's device was lost before the window was confirmed working: the +/// backend it started on failed to draw. Record that, and end this launch's +/// marker, so the replacement drawing later is not counted for it. +fn lost_before_drawing() { + if PRESENT.lock().is_ok_and(|tracker| tracker.confirmed) { + return; + } + let marker = STARTED.lock().ok().and_then(|mut started| started.take()); + if let (Some(directory), Some(marker)) = (STATE_DIR.get(), marker) { + warn!(gpu = %marker.gpu, backend = %marker.backend, "Window device lost before the window worked"); + lose_launch(directory, &marker, ProcessId::current()); + } +} + +/// Record `marker`'s backend as failed and remove the marker. While another +/// instance holds the history the marker stays, and the next launch counts it. +fn lose_launch(directory: &Path, marker: &Marker, owner: ProcessId) { + if update_history(directory, |history| { + record_failure(history, marker, DEVICE_LOST) + }) + .is_some() + { + let _ = fs::remove_file(marker_path(directory, owner)); + } +} + /// Drop eframe's copy of the window's first device, which is kept for paint /// callbacks the window has none of. A lost device is only replaced once /// nothing holds it: D3D12 keeps one device per GPU in a process, and the @@ -601,10 +639,13 @@ fn select( rules: &[RendererRule], overruled: Option<&'static str>, ) -> Result { - // A replacement for a lost device chooses as the start did, but this - // launch's marker stays as it is: the window started, so a later end is - // not a failed start. + // A replacement for a lost device chooses as the start did, and starts no + // marker. The window may have worked, so a later end is not a failed + // start; if it hadn't yet, the backend it started on failed. let replacing = SELECTIONS.fetch_add(1, Ordering::SeqCst) > 0; + if replacing { + lost_before_drawing(); + } let usable: Vec<&wgpu::Adapter> = adapters .iter() .filter(|adapter| surface.is_none_or(|surface| adapter.is_surface_supported(surface))) @@ -2064,6 +2105,43 @@ mod tests { assert!(run(&mut tracker, start, 0..20, 0)); } + #[test] + fn frames_skipped_for_a_lost_device_or_a_new_device_are_not_a_working_window() { + use eframe::egui_wgpu::winit::SKIPPED_FRAMES; + let start = Instant::now(); + let mut tracker = PresentTracker::new(); + assert!(!run(&mut tracker, start, 0..10, failures())); + // The device is lost: egui-wgpu skips frames without a surface error. + SKIPPED_FRAMES.fetch_add(3, Ordering::SeqCst); + assert!(!run(&mut tracker, start, 10..24, failures())); + // A new device is made, and it must draw for a while too. + SELECTIONS.fetch_add(1, Ordering::SeqCst); + assert!(!run(&mut tracker, start, 24..38, failures())); + assert!(run(&mut tracker, start, 38..60, failures())); + } + + #[test] + fn a_device_lost_before_the_window_works_fails_its_backend() { + let directory = temp_dir("lost-start"); + let owner = ProcessId::current(); + let gpu = "10de:2c02 32.0.15.9186"; + let started = marker(gpu, "dx12"); + assert!(save_marker(&directory, owner, &started)); + lose_launch(&directory, &started, owner); + let history = read_history(&directory); + assert_eq!(history.len(), 1); + assert_eq!( + (history[0].gpu.as_str(), history[0].backend.as_str()), + (gpu, "dx12") + ); + assert_eq!(history[0].failures, 1); + assert_eq!(history[0].fault.as_deref(), Some(DEVICE_LOST)); + assert!(!marker_path(&directory, owner).exists()); + // Not a driver crash: retried after clean launches, like any other end. + assert!(!history[0].in_driver()); + fs::remove_dir_all(directory).unwrap(); + } + #[test] fn a_minimized_window_is_not_confirmed() { let start = Instant::now(); From c2715bf20a2bc0dc24a2a9b3cf96a8c51a503f8f Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 15:58:03 -0500 Subject: [PATCH 057/226] Keep the device-loss test hook out of release builds The simulation in release_render_state, and the DX12 RemoveDevice behind it, now build only in debug builds and with the fault-injection feature, which also brings in the D3D12 bindings. A debug build without the feature loses the device with Device::destroy. --- host/Cargo.toml | 3 +-- host/src/gui/renderer.rs | 36 +++++++++++++++++++++--------------- host/src/startup/mod.rs | 2 ++ 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/host/Cargo.toml b/host/Cargo.toml index 6220732d..483eab72 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -51,7 +51,7 @@ socket2 = "0.5" [features] # Lets a release build obey ETERNAL_SIMULATE_STARTUP_FAILURE, for native # failure-path checks. Debug builds always do; shipped builds never do. -fault-injection = [] +fault-injection = ["windows/Win32_Graphics_Direct3D12"] # Lets a release build obey ETERNAL_USB_DIRECT, for the end-to-end scripts. # Debug builds always do; shipped builds never do. usb-direct = [] @@ -100,7 +100,6 @@ features = [ "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", - "Win32_Graphics_Direct3D12", "Win32_Graphics_Gdi", "Win32_Globalization", "Win32_System_Registry", diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 27136d70..7f4f3aaa 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -497,30 +497,36 @@ fn lose_launch(directory: &Path, marker: &Marker, owner: ProcessId) { /// Drop eframe's copy of the window's first device, which is kept for paint /// callbacks the window has none of. A lost device is only replaced once /// nothing holds it: D3D12 keeps one device per GPU in a process, and the -/// window's old swapchain stays with it. When [`crate::startup::SIMULATE_ENV`] -/// asks for a device loss, the copy is kept until then, and the device is lost -/// on purpose: DX12 removes it, as a driver reset does; other backends destroy -/// it. +/// window's old swapchain stays with it. pub(super) fn release_render_state(frame: &mut eframe::Frame) { - if crate::startup::device_loss_requested() && !crate::startup::simulate_device_loss_now() { - return; - } - let Some(render_state) = frame.wgpu_render_state.take() else { - return; - }; - if !crate::startup::device_loss_requested() { + #[cfg(any(debug_assertions, feature = "fault-injection"))] + if crate::startup::device_loss_requested() { + // The copy is kept until the simulated loss. + if crate::startup::simulate_device_loss_now() { + if let Some(render_state) = frame.wgpu_render_state.take() { + lose_device(&render_state.device); + } + } return; } + frame.wgpu_render_state = None; +} + +/// Lose the window's device on purpose, when [`crate::startup::SIMULATE_ENV`] +/// asks: with the `fault-injection` feature DX12 removes it, as a driver reset +/// does; otherwise it is destroyed. +#[cfg(any(debug_assertions, feature = "fault-injection"))] +fn lose_device(device: &wgpu::Device) { warn!("Simulated loss of the window's graphics device"); - #[cfg(windows)] - if remove_dx12_device(&render_state.device) { + #[cfg(all(windows, feature = "fault-injection"))] + if remove_dx12_device(device) { return; } - render_state.device.destroy(); + device.destroy(); } /// `ID3D12Device5::RemoveDevice` on a DX12 device; false on other backends. -#[cfg(windows)] +#[cfg(all(windows, feature = "fault-injection"))] fn remove_dx12_device(device: &wgpu::Device) -> bool { use windows::core::Interface; use windows::Win32::Graphics::Direct3D12::{ID3D12Device, ID3D12Device5}; diff --git a/host/src/startup/mod.rs b/host/src/startup/mod.rs index a4977226..24984341 100644 --- a/host/src/startup/mod.rs +++ b/host/src/startup/mod.rs @@ -586,12 +586,14 @@ pub(crate) fn simulate_window_failure() { } /// [`SIMULATE_ENV`] asks for `device-lost-later`. +#[cfg(any(debug_assertions, feature = "fault-injection"))] pub(crate) fn device_loss_requested() -> bool { simulated() == "device-lost-later" } /// True once, 15 s after the window works, when [`device_loss_requested`]: /// the window's graphics device is to be lost then. +#[cfg(any(debug_assertions, feature = "fault-injection"))] pub(crate) fn simulate_device_loss_now() -> bool { static DONE: AtomicBool = AtomicBool::new(false); device_loss_requested() From 1a3a452e8b63125e2b42890e7844b051be5dbd98 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 16:15:56 -0500 Subject: [PATCH 058/226] Make log-capturing host tests deterministic under parallel runs a_session_ended_with_a_fallback_in_one_pass_names_it failed in about 1 of 75 runs next to other tests, for two reasons. tracing caches, once per process, whether each log line is wanted. While only one subscriber is registered, a line's first run asks only the thread running it, so another test thread with no subscriber caches the summary line as unwanted and the capturing test records nothing. Every test that captures log lines now first calls logging::prepare_log_capture, which keeps a second subscriber registered so tracing asks all of them. The cable test starts and ends a session through act, which replaces the process-wide fallback slot; the summary then said fallback=none. It now holds FALLBACK_SLOT like the other tests that use the slot. --- host/src/audio/stage.rs | 1 + host/src/capture/recovery.rs | 1 + host/src/capture/second_screen.rs | 1 + host/src/capture/second_screen/memory.rs | 1 + host/src/diagnostics.rs | 2 ++ host/src/encoder/mod.rs | 2 ++ host/src/gpu_preference.rs | 1 + host/src/logging.rs | 41 ++++++++++++++++++++++++ host/src/network.rs | 1 + host/src/pairing.rs | 1 + host/src/stall.rs | 2 ++ host/src/startup/port.rs | 1 + host/src/telemetry/redact.rs | 1 + host/src/transport/mod.rs | 7 ++++ host/src/transport/session.rs | 1 + host/src/updates/tests.rs | 1 + host/src/vdd.rs | 1 + 17 files changed, 66 insertions(+) diff --git a/host/src/audio/stage.rs b/host/src/audio/stage.rs index 8b3dab70..8659b5ba 100644 --- a/host/src/audio/stage.rs +++ b/host/src/audio/stage.rs @@ -708,6 +708,7 @@ mod tests { .with_ansi(false) .with_writer(move || Captured(writer.clone())) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, run); let text = String::from_utf8_lossy(&buffer.lock()).into_owned(); text diff --git a/host/src/capture/recovery.rs b/host/src/capture/recovery.rs index 61f8cf03..1cc4232b 100644 --- a/host/src/capture/recovery.rs +++ b/host/src/capture/recovery.rs @@ -776,6 +776,7 @@ mod tests { let waits = RefCell::new(Vec::new()); let warnings = Arc::new(AtomicUsize::new(0)); let subscriber = tracing_subscriber::registry().with(Warnings(warnings.clone())); + crate::logging::prepare_log_capture(); let result = tracing::subscriber::with_default(subscriber, || { run_until_unblocked( &shared, diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 785d4fdd..c3844986 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -2948,6 +2948,7 @@ mod tests { .with_ansi(false) .with_writer(move || Sink(Arc::clone(&writer))) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { attach(&shared, &driver, ATTACH_TIMEOUT) }); diff --git a/host/src/capture/second_screen/memory.rs b/host/src/capture/second_screen/memory.rs index 72773103..88bfd7ac 100644 --- a/host/src/capture/second_screen/memory.rs +++ b/host/src/capture/second_screen/memory.rs @@ -3002,6 +3002,7 @@ mod tests { .with_ansi(false) .with_writer(move || sink.clone()) .finish(); + crate::logging::prepare_log_capture(); let value = tracing::subscriber::with_default(subscriber, act); let log = String::from_utf8(out.lock().clone()).unwrap(); (log, value) diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 999b6698..abe07cfa 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -942,6 +942,7 @@ pub(crate) mod tests { let shared = SharedControl::new(9876, 15_000_000); *shared.capture_target.lock() = CaptureTarget::VirtualExtended; let subscriber = tracing_subscriber::registry().with(FallbackWatch); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { tracing::info!(second_screen_fallback = "remembered", "before the session"); note_session_start(1, &shared); @@ -981,6 +982,7 @@ pub(crate) mod tests { session_id: Some(id), ..end(EndReason::Goodbye(ByeReason::UserDisconnect), None, 1_000) }; + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { *shared.capture_target.lock() = CaptureTarget::VirtualExtended; note_session_start(1, &shared); diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index a865edf5..9d600a92 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -2835,6 +2835,7 @@ mod tests { .with_ansi(false) .with_writer(move || Sink(Arc::clone(&writer))) .finish(); + crate::logging::prepare_log_capture(); let result = tracing::subscriber::with_default(subscriber, run); let lines = String::from_utf8(out.lock().clone()).unwrap(); (result, lines.lines().map(str::to_string).collect()) @@ -4156,6 +4157,7 @@ mod tests { let start = Instant::now(); let reads = Arc::new(AtomicU64::new(0)); let clock_reads = Arc::clone(&reads); + crate::logging::prepare_log_capture(); let opened = tracing::subscriber::with_default(subscriber, || { run_in( lanes, diff --git a/host/src/gpu_preference.rs b/host/src/gpu_preference.rs index 26c5b950..43f12221 100644 --- a/host/src/gpu_preference.rs +++ b/host/src/gpu_preference.rs @@ -1439,6 +1439,7 @@ mod tests { .with_writer(move || Sink(Arc::clone(&writer))) .finish(); let mut store = key(None); + crate::logging::prepare_log_capture(); let error = tracing::subscriber::with_default(subscriber, || { switch( &mut store, diff --git a/host/src/logging.rs b/host/src/logging.rs index 56d1f1b6..e3628c08 100644 --- a/host/src/logging.rs +++ b/host/src/logging.rs @@ -543,12 +543,51 @@ impl Visit for MessageVisitor { } } +/// Call before a test sets the subscriber that captures its log lines. +/// +/// `tracing` decides once, for the whole process, whether a log line is +/// wanted: the first time any thread runs it. While only one subscriber is +/// registered, it asks only the thread running the line, so a test thread +/// with no subscriber caches the line as unwanted and the test capturing it +/// on another thread never sees it. A second registered subscriber, nobody's +/// default, makes `tracing` ask every live subscriber instead. +#[cfg(test)] +pub(crate) fn prepare_log_capture() { + static ANOTHER: Lazy = + Lazy::new(|| tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default())); + Lazy::force(&ANOTHER); +} + #[cfg(test)] mod tests { use super::*; use std::sync::mpsc; use std::time::Duration; + /// A line another thread runs first, with no subscriber, still reaches + /// the capturing test. + #[test] + fn a_capture_sees_a_line_another_thread_ran_first() { + use tracing_subscriber::prelude::*; + fn line() { + tracing::info!("first run elsewhere"); + } + struct Count(Arc>); + impl tracing_subscriber::Layer for Count { + fn on_event(&self, _: &Event<'_>, _: Context<'_, S>) { + *self.0.lock() += 1; + } + } + prepare_log_capture(); + let count = Arc::new(Mutex::new(0)); + let _capture = tracing::subscriber::set_default( + tracing_subscriber::registry().with(Count(Arc::clone(&count))), + ); + std::thread::spawn(line).join().unwrap(); + line(); + assert_eq!(*count.lock(), 1); + } + #[test] fn mdns_success_announcements_do_not_hide_real_errors() { use tracing_subscriber::prelude::*; @@ -565,6 +604,7 @@ mod tests { .with_writer(move || writer.clone()) .with_filter(MdnsDedupFilter::new()), ); + prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { // The log-to-tracing bridge stores the original module in log.target. tracing::error!(target: "log", { log.target = "mdns_sd::service_daemon" }, @@ -815,6 +855,7 @@ mod tests { .with_ansi(false) .with_writer(move || writer.clone()), ); + prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { crate::diagnostics::install_ffmpeg_logging(); let format = diff --git a/host/src/network.rs b/host/src/network.rs index be458d75..d4715d4a 100644 --- a/host/src/network.rs +++ b/host/src/network.rs @@ -1288,6 +1288,7 @@ mod tests { let warned = AtomicBool::new(false); let levels = Levels::default(); let subscriber = tracing_subscriber::registry().with(levels.clone()); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { for _ in 0..3 { assert_eq!(fill(&warned, |_, _| 232), None); diff --git a/host/src/pairing.rs b/host/src/pairing.rs index 28954b09..b836e02e 100644 --- a/host/src/pairing.rs +++ b/host/src/pairing.rs @@ -415,6 +415,7 @@ mod tests { .with_ansi(false) .with_writer(move || Captured(writer.clone())) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, run); let text = String::from_utf8_lossy(&buffer.lock()).into_owned(); text diff --git a/host/src/stall.rs b/host/src/stall.rs index 9d8b5a20..a1402500 100644 --- a/host/src/stall.rs +++ b/host/src/stall.rs @@ -940,6 +940,7 @@ mod tests { .with_ansi(false) .with_writer(move || Sink(Arc::clone(&writer))) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { log_timed_out( Call::D3d11CreateDevice, @@ -954,6 +955,7 @@ mod tests { ); // An encoder's line gives that encoder's own limit. out.lock().clear(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default( tracing_subscriber::fmt() .with_ansi(false) diff --git a/host/src/startup/port.rs b/host/src/startup/port.rs index ba082fb4..d58562e3 100644 --- a/host/src/startup/port.rs +++ b/host/src/startup/port.rs @@ -604,6 +604,7 @@ mod tests { .with_ansi(false) .with_writer(move || writer.clone()) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, run); let bytes = lines.0.lock().unwrap().clone(); String::from_utf8(bytes).unwrap() diff --git a/host/src/telemetry/redact.rs b/host/src/telemetry/redact.rs index f3bf8d59..683cfb45 100644 --- a/host/src/telemetry/redact.rs +++ b/host/src/telemetry/redact.rs @@ -1321,6 +1321,7 @@ mod tests { .with_ansi(false) .with_writer(move || Captured(writer.clone())), ); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, log); let line = String::from_utf8(out.lock().clone()).unwrap(); line.trim_end().to_owned() diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index f98e1881..f38ba27c 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -1015,6 +1015,7 @@ mod tests { use tracing_subscriber::prelude::*; let _slot = crate::diagnostics::tests::FALLBACK_SLOT.lock(); let summaries = std::sync::Arc::new(parking_lot::Mutex::new(Vec::::new())); + crate::logging::prepare_log_capture(); let _log = tracing::subscriber::set_default( tracing_subscriber::registry() .with(crate::diagnostics::FallbackWatch) @@ -1253,11 +1254,16 @@ mod tests { /// session, and both go up, as at every other end of a session. #[cfg(not(windows))] #[tokio::test] + // The slot only keeps other tests out; nothing on this test's runtime + // takes it. + #[allow(clippy::await_holding_lock)] async fn a_usb_session_ended_by_its_cable_releases_held_input() { use crate::input::{ recorder, CaptureGeometry, Injection, KIND_KEY, KIND_MOUSE_ABS, PHASE_BEGAN, }; use eternal_wire::v2::control::{InputEvent, FEATURE_WANTS_INPUT}; + // The session starts and ends through `act`, which uses the fallback slot. + let _slot = crate::diagnostics::tests::FALLBACK_SLOT.lock(); let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); let shared = SharedControl::new(socket.local_addr().unwrap().port(), 15_000_000); shared.pairing.lock().required = false; @@ -1758,6 +1764,7 @@ mod tests { .with_writer(move || Sink(std::sync::Arc::clone(&writer))) .finish(); let error = std::io::Error::from(std::io::ErrorKind::ConnectionReset); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { log_send_failed(LinkId::Usb { device_id: 3 }, &error); log_send_failed(LinkId::Usb { device_id: 3 }, &error); diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 3a3b7b1b..50be83ab 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -3403,6 +3403,7 @@ mod tests { use tracing_subscriber::prelude::*; let counts = LeftOut::default(); let seen = counts.0.clone(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(tracing_subscriber::registry().with(counts), || { session.tick(&config, false, now + Duration::from_secs(30)); session.tick(&config, false, now + REFUSAL_MINUTE); diff --git a/host/src/updates/tests.rs b/host/src/updates/tests.rs index 6665c6de..f7b0dac4 100644 --- a/host/src/updates/tests.rs +++ b/host/src/updates/tests.rs @@ -1716,6 +1716,7 @@ fn logged(run: impl FnOnce()) -> String { .with_ansi(false) .with_writer(move || sink.clone()) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, run); let text = String::from_utf8(out.lock().clone()).unwrap(); text diff --git a/host/src/vdd.rs b/host/src/vdd.rs index 23e3cb38..1ce65f16 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -819,6 +819,7 @@ mod tests { .with_max_level(tracing::Level::DEBUG) .with_writer(std::io::sink) .finish(); + crate::logging::prepare_log_capture(); let ran = tracing::subscriber::with_default(subscriber, || { tasks.set_enabled( true, From 903380fac5b4dc0cf49899b939e193e9a405acd3 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 16:22:59 -0500 Subject: [PATCH 059/226] Make the input restart and late-input tests wait on the queue, not the clock input_restart slept 300 ms and took it that the new stream's press was then queued behind the held one. On a busy machine it could release first and pass without testing anything. It now waits for a test-hooks count of queued input events to rise, and checks Windows still holds the old input. The late-input unit test treated anything older than 50 ms as late, so a slow pick-up of the fresh event could turn it late. Its stale events are now queued as though 20 s ago, against a 10 s limit. --- host/src/input/injector.rs | 41 +++++++++++++++++++++++++++++++++---- host/tests/input_restart.rs | 26 ++++++++++------------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs index 6fe7bc45..bf4c7958 100644 --- a/host/src/input/injector.rs +++ b/host/src/input/injector.rs @@ -182,6 +182,10 @@ struct State { /// Releases that came while the queue was full, each now part of a /// release of everything. folded: u64, + /// Input events queued since the thread started, for tests that must + /// know one is behind another. + #[cfg(any(test, feature = "test-hooks"))] + relays_queued: u64, } /// What the transport's handles and the thread share. @@ -233,6 +237,10 @@ impl Shared { } return; } + #[cfg(any(test, feature = "test-hooks"))] + { + state.relays_queued += u64::from(matches!(command, Command::Relay { .. })); + } state.commands.push_back(Queued { generation, command, @@ -288,6 +296,14 @@ impl InputThread { thread } + /// Input events queued so far, merged and dropped ones aside. + #[cfg(any(test, feature = "test-hooks"))] + pub fn relays_queued(&self) -> u64 { + self.shared + .get() + .map_or(0, |shared| shared.state.lock().relays_queued) + } + /// At quit: wait up to `limit` for what is queued, the last stream's /// releases among it, to reach Windows. Whether it did. pub fn finish(&self, limit: Duration) -> bool { @@ -692,6 +708,21 @@ mod tests { .relay(1, event(kind, phase, id), OUTPUT, false); } + /// Queued as though at `at`. + fn send_at(&self, kind: u8, phase: u8, id: u32, at: Instant) { + let injector = self.injector.as_ref().unwrap(); + injector.shared.push( + injector.generation, + Command::Relay { + session_id: 1, + event: event(kind, phase, id), + output: OUTPUT, + late: false, + }, + at, + ); + } + /// Holds Windows in the first event's call until `go`. fn stuck(&self) { self.send(KIND_KEY, PHASE_BEGAN, 0); @@ -832,14 +863,16 @@ mod tests { #[test] fn input_that_waited_too_long_goes_as_late_and_releases_still_land() { + // A limit no pick-up under load comes near, and input queued as if + // long ago: no sleep decides which is late. let harness = harness(Limits { - stale_after: Duration::from_millis(50), + stale_after: Duration::from_secs(10), ..Limits::default() }); harness.stuck(); - harness.send(KIND_KEY, PHASE_ENDED, 1); - harness.send(KIND_MOUSE_ABS, PHASE_BEGAN, 2); - std::thread::sleep(Duration::from_millis(100)); + let long_ago = Instant::now() - Duration::from_secs(20); + harness.send_at(KIND_KEY, PHASE_ENDED, 1, long_ago); + harness.send_at(KIND_MOUSE_ABS, PHASE_BEGAN, 2, long_ago); harness.go.send(()).unwrap(); // Fresh input after Windows came back is not late. harness.wait_for(3); diff --git a/host/tests/input_restart.rs b/host/tests/input_restart.rs index 95db5a51..fac0d767 100644 --- a/host/tests/input_restart.rs +++ b/host/tests/input_restart.rs @@ -211,25 +211,21 @@ fn a_restart_while_windows_holds_input_never_releases_the_new_generations_keys() stop(&shared, old); let new = start(&shared, port); - // The iPad, still in its session, presses A again on the new stream. - let pressed_again = ipad.event_id + 1; - let started = Instant::now(); - while shared.transport_beat.read().is_none() || ipad.event_id < pressed_again { - assert!( - started.elapsed() < Duration::from_secs(5), - "the new loop starts" - ); - if shared.transport_beat.read().is_some() { - ipad.press(KEY_A); - } - std::thread::sleep(Duration::from_millis(20)); - } - std::thread::sleep(Duration::from_millis(300)); + // The iPad, still in its session, presses A again on the new stream, + // and the press is queued while Windows still holds the old one's. + wait_until("the new loop starts", || { + shared.transport_beat.read().is_some() + }); + let queued = shared.input.relays_queued(); + ipad.press(KEY_A); + wait_until("the new press is queued", || { + shared.input.relays_queued() > queued + }); + assert_eq!(hold::waiting(), 1, "Windows still holds the old input"); // Windows answers again. Once everything queued has landed, A is down. hold::release(); assert!(shared.input.finish(Duration::from_secs(5))); - std::thread::sleep(Duration::from_millis(200)); let injected = recorder::take(); assert_eq!(a_is_down(&injected), Some(true), "{injected:?}"); From d57d646e3d48378f6323cc1d973dc450ff98e6a7 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 16:24:11 -0500 Subject: [PATCH 060/226] Merge a backed-up move into the newest of its kind, not only the last one A move merged only into a move of the same kind queued just before it. A Pencil hovering while the pointer moves sends the two kinds interleaved, about 250 a second, so nothing merged and a stuck Windows filled the queue within seconds. A move now replaces the newest queued move of the same kind, session and buttons, as long as no press, release, reset or other generation follows it. The merged move goes last, so each kind's newest moves keep the order they came in. interleaved_moves_of_different_kinds_each_merge_up_to_the_last_edge queues alternating Pencil and pointer hovers past the merge point, then a press and more of both. Before this change every one of them went to Windows. --- host/src/input/injector.rs | 95 +++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs index bf4c7958..9311015d 100644 --- a/host/src/input/injector.rs +++ b/host/src/input/injector.rs @@ -14,8 +14,9 @@ //! a stuck one. //! //! The queue is bounded. While Windows takes nothing, a pointer or Pencil -//! move replaces the move queued just before it, so a backlog never replays -//! a path the iPad already left. Input that waited longer than +//! move replaces the newest queued move of its kind that no press or +//! release follows, so a backlog never replays a path the iPad already +//! left. Input that waited longer than //! [`INPUT_LATE`] goes to the relay as late, which lands only a release of //! what the PC holds, as for input that arrived late from the iPad. Ends, //! key releases and session resets are never merged, and never lost: at @@ -38,12 +39,13 @@ use crate::transport::session::INPUT_LATE; /// Commands kept at most, besides one start and one end per stream /// generation. A second of a Pencil at 240 Hz with every other kind of -/// input; while Windows is stuck, moves merge and the queue grows only by -/// presses and releases. +/// input. While Windows is stuck, each kind of move keeps only its newest +/// after the last press or release, so the queue grows by presses and +/// releases. const CAPACITY: usize = 1024; -/// From this many waiting commands, a move replaces the move before it. -/// Below it every sample goes to Windows, as the Pencil's ink needs. +/// From this many waiting commands, a move replaces the newest move of its +/// kind. Below it every sample goes to Windows, as the Pencil's ink needs. const MERGE_FROM: usize = 32; /// How often a native pen contact is refreshed; Windows expires a still one. @@ -152,6 +154,15 @@ impl Command { && event.buttons == earlier_event.buttons } + /// A press, release, reset or generation change: what a move must not + /// pass. + fn is_edge(&self) -> bool { + match self { + Command::Relay { event, .. } => event.phase != PHASE_MOVED, + Command::Begin { .. } | Command::Reset | Command::End => true, + } + } + /// An end or reset: what keeps the PC from holding a key or button. fn releases(&self) -> bool { match self { @@ -207,17 +218,27 @@ impl Shared { let mut state = self.state.lock(); let waiting = state.commands.len(); if waiting >= self.limits.merge_from { - if let Some(last) = state.commands.back_mut() { - if last.generation == generation - && !last.then_reset - && command.merges_with(&last.command) - { - // The newer sample, in the older one's place: it has - // waited as long. - last.command = command; - state.merged += 1; - return; - } + // The newest move of this kind, unless a press or release, or + // another generation, came after it. + let merges = state + .commands + .iter() + .enumerate() + .rev() + .take_while(|(_, queued)| { + queued.generation == generation + && !queued.then_reset + && !queued.command.is_edge() + }) + .find(|(_, queued)| command.merges_with(&queued.command)) + .map(|(index, _)| index); + if let Some(index) = merges { + // It goes last, where it came: the other kinds' newest + // moves keep their order before it. + let earlier = state.commands.remove(index).expect("found above"); + state.commands.push_back(Queued { command, ..earlier }); + state.merged += 1; + return; } } // A generation's start and end always go in: there is one of each @@ -861,6 +882,46 @@ mod tests { harness.settled(expected.len()); } + /// A Pencil hovering while the pointer moves sends both kinds of move + /// interleaved, about 250 a second. While Windows is stuck each kind + /// keeps only its newest move after the last press or release, in the + /// order those newest moves came. + #[test] + fn interleaved_moves_of_different_kinds_each_merge_up_to_the_last_edge() { + let harness = harness(Limits { + merge_from: 4, + ..Limits::default() + }); + harness.stuck(); + for id in 1..=104 { + let kind = if id % 2 == 1 { KIND_PENCIL } else { KIND_HOVER }; + harness.send(kind, PHASE_MOVED, id); + } + harness.send(KIND_MOUSE_ABS, PHASE_BEGAN, 200); + for id in 201..=204 { + let kind = if id % 2 == 1 { KIND_PENCIL } else { KIND_HOVER }; + harness.send(kind, PHASE_MOVED, id); + } + harness.send(KIND_MOUSE_ABS, PHASE_ENDED, 300); + harness.injector.as_ref().unwrap().reset(); + harness.go.send(()).unwrap(); + let expected = vec![ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_PENCIL, PHASE_MOVED, 1), + relayed(KIND_HOVER, PHASE_MOVED, 2), + relayed(KIND_PENCIL, PHASE_MOVED, 103), + relayed(KIND_HOVER, PHASE_MOVED, 104), + // Moves never merge across a press or release. + relayed(KIND_MOUSE_ABS, PHASE_BEGAN, 200), + relayed(KIND_PENCIL, PHASE_MOVED, 203), + relayed(KIND_HOVER, PHASE_MOVED, 204), + relayed(KIND_MOUSE_ABS, PHASE_ENDED, 300), + Call::Reset, + ]; + assert!(harness.thread.finish(Duration::from_secs(5))); + assert_eq!(*harness.calls.lock(), expected); + } + #[test] fn input_that_waited_too_long_goes_as_late_and_releases_still_land() { // A limit no pick-up under load comes near, and input queued as if From 5d661f0568b2f1fea25fbadb920f7f22fb6c1410 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 16:25:33 -0500 Subject: [PATCH 061/226] Log a Windows input call that has been held for 5 s while it is still held A call that returned late was logged when it returned. One that never returns logged nothing, and the log only showed input going missing. The input thread notes when it enters Windows, for a command or a pen refresh. The next input the transport queues after 5 s logs one line, "Windows has held input for seconds and has not returned", with held_s and the commands waiting. It logs once per held call. a_call_windows_holds_for_seconds_is_logged_once checks that one line comes from the input queued 6 s in, and none more. --- host/src/input/injector.rs | 69 +++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs index 9311015d..6feb528e 100644 --- a/host/src/input/injector.rs +++ b/host/src/input/injector.rs @@ -54,6 +54,10 @@ const PEN_TICK: Duration = Duration::from_millis(50); /// A Windows call this slow is logged: it would have stalled the stream. const SLOW_CALL: Duration = Duration::from_millis(250); +/// A Windows call held this long is logged while it is still held, from +/// the transport's next input: one that never returns would log nothing. +const HELD_LOGGED: Duration = Duration::from_secs(5); + /// Waits this short are timed precisely, for pen frames held back to keep /// them `FRAME_SPACING` apart; a condition variable wakes in whole /// milliseconds or worse on Windows. @@ -186,8 +190,13 @@ struct Queued { #[derive(Debug, Default)] struct State { commands: VecDeque, - /// The thread is handing a command to Windows. - busy: bool, + /// Since when the thread has been in Windows, with a command or a pen + /// refresh. + in_windows_since: Option, + /// That call was logged as held. + held_logged: bool, + #[cfg(test)] + held_logs: u64, merged: u64, dropped: u64, /// Releases that came while the queue was full, each now part of a @@ -199,6 +208,13 @@ struct State { relays_queued: u64, } +impl State { + fn leave_windows(&mut self) { + self.in_windows_since = None; + self.held_logged = false; + } +} + /// What the transport's handles and the thread share. #[derive(Debug)] struct Shared { @@ -217,6 +233,20 @@ impl Shared { fn push(&self, generation: u64, command: Command, now: Instant) { let mut state = self.state.lock(); let waiting = state.commands.len(); + if let Some(since) = state.in_windows_since { + let held = now.saturating_duration_since(since); + if held >= HELD_LOGGED && !state.held_logged { + state.held_logged = true; + #[cfg(test)] + { + state.held_logs += 1; + } + warn!( + held_s = held.as_secs(), + waiting, "Windows has held input for seconds and has not returned" + ); + } + } if waiting >= self.limits.merge_from { // The newest move of this kind, unless a press or release, or // another generation, came after it. @@ -333,7 +363,7 @@ impl InputThread { }; let deadline = Instant::now() + limit; let mut state = shared.state.lock(); - while state.busy || !state.commands.is_empty() { + while state.in_windows_since.is_some() || !state.commands.is_empty() { if shared.idle.wait_until(&mut state, deadline).timed_out() { warn!( waiting = state.commands.len(), @@ -487,10 +517,10 @@ fn run(shared: &Shared, sink: &mut S) { loop { let next = { let mut state = shared.state.lock(); - state.busy = false; + state.leave_windows(); loop { if let Some(queued) = state.commands.pop_front() { - state.busy = true; + state.in_windows_since = Some(Instant::now()); break Next::Command(queued); } shared.idle.notify_all(); @@ -607,7 +637,9 @@ fn run(shared: &Shared, sink: &mut S) { // input datagram, and Windows can hold the refresh. let output = *current.geometry.lock(); let started = Instant::now(); + shared.state.lock().in_windows_since = Some(started); sink.keep_alive(output); + shared.state.lock().leave_windows(); note_slow(started, "pen_refresh"); } } @@ -1066,6 +1098,33 @@ mod tests { assert_eq!(harness.wait_for(4)[3], Call::Reset); } + /// A Windows call that never returns is logged once, from the input + /// that queues behind it, and again only for the next held call. + #[test] + fn a_call_windows_holds_for_seconds_is_logged_once() { + let harness = harness(Limits::default()); + harness.stuck(); + let logs = || { + harness + .injector + .as_ref() + .unwrap() + .shared + .state + .lock() + .held_logs + }; + let later = |seconds| Instant::now() + Duration::from_secs(seconds); + harness.send(KIND_HOVER, PHASE_MOVED, 1); + assert_eq!(logs(), 0); + harness.send_at(KIND_HOVER, PHASE_MOVED, 2, later(6)); + harness.send_at(KIND_HOVER, PHASE_MOVED, 3, later(60)); + assert_eq!(logs(), 1); + harness.go.send(()).unwrap(); + assert!(harness.thread.finish(Duration::from_secs(5))); + assert_eq!(logs(), 1); + } + #[test] fn quitting_waits_for_queued_releases_but_not_past_its_limit() { let harness = harness(Limits::default()); From d85993ec408c13b372b9d72cb3719bf2899bbec7 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 16:27:31 -0500 Subject: [PATCH 062/226] Release held input at quit even when the last stream never queued its end Quitting waited for the queue to drain, and relied on the last stream's end being queued. A transport that hadn't stopped within the supervisor's 3 s never queued it, and the app exited with keys or buttons down. finish now queues a release of everything first, behind whatever is queued, whichever stream it comes from, then waits up to its limit. Tests that only wait for the thread use wait_idle, which releases nothing. quitting_releases_what_a_running_stream_holds quits with a key down and the stream's handle alive. Before this change Windows got no release. --- host/src/input/injector.rs | 76 ++++++++++++++++++++++++++++++------- host/src/main.rs | 4 +- host/tests/input_restart.rs | 2 +- 3 files changed, 66 insertions(+), 16 deletions(-) diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs index 6feb528e..e804355e 100644 --- a/host/src/input/injector.rs +++ b/host/src/input/injector.rs @@ -120,6 +120,9 @@ enum Command { Reset, /// The generation's stream stopped: release what it holds. End, + /// The app quits: release what the PC holds, whichever stream it is + /// from, and take no more input. + ReleaseAll, } impl Command { @@ -163,14 +166,14 @@ impl Command { fn is_edge(&self) -> bool { match self { Command::Relay { event, .. } => event.phase != PHASE_MOVED, - Command::Begin { .. } | Command::Reset | Command::End => true, + Command::Begin { .. } | Command::Reset | Command::End | Command::ReleaseAll => true, } } /// An end or reset: what keeps the PC from holding a key or button. fn releases(&self) -> bool { match self { - Command::Reset | Command::End => true, + Command::Reset | Command::End | Command::ReleaseAll => true, Command::Begin { .. } => false, Command::Relay { event, .. } => event.phase > PHASE_MOVED, } @@ -273,7 +276,10 @@ impl Shared { } // A generation's start and end always go in: there is one of each // per stream. - let boundary = matches!(command, Command::Begin { .. } | Command::End); + let boundary = matches!( + command, + Command::Begin { .. } | Command::End | Command::ReleaseAll + ); if waiting >= self.limits.capacity && !boundary { let barrier = state .commands @@ -355,9 +361,32 @@ impl InputThread { .map_or(0, |shared| shared.state.lock().relays_queued) } - /// At quit: wait up to `limit` for what is queued, the last stream's - /// releases among it, to reach Windows. Whether it did. + /// At quit: release what the PC holds, after everything queued, and + /// wait up to `limit` for it to reach Windows. Whether it did. A stream + /// still running need not have queued its end. pub fn finish(&self, limit: Duration) -> bool { + let Some(shared) = self.shared.get() else { + return true; + }; + shared.push(0, Command::ReleaseAll, Instant::now()); + let done = self.idle(limit); + if !done { + warn!( + waiting = shared.state.lock().commands.len(), + "Quitting before Windows took the last input" + ); + } + done + } + + /// Wait up to `limit` for everything queued to reach Windows, without + /// releasing anything. + #[cfg(any(test, feature = "test-hooks"))] + pub fn wait_idle(&self, limit: Duration) -> bool { + self.idle(limit) + } + + fn idle(&self, limit: Duration) -> bool { let Some(shared) = self.shared.get() else { return true; }; @@ -365,10 +394,6 @@ impl InputThread { let mut state = shared.state.lock(); while state.in_windows_since.is_some() || !state.commands.is_empty() { if shared.idle.wait_until(&mut state, deadline).timed_out() { - warn!( - waiting = state.commands.len(), - "Quitting before Windows took the last input" - ); return false; } } @@ -608,6 +633,10 @@ fn run(shared: &Shared, sink: &mut S) { current.relayed = true; } } + Command::ReleaseAll => { + current = None; + sink.reset(); + } // An ended generation's: its end released what it held. Command::End | Command::Reset | Command::Relay { .. } => {} } @@ -799,7 +828,7 @@ mod tests { /// Nothing more reaches Windows. fn settled(&self, count: usize) { - assert!(self.thread.finish(Duration::from_secs(5))); + assert!(self.thread.wait_idle(Duration::from_secs(5))); assert_eq!(self.calls.lock().len(), count, "{:?}", self.calls.lock()); } } @@ -950,7 +979,7 @@ mod tests { relayed(KIND_MOUSE_ABS, PHASE_ENDED, 300), Call::Reset, ]; - assert!(harness.thread.finish(Duration::from_secs(5))); + assert!(harness.thread.wait_idle(Duration::from_secs(5))); assert_eq!(*harness.calls.lock(), expected); } @@ -1125,6 +1154,20 @@ mod tests { assert_eq!(logs(), 1); } + /// A quit while a stream still runs, its end not yet queued, still + /// releases what the PC holds. + #[test] + fn quitting_releases_what_a_running_stream_holds() { + let harness = harness(Limits::default()); + harness.stuck(); + harness.go.send(()).unwrap(); + assert!(harness.thread.finish(Duration::from_secs(5))); + assert_eq!( + *harness.calls.lock(), + [relayed(KIND_KEY, PHASE_BEGAN, 0), Call::Reset] + ); + } + #[test] fn quitting_waits_for_queued_releases_but_not_past_its_limit() { let harness = harness(Limits::default()); @@ -1134,8 +1177,15 @@ mod tests { assert!(!harness.thread.finish(Duration::from_millis(50))); assert!(started.elapsed() < Duration::from_secs(1)); harness.go.send(()).unwrap(); - assert!(harness.thread.finish(Duration::from_secs(5))); - assert_eq!(harness.calls.lock().len(), 2); + assert!(harness.thread.wait_idle(Duration::from_secs(5))); + assert_eq!( + *harness.calls.lock(), + [ + relayed(KIND_KEY, PHASE_BEGAN, 0), + relayed(KIND_KEY, PHASE_ENDED, 1), + Call::Reset, + ] + ); } /// The hop the thread adds: from the transport queueing an event to the diff --git a/host/src/main.rs b/host/src/main.rs index 18e3bfee..f27673e9 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -367,8 +367,8 @@ fn main() -> Result<(), Box> { tracing::warn!("Supervisor did not shut down within 3s — forcing exit"); } - // The last stream's releases of held keys and buttons reach Windows, - // unless Windows is stuck. + // Release every key and button the iPad left held, behind whatever the + // last stream queued, unless Windows is stuck. gui_control .shared .input diff --git a/host/tests/input_restart.rs b/host/tests/input_restart.rs index fac0d767..14e5c5a8 100644 --- a/host/tests/input_restart.rs +++ b/host/tests/input_restart.rs @@ -225,7 +225,7 @@ fn a_restart_while_windows_holds_input_never_releases_the_new_generations_keys() // Windows answers again. Once everything queued has landed, A is down. hold::release(); - assert!(shared.input.finish(Duration::from_secs(5))); + assert!(shared.input.wait_idle(Duration::from_secs(5))); let injected = recorder::take(); assert_eq!(a_is_down(&injected), Some(true), "{injected:?}"); From 63057977f3e1cb6f521f51548ca1242e8c2f245d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 16:34:47 -0500 Subject: [PATCH 063/226] Format the startup Disable test --- host/src/vdd.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/host/src/vdd.rs b/host/src/vdd.rs index d0f2b429..2ed43730 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -599,9 +599,12 @@ mod tests { let tasks = DriverTasks::new(); assert!(tasks.found_off(DeviceState::Problem(22))); // Neither the startup cleanup nor a quit before any stream runs one. - assert!(tasks.set_enabled(false, Purpose::Cleanup, || {}, |_, _| { - panic!("ran the Disable task for a display that is off") - })); + assert!(tasks.set_enabled( + false, + Purpose::Cleanup, + || {}, + |_, _| { panic!("ran the Disable task for a display that is off") } + )); // The first Enable runs, and so does the Disable after it. let ran = Cell::new(Vec::new()); let run = |enabled: bool, _: &mut dyn FnMut()| { @@ -627,10 +630,15 @@ mod tests { let tasks = DriverTasks::new(); assert!(!tasks.found_off(device), "{device:?}"); let ran = Cell::new(false); - tasks.set_enabled(false, Purpose::Cleanup, || {}, |_, _| { - ran.set(true); - TaskOutcome::Done - }); + tasks.set_enabled( + false, + Purpose::Cleanup, + || {}, + |_, _| { + ran.set(true); + TaskOutcome::Done + }, + ); assert!(ran.get(), "no Disable for {device:?}"); } } From 732e9fa97f40399da7a465511a23a4ee7a60cb27 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 16:34:47 -0500 Subject: [PATCH 064/226] Read a UTF-16 virtual display settings file and write it back as UTF-8 The host read vdd_settings.xml as UTF-8 only. A file another tool saved as UTF-16 (Windows PowerShell's Out-File does) failed with "stream did not contain valid UTF-8": the second screen kept the driver's old modes (800x600 on one PC) and the graphics card was never named, on a hybrid laptop. The file is now read as UTF-8, with or without a byte order mark, or as UTF-16 little or big endian, with a byte order mark or, as XML allows, without one. When the host writes it, it writes UTF-8, which the driver's own file and writer use. A UTF-16 document's encoding="utf-16" declaration becomes utf-8 with it, so the declaration names what is on disk. A file in any other encoding, or broken UTF-16, is still left alone, and the error says so. --- host/src/vdd_settings.rs | 186 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 183 insertions(+), 3 deletions(-) diff --git a/host/src/vdd_settings.rs b/host/src/vdd_settings.rs index 9f132195..02487971 100644 --- a/host/src/vdd_settings.rs +++ b/host/src/vdd_settings.rs @@ -389,7 +389,7 @@ pub fn with_gpu(xml: &str, gpu: &str) -> Option { /// would leave it none. A file that does not parse as expected is left /// alone, and the error says why. pub fn pin_gpu_in(path: &Path, gpu: &str) -> io::Result { - let current = std::fs::read_to_string(path)?; + let current = read_settings(path)?; let updated = gpu_edit(¤t, gpu) .map_err(|reason| io::Error::new(io::ErrorKind::InvalidData, reason))?; if updated == current { @@ -551,7 +551,7 @@ pub fn apply_pin( wanted: Option<&str>, ) -> io::Result { let invalid = |reason| io::Error::new(io::ErrorKind::InvalidData, reason); - let current = std::fs::read_to_string(path)?; + let current = read_settings(path)?; let named = named_gpu(¤t).map_err(invalid)?; let ours = recorded_pins(record); let ours: Vec<&str> = ours.iter().map(String::as_str).collect(); @@ -622,7 +622,7 @@ pub fn write_with_pin( modes: &[VddMode], wanted: Option<&str>, ) -> io::Result { - let named = match std::fs::read_to_string(path) { + let named = match read_settings(path) { Ok(current) => match named_gpu(¤t) { Ok(named) => named, Err(_) if owned && wanted.is_some() => None, @@ -662,6 +662,68 @@ pub fn pin_result( } } +/// The settings file as text. Another tool may have saved it as UTF-16 +/// (Windows PowerShell's `Out-File` does), with a byte order mark or, as XML +/// allows, without one; it is read and, when this host writes it, written +/// back as UTF-8, the encoding the driver's own file has and every file this +/// host writes. A UTF-16 document's `encoding="utf-16"` declaration becomes +/// `utf-8` with it, so the declaration always names what is on disk. UTF-8 +/// keeps any byte order mark. Any other encoding is left alone, and the +/// error says so. +fn read_settings(path: &Path) -> io::Result { + decode_settings(&std::fs::read(path)?).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "the settings file is neither UTF-8 nor UTF-16", + ) + }) +} + +fn decode_settings(bytes: &[u8]) -> Option { + let (units, big_endian): (&[u8], bool) = match bytes { + [0xFF, 0xFE, rest @ ..] => (rest, false), + [0xFE, 0xFF, rest @ ..] => (rest, true), + // " (bytes, false), + [0, b'<', 0, b'?', ..] => (bytes, true), + _ => return String::from_utf8(bytes.to_vec()).ok(), + }; + if units.len() % 2 != 0 { + return None; + } + let units = units.chunks_exact(2).map(|pair| { + let pair = [pair[0], pair[1]]; + if big_endian { + u16::from_be_bytes(pair) + } else { + u16::from_le_bytes(pair) + } + }); + let text = char::decode_utf16(units) + .collect::>() + .ok()?; + Some(declared_utf8(text)) +} + +/// `xml` with its declaration's UTF-16 encoding named as UTF-8, quote and +/// all else unchanged. +fn declared_utf8(xml: String) -> String { + let Some(end) = xml.starts_with("")).flatten() else { + return xml; + }; + let declaration = &xml[..end]; + for quote in ['"', '\''] { + for label in ["utf-16", "utf-16le", "utf-16be"] { + let pattern = format!("encoding={quote}{label}{quote}"); + if let Some(at) = declaration.to_ascii_lowercase().find(&pattern) { + let utf8 = format!("encoding={quote}utf-8{quote}"); + return format!("{}{utf8}{}", &xml[..at], &xml[at + pattern.len()..]); + } + } + } + xml +} + /// Write atomically (tmp + rename) so the driver never reads a torn file. /// Creates the directory if the driver hasn't made it yet. pub fn write_to(path: &Path, modes: &[VddMode], gpu: Option<&str>) -> io::Result<()> { @@ -1552,4 +1614,122 @@ mod tests { assert!(!path.with_extension("xml.tmp").exists(), "tmp cleaned up"); let _ = std::fs::remove_dir_all(&dir); } + + /// `text` as UTF-16, little or big endian, with or without a byte order + /// mark. + fn utf16(text: &str, big_endian: bool, bom: bool) -> Vec { + let units = bom.then_some(0xFEFF).into_iter().chain(text.encode_utf16()); + units + .flat_map(|unit| { + if big_endian { + unit.to_be_bytes() + } else { + unit.to_le_bytes() + } + }) + .collect() + } + + /// Field triage round 2: a settings file saved as UTF-16 was "not valid + /// UTF-8", so the second screen kept the driver's old modes and the card + /// was never named. + #[test] + fn a_utf16_or_bom_settings_file_is_read_and_written_back_as_utf8() { + let dir = std::env::temp_dir().join(format!("vdd-encoding-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let (path, record) = ( + dir.join("vdd_settings.xml"), + dir.join("app").join("vdd-pin.txt"), + ); + const INTEL: &str = "Intel(R) HD Graphics 630"; + let declared16 = VENDOR.replace("encoding='utf-8'", "encoding='UTF-16'"); + let fixtures: Vec<(&str, Vec)> = vec![ + ("utf-16 le, bom", utf16(&declared16, false, true)), + ("utf-16 be, bom", utf16(&declared16, true, true)), + ("utf-16 le, no bom", utf16(&declared16, false, false)), + ("utf-16 be, no bom", utf16(&declared16, true, false)), + ("utf-16 le, bom, utf-8 declared", utf16(VENDOR, false, true)), + ( + "utf-8, bom", + [&[0xEF, 0xBB, 0xBF][..], VENDOR.as_bytes()].concat(), + ), + ]; + for (name, bytes) in &fixtures { + // Only the card changes, as the driver will read it. + std::fs::write(&path, bytes).unwrap(); + let _ = std::fs::remove_file(&record); + let pinned = apply_pin(&path, &record, false, Some(INTEL)); + assert_eq!(pinned.ok(), Some(PinOutcome::Named), "{name}"); + let written = std::fs::read(&path).unwrap(); + let text = String::from_utf8(written.clone()).expect(name); + let body = text.strip_prefix('\u{feff}').unwrap_or(&text); + assert!( + body.starts_with("\r\n"), + "{name}: {body:?}" + ); + assert_eq!(named_gpu(&text), Ok(Some(INTEL.to_string())), "{name}"); + assert_eq!( + text.starts_with('\u{feff}'), + name.starts_with("utf-8"), + "{name}" + ); + assert_eq!( + body, + VENDOR.replace(">default<", &format!(">{INTEL}<")), + "{name}" + ); + + // Match the iPad's resolution writes the whole file, as UTF-8. + std::fs::write(&path, bytes).unwrap(); + let _ = std::fs::remove_file(&record); + let modes = modes_for_client(1640, 2360, 60); + let written = write_with_pin(&path, &record, false, &modes, Some(INTEL)); + assert_eq!(written.ok(), Some(PinOutcome::Named), "{name}"); + let text = std::fs::read_to_string(&path).expect(name); + assert_eq!(text, render(&modes, Some(INTEL)), "{name}"); + } + std::fs::remove_dir_all(&dir).unwrap(); + } + + /// A file in another encoding, or broken UTF-16, may hold anything: + /// it stays byte for byte, and the error says why. + #[test] + fn a_settings_file_in_another_encoding_is_left_alone() { + let dir = std::env::temp_dir().join(format!("vdd-encoding-other-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let (path, record) = ( + dir.join("vdd_settings.xml"), + dir.join("app").join("vdd-pin.txt"), + ); + let ansi = VENDOR + .replace(">default<", ">Radeon\u{AE}<") + .replace('\u{AE}', ""); + let mut ansi = ansi.into_bytes(); + let at = ansi.windows(7).position(|w| w == b"Radeon<").unwrap() + 6; + ansi.insert(at, 0xAE); + let mut odd = utf16(VENDOR, false, true); + odd.push(b' '); + let mut lone = utf16(VENDOR, false, true); + lone.extend_from_slice(&0xD800u16.to_le_bytes()); + for (name, bytes) in [ + ("ansi", ansi), + ("odd utf-16", odd), + ("lone surrogate", lone), + ] { + std::fs::write(&path, &bytes).unwrap(); + let pinned = apply_pin(&path, &record, false, Some("Intel(R) HD Graphics 630")); + let error = pinned.expect_err(name); + assert_eq!(error.kind(), io::ErrorKind::InvalidData, "{name}"); + assert_eq!( + error.to_string(), + "the settings file is neither UTF-8 nor UTF-16" + ); + let written = write_with_pin(&path, &record, false, &default_modes(), Some("x")); + assert!(written.is_err(), "{name}"); + assert_eq!(std::fs::read(&path).unwrap(), bytes, "{name}"); + } + std::fs::remove_dir_all(&dir).unwrap(); + } } From e97697a0606fae419bd63c6feb2d4b30d93343e3 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 12:30:00 -0500 Subject: [PATCH 065/226] Keep a stuck graphics driver from holding the window at start Startup's GPU detection runs under the stall guard in a new gpu_detect lane (1 s warning, 4 s limit), as in #166. The same lane now also holds every other graphics call before the window's first paint: the banner's and the window's screen lists, the renderer's DXGI adapter list, and a wgpu probe that lists the window's adapters on another thread before the window's thread makes its first graphics call. After one call hangs, the others are refused at once. While the probe does not return, or an earlier call is still stuck, a message box that Windows draws without the graphics driver says why the window is not there yet, and the window opens once the driver answers. The marker names the probe's thread while it lists adapters, so a driver crash there still counts against the window's backend. A detection given up on keeps what it finds when it returns. The next pipeline, as after Restart, uses that graphics card. Until then the processor encodes, and the Stream page shows the software-encoding notice, saying the driver did not respond at startup. --- host/src/capture/mod.rs | 8 + host/src/encoder/mod.rs | 9 +- host/src/gpu.rs | 183 +++++++++++++++- host/src/gpu_other.rs | 9 + host/src/gui/driver_wait.rs | 333 ++++++++++++++++++++++++++++++ host/src/gui/mod.rs | 21 +- host/src/gui/page_stream.rs | 31 ++- host/src/gui/renderer.rs | 69 +++++++ host/src/gui/tests.rs | 23 ++- host/src/main.rs | 5 +- host/src/stall.rs | 17 +- host/src/stats.rs | 5 + host/src/supervisor.rs | 61 +++++- host/tests/gpu_detect_fallback.rs | 75 +++++++ 14 files changed, 820 insertions(+), 29 deletions(-) create mode 100644 host/src/gui/driver_wait.rs create mode 100644 host/tests/gpu_detect_fallback.rs diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 1473f9a9..035e8fb9 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -237,6 +237,14 @@ pub fn enumerate_outputs() -> Vec { } } +/// [`enumerate_outputs`] on the way to the window's first paint, in the +/// startup graphics lane ([`crate::stall::Call::GpuDetect`]): empty if it +/// does not return within the limit, or while an earlier startup call is +/// still stuck on the same driver. +pub fn enumerate_outputs_at_startup() -> Vec { + crate::stall::run(crate::stall::Call::GpuDetect, enumerate_outputs, || {}).unwrap_or_default() +} + /// Resolve which output (index into `outputs`) the capture loop should duplicate for /// `target`. `PrimaryAuto` picks the (0,0) output, else the first attached. `Output(name)` /// matches the `DeviceName`, falling back to the primary (with a warning) when not found — diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index a865edf5..a6d5c78c 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -106,13 +106,18 @@ fn run_encode_loop( return Err(format!("{} codec not found in FFmpeg", encoder_name).into()); } info!(encoder = %encoder_name, "Found encoder codec"); + // The processor encodes only because startup's GPU detection did not + // return in time, and no override chose the encoder. + let detection_stalled = gpu.detection_stalled && shared.encoder_override.lock().is_none(); { let mut stats = PIPELINE_STATS.lock(); stats.set_codec_name(&codec_display_name); // Fresh pipeline start: clear any stale software-fallback flag. The encode loop // re-sets it below only if a hardware encoder actually fails to open. A deliberate - // libx264 override is not a fallback, so it intentionally leaves this false. - stats.set_software_fallback(false); + // libx264 override is not a fallback, so it intentionally leaves this false. A + // stalled detection is one from the start. + stats.set_software_fallback(detection_stalled); + stats.gpu_detection_stalled = detection_stalled; stats.nvidia_driver_outdated = false; } diff --git a/host/src/gpu.rs b/host/src/gpu.rs index bcc382d8..f19fadcf 100644 --- a/host/src/gpu.rs +++ b/host/src/gpu.rs @@ -4,6 +4,8 @@ use windows::Win32::Graphics::Dxgi::{ CreateDXGIFactory1, IDXGIAdapter1, IDXGIFactory1, DXGI_ERROR_NOT_FOUND, }; +use crate::stall::Call; + const VENDOR_NVIDIA: u32 = 0x10DE; const VENDOR_AMD: u32 = 0x1002; const VENDOR_INTEL: u32 = 0x8086; @@ -49,19 +51,24 @@ pub struct GpuInfo { pub dedicated_vram_mb: u64, pub encoder_name: String, pub codec_display_name: String, + /// Detection did not return within its limit: these are the software + /// facts, and the processor encodes only because of that. A later + /// pipeline takes [`take_late_detection`]'s facts if it has returned. + pub detection_stalled: bool, } impl GpuInfo { /// Detect the best GPU via DXGI and resolve the preferred encoder. - /// Falls back to software encoding if detection fails. + /// Falls back to software encoding if detection fails, or does not + /// return within its limit ([`Call::GpuDetect`]), so a stuck driver + /// cannot hold startup and the window. pub fn detect() -> Self { - match detect_from_dxgi() { - Ok(info) => info, - Err(e) => { - warn!(error = %e, "DXGI GPU detection failed, falling back to software encoding"); - Self::software_fallback() - } - } + detect_within( + crate::stall::lanes(), + Call::GpuDetect.limits(), + &FOUND, + || detect_from_dxgi().map_err(|e| e.to_string()), + ) } pub fn software_fallback() -> Self { @@ -72,10 +79,49 @@ impl GpuInfo { dedicated_vram_mb: 0, encoder_name: "libx264".to_string(), codec_display_name: "H.264 (x264)".to_string(), + detection_stalled: false, } } } +/// The graphics card the last detection found, kept when it returns, also +/// after startup gave up on it. +static FOUND: parking_lot::Mutex> = parking_lot::const_mutex(None); + +/// What a startup detection that was given up on found once it returned: +/// None until then, or if it found no graphics card. Taken once. +pub fn take_late_detection() -> Option { + FOUND.lock().take() +} + +/// `detect`'s result, run on a thread of its own in `lanes` under `limits`. +/// What it finds is also kept in `found`, so a detection given up on still +/// leaves its facts for later pipelines when it returns. The stall guard +/// logs the one line for a timeout. +fn detect_within( + lanes: &'static crate::stall::Lanes, + limits: crate::stall::Limits, + found: &'static parking_lot::Mutex>, + detect: impl FnOnce() -> Result + Send + 'static, +) -> GpuInfo { + let detect = move || { + let result = detect(); + *found.lock() = result.as_ref().ok().cloned(); + result + }; + match crate::stall::run_on(lanes, Call::GpuDetect, limits, detect, || {}) { + Ok(Ok(info)) => info, + Ok(Err(e)) => { + warn!(error = %e, "DXGI GPU detection failed, falling back to software encoding"); + GpuInfo::software_fallback() + } + Err(_) => GpuInfo { + detection_stalled: true, + ..GpuInfo::software_fallback() + }, + } +} + /// Vendors of the hardware adapters present now. Read at each encoder /// fallback rather than at launch, so a card connected since (an eGPU, a /// dock) or a driver installed since comes first. None unless DXGI listed @@ -230,6 +276,7 @@ fn detect_from_dxgi() -> Result> { dedicated_vram_mb: vram_mb, encoder_name, codec_display_name, + detection_stalled: false, }) } @@ -324,6 +371,126 @@ mod tests { }) } + /// `detect_within` over `detect` in lanes and a slot of its own, under + /// a 300 ms limit: what it found, how long it took, what it logged, the + /// lanes and the slot. + fn detect_held( + detect: impl FnOnce() -> Result + Send + 'static, + ) -> ( + GpuInfo, + std::time::Duration, + String, + &'static crate::stall::Lanes, + &'static parking_lot::Mutex>, + ) { + use std::sync::Arc; + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let found: &'static parking_lot::Mutex> = + Box::leak(Box::new(parking_lot::Mutex::new(None))); + let limits = crate::stall::Limits { + warn: std::time::Duration::from_millis(100), + limit: std::time::Duration::from_millis(300), + }; + let out = Arc::new(parking_lot::Mutex::new(Vec::::new())); + struct Sink(Arc>>); + impl std::io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let writer = Arc::clone(&out); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || Sink(Arc::clone(&writer))) + .finish(); + let started = std::time::Instant::now(); + let info = tracing::subscriber::with_default(subscriber, || { + detect_within(lanes, limits, found, detect) + }); + let took = started.elapsed(); + let text = String::from_utf8(out.lock().clone()).unwrap(); + (info, took, text, lanes, found) + } + + fn found(name: &str) -> GpuInfo { + GpuInfo { + vendor: GpuVendor::Nvidia, + name: name.to_string(), + encoder_name: "h264_nvenc".to_string(), + codec_display_name: "H.264 (NVENC)".to_string(), + ..GpuInfo::software_fallback() + } + } + + #[test] + fn a_stuck_detection_starts_at_its_limit_and_leaves_its_late_facts() { + // The detector waits on a driver that does not return until the + // test lets it. + let (release, held) = std::sync::mpsc::channel::<()>(); + let (info, took, text, lanes, slot) = detect_held(move || { + let _ = held.recv(); + Ok(found("late")) + }); + // The software facts at the limit, marked as a stall so the + // processor's encoding is explained. + assert_eq!(info.name, "Software"); + assert_eq!(info.encoder_name, "libx264"); + assert_eq!(info.vendor, GpuVendor::Unknown); + assert!(info.detection_stalled); + assert!( + took >= std::time::Duration::from_millis(300) + && took < std::time::Duration::from_secs(3), + "{took:?}" + ); + // One WARN, saying detection timed out and after how long. + let warns: Vec<_> = text.lines().filter(|l| l.contains("WARN")).collect(); + assert_eq!(warns.len(), 1, "{text}"); + assert!(warns[0].contains("stall=gpu_detect after_ms="), "{text}"); + assert!(warns[0].contains("limit_ms=300"), "{text}"); + // Until it returns, the lane holds it and nothing was found. + assert_eq!(lanes.abandoned(Call::GpuDetect), 1); + assert!(slot.lock().is_none()); + release.send(()).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while lanes.abandoned(Call::GpuDetect) != 0 { + assert!( + std::time::Instant::now() < deadline, + "the lane did not free" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + // What it found once it returned is there for later pipelines. + let late = slot.lock().take().expect("the late facts were not kept"); + assert_eq!( + (late.name.as_str(), late.encoder_name.as_str()), + ("late", "h264_nvenc") + ); + assert!(!late.detection_stalled); + } + + #[test] + fn a_detection_that_returns_in_time_keeps_its_facts() { + let (info, _, text, _, _) = detect_held(|| Ok(found("RTX"))); + assert_eq!( + (info.name.as_str(), info.encoder_name.as_str()), + ("RTX", "h264_nvenc") + ); + assert!(!info.detection_stalled); + assert!(!text.contains("WARN"), "{text}"); + // A failed one falls back with its own line, as before, and is not + // a stall: no notice blames a driver that answered. + let (info, _, text, _, slot) = detect_held(|| Err("no adapters".to_string())); + assert_eq!(info.encoder_name, "libx264"); + assert!(!info.detection_stalled); + assert!(slot.lock().is_none()); + assert!(text.contains("DXGI GPU detection failed"), "{text}"); + assert!(!text.contains("stall="), "{text}"); + } + #[test] fn every_adapter_is_listed_or_there_is_no_list() { use GpuVendor::{Amd, Intel, Nvidia, Unknown}; diff --git a/host/src/gpu_other.rs b/host/src/gpu_other.rs index 82d009d2..a11f1c91 100644 --- a/host/src/gpu_other.rs +++ b/host/src/gpu_other.rs @@ -33,6 +33,8 @@ pub struct GpuInfo { pub dedicated_vram_mb: u64, pub encoder_name: String, pub codec_display_name: String, + /// Always false: detection here only asks FFmpeg. + pub detection_stalled: bool, } impl GpuInfo { @@ -53,6 +55,7 @@ impl GpuInfo { dedicated_vram_mb: 0, encoder_name: name.to_string(), codec_display_name: display_name.to_string(), + detection_stalled: false, }; } info!(encoder = name, "Encoder not available, trying next"); @@ -70,10 +73,16 @@ impl GpuInfo { dedicated_vram_mb: 0, encoder_name: "libx264".to_string(), codec_display_name: "H.264 (x264)".to_string(), + detection_stalled: false, } } } +/// Detection here never stalls, so there is never a late one. +pub fn take_late_detection() -> Option { + None +} + /// No NVIDIA drivers to read outside Windows. pub fn nvidia_driver_versions() -> Vec { Vec::new() diff --git a/host/src/gui/driver_wait.rs b/host/src/gui/driver_wait.rs new file mode 100644 index 00000000..c3bab2a4 --- /dev/null +++ b/host/src/gui/driver_wait.rs @@ -0,0 +1,333 @@ +//! The window waits for a graphics driver that answers before it opens. +//! +//! The window's thread lists adapters, creates its device and draws through +//! the graphics driver, and Windows cannot stop a thread stuck inside it. So +//! before that thread makes its first graphics call, the same calls are made +//! once on a thread of their own, in the startup graphics lane +//! ([`Call::GpuDetect`]), under its limit. While they do not return, or an +//! earlier startup call in that lane is still stuck (GPU detection, the +//! screen list), a notice drawn by Windows without the graphics card says +//! why the window is not there yet, and the window opens once they return. +// Only Windows waits; the logic is tested everywhere. +#![cfg_attr(not(windows), allow(dead_code))] + +use std::time::{Duration, Instant}; + +use tracing::{info, warn}; + +use crate::stall::{Call, Lanes, Limits}; + +/// How often a waiting launch looks whether the stuck call has returned. +pub(super) const POLL: Duration = Duration::from_millis(250); + +/// What the user sees while the window waits. +pub(super) trait Notice { + fn show(&mut self); + /// The user chose to quit instead of waiting. + fn quit(&mut self) -> bool; + fn close(&mut self); +} + +/// Runs a fresh `probe` in `lanes` until one returns within `limits`, with +/// `notice` up from the first one that does not. False when the user quit +/// from the notice. +pub(super) fn wait_for_driver( + lanes: &'static Lanes, + limits: Limits, + mut probe: impl FnMut() -> F, + notice: &mut impl Notice, + poll: Duration, +) -> bool +where + F: FnOnce() + Send + 'static, +{ + let started = Instant::now(); + let mut shown = false; + loop { + // While an earlier call is stuck, a new one would only be refused. + if lanes.abandoned(Call::GpuDetect) == 0 + && crate::stall::run_on(lanes, Call::GpuDetect, limits, probe(), || {}).is_ok() + { + if shown { + notice.close(); + info!( + waited_ms = started.elapsed().as_millis() as u64, + "The graphics driver answered; opening the window" + ); + } + return true; + } + if !shown { + shown = true; + notice.show(); + warn!("The graphics driver is not answering; the window opens once it does"); + } + if notice.quit() { + info!( + waited_ms = started.elapsed().as_millis() as u64, + "Quit while the window waited for the graphics driver" + ); + return false; + } + std::thread::sleep(poll); + } +} + +/// A message box: Windows draws it without the graphics card's driver. +#[cfg(windows)] +#[derive(Default)] +pub(super) struct MessageNotice { + shown: Option, +} + +#[cfg(windows)] +struct Shown { + /// The box's thread, to find its window by. + thread: std::sync::Arc, + answer: std::thread::JoinHandle, +} + +#[cfg(windows)] +const NOTICE: &str = "The graphics driver is not responding, so the EternalMonitor window can't open yet. It opens on its own as soon as the driver answers.\n\nIf this stays, restart the PC or update the graphics driver from NVIDIA, AMD or Intel.\n\nClick OK to keep waiting, or Cancel to quit EternalMonitor."; + +#[cfg(windows)] +impl Notice for MessageNotice { + fn show(&mut self) { + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::Arc; + use windows::core::HSTRING; + use windows::Win32::UI::WindowsAndMessaging::{ + MessageBoxW, MB_ICONWARNING, MB_OKCANCEL, MB_SETFOREGROUND, + }; + let thread = Arc::new(AtomicU32::new(0)); + let id = Arc::clone(&thread); + let spawned = std::thread::Builder::new() + .name("driver-wait-notice".into()) + .spawn(move || unsafe { + id.store( + windows::Win32::System::Threading::GetCurrentThreadId(), + Ordering::SeqCst, + ); + MessageBoxW( + None, + &HSTRING::from(NOTICE), + &HSTRING::from("EternalMonitor"), + MB_OKCANCEL | MB_ICONWARNING | MB_SETFOREGROUND, + ) + }); + match spawned { + Ok(answer) => self.shown = Some(Shown { thread, answer }), + Err(error) => warn!(%error, "Could not show the graphics driver notice"), + } + } + + fn quit(&mut self) -> bool { + use windows::Win32::UI::WindowsAndMessaging::IDCANCEL; + // OK, or a box that could not be shown, keeps waiting. + if !self.shown.as_ref().is_some_and(|s| s.answer.is_finished()) { + return false; + } + let shown = self.shown.take().expect("checked above"); + shown.answer.join().is_ok_and(|answer| answer == IDCANCEL) + } + + fn close(&mut self) { + use std::sync::atomic::Ordering; + use windows::Win32::Foundation::{BOOL, HWND, LPARAM, WPARAM}; + use windows::Win32::UI::WindowsAndMessaging::{EnumThreadWindows, PostMessageW, WM_CLOSE}; + unsafe extern "system" fn close_window(hwnd: HWND, _: LPARAM) -> BOOL { + let _ = PostMessageW(hwnd, WM_CLOSE, WPARAM(0), LPARAM(0)); + true.into() + } + let Some(shown) = self.shown.take() else { + return; + }; + // The box may not be up yet when the driver answers at once, so ask + // again until its thread ends. Bounded: the window opens regardless. + let deadline = Instant::now() + Duration::from_secs(2); + while !shown.answer.is_finished() && Instant::now() < deadline { + let thread = shown.thread.load(Ordering::SeqCst); + if thread != 0 { + unsafe { + let _ = EnumThreadWindows(thread, Some(close_window), LPARAM(0)); + } + } + std::thread::sleep(Duration::from_millis(50)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; + use std::sync::{mpsc, Arc}; + + const LIMITS: Limits = Limits { + warn: Duration::from_millis(100), + limit: Duration::from_millis(300), + }; + const SLICE: Duration = Duration::from_millis(10); + + fn lanes() -> &'static Lanes { + Box::leak(Box::new(Lanes::new())) + } + + /// When the notice was shown and closed, from the start of the test. + #[derive(Default)] + struct Recorded { + started: Option, + shown: Option, + closed: Option, + quit_after_shown: bool, + } + + impl Notice for Recorded { + fn show(&mut self) { + self.shown = Some(self.started.unwrap().elapsed()); + } + fn quit(&mut self) -> bool { + self.quit_after_shown + } + fn close(&mut self) { + self.closed = Some(self.started.unwrap().elapsed()); + } + } + + fn notice() -> Recorded { + Recorded { + started: Some(Instant::now()), + ..Default::default() + } + } + + /// Something that blocks in `lanes`' startup lane until the returned + /// sender sends, already given up on: a stuck GPU detection. + fn stuck_detection(lanes: &'static Lanes) -> mpsc::Sender<()> { + let (release, held) = mpsc::channel::<()>(); + let result = crate::stall::run_on( + lanes, + Call::GpuDetect, + LIMITS, + move || { + let _ = held.recv(); + }, + || {}, + ); + assert!(result.is_err()); + release + } + + #[test] + fn a_driver_that_answers_opens_the_window_without_a_notice() { + let mut notice = notice(); + let probes = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&probes); + let opened = wait_for_driver( + lanes(), + LIMITS, + || { + let count = Arc::clone(&count); + move || { + count.fetch_add(1, SeqCst); + } + }, + &mut notice, + SLICE, + ); + assert!(opened); + assert_eq!(probes.load(SeqCst), 1); + assert!(notice.shown.is_none() && notice.closed.is_none()); + } + + #[test] + fn while_detection_is_stuck_the_notice_shows_at_once_and_the_window_follows_it() { + let lanes = lanes(); + let release = stuck_detection(lanes); + let probes = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&probes); + let waiter = std::thread::spawn(move || { + let mut notice = notice(); + let opened = wait_for_driver( + lanes, + LIMITS, + || { + let count = Arc::clone(&count); + move || { + count.fetch_add(1, SeqCst); + } + }, + &mut notice, + SLICE, + ); + (opened, notice) + }); + // No probe joins the stuck driver. + std::thread::sleep(LIMITS.limit * 2); + assert_eq!(probes.load(SeqCst), 0); + release.send(()).unwrap(); + let (opened, notice) = waiter.join().unwrap(); + assert!(opened); + // Shown without waiting out another limit. + let shown = notice.shown.expect("no notice"); + assert!(shown < LIMITS.limit, "{shown:?}"); + // Closed once the driver answered, and the window's probe ran then. + assert!(notice.closed.expect("not closed") >= LIMITS.limit); + assert_eq!(probes.load(SeqCst), 1); + } + + #[test] + fn a_probe_that_hangs_shows_the_notice_at_its_limit() { + let lanes = lanes(); + let (release, held) = mpsc::channel::<()>(); + let held = Arc::new(parking_lot::Mutex::new(Some(held))); + let waiter = std::thread::spawn(move || { + let mut notice = notice(); + let opened = wait_for_driver( + lanes, + LIMITS, + || { + // The first probe blocks in the driver; the next answers. + let held = held.lock().take(); + move || { + if let Some(held) = held { + let _ = held.recv(); + } + } + }, + &mut notice, + SLICE, + ); + (opened, notice) + }); + std::thread::sleep(LIMITS.limit * 3); + release.send(()).unwrap(); + let (opened, notice) = waiter.join().unwrap(); + assert!(opened); + let shown = notice.shown.expect("no notice"); + assert!( + shown >= LIMITS.limit && shown < LIMITS.limit + Duration::from_secs(1), + "{shown:?}" + ); + assert!(notice.closed.expect("not closed") >= LIMITS.limit * 2); + } + + #[test] + fn quitting_from_the_notice_opens_no_window() { + let lanes = lanes(); + let _release = stuck_detection(lanes); + let mut notice = Recorded { + quit_after_shown: true, + ..notice() + }; + assert!(!wait_for_driver( + lanes, + LIMITS, + || || {}, + &mut notice, + SLICE + )); + assert!(notice.shown.is_some()); + assert!(notice.closed.is_none()); + } +} diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 53d7cf8e..a9c49f00 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -19,6 +19,7 @@ //! `author_id`, see `widgets::automation_id`): scripts and tests find them by //! that id first, then by name, so the same name works in every language. +mod driver_wait; mod feedback_panel; mod fonts; mod icons; @@ -44,7 +45,7 @@ use tracing::{info, warn}; use crate::autostart::{first_lan_ipv4, read_startup_registry, set_startup_registry}; use crate::capture::recovery::CaptureBlock; -use crate::capture::{enumerate_outputs, OutputInfo}; +use crate::capture::{enumerate_outputs, enumerate_outputs_at_startup, OutputInfo}; use crate::control::{CaptureTarget, GuiControl}; use crate::gpu_preference::{Change, Preference, SwitchRecord}; use crate::handoff::Handoff; @@ -81,6 +82,7 @@ struct StatsSnapshot { codec_name: String, using_software_fallback: bool, nvidia_driver_outdated: bool, + gpu_detection_stalled: bool, gpu_name: String, capture_display: String, transport_fps: f64, @@ -119,6 +121,7 @@ impl StatsSnapshot { codec_name: s.codec_name.clone(), using_software_fallback: s.using_software_fallback, nvidia_driver_outdated: s.nvidia_driver_outdated, + gpu_detection_stalled: s.gpu_detection_stalled, gpu_name: s.gpu_name.clone(), capture_display: s.capture_display.clone(), transport_fps: s.transport_fps, @@ -441,7 +444,8 @@ impl AnalyzerApp { String::new() } }; - let available_outputs = enumerate_outputs(); + // Before the first paint, so a stuck driver must not hold it. + let available_outputs = enumerate_outputs_at_startup(); // Same model as encoder_override: push the persisted preference into // SharedControl so the encoder honours it without a restart. @@ -1774,6 +1778,19 @@ pub fn run_gui(control: GuiControl) -> eframe::Result<()> { ..Default::default() }; renderer::configure(&mut options.wgpu_options); + // The window's thread makes its first graphics call only once the + // driver has answered the same calls on another thread. + #[cfg(windows)] + if !driver_wait::wait_for_driver( + crate::stall::lanes(), + crate::stall::Call::GpuDetect.limits(), + || renderer::adapter_probe(&options.wgpu_options), + &mut driver_wait::MessageNotice::default(), + driver_wait::POLL, + ) { + renderer::window_closed(); + return Ok(()); + } let result = eframe::run_native( "EternalMonitor", diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index 405e3d7f..92616572 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -75,9 +75,12 @@ impl AnalyzerApp { } if snap.using_software_fallback { // Choices made while troubleshooting are the usual reason a - // working graphics card is skipped, so offer to undo them first. - let customized = self.settings_encoder_choice != ENCODER_AUTO_LABEL - || self.settings_encoder_input != crate::encoder::input::EncoderInput::default(); + // working graphics card is skipped, so offer to undo them first, + // unless the driver did not answer at startup: then Restart. + let customized = !snap.gpu_detection_stalled + && (self.settings_encoder_choice != ENCODER_AUTO_LABEL + || self.settings_encoder_input + != crate::encoder::input::EncoderInput::default()); let mut reset = false; let mut restart = false; widgets::banner( @@ -85,7 +88,11 @@ impl AnalyzerApp { WARNING, Icon::Warning, SOFTWARE_FALLBACK_TITLE, - &software_fallback_body(customized, snap.nvidia_driver_outdated), + &software_fallback_body( + customized, + snap.nvidia_driver_outdated, + snap.gpu_detection_stalled, + ), if customized { 230.0 } else { 110.0 }, |ui| { if customized { @@ -830,11 +837,17 @@ enum HeaderAction { pub(super) const SOFTWARE_FALLBACK_TITLE: &str = "Video is using the processor instead of the graphics card"; -/// What the CPU fallback notice says: undo custom settings first, say the -/// NVIDIA driver is too old when that is what the graphics card refused, or -/// suggest a driver update. -pub(super) fn software_fallback_body(customized: bool, nvidia_driver_outdated: bool) -> String { - if customized { +/// What the CPU fallback notice says: that the graphics driver did not +/// answer at startup, undo custom settings, say the NVIDIA driver is too old +/// when that is what the graphics card refused, or suggest a driver update. +pub(super) fn software_fallback_body( + customized: bool, + nvidia_driver_outdated: bool, + detection_stalled: bool, +) -> String { + if detection_stalled { + "The graphics driver did not respond when EternalMonitor started, so the processor encodes the video for now. The picture may lag and the PC may run hot. Click Restart to try the graphics card again.".to_string() + } else if customized { "The graphics card could not encode video with the options chosen in Settings, so the picture may lag and the PC may run hot. Go back to the recommended options first.".to_string() } else if nvidia_driver_outdated { format!( diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 322c1d74..8326e3e7 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -305,6 +305,68 @@ pub(super) fn configure(config: &mut WgpuConfiguration) { }); } +/// The wgpu calls the window's thread makes first, for another thread to +/// make before it: an instance with this launch's backends lists its +/// adapters (DX12 creates a device on each to describe it). Built on the +/// window's thread, run on another by +/// [`super::driver_wait::wait_for_driver`]. While it lists, the marker +/// names that thread, so a driver crash there is still the window's. +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn adapter_probe(config: &WgpuConfiguration) -> impl FnOnce() + Send + 'static { + let window = current_thread(); + let descriptor = match &config.wgpu_setup { + WgpuSetup::CreateNew(setup) => Some(( + setup.instance_descriptor.backends, + setup.instance_descriptor.flags, + setup.instance_descriptor.backend_options.clone(), + setup.instance_descriptor.memory_budget_thresholds, + )), + WgpuSetup::Existing(_) => None, + }; + move || { + let Some((backends, flags, backend_options, memory_budget_thresholds)) = descriptor else { + return; + }; + mark_thread(current_thread()); + let started = Instant::now(); + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { + backends, + flags, + backend_options, + memory_budget_thresholds, + display: None, + }); + let listed = tokio::runtime::Builder::new_current_thread() + .build() + .map(|runtime| { + runtime + .block_on(instance.enumerate_adapters(backends)) + .len() + }); + drop(instance); + info!( + adapters = listed.unwrap_or(0), + took_ms = started.elapsed().as_millis() as u64, + "Window renderer: the graphics driver answered before the window" + ); + mark_thread(window); + } +} + +/// Name `thread` in this launch's marker and save it again. +fn mark_thread(thread: Option) { + let Ok(mut started) = STARTED.lock() else { + return; + }; + let Some(marker) = started.as_mut() else { + return; + }; + marker.thread = thread; + if let Some(directory) = STATE_DIR.get() { + let _ = save_marker(directory, ProcessId::current(), marker); + } +} + /// `WGPU_BACKEND` decides when set, as in wgpu, and nothing is narrowed or /// recorded then. Otherwise the saved choice, unless it failed before. Also /// returns why a remote rule can't apply, if one of them decides. @@ -814,8 +876,15 @@ fn system_gpus() -> String { } /// Vendor, device and driver version of each DXGI adapter. Creates no device. +/// Runs in the startup graphics lane: empty if it does not return within +/// the limit, or while an earlier call there is still stuck. #[cfg(windows)] fn dxgi_adapters() -> Vec<(u32, u32, String)> { + crate::stall::run(crate::stall::Call::GpuDetect, list_dxgi_adapters, || {}).unwrap_or_default() +} + +#[cfg(windows)] +fn list_dxgi_adapters() -> Vec<(u32, u32, String)> { use windows::core::Interface; use windows::Win32::Graphics::Dxgi::{CreateDXGIFactory1, IDXGIDevice, IDXGIFactory1}; let Ok(factory) = (unsafe { CreateDXGIFactory1::() }) else { diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index e74b54a8..22c47b69 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -424,7 +424,7 @@ fn display_choices_use_the_names_shared_with_the_ipad() { #[test] fn an_outdated_nvidia_driver_is_named_with_the_version_to_install() { - let body = software_fallback_body(false, true); + let body = software_fallback_body(false, true, false); assert!( body.contains("NVIDIA driver to version 570 or newer"), "{body}" @@ -441,10 +441,25 @@ fn an_outdated_nvidia_driver_is_named_with_the_version_to_install() { } // Custom settings are still the first thing to undo. assert_eq!( - software_fallback_body(true, true), - software_fallback_body(true, false) + software_fallback_body(true, true, false), + software_fallback_body(true, false, false) ); - assert!(!software_fallback_body(false, false).contains("570")); + assert!(!software_fallback_body(false, false, false).contains("570")); +} + +#[test] +fn a_driver_that_did_not_answer_at_startup_is_named_with_a_restart() { + let body = software_fallback_body(false, false, true); + assert!( + body.contains("graphics driver did not respond when EternalMonitor started"), + "{body}" + ); + assert!(body.contains("Click Restart"), "{body}"); + for jargon in ["DXGI", "libx264", "encoder", "timed out"] { + assert!(!body.contains(jargon), "{body}"); + } + // It is the reason, whatever else is true. + assert_eq!(body, software_fallback_body(true, true, true)); } #[test] diff --git a/host/src/main.rs b/host/src/main.rs index 0dbd39cb..b05b3263 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -130,9 +130,10 @@ fn main() -> Result<(), Box> { // Best-effort capture source for the banner — reflects the default (primary). The // authoritative per-run source (including a persisted/fallback selection) is logged - // inside the capture loop when the stream starts. + // inside the capture loop when the stream starts. Time-boxed with GPU + // detection: it would wait on the same driver. let capture_summary = { - let outputs = capture::enumerate_outputs(); + let outputs = capture::enumerate_outputs_at_startup(); outputs .iter() .find(|o| o.is_primary) diff --git a/host/src/stall.rs b/host/src/stall.rs index 9d8b5a20..bc5f07e2 100644 --- a/host/src/stall.rs +++ b/host/src/stall.rs @@ -40,9 +40,15 @@ pub enum Call { /// Reading the graphics cards and the screens' wiring for the second /// screen's decisions (`capture::second_screen::facts`). GpuFacts, + /// The graphics calls before the window's first paint: finding the + /// graphics cards (`gpu::GpuInfo::detect`), listing the screens, the + /// window renderer's adapter list and its wgpu probe. They share one + /// lane: they wait on the same driver, so after one hangs the others + /// are refused at once instead of each waiting out its own limit. + GpuDetect, } -const CALLS: usize = 6; +const CALLS: usize = 7; /// When a call is slow enough to log, and when it is given up on. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -70,6 +76,7 @@ impl Call { Self::VddEnable => "vdd_enable", Self::VddDisable => "vdd_disable", Self::GpuFacts => "gpu_facts", + Self::GpuDetect => "gpu_detect", } } @@ -99,6 +106,12 @@ impl Call { // adapters, as device creation begins, then asks DXCore and reads // the display configuration; it keeps device creation's limits. Self::GpuFacts => Limits::ms(1_000, 4_000), + // 34 startups on 3 PCs (two test PCs, one field PC's uploads): + // median 10 ms, slowest 44 ms, from "Support system information" + // to "Selected encoder". It makes the same DXGI calls as + // GpuFacts and keeps its limits. One AMD laptop's dxgkrnl + // deadlock held it 62 s, and the window with it. + Self::GpuDetect => Limits::ms(1_000, 4_000), } } } @@ -781,6 +794,7 @@ mod tests { Call::VddEnable, Call::VddDisable, Call::GpuFacts, + Call::GpuDetect, ] .map(Call::name) .into(); @@ -798,6 +812,7 @@ mod tests { Call::EncoderOpen, Call::VddEnable, Call::GpuFacts, + Call::GpuDetect, ] { let limits = call.limits(); assert!(limits.limit >= Duration::from_secs(3), "{call:?}"); diff --git a/host/src/stats.rs b/host/src/stats.rs index 98c9b58a..114e4808 100644 --- a/host/src/stats.rs +++ b/host/src/stats.rs @@ -46,6 +46,9 @@ pub struct PipelineStats { /// linked FFmpeg's NVENC needs. The fallback notice then says to update /// it. pub nvidia_driver_outdated: bool, + /// The processor encodes because startup's GPU detection did not return + /// in time. The fallback notice then says the driver did not answer. + pub gpu_detection_stalled: bool, encode_timestamps: VecDeque, // Transport @@ -109,6 +112,7 @@ impl PipelineStats { codec_name: String::new(), using_software_fallback: false, nvidia_driver_outdated: false, + gpu_detection_stalled: false, encode_timestamps: VecDeque::with_capacity(128), transport_fps: 0.0, @@ -150,6 +154,7 @@ impl PipelineStats { self.codec_name = previous.codec_name; self.using_software_fallback = previous.using_software_fallback; self.nvidia_driver_outdated = previous.nvidia_driver_outdated; + self.gpu_detection_stalled = previous.gpu_detection_stalled; self.mdns_active = previous.mdns_active; previous.audio } diff --git a/host/src/supervisor.rs b/host/src/supervisor.rs index e12e4d0e..ca118cb3 100644 --- a/host/src/supervisor.rs +++ b/host/src/supervisor.rs @@ -292,7 +292,7 @@ impl HealthReporter { pub fn run( listen_port: u16, shared: SharedControl, - gpu_info: GpuInfo, + mut gpu_info: GpuInfo, command_tx: mpsc::Sender, command_rx: mpsc::Receiver, ) { @@ -382,6 +382,11 @@ pub fn run( generation = CURRENT_GENERATION.fetch_add(1, Ordering::SeqCst) + 1; machine.note_started(generation); healthy_since = Instant::now(); + if take_late_detection(&mut gpu_info, crate::gpu::take_late_detection) { + let mut stats = PIPELINE_STATS.lock(); + stats.set_gpu_name(gpu_info.name.clone()); + stats.set_codec_name(gpu_info.codec_display_name.clone()); + } pipeline = Some(spawn_generation( generation, listen_port, @@ -405,6 +410,25 @@ pub fn run( info!("Supervisor loop ended"); } +/// Swap startup's software facts for what a detection given up on at +/// startup found once it returned (`late`), so this and later pipelines +/// encode on the graphics card. True when they changed. +fn take_late_detection(gpu: &mut GpuInfo, late: impl FnOnce() -> Option) -> bool { + if !gpu.detection_stalled { + return false; + } + let Some(found) = late() else { + return false; + }; + info!( + gpu = %found.name, + encoder = %found.encoder_name, + "GPU detection returned after startup went on without it; this pipeline uses its graphics card" + ); + *gpu = found; + true +} + /// Watchdog rules over the heartbeat atomics. Only meaningful while Running /// and after the pipeline has had a moment to warm up. fn wedge_reason(shared: &SharedControl, machine: &Machine) -> Option { @@ -560,6 +584,41 @@ fn bounded_join(pipeline: &mut Option>, generation: mod tests { use super::*; + #[test] + fn a_late_detection_is_used_by_the_next_pipeline_only_after_a_stall() { + let late = || { + Some(GpuInfo { + name: "late".to_string(), + encoder_name: "h264_amf".to_string(), + ..GpuInfo::software_fallback() + }) + }; + // Startup gave up: the next pipeline takes what it found since. + let mut gpu = GpuInfo { + detection_stalled: true, + ..GpuInfo::software_fallback() + }; + assert!(take_late_detection(&mut gpu, late)); + assert_eq!( + ( + gpu.name.as_str(), + gpu.encoder_name.as_str(), + gpu.detection_stalled + ), + ("late", "h264_amf", false) + ); + // Not returned yet: the processor goes on encoding, still marked. + let mut gpu = GpuInfo { + detection_stalled: true, + ..GpuInfo::software_fallback() + }; + assert!(!take_late_detection(&mut gpu, || None)); + assert!(gpu.detection_stalled); + // A detection that answered at startup is never replaced. + let mut gpu = GpuInfo::software_fallback(); + assert!(!take_late_detection(&mut gpu, late)); + assert_eq!(gpu.name, "Software"); + } #[test] fn audio_failures_never_restart_or_exhaust_the_video_failure_budget() { let mut machine = Machine::new(7); diff --git a/host/tests/gpu_detect_fallback.rs b/host/tests/gpu_detect_fallback.rs new file mode 100644 index 00000000..ed24700f --- /dev/null +++ b/host/tests/gpu_detect_fallback.rs @@ -0,0 +1,75 @@ +//! The real encode stage on the software facts startup uses when GPU +//! detection did not return in time: the processor encodes, and the window +//! says why. + +use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use eternal_host::capture::synthetic::render_synthetic_frame; +use eternal_host::capture::{frame_buffer, frame_slot, RawFrame}; +use eternal_host::control::SharedControl; +use eternal_host::encoder::{run_encode_stage, NAL_CHANNEL_CAPACITY}; +use eternal_host::gpu::GpuInfo; +use eternal_host::stats::PIPELINE_STATS; + +/// The stage and PIPELINE_STATS are process-global, so stages run one at a time. +static STAGE_LOCK: Mutex<()> = Mutex::new(()); + +/// Runs a few frames through a fresh encode stage with `gpu` and the +/// encoder override `chosen`, and returns the fallback notice's two flags. +fn notice_after_stage(gpu: GpuInfo, chosen: Option<&str>) -> (bool, bool) { + let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + ffmpeg_next::init().unwrap(); + let (producer, consumer) = frame_slot(); + let (tx, mut rx) = tokio::sync::mpsc::channel(NAL_CHANNEL_CAPACITY); + let shared = SharedControl::new(0, 8_000_000); + shared.running.store(true, Ordering::SeqCst); + *shared.encoder_override.lock() = chosen.map(str::to_string); + let generation = eternal_host::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let stage = std::thread::spawn(move || { + run_encode_stage(consumer, tx, shared, gpu, generation).map_err(|e| e.to_string()) + }); + let collector = std::thread::spawn(move || { + let mut packets = 0; + while rx.blocking_recv().is_some() { + packets += 1; + } + packets + }); + let (width, height) = (640, 360); + for number in 0..10 { + let mut data = frame_buffer((width * height * 4) as usize); + render_synthetic_frame(&mut data, width, height, number); + producer.publish(RawFrame { + frame_number: number, + timestamp: Instant::now(), + data: Arc::new(data), + width, + height, + }); + std::thread::sleep(Duration::from_millis(20)); + } + producer.close(); + stage.join().unwrap().expect("the encode stage failed"); + assert!(collector.join().unwrap() > 0, "no packets"); + let stats = PIPELINE_STATS.lock(); + (stats.using_software_fallback, stats.gpu_detection_stalled) +} + +#[test] +fn a_stalled_detection_encodes_on_the_processor_with_the_notice() { + let stalled = GpuInfo { + detection_stalled: true, + ..GpuInfo::software_fallback() + }; + assert_eq!(notice_after_stage(stalled.clone(), None), (true, true)); + // libx264 chosen in Settings is the user's choice, not a fallback. + assert_eq!(notice_after_stage(stalled, Some("libx264")), (false, false)); + // A detection that answered and found no graphics card: no notice, as + // before. + assert_eq!( + notice_after_stage(GpuInfo::software_fallback(), None), + (false, false) + ); +} From f732b2559b98ca9dcaad071354ddfd5326fa1253 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 16:52:25 -0500 Subject: [PATCH 066/226] Offer Beta builds the newer Beta A Beta build (x.y.z-beta.*) now also reads the nightly's fixed `beta` GitHub release and is offered the newer of that Beta and the stable release. Stable builds never read or see a Beta. The Beta's version and SHA-256 come from the release body nightly.py writes, and GitHub's digest of the installer must match that hash, since no website manifest names a Beta. That also refuses the release while the nightly swaps the installer before it rewrites the body. A missing or malformed version, hash or digest means no offer, and replaces a saved Beta; offline keeps it. The download, HTTPS host allowlist and helper are the stable path's. A Beta published between the check and Update now replaces the offer before anything downloads. A Beta is announced at most once a day and never during a stream; its popup shows at most once a day and Later covers every Beta for a day. The relaunch report compares pre-release versions too, so a Beta update that did not take is reported as failed. Settings, General gets a "Get Beta updates" toggle, shown only on a Beta build, to go back to stable releases only. nightly.py's beta body moves into beta_body(), and a shared fixture of that body is checked by both test_nightly.py and the Rust parser. --- .github/workflows/nightly.yml | 6 +- SECURITY.md | 5 +- docs/beta-testing.md | 9 + host/src/gui/mod.rs | 10 + host/src/gui/update_banner.rs | 16 + host/src/settings.rs | 5 + host/src/updates.rs | 254 ++++++++++- host/src/updates/tests.rs | 630 +++++++++++++++++++++++++- scripts/fixtures/beta-release-body.md | 13 + scripts/nightly.py | 14 +- scripts/test_nightly.py | 8 + 11 files changed, 946 insertions(+), 24 deletions(-) create mode 100644 scripts/fixtures/beta-release-body.md diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ced5da86..0be20afe 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -7,8 +7,10 @@ name: Beta builds # https://github.com/sidebandstudio/EternalMonitor/releases/download/beta/EternalMonitor-Setup-beta.exe # never changes (docs/download.html?channel=beta links it). No tag here starts # with v, so release.yml and testflight.yml never run for a beta build, and -# docs/latest.json, /releases/latest and the apps' update checks never see -# one. scripts/nightly.py has the version, branch and retention rules. +# docs/latest.json, /releases/latest and stable installs never see one. A +# Beta install reads the beta release's body for the newest Beta +# (host/src/updates.rs). scripts/nightly.py has the version, branch and +# retention rules. # # Only a run on main publishes. A manual run with dry_run, on any other # branch or for a pull request that changes this pipeline, builds and scans diff --git a/SECURITY.md b/SECURITY.md index e058dcc1..c19ac7c8 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,7 +33,10 @@ Knowing these limits helps you judge whether a finding is new. - The installer is not code-signed yet. Check its SHA-256 against the release page. - In-app updates (Windows) trust the network for nothing: the host fetches over HTTPS from an allowlist of GitHub hosts, and the installer must match the - SHA-256 in both eternalmonitor.dev/latest.json and the GitHub release. Against + SHA-256 in both eternalmonitor.dev/latest.json and the GitHub release. A Beta + build may also update to the newer Beta from the fixed `beta` GitHub release; + that installer must match both the SHA-256 in the release body and GitHub's own + digest of the file. Stable builds never take a Beta. Against other software already running as your Windows user, the updater adds defense in depth rather than a guarantee. Its helper loads DLLs only from System32 and imports only Windows system libraries. It inherits only its output pipe from diff --git a/docs/beta-testing.md b/docs/beta-testing.md index e4e23a7b..ea2591ac 100644 --- a/docs/beta-testing.md +++ b/docs/beta-testing.md @@ -32,6 +32,15 @@ back to Stable by running the Stable installer over it. `docs/latest.json`, else that release's next patch. It sorts below `0.3.4`, so Beta users get the next Stable release from the normal update prompt, and Stable users are never offered a Beta build. +- **Updates on Beta:** a Beta build also reads the `beta` release below and + offers the newer of that Beta and the next Stable release, with the same + banner, popup and helper as a Stable update. The version and SHA-256 come + from the release body `nightly.py` writes, and GitHub's digest of the + installer must match it. A new Beta is announced at most once a day, never + during a stream. Settings, General, **Get Beta updates** (shown only on a + Beta build) turns this off. Taking a Stable release leaves the Beta channel. + Betas built before this arrived can't update themselves, so their testers + install one newer Beta by hand. - **On GitHub:** each build is a pre-release tagged `beta--`, titled like "EternalMonitor 0.3.4 beta (Oct 30, 2026)". The newest 7 are kept. The pre-release `beta` always holds the newest, so this link never diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index e49d475b..7ba44159 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -303,6 +303,7 @@ pub struct AnalyzerApp { settings_fps_target: u32, settings_encoder_input: crate::encoder::input::EncoderInput, settings_check_updates: bool, + settings_beta_updates: bool, updates: crate::updates::Updater, /// The update popup is on screen; it stays until a button is chosen. update_popup_open: bool, @@ -473,6 +474,7 @@ impl AnalyzerApp { let ctx = cc.egui_ctx.clone(); crate::updates::Updater::start( persisted.check_for_updates, + persisted.beta_updates, control.shared.session.clone(), control.shared.handoff.clone(), std::sync::Arc::new(move |version: Option<&str>| { @@ -489,6 +491,7 @@ impl AnalyzerApp { Self { settings_encoder_input: persisted.encoder_input, settings_check_updates: persisted.check_for_updates, + settings_beta_updates: persisted.beta_updates, updates, update_popup_open: false, ipad_last_seen: Some(popup_now()), @@ -565,6 +568,7 @@ impl AnalyzerApp { encoder_override, encoder_input: self.settings_encoder_input, check_for_updates: self.settings_check_updates, + beta_updates: self.settings_beta_updates, capture_display: if self.settings_capture_display.is_empty() { None } else { @@ -787,6 +791,12 @@ impl AnalyzerApp { self.persist_settings(); } + fn set_beta_updates(&mut self, enabled: bool) { + self.settings_beta_updates = enabled; + self.updates.set_beta(enabled); + self.persist_settings(); + } + fn set_require_pairing(&mut self, required: bool) { { let mut pairing = self.control.shared.pairing.lock(); diff --git a/host/src/gui/update_banner.rs b/host/src/gui/update_banner.rs index 84443e47..7d13c20a 100644 --- a/host/src/gui/update_banner.rs +++ b/host/src/gui/update_banner.rs @@ -457,6 +457,22 @@ impl AnalyzerApp { self.set_check_updates(automatic); } widgets::divider(ui); + if crate::updates::is_beta(env!("CARGO_PKG_VERSION")) { + let mut beta = self.settings_beta_updates; + widgets::setting_row( + ui, + "Get Beta updates", + "This is a Beta build. Turn this off to be offered only stable releases.", + 64.0, + |ui| { + widgets::toggle(ui, &mut beta, "Get Beta updates"); + }, + ); + if beta != self.settings_beta_updates { + self.set_beta_updates(beta); + } + widgets::divider(ui); + } let status = self.updates.check_status(); let available = self.updates.available_version(); let description = match (status, &available) { diff --git a/host/src/settings.rs b/host/src/settings.rs index d1214233..79bfda13 100644 --- a/host/src/settings.rs +++ b/host/src/settings.rs @@ -24,6 +24,10 @@ pub struct SettingsFile { pub encoder_input: crate::encoder::input::EncoderInput, #[serde(default = "default_true")] pub check_for_updates: bool, + /// On a Beta build: newer Betas are offered too. Off: only stable + /// releases. + #[serde(default = "default_true")] + pub beta_updates: bool, /// DXGI `DeviceName` of the capture display (e.g. `\\.\DISPLAY3`). /// `None` means auto (primary monitor). #[serde(default)] @@ -147,6 +151,7 @@ impl Default for SettingsFile { encoder_override: None, encoder_input: Default::default(), check_for_updates: true, + beta_updates: true, capture_display: None, hevc_enabled: false, stream_audio: true, diff --git a/host/src/updates.rs b/host/src/updates.rs index 6755c848..e56cfcc8 100644 --- a/host/src/updates.rs +++ b/host/src/updates.rs @@ -6,6 +6,10 @@ //! iPad's while its TestFlight build awaits Apple. Before //! installing, the installer's SHA-256 must match both that manifest and the //! GitHub release body, which are published by different steps. +//! +//! A Beta build (`x.y.z-beta.…`) also reads the nightly's fixed `beta` +//! GitHub release (scripts/nightly.py) and is offered the newer of that Beta +//! and the stable release. Stable builds never see a Beta. use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::sync::{mpsc, Arc}; @@ -22,6 +26,14 @@ use crate::remote_config::Rollout; pub const WEBSITE_DOWNLOAD: &str = "https://eternalmonitor.dev/download.html"; pub use eternal_updater::SETUP_FLAGS; const INSTALLER_NAME: &str = "EternalMonitor-Setup.exe"; +/// The nightly's fixed release and its installer (scripts/nightly.py). +const BETA_TAG: &str = "beta"; +const BETA_INSTALLER_NAME: &str = "EternalMonitor-Setup-beta.exe"; +/// What the popup says under a Beta's title. +const BETA_NOTE: &str = "A Beta build with the newest fixes, for testing."; +/// A new Beta comes out most nights. Its notification and popup come at +/// most this often, and its Later covers every Beta for this long. +const BETA_INTERVAL: u64 = 24 * 60 * 60; const CHECK_INTERVAL: u64 = 24 * 60 * 60; /// A relaunch checks again only when the last check is this old, so a crash /// loop or repeated restarts cannot hammer the website. @@ -117,6 +129,23 @@ pub fn newer_stable(candidate: &str, current: &str) -> Option { (release > current).then(|| release.to_string()) } +/// Whether `version` is a Beta channel build: `x.y.z-beta.…`, either the +/// nightly's `-beta..` or a hand-made `-beta.1`. +pub fn is_beta(version: &str) -> bool { + semver::Version::parse(version).is_ok_and(|version| version.pre.as_str().starts_with("beta.")) +} + +/// Whether `candidate` is newer than `current` by semver, previews included. +/// Build metadata is ignored; malformed versions are never newer. +fn newer(candidate: &str, current: &str) -> bool { + let parse = |version: &str| { + let mut version = semver::Version::parse(version).ok()?; + version.build = semver::BuildMetadata::EMPTY; + Some(version) + }; + matches!((parse(candidate), parse(current)), (Some(a), Some(b)) if a > b) +} + /// The Windows release named by the manifest, already validated. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct Release { @@ -139,6 +168,12 @@ fn recommended() -> bool { true } +impl Release { + fn is_beta(&self) -> bool { + is_beta(&self.version) + } +} + /// Unknown fields are ignored, so a newer manifest still parses here. #[derive(Deserialize)] struct Manifest { @@ -280,6 +315,88 @@ fn published_sha256(body: &str) -> Option { Some(pattern.captures(body)?[1].to_ascii_lowercase()) } +/// The Beta channel's fixed `beta` release as the GitHub API returns it, and +/// its installer's size. scripts/nightly.py writes the body: a line +/// `Version , from the Beta channel…` and the installer's SHA-256 +/// in a code block. No website manifest names a Beta, so GitHub's own digest +/// of the installer must match the body instead, which also fails closed +/// while the nightly swaps the installer before it rewrites the body. +pub fn parse_beta(data: &[u8], endpoints: &Endpoints) -> Result<(Release, u64), UpdateError> { + let bad = |why: &str| UpdateError::BadManifest(why.into()); + let unsafe_download = |why: &str| UpdateError::UnsafeDownload(why.into()); + let github: GithubRelease = + serde_json::from_slice(data).map_err(|error| bad(&error.to_string()))?; + if github.tag_name != BETA_TAG || github.draft || !github.prerelease { + return Err(unsafe_download("release is not the published Beta")); + } + let body = github.body.as_deref().unwrap_or_default(); + let version = only_match(r"(?m)^Version (\S+), from the Beta channel", body) + .and_then(|version| semver::Version::parse(&version).ok()) + .filter(|version| is_beta(&version.to_string()) && version.build.is_empty()) + .ok_or_else(|| bad("Beta has no version"))?; + let sha256 = only_match( + r"(?m)^SHA-256 of EternalMonitor-Setup-beta\.exe:\s*```\s*([0-9A-Fa-f]{64})\s*```", + body, + ) + .ok_or_else(|| bad("Beta has no SHA-256"))? + .to_ascii_lowercase(); + let installer = format!( + "{}{BETA_TAG}/{BETA_INSTALLER_NAME}", + endpoints.installer_prefix + ); + let asset = github + .assets + .iter() + .find(|asset| asset.name == BETA_INSTALLER_NAME) + .ok_or_else(|| unsafe_download("release has no installer"))?; + if asset.browser_download_url != installer { + return Err(unsafe_download("unexpected installer URL")); + } + if asset.digest.as_deref().map(str::to_ascii_lowercase) != Some(format!("sha256:{sha256}")) { + return Err(unsafe_download("GitHub digest is missing or differs")); + } + if asset.size == 0 || asset.size > MAX_INSTALLER_BYTES { + return Err(unsafe_download("installer size out of range")); + } + let release = Release { + version: version.to_string(), + installer, + sha256, + recommended: false, + note: Some(BETA_NOTE.into()), + rollout: None, + }; + Ok((release, asset.size)) +} + +/// The first group of `pattern`'s one match in `text`. None, or two, is +/// `None`. +fn only_match(pattern: &str, text: &str) -> Option { + let pattern = regex::Regex::new(pattern).ok()?; + let mut matches = pattern.captures_iter(text); + let first = matches.next()?[1].to_string(); + matches.next().is_none().then_some(first) +} + +/// Checks the Beta release against the one chosen and returns the +/// installer's size. A Beta published since then is refused: its hash is not +/// the one the helper will check. +fn check_beta_release( + data: &[u8], + release: &Release, + endpoints: &Endpoints, +) -> Result { + let (published, size) = parse_beta(data, endpoints)?; + if (&published.version, &published.installer, &published.sha256) + != (&release.version, &release.installer, &release.sha256) + { + return Err(UpdateError::UnsafeDownload( + "the Beta release changed".into(), + )); + } + Ok(size) +} + fn is_sha256(value: &str) -> bool { value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) } @@ -383,6 +500,13 @@ pub fn fetch_manifest(endpoints: &Endpoints) -> Result { parse_manifest(&data, endpoints) } +const GITHUB_JSON: &str = "application/vnd.github+json"; + +pub fn fetch_beta(endpoints: &Endpoints) -> Result { + let data = get_json(&format!("{}{BETA_TAG}", endpoints.release_api), GITHUB_JSON)?; + parse_beta(&data, endpoints).map(|(release, _)| release) +} + // ── The failure message ───────────────────────────────────────────────────── /// Whether the failure message can offer a newer version, and how that was @@ -512,9 +636,14 @@ pub fn download( progress: impl FnMut(u64, u64), ) -> Result { let io = |error: std::io::Error| UpdateError::Network(error.to_string()); - let api = format!("{}v{}", endpoints.release_api, release.version); - let github = get_json(&api, "application/vnd.github+json")?; - let size = check_github_release(&github, release)?; + let size = if release.is_beta() { + let github = get_json(&format!("{}{BETA_TAG}", endpoints.release_api), GITHUB_JSON)?; + check_beta_release(&github, release, endpoints)? + } else { + let api = format!("{}v{}", endpoints.release_api, release.version); + let github = get_json(&api, GITHUB_JSON)?; + check_github_release(&github, release)? + }; // Earlier updates are finished; their folders can go. let _ = std::fs::remove_dir_all(root); std::fs::create_dir_all(root).map_err(io)?; @@ -866,6 +995,12 @@ struct Saved { /// showing for that version, whatever the clock does later. #[serde(default)] rollout_reached: Option, + /// The newest Beta, as last read. Offered only to a Beta build. + #[serde(default)] + beta: Option, + /// When a notification last announced a Beta. + #[serde(default)] + beta_notified: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -936,6 +1071,9 @@ pub enum Command { pub struct State { current: String, auto: bool, + /// A Beta build also reads the Beta channel unless this is off + /// (Settings). + beta_updates: bool, manual_check: bool, saved: Saved, /// The release being downloaded, waited for or installed. The helper @@ -988,7 +1126,7 @@ fn take_pending(current: &str, saved: &mut Saved) -> Option { Some(UpdateResult::Setup { exit, .. }) => Some(*exit), _ => None, }; - if newer_stable(&to, current).is_none() { + if !newer(&to, current) { // Setup replaces the app before its last steps, so the new version // can run after Setup failed. return Some(match exit { @@ -1092,6 +1230,7 @@ impl State { let mut state = Self { current: current.into(), auto, + beta_updates: true, manual_check: false, saved, chosen: None, @@ -1134,11 +1273,57 @@ impl State { .is_none_or(|last| now.saturating_sub(last) >= RECHECK_OFFER || last > now) } + /// The newer of the stable release and, on a Beta build, the Beta, if + /// it is newer than this version. fn available(&self) -> Option<&Release> { - self.saved + let stable = self + .saved .latest .as_ref() - .filter(|latest| newer_stable(&latest.version, &self.current).is_some()) + .filter(|latest| newer_stable(&latest.version, &self.current).is_some()); + let beta = self.saved.beta.as_ref().filter(|beta| { + self.beta_channel() && beta.is_beta() && newer(&beta.version, &self.current) + }); + match (stable, beta) { + (Some(stable), Some(beta)) if newer(&beta.version, &stable.version) => Some(beta), + (stable, beta) => stable.or(beta), + } + } + + /// This is a Beta build and Beta updates are on. + fn beta_channel(&self) -> bool { + self.beta_updates && is_beta(&self.current) + } + + /// The Beta channel's answer, taken before [`Self::finish_check`] of the + /// same check. Offline, the saved Beta stays; a Beta that fails its + /// checks is not offered, and neither is the one saved before it. + fn finish_beta(&mut self, result: Result) { + match result { + Ok(release) => self.saved.beta = Some(release), + Err(error @ UpdateError::Network(_)) => { + tracing::info!(%error, "Beta check unavailable"); + } + Err(error) => { + tracing::warn!(%error, "Beta release refused"); + self.saved.beta = None; + } + } + } + + /// A Beta is not announced while a stream runs, nor within a day of + /// the last Beta notification. + fn beta_quiet(&mut self, now: u64, version: &str) -> bool { + // A time ahead of the clock counts as now, as for the popup. + if let Some(at) = self.saved.beta_notified.as_mut() { + *at = (*at).min(now); + } + is_beta(version) + && (self.streaming + || self + .saved + .beta_notified + .is_some_and(|at| now - at < BETA_INTERVAL)) } /// The available release is staged and this install's turn has not come: @@ -1230,17 +1415,24 @@ impl State { || self.snoozed(now) || self.held_back(now) || self.saved.notified.as_deref() == Some(&version) + || self.beta_quiet(now, &version) { return None; } self.saved.notified = Some(version.clone()); + if is_beta(&version) { + self.saved.beta_notified = Some(now); + } Some(version) } + /// Later covers its version, and on a Beta every newer Beta, for a day. fn snoozed(&self, now: u64) -> bool { match (&self.saved.later, self.available()) { (Some((version, at)), Some(latest)) => { - *version == latest.version && now.saturating_sub(*at) < LATER_INTERVAL && *at <= now + (*version == latest.version || (is_beta(version) && latest.is_beta())) + && now.saturating_sub(*at) < LATER_INTERVAL + && *at <= now } _ => false, } @@ -1278,7 +1470,8 @@ impl State { } /// Once per version while the banner offers it, then at most every - /// three days after it is closed. Never when automatic checks are off. + /// three days after it is closed; a Beta at most once a day, whichever + /// Beta was closed. Never when automatic checks are off. fn popup(&mut self, now: u64) -> Option { self.reach_rollout(now); // Closed "in the future" means the clock was set back: count it as @@ -1291,10 +1484,10 @@ impl State { return None; } let release = self.available()?; - let closed = - self.saved.popup.as_ref().is_some_and(|(version, at)| { - *version == release.version && now - at < POPUP_INTERVAL - }); + let closed = self.saved.popup.as_ref().is_some_and(|(version, at)| { + (*version == release.version && now - at < POPUP_INTERVAL) + || (is_beta(version) && release.is_beta() && now - at < BETA_INTERVAL) + }); (!closed).then(|| Popup { version: release.version.clone(), recommended: release.recommended, @@ -1727,8 +1920,8 @@ fn wait_while_checks_held() { impl Updater { /// `gate` is the process's handoff gate, shared with the window's - /// restart. - pub fn start(auto: bool, session: SharedSession, gate: Gate, wake: Wake) -> Self { + /// restart. `beta`: a Beta build also offers newer Betas. + pub fn start(auto: bool, beta: bool, session: SharedSession, gate: Gate, wake: Wake) -> Self { let path = state_path(); let saved = load(path.as_deref()); let mut state = State::resume( @@ -1738,6 +1931,7 @@ impl Updater { PREVIOUS.lock().take(), ); state.gate = gate; + state.beta_updates = beta; let state = Arc::new(Mutex::new(state)); save(path.as_deref(), &state.lock().saved); let (tx, rx) = mpsc::channel(); @@ -1832,6 +2026,17 @@ impl Updater { } } + /// Turning Beta updates back on reads the Beta channel again. + pub fn set_beta(&self, beta: bool) { + let check = self.with(|state| { + state.beta_updates = beta; + state.auto && state.beta_channel() + }); + if check { + self.send(Some(Command::Check { manual: false })); + } + } + pub fn check_now(&self) { let checking = self.with(|state| { let busy = state.check == CheckStatus::Checking; @@ -1908,20 +2113,35 @@ impl Worker { }; match command { Command::Check { manual } => { - self.state.lock().begin_check(manual); + let beta = { + let mut state = self.state.lock(); + state.begin_check(manual); + state.beta_channel() + }; (self.wake)(None); wait_while_checks_held(); + let beta = beta.then(|| fetch_beta(&self.endpoints)); let result = get_json(&self.endpoints.manifest, "application/json").and_then(|data| { // Per-PC settings apply from the next app run. crate::remote_config::save_fetched(&data); parse_manifest(&data, &self.endpoints) }); - let notify = - self.update(|state| state.finish_check(now_secs(), manual, result)); + let notify = self.update(|state| { + if let Some(beta) = beta { + state.finish_beta(beta); + } + state.finish_check(now_secs(), manual, result) + }); (self.wake)(notify.as_deref()); } Command::Download(release) => { + // A Beta published since the check replaces the offer + // here, rather than failing the download's hash check. + if release.is_beta() { + let latest = fetch_beta(&self.endpoints); + self.update(|state| state.finish_beta(latest)); + } if !self.update(|state| state.download_starts(&release)) { (self.wake)(None); continue; diff --git a/host/src/updates/tests.rs b/host/src/updates/tests.rs index 6665c6de..0231af76 100644 --- a/host/src/updates/tests.rs +++ b/host/src/updates/tests.rs @@ -58,8 +58,10 @@ fn versions_compare_as_semver_and_ignore_previews_build_metadata_and_garbage() { assert_eq!(newer_stable("0.4.0", "not a version"), None); } +/// The stable comparison never names a Beta; a Beta build reads the Beta +/// channel separately (see the Beta channel tests at the end). #[test] -fn beta_channel_builds_are_never_offered_and_get_the_next_stable_release() { +fn newer_stable_never_names_a_beta_and_a_beta_gets_the_next_stable_release() { // scripts/nightly.py versions the nightly Beta channel -beta... let beta = "0.3.4-beta.20261030.12"; assert_eq!(newer_stable(beta, "0.3.3"), None); @@ -2125,3 +2127,629 @@ fn the_update_funnel_is_logged_without_identifiers() { "{log}" ); } + +// ── The Beta channel ──────────────────────────────────────────────────────── + +/// The newest Beta in these tests, and the Beta build they run as. +const BETA: &str = "0.3.5-beta.20261107.26"; +const OLD_BETA: &str = "0.3.5-beta.20261105.20"; + +fn beta_installer() -> String { + "https://github.com/sidebandstudio/EternalMonitor/releases/download/beta/EternalMonitor-Setup-beta.exe".into() +} + +fn beta(version: &str) -> Release { + Release { + version: version.into(), + installer: beta_installer(), + sha256: SHA.into(), + recommended: false, + note: Some(BETA_NOTE.into()), + rollout: None, + } +} + +/// The beta release's body exactly as scripts/nightly.py publishes it, for +/// `version` and `sha` (test_nightly.py checks the fixture is nightly.py's). +fn beta_body(version: &str, sha: &str) -> String { + let fixture = + include_str!("../../../scripts/fixtures/beta-release-body.md").replace("\r\n", "\n"); + assert_eq!(fixture.matches("0.3.5-beta.20261107.26").count(), 1); + assert_eq!(fixture.matches(&"ab".repeat(32)).count(), 1); + fixture + .replace("0.3.5-beta.20261107.26", version) + .replace(&"ab".repeat(32), sha) +} + +/// The fixed `beta` release as the GitHub API returns it, with one installer +/// at `url` whose GitHub digest is `digest`. `extra` overrides release +/// fields; its `asset` entry overrides the installer's. +fn beta_github( + body: &str, + url: &str, + size: u64, + digest: &str, + extra: serde_json::Value, +) -> Vec { + let mut asset = serde_json::json!({ + "name": "EternalMonitor-Setup-beta.exe", + "size": size, + "browser_download_url": url, + "digest": format!("sha256:{digest}"), + }); + let mut release = serde_json::json!({ + "tag_name": "beta", "draft": false, "prerelease": true, "body": body, + }); + for (key, value) in extra.as_object().unwrap() { + if key == "asset" { + for (key, value) in value.as_object().unwrap() { + asset[key] = value.clone(); + } + } else { + release[key] = value.clone(); + } + } + release["assets"] = serde_json::json!([asset]); + release.to_string().into_bytes() +} + +fn beta_state(saved: Saved) -> State { + State::new(OLD_BETA, true, saved) +} + +#[test] +fn beta_versions_order_by_semver_across_beta_stable_and_dated_forms() { + for version in [ + BETA, + "0.3.4-beta.1", + "1.0.0-beta.2", + "0.3.5-beta.20261107.26+ci.1", + ] { + assert!(is_beta(version), "{version}"); + } + for version in [ + "0.3.5", + "0.3.5-rc.1", + "0.3.5-betas.1", + "0.3.5-beta", + "v0.3.5-beta.1", + "0.3-beta.1", + "", + "garbage", + ] { + assert!(!is_beta(version), "{version}"); + } + let newer_than = [ + // A later night, or a later run the same night. + ("0.3.5-beta.20261108.27", BETA), + (BETA, "0.3.5-beta.20261107.25"), + // Numbers compare as numbers, not text. + ("0.3.5-beta.20261107.100", BETA), + // Dated builds sort above the hand-made -beta.1, below the release. + (BETA, "0.3.5-beta.1"), + ("0.3.5", BETA), + (BETA, "0.3.4"), + ("0.3.6-beta.20261201.40", "0.3.5"), + ]; + for (candidate, current) in newer_than { + assert!(newer(candidate, current), "{candidate} > {current}"); + assert!(!newer(current, candidate), "{current} < {candidate}"); + } + for (candidate, current) in [ + (BETA, BETA), + // A higher run on an earlier night is older. + ("0.3.5-beta.20261106.30", BETA), + // Build metadata does not make a version newer. + ("0.3.5-beta.20261107.26+ci.2", BETA), + ("garbage", BETA), + (BETA, "garbage"), + ] { + assert!(!newer(candidate, current), "{candidate} vs {current}"); + } +} + +#[test] +fn the_beta_release_names_its_version_and_hash_in_the_body_nightly_writes() { + let endpoints = Endpoints::production(); + let body = beta_body(BETA, &SHA.to_uppercase()); + // A body edited on Windows may have CRLF line endings. + for body in [body.clone(), body.replace('\n', "\r\n")] { + let data = beta_github(&body, &beta_installer(), 9, SHA, serde_json::json!({})); + assert_eq!(parse_beta(&data, &endpoints), Ok((beta(BETA), 9))); + } + let shown = Popup { + version: BETA.into(), + recommended: false, + note: Some(BETA_NOTE.into()), + }; + assert_eq!(shown.message(), "An update is available."); +} + +#[test] +fn a_beta_with_a_missing_or_malformed_version_or_hash_is_refused() { + let endpoints = Endpoints::production(); + let url = beta_installer(); + let none = || serde_json::json!({}); + let body = beta_body(BETA, SHA); + let no_version = body.replace( + "Version 0.3.5-beta.20261107.26, from the Beta channel", + "From the Beta channel", + ); + let two_versions = format!("Version 0.3.5-beta.20261108.27, from the Beta channel\n{body}"); + let no_hash = body.replace(SHA, ""); + let two_hashes = format!( + "{body}\nSHA-256 of EternalMonitor-Setup-beta.exe:\n\n```\n{}\n```\n", + "0".repeat(64) + ); + let malformed = [ + beta_github(&beta_body("0.3.5", SHA), &url, 9, SHA, none()), + beta_github(&beta_body("0.3.5-rc.1", SHA), &url, 9, SHA, none()), + beta_github(&beta_body("0.3.5-beta.1+ci.2", SHA), &url, 9, SHA, none()), + beta_github(&beta_body("0.3.5-beta.01", SHA), &url, 9, SHA, none()), + beta_github( + &beta_body("v0.3.5-beta.20261107.26", SHA), + &url, + 9, + SHA, + none(), + ), + beta_github(&no_version, &url, 9, SHA, none()), + beta_github(&two_versions, &url, 9, SHA, none()), + beta_github(&no_hash, &url, 9, SHA, none()), + beta_github(&beta_body(BETA, &SHA[..63]), &url, 9, SHA, none()), + beta_github( + &beta_body(BETA, &SHA.replace('b', "g")), + &url, + 9, + SHA, + none(), + ), + beta_github(&two_hashes, &url, 9, SHA, none()), + beta_github(&body, &url, 9, SHA, serde_json::json!({ "body": null })), + b"rate limited".to_vec(), + ]; + for data in malformed { + assert!( + matches!( + parse_beta(&data, &endpoints), + Err(UpdateError::BadManifest(_)) + ), + "{}", + String::from_utf8_lossy(&data) + ); + } + let other = "0".repeat(64); + let unsafe_releases = [ + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "tag_name": "v0.3.5" }), + ), + beta_github(&body, &url, 9, SHA, serde_json::json!({ "draft": true })), + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "prerelease": false }), + ), + // GitHub's digest stands in for latest.json: it must be there and agree. + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "asset": { "digest": null } }), + ), + beta_github(&body, &url, 9, &other, none()), + // Mid-publish: the new installer is up, the body still names the old one. + beta_github(&beta_body(BETA, &other), &url, 9, SHA, none()), + beta_github(&body, &url.replace("https:", "http:"), 9, SHA, none()), + beta_github(&body, &production_installer("0.3.5"), 9, SHA, none()), + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "asset": { "name": "EternalMonitor-Setup.exe" } }), + ), + beta_github(&body, &url, 0, SHA, none()), + beta_github(&body, &url, MAX_INSTALLER_BYTES + 1, SHA, none()), + ]; + for data in unsafe_releases { + assert!( + matches!( + parse_beta(&data, &endpoints), + Err(UpdateError::UnsafeDownload(_)) + ), + "{}", + String::from_utf8_lossy(&data) + ); + } +} + +#[test] +fn a_stable_install_is_never_offered_a_beta() { + for current in ["0.3.4", "0.3.4-rc.1"] { + let mut s = State::new(current, true, Saved::default()); + assert!(!s.beta_channel(), "{current}"); + // Even with a Beta saved (an install that was a Beta before). + s.finish_beta(Ok(beta(BETA))); + assert_eq!(s.finish_check(1000, false, Ok(release("0.3.3"))), None); + assert_eq!(s.notice(1000), None); + assert_eq!(s.popup(1000), None); + assert!(s.update_now(false).is_none()); + // The next stable release is offered as before. + assert_eq!( + s.finish_check(1000 + DAY, false, Ok(release("0.3.5"))), + Some("0.3.5".into()) + ); + assert_eq!( + s.notice(1000 + DAY), + Some(Notice::Available("0.3.5".into())) + ); + } +} + +#[test] +fn a_beta_install_is_offered_the_newer_beta_and_never_an_older_one() { + let mut s = beta_state(Saved::default()); + assert!(s.beta_channel()); + s.finish_beta(Ok(beta(BETA))); + assert_eq!( + s.finish_check(1000, false, Ok(release("0.3.4"))), + Some(BETA.into()) + ); + assert_eq!(s.check, CheckStatus::Available); + assert_eq!(s.notice(1000), Some(Notice::Available(BETA.into()))); + let popup = s.popup(1000).unwrap(); + assert_eq!(popup.title(), format!("EternalMonitor {BETA} is ready")); + assert_eq!(popup.note.as_deref(), Some(BETA_NOTE)); + match s.update_now(false) { + Some(Command::Download(release)) => assert_eq!(release, beta(BETA)), + _ => panic!("Update now downloads the Beta"), + } + + for version in [ + OLD_BETA, + "0.3.5-beta.20261104.30", + "0.3.4-beta.20261020.3", + "0.3.5-beta.1", + ] { + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(version))); + assert_eq!( + s.finish_check(1000, false, Ok(release("0.3.4"))), + None, + "{version}" + ); + assert_eq!(s.check, CheckStatus::UpToDate, "{version}"); + assert_eq!(s.notice(1000), None, "{version}"); + assert!(s.update_now(false).is_none(), "{version}"); + } +} + +#[test] +fn a_newer_stable_release_wins_over_the_beta() { + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + assert_eq!( + s.finish_check(1000, false, Ok(release("0.3.5"))), + Some("0.3.5".into()) + ); + assert_eq!(s.notice(1000), Some(Notice::Available("0.3.5".into()))); + assert_eq!( + s.available().map(|release| release.installer.clone()), + Some(production_installer("0.3.5")) + ); + // A Beta of the release after it is newer still. + s.finish_beta(Ok(beta("0.3.6-beta.20261201.40"))); + s.finish_check(1000 + DAY, false, Ok(release("0.3.5"))); + assert_eq!( + s.notice(1000 + DAY), + Some(Notice::Available("0.3.6-beta.20261201.40".into())) + ); +} + +#[test] +fn a_refused_beta_is_not_offered_and_offline_keeps_the_last_one() { + let endpoints = Endpoints::production(); + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + s.finish_check(1000, false, Ok(release("0.3.4"))); + // Offline: the saved Beta is still offered. + s.finish_beta(Err(UpdateError::Network("offline".into()))); + s.finish_check( + 1000 + DAY, + false, + Err(UpdateError::Network("offline".into())), + ); + assert_eq!(s.notice(1000 + DAY), Some(Notice::Available(BETA.into()))); + // A release with no hash replaces it: neither is offered. + let no_hash = beta_github( + &beta_body(BETA, ""), + &beta_installer(), + 9, + SHA, + serde_json::json!({}), + ); + s.finish_beta(parse_beta(&no_hash, &endpoints).map(|(release, _)| release)); + s.finish_check(1000 + 2 * DAY, false, Ok(release("0.3.4"))); + assert_eq!(s.saved.beta, None); + assert_eq!(s.check, CheckStatus::UpToDate); + assert_eq!(s.notice(1000 + 2 * DAY), None); + // So does one whose installer GitHub hashes differently. + s.finish_beta(Ok(beta(BETA))); + let swapped = beta_github( + &beta_body(BETA, SHA), + &beta_installer(), + 9, + &"0".repeat(64), + serde_json::json!({}), + ); + s.finish_beta(parse_beta(&swapped, &endpoints).map(|(release, _)| release)); + assert_eq!(s.saved.beta, None); +} + +#[test] +fn turning_beta_updates_off_offers_only_stable_releases() { + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + s.finish_check(1000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(1000), Some(Notice::Available(BETA.into()))); + s.beta_updates = false; + assert!(!s.beta_channel()); + assert_eq!(s.notice(1000), None); + assert_eq!( + s.finish_check(1000 + DAY, false, Ok(release("0.3.5"))), + Some("0.3.5".into()) + ); + assert_eq!( + s.notice(1000 + DAY), + Some(Notice::Available("0.3.5".into())) + ); +} + +#[test] +fn betas_are_announced_at_most_once_a_day_and_never_during_a_stream() { + const HOUR: u64 = 60 * 60; + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + assert_eq!( + s.finish_check(1000, false, Ok(release("0.3.4"))), + Some(BETA.into()) + ); + s.popup_closed(1000); + // A newer Beta the same day is on the banner, without a notification or + // the popup. + let next = "0.3.5-beta.20261107.27"; + s.finish_beta(Ok(beta(next))); + assert_eq!( + s.finish_check(1000 + HOUR, false, Ok(release("0.3.4"))), + None + ); + assert_eq!(s.notice(1000 + HOUR), Some(Notice::Available(next.into()))); + assert_eq!(s.popup(1000 + DAY - 1), None); + // A day on, both come back, also after a restart. + let mut s = beta_state(s.saved.clone()); + assert!(s.popup(1000 + DAY).is_some()); + assert_eq!( + s.finish_check(1000 + DAY, false, Ok(release("0.3.4"))), + Some(next.into()) + ); + // Not while a stream runs; a check after it announces it. + let third = "0.3.5-beta.20261109.28"; + s.finish_beta(Ok(beta(third))); + s.streaming = true; + assert_eq!( + s.finish_check(1000 + 2 * DAY, false, Ok(release("0.3.4"))), + None + ); + s.streaming = false; + assert_eq!( + s.finish_check(1000 + 2 * DAY + HOUR, false, Ok(release("0.3.4"))), + Some(third.into()) + ); + // Announced "in the future" (the clock went back) counts as now. + s.saved.beta_notified = Some(u64::MAX / 2); + s.finish_beta(Ok(beta("0.3.5-beta.20261120.40"))); + assert_eq!(s.finish_check(20 * DAY, false, Ok(release("0.3.4"))), None); + assert_eq!(s.saved.beta_notified, Some(20 * DAY)); + assert_eq!( + s.finish_check(21 * DAY, false, Ok(release("0.3.4"))), + Some("0.3.5-beta.20261120.40".into()) + ); + // A stable release keeps its own rules, streaming or not. + s.streaming = true; + assert_eq!( + s.finish_check(21 * DAY + HOUR, false, Ok(release("0.3.5"))), + Some("0.3.5".into()) + ); +} + +#[test] +fn later_on_a_beta_covers_newer_betas_for_a_day_but_not_a_stable_release() { + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + s.finish_check(1000, false, Ok(release("0.3.4"))); + s.later(1000); + assert_eq!(s.notice(1000), None); + let next = "0.3.5-beta.20261108.27"; + s.finish_beta(Ok(beta(next))); + assert_eq!(s.finish_check(2000, false, Ok(release("0.3.4"))), None); + assert_eq!(s.notice(1000 + DAY - 1), None); + assert_eq!(s.notice(1000 + DAY), Some(Notice::Available(next.into()))); + s.later(1000 + DAY); + s.finish_check(2000 + DAY, false, Ok(release("0.3.5"))); + assert_eq!( + s.notice(2000 + DAY), + Some(Notice::Available("0.3.5".into())) + ); +} + +#[test] +fn relaunch_after_a_beta_update_reports_whether_it_finished() { + let saved = Saved { + pending: Some((OLD_BETA.into(), BETA.into())), + beta: Some(beta(BETA)), + ..Saved::default() + }; + let updated = State::new(BETA, true, saved.clone()); + assert_eq!(updated.notice(0), Some(Notice::Updated(BETA.into()))); + let failed = State::new(OLD_BETA, true, saved); + assert!( + matches!(failed.notice(0), Some(Notice::Failed(_))), + "{:?}", + failed.notice(0) + ); +} + +/// A fake GitHub serving the `beta` release for `published`, whose installer +/// redirects to a blob that serves `served`. +fn fake_beta(published: &[u8], served: Vec) -> (Endpoints, Release) { + let sha = sha_of(published); + let size = published.len() as u64; + let api_sha = sha.clone(); + let base = serve(Box::new(move |base, path| { + let installer = format!("{base}/releases/download/beta/EternalMonitor-Setup-beta.exe"); + match path { + "/repos/sidebandstudio/EternalMonitor/releases/tags/beta" => { + let body = beta_body(BETA, &api_sha); + ( + 200, + vec![], + beta_github(&body, &installer, size, &api_sha, serde_json::json!({})), + ) + } + "/releases/download/beta/EternalMonitor-Setup-beta.exe" => ( + 302, + vec![("Location".into(), "/assets/blob".into())], + Vec::new(), + ), + "/assets/blob" => (200, vec![], served.clone()), + _ => (404, vec![], Vec::new()), + } + })); + let endpoints = Endpoints::local(&base).unwrap(); + let release = fetch_beta(&endpoints).unwrap(); + assert_eq!( + release.installer, + format!("{base}/releases/download/beta/EternalMonitor-Setup-beta.exe") + ); + (endpoints, release) +} + +#[test] +fn a_beta_downloads_and_is_verified_like_a_stable_release() { + let payload = b"MZ fake beta installer".to_vec(); + let (endpoints, release) = fake_beta(&payload, payload.clone()); + assert_eq!( + (release.version.as_str(), release.sha256.as_str()), + (BETA, sha_of(&payload).as_str()) + ); + let path = download_case(&endpoints, &release).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), payload); + assert!(eternal_updater::open_verified(&path, &release.sha256).is_ok()); + std::fs::remove_dir_all(path.parent().unwrap().parent().unwrap()).unwrap(); + + let (endpoints, release) = fake_beta( + b"MZ fake beta installer", + b"MZ evil beta installer".to_vec(), + ); + let error = download_case(&endpoints, &release).unwrap_err(); + assert!( + matches!(&error, UpdateError::UnsafeDownload(why) if why.contains("SHA-256")), + "{error}" + ); + + // The release now names another Beta than the one chosen. + let (endpoints, release) = fake_beta(&payload, payload.clone()); + let chosen = Release { + version: OLD_BETA.into(), + ..release + }; + let error = download_case(&endpoints, &chosen).unwrap_err(); + assert!( + matches!(&error, UpdateError::UnsafeDownload(why) if why.contains("changed")), + "{error}" + ); +} + +/// Through the worker against a fake website and GitHub: a stable install +/// never asks for the Beta, a Beta install offers it, and a Beta published +/// after the check replaces the offer before anything downloads. +#[test] +fn the_worker_reads_the_beta_only_for_a_beta_build_and_again_before_downloading() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let served = Arc::new(Mutex::new(BETA.to_string())); + let beta_reads = Arc::new(AtomicUsize::new(0)); + let (route_served, route_reads) = (served.clone(), beta_reads.clone()); + let base = serve(Box::new(move |base, path| match path { + "/latest.json" => ( + 200, + vec![], + manifest( + "0.3.4", + &format!("{base}/releases/download/v0.3.4/EternalMonitor-Setup.exe"), + SHA, + ), + ), + "/repos/sidebandstudio/EternalMonitor/releases/tags/beta" => { + route_reads.fetch_add(1, Ordering::SeqCst); + let installer = format!("{base}/releases/download/beta/EternalMonitor-Setup-beta.exe"); + let body = beta_body(&route_served.lock(), SHA); + ( + 200, + vec![], + beta_github(&body, &installer, 9, SHA, serde_json::json!({})), + ) + } + _ => (404, vec![], Vec::new()), + })); + let endpoints = Endpoints::local(&base).unwrap(); + let run = |state: &Arc>, command: Command| { + let (tx, rx) = mpsc::channel(); + tx.send(command).unwrap(); + drop(tx); + Worker { + state: state.clone(), + path: None, + endpoints: endpoints.clone(), + session: Arc::new(Mutex::new(crate::transport::session::Session::new(1))), + wake: Arc::new(|_| {}), + } + .run(rx); + }; + + let stable = Arc::new(Mutex::new(State::new("0.3.4", true, Saved::default()))); + run(&stable, Command::Check { manual: false }); + assert_eq!(stable.lock().check, CheckStatus::UpToDate); + assert_eq!( + beta_reads.load(Ordering::SeqCst), + 0, + "a stable install never asks" + ); + + let state = Arc::new(Mutex::new(beta_state(Saved::default()))); + run(&state, Command::Check { manual: false }); + assert_eq!(beta_reads.load(Ordering::SeqCst), 1); + assert_eq!( + state.lock().notice(now_secs()), + Some(Notice::Available(BETA.into())) + ); + let Some(Command::Download(chosen)) = state.lock().update_now(false) else { + panic!("Update now downloads the Beta"); + }; + let next = "0.3.5-beta.20261108.27"; + *served.lock() = next.into(); + run(&state, Command::Download(chosen)); + assert_eq!(beta_reads.load(Ordering::SeqCst), 2); + let state = state.lock(); + assert_eq!(state.phase, Phase::Idle, "nothing downloaded"); + assert_eq!( + state.notice(now_secs()), + Some(Notice::Available(next.into())) + ); +} diff --git a/scripts/fixtures/beta-release-body.md b/scripts/fixtures/beta-release-body.md new file mode 100644 index 00000000..e00fee1d --- /dev/null +++ b/scripts/fixtures/beta-release-body.md @@ -0,0 +1,13 @@ +Version 0.3.5-beta.20261107.26, from the Beta channel: the newest fixes, built every night. For testing; you can go back to Stable anytime. + +Built from `release/0.3.5-candidate` at [`62b9d9d`](https://github.com/sidebandstudio/EternalMonitor/commit/62b9d9d000000000000000000000000000000000). [Changes since v0.3.4](https://github.com/sidebandstudio/EternalMonitor/compare/v0.3.4...62b9d9d000000000000000000000000000000000). + +To go back to Stable, run the installer from https://eternalmonitor.dev/download.html over this one. The app offers the next Stable version when it comes out. iPad betas come through TestFlight: https://eternalmonitor.dev/download.html#ipad + +SHA-256 of EternalMonitor-Setup-beta.exe: + +``` +abababababababababababababababababababababababababababababababab +``` + + diff --git a/scripts/nightly.py b/scripts/nightly.py index df60c1f3..1275ec6a 100644 --- a/scripts/nightly.py +++ b/scripts/nightly.py @@ -20,8 +20,9 @@ Users see these builds as the Beta channel. Each is a semver pre-release, -beta.., which semver orders below and above a -hand-made -beta.1. The apps never offer a pre-release, and a Beta user -is offered the next stable release as usual (host/src/updates.rs). +hand-made -beta.1. A stable install is never offered one. A Beta +install reads the beta release's body (beta_body()) and is offered the newer +of that Beta and the next stable release (host/src/updates.rs). """ import argparse import datetime @@ -121,6 +122,8 @@ def stamp(cargo_toml, cargo_lock, version): def notes(version, branch, sha, stable, sha256, repo=REPO): + """A Beta install reads the first line's "Version , from the Beta + channel" and the hash after "SHA-256 of :" (scripts/fixtures/beta-release-body.md).""" if not re.fullmatch(r"[0-9a-f]{40}", sha) or not re.fullmatch(r"[0-9A-Fa-f]{64}", sha256): raise ValueError("notes need the full commit and the installer's SHA-256") return ( @@ -159,6 +162,11 @@ def marker(sha): return f"" +def beta_body(notes, sha): + """The beta release's body once sha's build has finished publishing.""" + return f"{notes.rstrip()}\n\n{marker(sha)}\n" + + def published_commit(body): match = MARKER.search(body or "") return match[1] if match else "" @@ -344,7 +352,7 @@ def publish(github, tag, sha, title, notes, path, sleep=time.sleep): # 3. The tag, then the marker that says this commit is done. github.move_tag(MOVING_TAG, sha) - github.edit(MOVING_TAG, title, f"{notes.rstrip()}\n\n{marker(sha)}\n") + github.edit(MOVING_TAG, title, beta_body(notes, sha)) def git(*args): diff --git a/scripts/test_nightly.py b/scripts/test_nightly.py index 111d8016..6d9e2d85 100644 --- a/scripts/test_nightly.py +++ b/scripts/test_nightly.py @@ -140,6 +140,14 @@ def test_notes_name_the_build_and_the_way_back(self): self.assertIn("TestFlight", text) self.assertIn(f"```\n{HASH}\n```", text) + def test_the_beta_body_is_the_one_the_windows_app_reads(self): + # host/src/updates.rs reads the version and SHA-256 from this fixture too. + fixture = (ROOT / "scripts/fixtures/beta-release-body.md").read_text(encoding="utf-8") + text = nightly.beta_body( + nightly.notes("0.3.5-beta.20261107.26", "release/0.3.5-candidate", SHA, "0.3.4", HASH), SHA) + self.assertEqual(text, fixture.replace("\r\n", "\n")) + self.assertEqual(nightly.published_commit(text), SHA) + def test_notes_need_the_full_commit_and_hash(self): with self.assertRaises(ValueError): nightly.notes("v", "b", SHA[:7], "0.3.3", HASH) From 5dd5b9ceea837ac320e073e78e4d45867959f435 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 17:12:11 -0500 Subject: [PATCH 067/226] Require a signed manifest before offering a Beta The release body and the installer's GitHub digest are both written by whoever can publish releases, so they alone can't vouch for a Beta. A Beta is now offered only when the release carries beta-manifest.txt with a valid Ed25519 signature, beta-manifest.sig, by the public key pinned in the app, and the manifest's version, SHA-256 and size match the release body, the installer's digest and size, and so the downloaded file. The manifest is canonical text (version 1, UTF-8, LF): it parses only if its fields render back to the same bytes. A missing or invalid manifest or signature means no offer and drops a saved Beta, which also covers the nightly's swap window: nothing is offered until the build is signed. The manifest and signature download over HTTPS from the allowed hosts, as the installer does; ring verifies. scripts/sign_beta_manifest.py reads the beta release with gh, checks its body, asset and digest agree, writes the manifest, signs it with openssl, the private key piped through stdin only, refuses a key that isn't the pinned one, uploads both files and verifies them. --verify-only checks what is published. Fixtures share the manifest text, and a signature made by OpenSSL from a published test-only seed that the host's tests reproduce with ring. --- SECURITY.md | 10 +- docs/beta-testing.md | 14 +- host/src/updates.rs | 215 ++++++++++-- host/src/updates/tests.rs | 429 ++++++++++++++++++++---- scripts/fixtures/.gitattributes | 2 + scripts/fixtures/beta-manifest.test.sig | 2 + scripts/fixtures/beta-manifest.txt | 5 + scripts/nightly.py | 3 +- scripts/sign_beta_manifest.py | 192 +++++++++++ scripts/test_sign_beta_manifest.py | 212 ++++++++++++ 10 files changed, 965 insertions(+), 119 deletions(-) create mode 100644 scripts/fixtures/.gitattributes create mode 100644 scripts/fixtures/beta-manifest.test.sig create mode 100644 scripts/fixtures/beta-manifest.txt create mode 100755 scripts/sign_beta_manifest.py create mode 100644 scripts/test_sign_beta_manifest.py diff --git a/SECURITY.md b/SECURITY.md index c19ac7c8..6e8b08be 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -34,9 +34,13 @@ Knowing these limits helps you judge whether a finding is new. - In-app updates (Windows) trust the network for nothing: the host fetches over HTTPS from an allowlist of GitHub hosts, and the installer must match the SHA-256 in both eternalmonitor.dev/latest.json and the GitHub release. A Beta - build may also update to the newer Beta from the fixed `beta` GitHub release; - that installer must match both the SHA-256 in the release body and GitHub's own - digest of the file. Stable builds never take a Beta. Against + build may also update to the newer Beta from the fixed `beta` GitHub release. + Beta updates are signed: the release must carry a manifest with a valid + Ed25519 signature by a public key pinned in the app, made with a key the + release workflows don't have, and the manifest's version, SHA-256 and size + must match the release body, GitHub's digest of the installer and the + downloaded file. Without a valid signature no Beta is offered. Stable builds + never take a Beta. Against other software already running as your Windows user, the updater adds defense in depth rather than a guarantee. Its helper loads DLLs only from System32 and imports only Windows system libraries. It inherits only its output pipe from diff --git a/docs/beta-testing.md b/docs/beta-testing.md index ea2591ac..46248acb 100644 --- a/docs/beta-testing.md +++ b/docs/beta-testing.md @@ -34,10 +34,16 @@ back to Stable by running the Stable installer over it. Stable users are never offered a Beta build. - **Updates on Beta:** a Beta build also reads the `beta` release below and offers the newer of that Beta and the next Stable release, with the same - banner, popup and helper as a Stable update. The version and SHA-256 come - from the release body `nightly.py` writes, and GitHub's digest of the - installer must match it. A new Beta is announced at most once a day, never - during a stream. Settings, General, **Get Beta updates** (shown only on a + banner, popup and helper as a Stable update. Beta updates are signed and + verified against a public key pinned in the app: the release must carry + `beta-manifest.txt` and its Ed25519 signature `beta-manifest.sig`, and the + manifest's version, SHA-256 and size must match the release body + `nightly.py` writes, GitHub's digest of the installer and the downloaded + file. The nightly does not sign; until a build is signed, Beta installs are + offered no Beta. After a nightly publishes, pipe the signing key into + `scripts/sign_beta_manifest.py`, which checks the release, signs, uploads + both files and verifies them; `--verify-only` checks what is published. + A new Beta is announced at most once a day, never during a stream. Settings, General, **Get Beta updates** (shown only on a Beta build) turns this off. Taking a Stable release leaves the Beta channel. Betas built before this arrived can't update themselves, so their testers install one newer Beta by hand. diff --git a/host/src/updates.rs b/host/src/updates.rs index e56cfcc8..77b96038 100644 --- a/host/src/updates.rs +++ b/host/src/updates.rs @@ -9,7 +9,10 @@ //! //! A Beta build (`x.y.z-beta.…`) also reads the nightly's fixed `beta` //! GitHub release (scripts/nightly.py) and is offered the newer of that Beta -//! and the stable release. Stable builds never see a Beta. +//! and the stable release. Stable builds never see a Beta. A Beta needs a +//! manifest signed with a pinned Ed25519 key that GitHub does not hold +//! (scripts/sign_beta_manifest.py): the release body and the installer's +//! digest are both written by whoever can publish releases. use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::sync::{mpsc, Arc}; @@ -29,6 +32,16 @@ const INSTALLER_NAME: &str = "EternalMonitor-Setup.exe"; /// The nightly's fixed release and its installer (scripts/nightly.py). const BETA_TAG: &str = "beta"; const BETA_INSTALLER_NAME: &str = "EternalMonitor-Setup-beta.exe"; +/// The Beta's signed manifest and its raw Ed25519 signature, assets of the +/// same release. +const BETA_MANIFEST_NAME: &str = "beta-manifest.txt"; +const BETA_SIGNATURE_NAME: &str = "beta-manifest.sig"; +const MAX_MANIFEST_BYTES: u64 = 4 * 1024; +/// The Ed25519 public key Beta manifests are signed with, raw. +const BETA_PUBLIC_KEY: [u8; 32] = [ + 0x82, 0xc2, 0xaf, 0x21, 0xc6, 0x85, 0xd2, 0xc0, 0x42, 0xc5, 0x1f, 0x72, 0xa3, 0x4c, 0x78, 0xbd, + 0xb5, 0xd0, 0x04, 0xba, 0xd8, 0x9e, 0x75, 0xa8, 0x30, 0x6a, 0x43, 0x22, 0x4d, 0x16, 0xec, 0x5a, +]; /// What the popup says under a Beta's title. const BETA_NOTE: &str = "A Beta build with the newest fixes, for testing."; /// A new Beta comes out most nights. Its notification and popup come at @@ -68,6 +81,8 @@ pub struct Endpoints { /// Hosts the download may start at or be redirected to. pub download_hosts: Vec, pub https_only: bool, + /// The Ed25519 public key a Beta manifest must be signed with. + pub beta_key: [u8; 32], } impl Endpoints { @@ -84,6 +99,7 @@ impl Endpoints { "release-assets.githubusercontent.com".into(), ], https_only: true, + beta_key: BETA_PUBLIC_KEY, } } @@ -109,6 +125,7 @@ impl Endpoints { installer_prefix: format!("{base}/releases/download/"), download_hosts: vec![host], https_only: false, + beta_key: BETA_PUBLIC_KEY, }) } } @@ -315,13 +332,22 @@ fn published_sha256(body: &str) -> Option { Some(pattern.captures(body)?[1].to_ascii_lowercase()) } -/// The Beta channel's fixed `beta` release as the GitHub API returns it, and -/// its installer's size. scripts/nightly.py writes the body: a line -/// `Version , from the Beta channel…` and the installer's SHA-256 -/// in a code block. No website manifest names a Beta, so GitHub's own digest -/// of the installer must match the body instead, which also fails closed -/// while the nightly swaps the installer before it rewrites the body. -pub fn parse_beta(data: &[u8], endpoints: &Endpoints) -> Result<(Release, u64), UpdateError> { +/// The `beta` release as GitHub lists it, checked: the Beta its body names, +/// the installer's size, and where its signed manifest and signature are. +#[derive(Debug, PartialEq, Eq)] +pub struct BetaListing { + pub release: Release, + pub size: u64, + manifest: String, + signature: String, +} + +/// The Beta channel's fixed `beta` release as the GitHub API returns it. +/// scripts/nightly.py writes the body: a line `Version , from the +/// Beta channel…` and the installer's SHA-256 in a code block. GitHub's own +/// digest of the installer must match it, and the signed manifest's assets +/// must be there; [`check_beta_manifest`] checks those against the listing. +pub fn parse_beta(data: &[u8], endpoints: &Endpoints) -> Result { let bad = |why: &str| UpdateError::BadManifest(why.into()); let unsafe_download = |why: &str| UpdateError::UnsafeDownload(why.into()); let github: GithubRelease = @@ -340,33 +366,36 @@ pub fn parse_beta(data: &[u8], endpoints: &Endpoints) -> Result<(Release, u64), ) .ok_or_else(|| bad("Beta has no SHA-256"))? .to_ascii_lowercase(); - let installer = format!( - "{}{BETA_TAG}/{BETA_INSTALLER_NAME}", - endpoints.installer_prefix - ); - let asset = github - .assets - .iter() - .find(|asset| asset.name == BETA_INSTALLER_NAME) - .ok_or_else(|| unsafe_download("release has no installer"))?; - if asset.browser_download_url != installer { - return Err(unsafe_download("unexpected installer URL")); - } - if asset.digest.as_deref().map(str::to_ascii_lowercase) != Some(format!("sha256:{sha256}")) { + // Each asset must be the release's own file of that name. + let asset = |name: &str| { + let url = format!("{}{BETA_TAG}/{name}", endpoints.installer_prefix); + match github.assets.iter().find(|asset| asset.name == name) { + Some(asset) if asset.browser_download_url == url => Ok(asset), + Some(_) => Err(unsafe_download("unexpected Beta asset URL")), + None => Err(bad("Beta asset missing")), + } + }; + let installer = asset(BETA_INSTALLER_NAME)?; + if installer.digest.as_deref().map(str::to_ascii_lowercase) != Some(format!("sha256:{sha256}")) + { return Err(unsafe_download("GitHub digest is missing or differs")); } - if asset.size == 0 || asset.size > MAX_INSTALLER_BYTES { + if installer.size == 0 || installer.size > MAX_INSTALLER_BYTES { return Err(unsafe_download("installer size out of range")); } - let release = Release { - version: version.to_string(), - installer, - sha256, - recommended: false, - note: Some(BETA_NOTE.into()), - rollout: None, - }; - Ok((release, asset.size)) + Ok(BetaListing { + release: Release { + version: version.to_string(), + installer: installer.browser_download_url.clone(), + sha256, + recommended: false, + note: Some(BETA_NOTE.into()), + rollout: None, + }, + size: installer.size, + manifest: asset(BETA_MANIFEST_NAME)?.browser_download_url.clone(), + signature: asset(BETA_SIGNATURE_NAME)?.browser_download_url.clone(), + }) } /// The first group of `pattern`'s one match in `text`. None, or two, is @@ -378,15 +407,84 @@ fn only_match(pattern: &str, text: &str) -> Option { matches.next().is_none().then_some(first) } -/// Checks the Beta release against the one chosen and returns the -/// installer's size. A Beta published since then is refused: its hash is not -/// the one the helper will check. +/// What `beta-manifest.txt` says, signed outside GitHub. +#[derive(Debug, PartialEq, Eq)] +pub struct BetaManifest { + pub version: String, + pub sha256: String, + pub size: u64, +} + +impl BetaManifest { + /// The canonical text, version 1: UTF-8, every line ending in LF. + pub fn text(&self) -> String { + format!( + "eternalmonitor-beta-manifest 1\nversion={}\nsha256={}\nsize={}\nasset={BETA_INSTALLER_NAME}\n", + self.version, self.sha256, self.size + ) + } + + /// Only the canonical text parses: its fields must render back to the + /// same bytes, so no spacing, case or line ending varies. + pub fn parse(data: &[u8]) -> Option { + let text = std::str::from_utf8(data).ok()?; + // The header and asset lines are checked by rendering back. + let mut lines = text.strip_suffix('\n')?.split('\n').skip(1); + let mut field = |key: &str| { + let line = lines.next()?; + line.strip_prefix(key)? + .strip_prefix('=') + .map(str::to_string) + }; + let manifest = Self { + version: field("version")?, + sha256: field("sha256")?, + size: field("size")?.parse().ok()?, + }; + let valid = semver::Version::parse(&manifest.version).is_ok_and(|version| { + version.to_string() == manifest.version && version.build.is_empty() + }) && is_beta(&manifest.version) + && is_sha256(&manifest.sha256) + && manifest.sha256 == manifest.sha256.to_ascii_lowercase(); + (valid && manifest.text().as_bytes() == data).then_some(manifest) + } +} + +/// The manifest must carry a valid signature by `key` and name the Beta, +/// hash and size the listing does. Anything else means no offer. +pub fn check_beta_manifest( + listing: &BetaListing, + manifest: &[u8], + signature: &[u8], + key: &[u8; 32], +) -> Result<(), UpdateError> { + use ring::signature::{UnparsedPublicKey, ED25519}; + UnparsedPublicKey::new(&ED25519, key) + .verify(manifest, signature) + .map_err(|_| UpdateError::UnsafeDownload("Beta signature is not valid".into()))?; + let manifest = BetaManifest::parse(manifest) + .ok_or_else(|| UpdateError::BadManifest("malformed Beta manifest".into()))?; + let release = &listing.release; + if (&manifest.version, &manifest.sha256, manifest.size) + != (&release.version, &release.sha256, listing.size) + { + return Err(UpdateError::UnsafeDownload( + "Beta manifest differs from the release".into(), + )); + } + Ok(()) +} + +/// Checks the Beta release against the one chosen, whose manifest was +/// verified, and returns the installer's size. A Beta published since then +/// is refused: its hash is not the one the helper will check. fn check_beta_release( data: &[u8], release: &Release, endpoints: &Endpoints, ) -> Result { - let (published, size) = parse_beta(data, endpoints)?; + let listing = parse_beta(data, endpoints)?; + let published = &listing.release; if (&published.version, &published.installer, &published.sha256) != (&release.version, &release.installer, &release.sha256) { @@ -394,7 +492,7 @@ fn check_beta_release( "the Beta release changed".into(), )); } - Ok(size) + Ok(listing.size) } fn is_sha256(value: &str) -> bool { @@ -502,9 +600,52 @@ pub fn fetch_manifest(endpoints: &Endpoints) -> Result { const GITHUB_JSON: &str = "application/vnd.github+json"; +/// The newest Beta, offered only with a valid signed manifest. pub fn fetch_beta(endpoints: &Endpoints) -> Result { let data = get_json(&format!("{}{BETA_TAG}", endpoints.release_api), GITHUB_JSON)?; - parse_beta(&data, endpoints).map(|(release, _)| release) + let listing = parse_beta(&data, endpoints)?; + let manifest = get_asset(&listing.manifest, endpoints)?; + let signature = get_asset(&listing.signature, endpoints)?; + check_beta_manifest(&listing, &manifest, &signature, &endpoints.beta_key)?; + Ok(listing.release) +} + +/// A small release file, following redirects only to allowed hosts. +fn get_asset(url: &str, endpoints: &Endpoints) -> Result, UpdateError> { + let network = |error: &dyn std::fmt::Display| UpdateError::Network(error.to_string()); + let unsafe_download = |why: String| UpdateError::UnsafeDownload(why); + let agent = agent(Duration::from_secs(10)); + let mut url = url.to_string(); + for _ in 0..=MAX_REDIRECTS { + if !allowed_url(&url, endpoints) { + return Err(unsafe_download(format!("host not allowed: {url}"))); + } + let mut response = agent + .get(&url) + .header("User-Agent", USER_AGENT) + .call() + .map_err(|e| network(&e))?; + if response.status().is_redirection() { + let location = response + .headers() + .get("location") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| unsafe_download("redirect without a location".into()))?; + url = resolve_redirect(&url, location) + .ok_or_else(|| unsafe_download(format!("bad redirect: {location}")))?; + continue; + } + if !response.status().is_success() { + return Err(UpdateError::Network(format!("HTTP {}", response.status()))); + } + return response + .body_mut() + .with_config() + .limit(MAX_MANIFEST_BYTES) + .read_to_vec() + .map_err(|e| network(&e)); + } + Err(unsafe_download("too many redirects".into())) } // ── The failure message ───────────────────────────────────────────────────── diff --git a/host/src/updates/tests.rs b/host/src/updates/tests.rs index 0231af76..e412f9e5 100644 --- a/host/src/updates/tests.rs +++ b/host/src/updates/tests.rs @@ -2162,8 +2162,9 @@ fn beta_body(version: &str, sha: &str) -> String { } /// The fixed `beta` release as the GitHub API returns it, with one installer -/// at `url` whose GitHub digest is `digest`. `extra` overrides release -/// fields; its `asset` entry overrides the installer's. +/// at `url` whose GitHub digest is `digest`, and the manifest and signature +/// beside it. `extra` overrides release fields; its `asset` entry overrides +/// the installer's, and `drop` lists assets to leave out. fn beta_github( body: &str, url: &str, @@ -2180,16 +2181,28 @@ fn beta_github( let mut release = serde_json::json!({ "tag_name": "beta", "draft": false, "prerelease": true, "body": body, }); + let mut drop = Vec::new(); for (key, value) in extra.as_object().unwrap() { if key == "asset" { for (key, value) in value.as_object().unwrap() { asset[key] = value.clone(); } + } else if key == "drop" { + drop = value.as_array().unwrap().clone(); } else { release[key] = value.clone(); } } - release["assets"] = serde_json::json!([asset]); + let folder = url.rsplit_once('/').unwrap().0; + let mut assets = vec![asset]; + for name in [BETA_MANIFEST_NAME, BETA_SIGNATURE_NAME] { + if !drop.contains(&name.into()) { + assets.push(serde_json::json!({ + "name": name, "size": 64, "browser_download_url": format!("{folder}/{name}"), + })); + } + } + release["assets"] = serde_json::json!(assets); release.to_string().into_bytes() } @@ -2255,7 +2268,8 @@ fn the_beta_release_names_its_version_and_hash_in_the_body_nightly_writes() { // A body edited on Windows may have CRLF line endings. for body in [body.clone(), body.replace('\n', "\r\n")] { let data = beta_github(&body, &beta_installer(), 9, SHA, serde_json::json!({})); - assert_eq!(parse_beta(&data, &endpoints), Ok((beta(BETA), 9))); + let listing = parse_beta(&data, &endpoints).unwrap(); + assert_eq!((listing.release, listing.size), (beta(BETA), 9)); } let shown = Popup { version: BETA.into(), @@ -2306,6 +2320,28 @@ fn a_beta_with_a_missing_or_malformed_version_or_hash_is_refused() { ), beta_github(&two_hashes, &url, 9, SHA, none()), beta_github(&body, &url, 9, SHA, serde_json::json!({ "body": null })), + // The installer and the signed manifest's files must be there. + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "asset": { "name": "EternalMonitor-Setup.exe" } }), + ), + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "drop": ["beta-manifest.txt"] }), + ), + beta_github( + &body, + &url, + 9, + SHA, + serde_json::json!({ "drop": ["beta-manifest.sig"] }), + ), b"rate limited".to_vec(), ]; for data in malformed { @@ -2335,7 +2371,7 @@ fn a_beta_with_a_missing_or_malformed_version_or_hash_is_refused() { SHA, serde_json::json!({ "prerelease": false }), ), - // GitHub's digest stands in for latest.json: it must be there and agree. + // GitHub's digest of the installer must be there and agree. beta_github( &body, &url, @@ -2348,13 +2384,6 @@ fn a_beta_with_a_missing_or_malformed_version_or_hash_is_refused() { beta_github(&beta_body(BETA, &other), &url, 9, SHA, none()), beta_github(&body, &url.replace("https:", "http:"), 9, SHA, none()), beta_github(&body, &production_installer("0.3.5"), 9, SHA, none()), - beta_github( - &body, - &url, - 9, - SHA, - serde_json::json!({ "asset": { "name": "EternalMonitor-Setup.exe" } }), - ), beta_github(&body, &url, 0, SHA, none()), beta_github(&body, &url, MAX_INSTALLER_BYTES + 1, SHA, none()), ]; @@ -2475,7 +2504,7 @@ fn a_refused_beta_is_not_offered_and_offline_keeps_the_last_one() { SHA, serde_json::json!({}), ); - s.finish_beta(parse_beta(&no_hash, &endpoints).map(|(release, _)| release)); + s.finish_beta(parse_beta(&no_hash, &endpoints).map(|listing| listing.release)); s.finish_check(1000 + 2 * DAY, false, Ok(release("0.3.4"))); assert_eq!(s.saved.beta, None); assert_eq!(s.check, CheckStatus::UpToDate); @@ -2489,7 +2518,7 @@ fn a_refused_beta_is_not_offered_and_offline_keeps_the_last_one() { &"0".repeat(64), serde_json::json!({}), ); - s.finish_beta(parse_beta(&swapped, &endpoints).map(|(release, _)| release)); + s.finish_beta(parse_beta(&swapped, &endpoints).map(|listing| listing.release)); assert_eq!(s.saved.beta, None); } @@ -2606,69 +2635,342 @@ fn relaunch_after_a_beta_update_reports_whether_it_finished() { ); } -/// A fake GitHub serving the `beta` release for `published`, whose installer -/// redirects to a blob that serves `served`. -fn fake_beta(published: &[u8], served: Vec) -> (Endpoints, Release) { - let sha = sha_of(published); - let size = published.len() as u64; - let api_sha = sha.clone(); +/// A published, test-only seed (scripts/test_sign_beta_manifest.py signs +/// with it too). +const TEST_SEED: [u8; 32] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, +]; + +fn test_key() -> ring::signature::Ed25519KeyPair { + ring::signature::Ed25519KeyPair::from_seed_unchecked(&TEST_SEED).unwrap() +} + +fn test_public_key() -> [u8; 32] { + use ring::signature::KeyPair; + test_key().public_key().as_ref().try_into().unwrap() +} + +/// What the fake GitHub publishes for the Beta: `version` with `published` +/// as its installer, which downloads as `served`; the manifest and +/// signature, or `None` for no such asset. +#[derive(Clone)] +struct BetaSite { + version: String, + published: Vec, + served: Vec, + manifest: Option>, + signature: Option>, +} + +impl BetaSite { + /// Signed with the test key. + fn signed(version: &str, published: &[u8]) -> Self { + let mut site = Self { + version: version.into(), + published: published.to_vec(), + served: published.to_vec(), + manifest: None, + signature: None, + }; + site.sign(BetaManifest { + version: version.into(), + sha256: sha_of(published), + size: published.len() as u64, + }); + site + } + + fn sign(&mut self, manifest: BetaManifest) { + let text = manifest.text().into_bytes(); + self.signature = Some(test_key().sign(&text).as_ref().to_vec()); + self.manifest = Some(text); + } +} + +/// Serves `site` like GitHub, and a latest.json naming 0.3.4, until the +/// test ends. The endpoints trust the test key. Counts reads of the Beta. +fn serve_beta(site: Arc>) -> (Endpoints, Arc) { + use std::sync::atomic::{AtomicUsize, Ordering}; + let reads = Arc::new(AtomicUsize::new(0)); + let counted = reads.clone(); let base = serve(Box::new(move |base, path| { - let installer = format!("{base}/releases/download/beta/EternalMonitor-Setup-beta.exe"); + let site = site.lock().clone(); + let folder = format!("{base}/releases/download/beta"); + let found = |data: Option>| match data { + Some(data) => (200, vec![], data), + None => (404, vec![], Vec::new()), + }; match path { + "/latest.json" => ( + 200, + vec![], + manifest( + "0.3.4", + &format!("{base}/releases/download/v0.3.4/EternalMonitor-Setup.exe"), + SHA, + ), + ), "/repos/sidebandstudio/EternalMonitor/releases/tags/beta" => { - let body = beta_body(BETA, &api_sha); - ( - 200, - vec![], - beta_github(&body, &installer, size, &api_sha, serde_json::json!({})), - ) + counted.fetch_add(1, Ordering::SeqCst); + let sha = sha_of(&site.published); + let mut drop = Vec::new(); + if site.manifest.is_none() { + drop.push(BETA_MANIFEST_NAME); + } + if site.signature.is_none() { + drop.push(BETA_SIGNATURE_NAME); + } + let listing = beta_github( + &beta_body(&site.version, &sha), + &format!("{folder}/EternalMonitor-Setup-beta.exe"), + site.published.len() as u64, + &sha, + serde_json::json!({ "drop": drop }), + ); + (200, vec![], listing) } + // Downloads redirect to an asset host, as on GitHub. "/releases/download/beta/EternalMonitor-Setup-beta.exe" => ( 302, - vec![("Location".into(), "/assets/blob".into())], + vec![("Location".into(), "/assets/installer".into())], Vec::new(), ), - "/assets/blob" => (200, vec![], served.clone()), + "/releases/download/beta/beta-manifest.txt" => ( + 302, + vec![("Location".into(), "/assets/manifest".into())], + Vec::new(), + ), + "/assets/installer" => (200, vec![], site.served), + "/assets/manifest" => found(site.manifest), + "/releases/download/beta/beta-manifest.sig" => found(site.signature), _ => (404, vec![], Vec::new()), } })); - let endpoints = Endpoints::local(&base).unwrap(); - let release = fetch_beta(&endpoints).unwrap(); + let endpoints = Endpoints { + beta_key: test_public_key(), + ..Endpoints::local(&base).unwrap() + }; + (endpoints, reads) +} + +fn fetch_site(site: BetaSite) -> (Endpoints, Result) { + let (endpoints, _) = serve_beta(Arc::new(Mutex::new(site))); + let result = fetch_beta(&endpoints); + (endpoints, result) +} + +#[test] +fn the_beta_manifest_parses_only_in_its_canonical_form() { + let fixture = include_str!("../../../scripts/fixtures/beta-manifest.txt").replace("\r\n", "\n"); + let parsed = BetaManifest::parse(fixture.as_bytes()).unwrap(); assert_eq!( - release.installer, - format!("{base}/releases/download/beta/EternalMonitor-Setup-beta.exe") + parsed, + BetaManifest { + version: BETA.into(), + sha256: "ab".repeat(32), + size: 63_301_683, + } ); - (endpoints, release) + assert_eq!(parsed.text(), fixture); + let variants = [ + fixture.replace('\n', "\r\n"), + fixture.trim_end().to_string(), + format!("{fixture}\n"), + format!("{fixture}note=x\n"), + fixture.replace("manifest 1", "manifest 2"), + fixture.replace(&"ab".repeat(32), &"AB".repeat(32)), + fixture.replace(&"ab".repeat(32), &"ab".repeat(31)), + fixture.replace("size=63301683", "size=063301683"), + fixture.replace("size=63301683", "size= 63301683"), + fixture.replace("size=63301683", "size=-1"), + fixture.replace("Setup-beta.exe", "Setup.exe"), + fixture.replace(BETA, "0.3.5"), + fixture.replace(BETA, "0.3.5-beta.01"), + fixture.replace(BETA, &format!("{BETA}+ci.1")), + fixture.replace("version=", "version = "), + fixture.replace("sha256=", "sha256:"), + ]; + for text in variants { + assert_eq!(BetaManifest::parse(text.as_bytes()), None, "{text:?}"); + } + assert_eq!(BetaManifest::parse(b"\xff\xfe"), None); } +/// Ed25519 is deterministic: ring signs the fixture to the bytes OpenSSL +/// made with the same test seed, the way scripts/sign_beta_manifest.py signs. #[test] -fn a_beta_downloads_and_is_verified_like_a_stable_release() { - let payload = b"MZ fake beta installer".to_vec(); - let (endpoints, release) = fake_beta(&payload, payload.clone()); +fn a_manifest_signed_by_the_release_script_verifies_here() { + let manifest = + include_str!("../../../scripts/fixtures/beta-manifest.txt").replace("\r\n", "\n"); + let signature = include_bytes!("../../../scripts/fixtures/beta-manifest.test.sig"); + assert_eq!(test_key().sign(manifest.as_bytes()).as_ref(), signature); + let listing = BetaListing { + release: Release { + sha256: "ab".repeat(32), + ..beta(BETA) + }, + size: 63_301_683, + manifest: String::new(), + signature: String::new(), + }; assert_eq!( - (release.version.as_str(), release.sha256.as_str()), - (BETA, sha_of(&payload).as_str()) + check_beta_manifest(&listing, manifest.as_bytes(), signature, &test_public_key()), + Ok(()) ); + // Not by the production key. + assert!(matches!( + check_beta_manifest(&listing, manifest.as_bytes(), signature, &BETA_PUBLIC_KEY), + Err(UpdateError::UnsafeDownload(_)) + )); +} + +#[test] +fn the_pinned_key_is_the_published_public_key() { + assert_eq!( + hex(&BETA_PUBLIC_KEY), + "82c2af21c685d2c042c51f72a34c78bdb5d004bad89e75a8306a43224d16ec5a" + ); + assert_eq!(Endpoints::production().beta_key, BETA_PUBLIC_KEY); + assert_eq!( + Endpoints::local("http://127.0.0.1:1").unwrap().beta_key, + BETA_PUBLIC_KEY + ); +} + +#[test] +fn a_beta_is_offered_only_with_a_valid_signed_manifest() { + let payload = b"MZ fake beta installer"; + let (_, fetched) = fetch_site(BetaSite::signed(BETA, payload)); + let fetched = fetched.unwrap(); + assert_eq!( + (fetched.version.as_str(), fetched.sha256.as_str()), + (BETA, sha_of(payload).as_str()) + ); + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(fetched)); + s.finish_check(1000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(1000), Some(Notice::Available(BETA.into()))); + + let good = BetaSite::signed(BETA, payload); + let manifest = |version: &str, sha256: String, size: usize| BetaManifest { + version: version.into(), + sha256, + size: size as u64, + }; + let mut flipped = good.clone(); + flipped.manifest.as_mut().unwrap()[40] ^= 1; + let mut other_sha = good.clone(); + other_sha.sign(manifest(BETA, "0".repeat(64), payload.len())); + let mut other_size = good.clone(); + other_size.sign(manifest(BETA, sha_of(payload), payload.len() + 1)); + let mut other_version = good.clone(); + other_version.sign(manifest( + "0.3.5-beta.20261108.27", + sha_of(payload), + payload.len(), + )); + let mut crlf = good.clone(); + let text = String::from_utf8(good.manifest.clone().unwrap()) + .unwrap() + .replace('\n', "\r\n"); + crlf.signature = Some(test_key().sign(text.as_bytes()).as_ref().to_vec()); + crlf.manifest = Some(text.into_bytes()); + let mut short = good.clone(); + short.signature.as_mut().unwrap().pop(); + let mut no_signature = good.clone(); + no_signature.signature = None; + let mut no_manifest = good.clone(); + no_manifest.manifest = None; + // The nightly published a new build that nobody has signed yet. + let mut unsigned_build = good.clone(); + unsigned_build.version = "0.3.5-beta.20261108.27".into(); + unsigned_build.published = b"MZ next beta installer".to_vec(); + let cases = [ + ("flipped byte", flipped), + ("manifest sha differs from the body", other_sha), + ("manifest size differs from the asset", other_size), + ("manifest version differs from the body", other_version), + ("signed, but not canonical", crlf), + ("63-byte signature", short), + ("missing signature", no_signature), + ("missing manifest", no_manifest), + ("unsigned new build", unsigned_build), + ]; + for (name, site) in cases { + let (_, fetched) = fetch_site(site); + assert!( + matches!( + fetched, + Err(UpdateError::UnsafeDownload(_) | UpdateError::BadManifest(_)) + ), + "{name}: {fetched:?}" + ); + // No offer, and the Beta saved before is not offered either. + let mut s = beta_state(Saved { + beta: Some(beta(BETA)), + ..Saved::default() + }); + s.finish_beta(fetched); + s.finish_check(1000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(1000), None, "{name}"); + } + + // Signed with another key: the production key does not trust the test key. + let (endpoints, _) = serve_beta(Arc::new(Mutex::new(good))); + let production = Endpoints { + beta_key: BETA_PUBLIC_KEY, + ..endpoints + }; + assert!( + matches!(fetch_beta(&production), Err(UpdateError::UnsafeDownload(why)) if why.contains("signature")), + ); +} + +#[test] +fn a_signed_beta_that_is_not_newer_is_not_offered() { + for version in [OLD_BETA, "0.3.5-beta.20261104.30", "0.3.4-beta.1"] { + let (_, fetched) = fetch_site(BetaSite::signed(version, b"MZ old beta installer")); + let mut s = beta_state(Saved::default()); + s.finish_beta(fetched); + assert_eq!( + s.saved.beta.as_ref().map(|r| r.version.as_str()), + Some(version) + ); + assert_eq!( + s.finish_check(1000, false, Ok(release("0.3.4"))), + None, + "{version}" + ); + assert_eq!(s.notice(1000), None, "{version}"); + } +} + +#[test] +fn a_beta_downloads_and_is_verified_like_a_stable_release() { + let payload = b"MZ fake beta installer".to_vec(); + let (endpoints, release) = fetch_site(BetaSite::signed(BETA, &payload)); + let release = release.unwrap(); let path = download_case(&endpoints, &release).unwrap(); assert_eq!(std::fs::read(&path).unwrap(), payload); assert!(eternal_updater::open_verified(&path, &release.sha256).is_ok()); std::fs::remove_dir_all(path.parent().unwrap().parent().unwrap()).unwrap(); - let (endpoints, release) = fake_beta( - b"MZ fake beta installer", - b"MZ evil beta installer".to_vec(), - ); - let error = download_case(&endpoints, &release).unwrap_err(); + // The downloaded file must be the one the signed manifest names. + let mut swapped = BetaSite::signed(BETA, &payload); + swapped.served = b"MZ evil beta installer".to_vec(); + let (endpoints, release) = fetch_site(swapped); + let error = download_case(&endpoints, &release.unwrap()).unwrap_err(); assert!( matches!(&error, UpdateError::UnsafeDownload(why) if why.contains("SHA-256")), "{error}" ); // The release now names another Beta than the one chosen. - let (endpoints, release) = fake_beta(&payload, payload.clone()); + let (endpoints, release) = fetch_site(BetaSite::signed(BETA, &payload)); let chosen = Release { version: OLD_BETA.into(), - ..release + ..release.unwrap() }; let error = download_case(&endpoints, &chosen).unwrap_err(); assert!( @@ -2682,33 +2984,12 @@ fn a_beta_downloads_and_is_verified_like_a_stable_release() { /// after the check replaces the offer before anything downloads. #[test] fn the_worker_reads_the_beta_only_for_a_beta_build_and_again_before_downloading() { - use std::sync::atomic::{AtomicUsize, Ordering}; - let served = Arc::new(Mutex::new(BETA.to_string())); - let beta_reads = Arc::new(AtomicUsize::new(0)); - let (route_served, route_reads) = (served.clone(), beta_reads.clone()); - let base = serve(Box::new(move |base, path| match path { - "/latest.json" => ( - 200, - vec![], - manifest( - "0.3.4", - &format!("{base}/releases/download/v0.3.4/EternalMonitor-Setup.exe"), - SHA, - ), - ), - "/repos/sidebandstudio/EternalMonitor/releases/tags/beta" => { - route_reads.fetch_add(1, Ordering::SeqCst); - let installer = format!("{base}/releases/download/beta/EternalMonitor-Setup-beta.exe"); - let body = beta_body(&route_served.lock(), SHA); - ( - 200, - vec![], - beta_github(&body, &installer, 9, SHA, serde_json::json!({})), - ) - } - _ => (404, vec![], Vec::new()), - })); - let endpoints = Endpoints::local(&base).unwrap(); + use std::sync::atomic::Ordering; + let site = Arc::new(Mutex::new(BetaSite::signed( + BETA, + b"MZ fake beta installer", + ))); + let (endpoints, beta_reads) = serve_beta(site.clone()); let run = |state: &Arc>, command: Command| { let (tx, rx) = mpsc::channel(); tx.send(command).unwrap(); @@ -2743,7 +3024,7 @@ fn the_worker_reads_the_beta_only_for_a_beta_build_and_again_before_downloading( panic!("Update now downloads the Beta"); }; let next = "0.3.5-beta.20261108.27"; - *served.lock() = next.into(); + *site.lock() = BetaSite::signed(next, b"MZ next beta installer"); run(&state, Command::Download(chosen)); assert_eq!(beta_reads.load(Ordering::SeqCst), 2); let state = state.lock(); diff --git a/scripts/fixtures/.gitattributes b/scripts/fixtures/.gitattributes new file mode 100644 index 00000000..5201f7da --- /dev/null +++ b/scripts/fixtures/.gitattributes @@ -0,0 +1,2 @@ +# Signatures are raw bytes; a CRLF checkout must not touch them. +*.sig binary diff --git a/scripts/fixtures/beta-manifest.test.sig b/scripts/fixtures/beta-manifest.test.sig new file mode 100644 index 00000000..c5970634 --- /dev/null +++ b/scripts/fixtures/beta-manifest.test.sig @@ -0,0 +1,2 @@ +}lqsr: /fܑ1I m_T6Ơʲ, +(lY[#gHGF*d \ No newline at end of file diff --git a/scripts/fixtures/beta-manifest.txt b/scripts/fixtures/beta-manifest.txt new file mode 100644 index 00000000..c06f8f1f --- /dev/null +++ b/scripts/fixtures/beta-manifest.txt @@ -0,0 +1,5 @@ +eternalmonitor-beta-manifest 1 +version=0.3.5-beta.20261107.26 +sha256=abababababababababababababababababababababababababababababababab +size=63301683 +asset=EternalMonitor-Setup-beta.exe diff --git a/scripts/nightly.py b/scripts/nightly.py index 1275ec6a..0e45d504 100644 --- a/scripts/nightly.py +++ b/scripts/nightly.py @@ -22,7 +22,8 @@ -beta.., which semver orders below and above a hand-made -beta.1. A stable install is never offered one. A Beta install reads the beta release's body (beta_body()) and is offered the newer -of that Beta and the next stable release (host/src/updates.rs). +of that Beta and the next stable release (host/src/updates.rs), once +scripts/sign_beta_manifest.py has signed the build; this script never signs. """ import argparse import datetime diff --git a/scripts/sign_beta_manifest.py b/scripts/sign_beta_manifest.py new file mode 100755 index 00000000..e0b79a8f --- /dev/null +++ b/scripts/sign_beta_manifest.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Sign the Beta channel's manifest, so Beta installs offer the newest Beta. + + sign_beta_manifest.py [--repo R] [--public-key PEM] < private-key.pem + Read the beta release, write beta-manifest.txt from it, sign that with + the Ed25519 private key read from stdin, upload both files to the + release, then verify them as a Beta install does. + sign_beta_manifest.py --verify-only [--repo R] [--public-key PEM] + Only verify the published manifest and signature. + +A Beta install offers a Beta only when beta-manifest.sig is a valid Ed25519 +signature, by the key pinned in host/src/updates.rs, over the exact bytes of +beta-manifest.txt, and the manifest names the version, SHA-256 and size the +release does. Until a build is signed, Beta installs are offered no Beta. + +The private key goes from stdin to openssl through a pipe. It is never an +argument or a file. Needs gh (signed in, for the upload) and OpenSSL 3 or +later; OPENSSL picks the binary. +""" +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile + +REPO = "sidebandstudio/EternalMonitor" +TAG = "beta" +ASSET = "EternalMonitor-Setup-beta.exe" +MANIFEST = "beta-manifest.txt" +SIGNATURE = "beta-manifest.sig" +# The public half of the key host/src/updates.rs pins (BETA_PUBLIC_KEY). +PUBLIC_KEY = b"""-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAgsKvIcaF0sBCxR9yo0x4vbXQBLrYnnWoMGpDIk0W7Fo= +-----END PUBLIC KEY----- +""" +VERSION_LINE = re.compile(r"(?m)^Version (\S+), from the Beta channel") +HASH_BLOCK = re.compile(r"(?m)^SHA-256 of EternalMonitor-Setup-beta\.exe:\s*```\s*([0-9A-Fa-f]{64})\s*```") +# x.y.z-beta., without build metadata; numbers have no leading zeros. +NUMBER = r"(?:0|[1-9]\d*)" +BETA_VERSION = re.compile(rf"{NUMBER}\.{NUMBER}\.{NUMBER}-beta(?:\.(?:{NUMBER}|\d*[A-Za-z-][0-9A-Za-z-]*))+") + + +class SigningError(Exception): + pass + + +def manifest_text(version, sha256, size): + """The canonical manifest, version 1 (host/src/updates.rs BetaManifest::text).""" + return (f"eternalmonitor-beta-manifest 1\nversion={version}\nsha256={sha256}\n" + f"size={size}\nasset={ASSET}\n") + + +def only(pattern, text, what): + found = pattern.findall(text) + if len(found) != 1: + raise SigningError(f"the release body names {len(found)} {what}, not one") + return found[0] + + +def read_release(release): + """(version, sha256, size) of the beta release's installer, once its body, asset + and GitHub digest agree.""" + if release.get("tag_name") != TAG or release.get("draft") or release.get("prerelease") is not True: + raise SigningError("not the published beta pre-release") + body = release.get("body") or "" + version = only(VERSION_LINE, body, "versions") + if not BETA_VERSION.fullmatch(version): + raise SigningError(f"not a Beta version: {version!r}") + sha256 = only(HASH_BLOCK, body, "installer hashes").lower() + assets = [a for a in release.get("assets", []) if a.get("name") == ASSET] + if len(assets) != 1: + raise SigningError(f"the release has no {ASSET}") + size, digest = assets[0].get("size"), (assets[0].get("digest") or "").lower() + if digest != f"sha256:{sha256}": + raise SigningError(f"GitHub's digest {digest or '(none)'} differs from the body's {sha256}") + if not isinstance(size, int) or size <= 0: + raise SigningError(f"bad installer size: {size!r}") + return version, sha256, size + + +class GitHub: + def __init__(self, repo=REPO, run=subprocess.run): + self.repo, self.run = repo, run + + def gh(self, *args): + return self.run(["gh", *args], check=True, capture_output=True, text=True).stdout + + def release(self): + return json.loads(self.gh("api", f"repos/{self.repo}/releases/tags/{TAG}")) + + def upload(self, paths): + self.gh("release", "upload", TAG, *paths, "--repo", self.repo, "--clobber") + + def download(self, names, directory): + patterns = [arg for name in names for arg in ("--pattern", name)] + self.gh("release", "download", TAG, "--repo", self.repo, "--dir", directory, "--clobber", *patterns) + + +def openssl(): + return os.environ.get("OPENSSL", "openssl") + + +def sign(manifest, signature, private_key, run=subprocess.run): + """Signs the file manifest into signature with the private key PEM (bytes), + which reaches openssl on its stdin.""" + result = run([openssl(), "pkeyutl", "-sign", "-rawin", "-inkey", "/dev/stdin", + "-in", manifest, "-out", signature], input=private_key, capture_output=True) + if result.returncode != 0: + raise SigningError(f"openssl could not sign: {result.stderr.decode(errors='replace').strip()}") + if os.path.getsize(signature) != 64: + raise SigningError("the signature is not 64 bytes: is the key Ed25519?") + + +def verified(manifest, signature, public_key, run=subprocess.run): + """Whether signature is public_key's (PEM bytes) signature over manifest.""" + with tempfile.TemporaryDirectory() as tmp: + key = os.path.join(tmp, "public.pem") + with open(key, "wb") as f: + f.write(public_key) + result = run([openssl(), "pkeyutl", "-verify", "-rawin", "-pubin", "-inkey", key, + "-in", manifest, "-sigfile", signature], capture_output=True) + return result.returncode == 0 and os.path.getsize(signature) == 64 + + +def verify_published(github, public_key=PUBLIC_KEY, run=subprocess.run): + """Checks the published manifest and signature as a Beta install does, and + returns the version they vouch for.""" + version, sha256, size = read_release(github.release()) + with tempfile.TemporaryDirectory() as tmp: + try: + github.download([MANIFEST, SIGNATURE], tmp) + except subprocess.CalledProcessError as error: + raise SigningError(f"could not download the manifest and signature: {error.stderr}") + manifest, signature = os.path.join(tmp, MANIFEST), os.path.join(tmp, SIGNATURE) + if not (os.path.isfile(manifest) and os.path.isfile(signature)): + raise SigningError("the release lacks the manifest or its signature") + if not verified(manifest, signature, public_key, run): + raise SigningError("the signature does not verify with the public key") + with open(manifest, "rb") as f: + if f.read() != manifest_text(version, sha256, size).encode(): + raise SigningError("the signed manifest does not match the release") + return version + + +def sign_and_publish(github, private_key, public_key=PUBLIC_KEY, run=subprocess.run): + """Writes, signs and uploads the manifest for the release as it is now, then + verifies what was published. Returns the version.""" + version, sha256, size = read_release(github.release()) + with tempfile.TemporaryDirectory() as tmp: + manifest, signature = os.path.join(tmp, MANIFEST), os.path.join(tmp, SIGNATURE) + with open(manifest, "w", encoding="utf-8", newline="\n") as f: + f.write(manifest_text(version, sha256, size)) + sign(manifest, signature, private_key, run) + # A wrong key must not replace a good signature. + if not verified(manifest, signature, public_key, run): + raise SigningError("the private key does not match the public key") + github.upload([manifest, signature]) + published = verify_published(github, public_key, run) + if published != version: + raise SigningError(f"the release moved to {published} while signing {version}; run again") + return version + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--repo", default=os.environ.get("GH_REPO", REPO)) + parser.add_argument("--public-key", help="PEM file to verify with instead of the pinned key") + parser.add_argument("--verify-only", action="store_true") + args = parser.parse_args(argv) + public_key = PUBLIC_KEY + if args.public_key: + with open(args.public_key, "rb") as f: + public_key = f.read() + github = GitHub(args.repo) + try: + if args.verify_only: + print(f"Verified the signed manifest for {verify_published(github, public_key)}") + else: + private_key = sys.stdin.buffer.read() + if b"PRIVATE KEY" not in private_key: + raise SigningError("pipe the Ed25519 private key PEM into stdin") + print(f"Signed and verified the manifest for {sign_and_publish(github, private_key, public_key)}") + except SigningError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_sign_beta_manifest.py b/scripts/test_sign_beta_manifest.py new file mode 100644 index 00000000..58f8f4d7 --- /dev/null +++ b/scripts/test_sign_beta_manifest.py @@ -0,0 +1,212 @@ +"""The Beta manifest is written from a release whose parts agree, signed without the key +touching an argument or a file, and verified the way a Beta install verifies it.""" +import base64 +import copy +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import nightly # noqa: E402 +import sign_beta_manifest as signer # noqa: E402 + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SHA = "62b9d9d" + "0" * 33 +HASH = "ab" * 32 +VERSION = "0.3.5-beta.20261107.26" +SIZE = 63301683 +# A published, test-only seed: host/src/updates/tests.rs signs with it too. +TEST_SEED = bytes(range(32)) + + +def run_openssl(*args, data=None): + return subprocess.run([signer.openssl(), *args], input=data, capture_output=True, check=True).stdout + + +def ed25519_available(): + try: + key = run_openssl("genpkey", "-algorithm", "ed25519") + with tempfile.TemporaryDirectory() as tmp: + manifest, signature = os.path.join(tmp, "m"), os.path.join(tmp, "s") + pathlib.Path(manifest).write_bytes(b"x\n") + signer.sign(manifest, signature, key) + return True + except (OSError, subprocess.CalledProcessError, signer.SigningError): + return False + + +def private_from_seed(seed): + der = bytes.fromhex("302e020100300506032b657004220420") + seed + return b"-----BEGIN PRIVATE KEY-----\n" + base64.b64encode(der) + b"\n-----END PRIVATE KEY-----\n" + + +def new_key(): + """A throwaway Ed25519 key pair: (private PEM, public PEM).""" + private = run_openssl("genpkey", "-algorithm", "ed25519") + return private, run_openssl("pkey", "-pubout", data=private) + + +def release(body=None, digest=None, size=SIZE, **fields): + notes = nightly.notes(VERSION, "release/0.3.5-candidate", SHA, "0.3.4", HASH) + data = { + "tag_name": "beta", "draft": False, "prerelease": True, + "body": nightly.beta_body(notes, SHA) if body is None else body, + "assets": [{"name": signer.ASSET, "size": size, + "digest": f"sha256:{HASH}" if digest is None else digest}], + } + data.update(fields) + return data + + +class FakeGitHub: + """The beta release and its uploaded assets, in memory.""" + + def __init__(self, data=None): + self.data, self.assets, self.uploads = data or release(), {}, 0 + + def release(self): + return copy.deepcopy(self.data) + + def upload(self, paths): + self.uploads += 1 + for path in paths: + self.assets[os.path.basename(path)] = pathlib.Path(path).read_bytes() + + def download(self, names, directory): + for name in names: + if name in self.assets: + pathlib.Path(directory, name).write_bytes(self.assets[name]) + + +class Recorder: + """subprocess.run, recording each call's arguments and whether it had input.""" + + def __init__(self): + self.calls = [] + + def __call__(self, args, **kwargs): + self.calls.append((list(args), kwargs.get("input"))) + return subprocess.run(args, **kwargs) + + +class ManifestTest(unittest.TestCase): + def test_the_manifest_is_the_canonical_text_the_host_parses(self): + fixture = (ROOT / "scripts/fixtures/beta-manifest.txt").read_bytes().replace(b"\r\n", b"\n") + self.assertEqual(signer.manifest_text(VERSION, HASH, SIZE).encode(), fixture) + + def test_the_release_parts_must_agree(self): + self.assertEqual(signer.read_release(release()), (VERSION, HASH, SIZE)) + good = release()["body"] + bad = [ + release(digest=f"sha256:{'0' * 64}"), + release(digest=""), + release(body=good.replace(HASH, "")), + release(body=good.replace(VERSION, "0.3.5")), + release(body=good.replace(VERSION, "0.3.5-beta.01")), + release(body=good.replace(VERSION, "0.3.5-beta.1+ci")), + release(body=f"Version 0.3.5-beta.20261108.27, from the Beta channel\n{good}"), + release(size=0), + release(tag_name="v0.3.5"), + release(prerelease=False), + release(draft=True), + release(assets=[]), + ] + for data in bad: + with self.assertRaises(signer.SigningError, msg=str(data)[:200]): + signer.read_release(data) + + +@unittest.skipUnless(ed25519_available(), "needs OpenSSL 3 with Ed25519") +class SigningTest(unittest.TestCase): + def test_signs_uploads_and_verifies(self): + private, public = new_key() + github, run = FakeGitHub(), Recorder() + self.assertEqual(signer.sign_and_publish(github, private, public, run), VERSION) + self.assertEqual(github.assets[signer.MANIFEST], signer.manifest_text(VERSION, HASH, SIZE).encode()) + self.assertEqual(len(github.assets[signer.SIGNATURE]), 64) + self.assertEqual(signer.verify_published(github, public), VERSION) + # The key went to openssl's stdin only: never an argument, never uploaded. + with_key = [args for args, given in run.calls if given == private] + self.assertEqual(len(with_key), 1) + self.assertIn("/dev/stdin", with_key[0]) + for args, _ in run.calls: + self.assertFalse(any("PRIVATE KEY" in arg for arg in args), args) + for data in github.assets.values(): + self.assertNotIn(b"PRIVATE KEY", data) + + def test_a_key_that_is_not_the_pinned_one_uploads_nothing(self): + private, _ = new_key() + _, other = new_key() + github = FakeGitHub() + with self.assertRaises(signer.SigningError): + signer.sign_and_publish(github, private, other) + self.assertEqual(github.uploads, 0) + # Against the production key, a throwaway key fails too. + with self.assertRaises(signer.SigningError): + signer.sign_and_publish(github, private) + self.assertEqual(github.uploads, 0) + + def test_a_release_whose_parts_disagree_is_not_signed(self): + private, public = new_key() + github = FakeGitHub(release(digest=f"sha256:{'0' * 64}")) + with self.assertRaises(signer.SigningError): + signer.sign_and_publish(github, private, public) + self.assertEqual(github.uploads, 0) + + def test_verify_only_refuses_a_bad_or_missing_manifest(self): + private, public = new_key() + github = FakeGitHub() + signer.sign_and_publish(github, private, public) + _, other = new_key() + with self.assertRaises(signer.SigningError): + signer.verify_published(github, other) + good = dict(github.assets) + flipped = bytearray(good[signer.MANIFEST]) + flipped[40] ^= 1 + cases = { + "flipped byte": {**good, signer.MANIFEST: bytes(flipped)}, + "no signature": {signer.MANIFEST: good[signer.MANIFEST]}, + "no manifest": {signer.SIGNATURE: good[signer.SIGNATURE]}, + } + for name, assets in cases.items(): + github.assets = assets + with self.assertRaises(signer.SigningError, msg=name): + signer.verify_published(github, public) + # Signed, but the nightly has moved the release on since. + github.assets = good + github.data = release(body=github.data["body"].replace(VERSION, "0.3.5-beta.20261108.27")) + with self.assertRaises(signer.SigningError): + signer.verify_published(github, public) + + def test_openssl_signs_what_the_host_verifies(self): + # Ed25519 is deterministic: the host's test signs this fixture with ring and + # gets the same bytes. + fixture = ROOT / "scripts/fixtures/beta-manifest.test.sig" + with tempfile.TemporaryDirectory() as tmp: + manifest = os.path.join(tmp, signer.MANIFEST) + signature = os.path.join(tmp, signer.SIGNATURE) + shutil.copyfile(ROOT / "scripts/fixtures/beta-manifest.txt", manifest) + pathlib.Path(manifest).write_bytes(pathlib.Path(manifest).read_bytes().replace(b"\r\n", b"\n")) + signer.sign(manifest, signature, private_from_seed(TEST_SEED)) + self.assertEqual(pathlib.Path(signature).read_bytes(), fixture.read_bytes()) + + +class CommandTest(unittest.TestCase): + def test_signing_needs_a_private_key_on_stdin(self): + class Stdin: + buffer = __import__("io").BytesIO(b"not a key") + + saved = sys.stdin + sys.stdin = Stdin() + try: + self.assertEqual(signer.main(["--repo", "example/none"]), 1) + finally: + sys.stdin = saved + + +if __name__ == "__main__": + unittest.main() From c6776bc99c450b05a34af3901af34738a48a52c3 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 17:28:54 -0500 Subject: [PATCH 068/226] Pin Beta signing to the nightly run and refuse replayed Betas The signer could only check that GitHub's parts agree, which a leaked publishing token controls. sign_beta_manifest.py now needs --sha256, and takes --version, from the nightly run, and refuses before signing when the release names another installer or version. The nightly's Publish job adds a "Beta build to sign" summary with the version and installer SHA-256 from the build job's output; nothing else in the workflow changes. Signing works only for sidebandstudio/EternalMonitor, whatever GH_REPO says. --public-key is accepted only with --verify-only, and signing always checks against the pinned key before uploading. In the app, a manifest file that isn't served or is over the size limit is a bad Beta, which drops the saved one, rather than a network error. The download checks the signed manifest again. The highest verified Beta version is saved, and an older signed Beta served later is refused, so replaying one can't hold testers back on it. --- .github/workflows/nightly.yml | 13 +++++ SECURITY.md | 6 ++- docs/beta-testing.md | 15 ++++-- host/src/updates.rs | 60 ++++++++++++++------- host/src/updates/tests.rs | 84 +++++++++++++++++++++++++++++- scripts/sign_beta_manifest.py | 79 ++++++++++++++++++++-------- scripts/test_sign_beta_manifest.py | 77 +++++++++++++++++++++++---- 7 files changed, 275 insertions(+), 59 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0be20afe..0d5ef1ac 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -165,6 +165,19 @@ jobs: python3 scripts/nightly.py notes --version "$VERSION" --branch "$BRANCH" \ --sha "$SHA" --stable "$STABLE" --sha256 "$SHA256" > notes.md + # What scripts/sign_beta_manifest.py --sha256/--version must be given: + # the hash the build computed, which the step above checked. + - name: Summarize the build to sign + run: | + { + echo "### Beta build to sign" + echo + echo "| | |" + echo "|---|---|" + echo "| Version | $VERSION |" + echo "| Installer SHA-256 | \`$SHA256\` |" + } >> "$GITHUB_STEP_SUMMARY" + # The dated release first, checked anonymously; then the fixed beta link, # swapped by renames and checked byte for byte before the old asset goes; # the beta tag; and last the marker plan reads to skip a finished commit, diff --git a/SECURITY.md b/SECURITY.md index 6e8b08be..825ec52c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -39,8 +39,10 @@ Knowing these limits helps you judge whether a finding is new. Ed25519 signature by a public key pinned in the app, made with a key the release workflows don't have, and the manifest's version, SHA-256 and size must match the release body, GitHub's digest of the installer and the - downloaded file. Without a valid signature no Beta is offered. Stable builds - never take a Beta. Against + downloaded file. Without a valid signature no Beta is offered, and a signed + Beta older than one the install has already verified is refused, so an old + signed release can't be replayed to hold testers back. Stable builds never + take a Beta. Against other software already running as your Windows user, the updater adds defense in depth rather than a guarantee. Its helper loads DLLs only from System32 and imports only Windows system libraries. It inherits only its output pipe from diff --git a/docs/beta-testing.md b/docs/beta-testing.md index 46248acb..95c8cad0 100644 --- a/docs/beta-testing.md +++ b/docs/beta-testing.md @@ -40,9 +40,18 @@ back to Stable by running the Stable installer over it. manifest's version, SHA-256 and size must match the release body `nightly.py` writes, GitHub's digest of the installer and the downloaded file. The nightly does not sign; until a build is signed, Beta installs are - offered no Beta. After a nightly publishes, pipe the signing key into - `scripts/sign_beta_manifest.py`, which checks the release, signs, uploads - both files and verifies them; `--verify-only` checks what is published. + offered no Beta. After a nightly publishes, open its run (Actions, Beta + builds) and copy **Installer SHA-256** and **Version** from the Publish + job's summary, "Beta build to sign": the hash the Windows installer job + computed for the build it scanned, which Publish checks against the file + it uploads. Then run + `scripts/sign_beta_manifest.py --sha256 --version ` with + the signing key on stdin. It refuses a release that names another + installer or version, checks the release's parts agree, signs, checks the + signature with the pinned key, uploads both files and verifies them; + `--verify-only` checks what is published. A signed Beta older than one an + install has already seen is refused, so taking a bad Beta back means + signing a newer build, not re-signing an older one. A new Beta is announced at most once a day, never during a stream. Settings, General, **Get Beta updates** (shown only on a Beta build) turns this off. Taking a Stable release leaves the Beta channel. Betas built before this arrived can't update themselves, so their testers diff --git a/host/src/updates.rs b/host/src/updates.rs index 77b96038..fc19dc71 100644 --- a/host/src/updates.rs +++ b/host/src/updates.rs @@ -475,15 +475,10 @@ pub fn check_beta_manifest( Ok(()) } -/// Checks the Beta release against the one chosen, whose manifest was -/// verified, and returns the installer's size. A Beta published since then -/// is refused: its hash is not the one the helper will check. -fn check_beta_release( - data: &[u8], - release: &Release, - endpoints: &Endpoints, -) -> Result { - let listing = parse_beta(data, endpoints)?; +/// Checks the verified Beta release against the one chosen and returns the +/// installer's size. A Beta published since then is refused: its hash is +/// not the one the helper will check. +fn check_beta_release(listing: &BetaListing, release: &Release) -> Result { let published = &listing.release; if (&published.version, &published.installer, &published.sha256) != (&release.version, &release.installer, &release.sha256) @@ -602,17 +597,29 @@ const GITHUB_JSON: &str = "application/vnd.github+json"; /// The newest Beta, offered only with a valid signed manifest. pub fn fetch_beta(endpoints: &Endpoints) -> Result { + verified_beta(endpoints).map(|listing| listing.release) +} + +/// The `beta` release, once its signed manifest checks out. +fn verified_beta(endpoints: &Endpoints) -> Result { let data = get_json(&format!("{}{BETA_TAG}", endpoints.release_api), GITHUB_JSON)?; let listing = parse_beta(&data, endpoints)?; let manifest = get_asset(&listing.manifest, endpoints)?; let signature = get_asset(&listing.signature, endpoints)?; check_beta_manifest(&listing, &manifest, &signature, &endpoints.beta_key)?; - Ok(listing.release) + Ok(listing) } -/// A small release file, following redirects only to allowed hosts. +/// A manifest file, following redirects only to allowed hosts. One that +/// isn't served or is too large is a bad Beta, not a network problem, so the +/// saved Beta goes too. fn get_asset(url: &str, endpoints: &Endpoints) -> Result, UpdateError> { - let network = |error: &dyn std::fmt::Display| UpdateError::Network(error.to_string()); + let bad = |why: String| UpdateError::BadManifest(why); + let network = |error: ureq::Error| match error { + ureq::Error::StatusCode(status) => bad(format!("Beta file: HTTP {status}")), + ureq::Error::BodyExceedsLimit(_) => bad("Beta file too large".into()), + error => UpdateError::Network(error.to_string()), + }; let unsafe_download = |why: String| UpdateError::UnsafeDownload(why); let agent = agent(Duration::from_secs(10)); let mut url = url.to_string(); @@ -624,7 +631,7 @@ fn get_asset(url: &str, endpoints: &Endpoints) -> Result, UpdateError> { .get(&url) .header("User-Agent", USER_AGENT) .call() - .map_err(|e| network(&e))?; + .map_err(network)?; if response.status().is_redirection() { let location = response .headers() @@ -636,14 +643,14 @@ fn get_asset(url: &str, endpoints: &Endpoints) -> Result, UpdateError> { continue; } if !response.status().is_success() { - return Err(UpdateError::Network(format!("HTTP {}", response.status()))); + return Err(bad(format!("Beta file: HTTP {}", response.status()))); } return response .body_mut() .with_config() .limit(MAX_MANIFEST_BYTES) .read_to_vec() - .map_err(|e| network(&e)); + .map_err(network); } Err(unsafe_download("too many redirects".into())) } @@ -778,8 +785,8 @@ pub fn download( ) -> Result { let io = |error: std::io::Error| UpdateError::Network(error.to_string()); let size = if release.is_beta() { - let github = get_json(&format!("{}{BETA_TAG}", endpoints.release_api), GITHUB_JSON)?; - check_beta_release(&github, release, endpoints)? + // The signed manifest is checked again, right before the download. + check_beta_release(&verified_beta(endpoints)?, release)? } else { let api = format!("{}v{}", endpoints.release_api, release.version); let github = get_json(&api, GITHUB_JSON)?; @@ -1142,6 +1149,11 @@ struct Saved { /// When a notification last announced a Beta. #[serde(default)] beta_notified: Option, + /// The highest Beta whose signed manifest checked out. An older signed + /// Beta served later is refused, so replaying one can't hold testers + /// back on it. + #[serde(default)] + beta_highest: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -1438,10 +1450,20 @@ impl State { /// The Beta channel's answer, taken before [`Self::finish_check`] of the /// same check. Offline, the saved Beta stays; a Beta that fails its - /// checks is not offered, and neither is the one saved before it. + /// checks, or is older than one verified before, is not offered, and + /// neither is the one saved before it. fn finish_beta(&mut self, result: Result) { match result { - Ok(release) => self.saved.beta = Some(release), + Ok(release) => match &self.saved.beta_highest { + Some(highest) if newer(highest, &release.version) => { + tracing::warn!(version = %release.version, %highest, "Older Beta refused"); + self.saved.beta = None; + } + _ => { + self.saved.beta_highest = Some(release.version.clone()); + self.saved.beta = Some(release); + } + }, Err(error @ UpdateError::Network(_)) => { tracing::info!(%error, "Beta check unavailable"); } diff --git a/host/src/updates/tests.rs b/host/src/updates/tests.rs index e412f9e5..393801dc 100644 --- a/host/src/updates/tests.rs +++ b/host/src/updates/tests.rs @@ -2653,7 +2653,8 @@ fn test_public_key() -> [u8; 32] { /// What the fake GitHub publishes for the Beta: `version` with `published` /// as its installer, which downloads as `served`; the manifest and -/// signature, or `None` for no such asset. +/// signature, or `None` for no such asset. With `status`, the listed +/// manifest's download answers that HTTP status instead. #[derive(Clone)] struct BetaSite { version: String, @@ -2661,6 +2662,7 @@ struct BetaSite { served: Vec, manifest: Option>, signature: Option>, + status: Option, } impl BetaSite { @@ -2672,6 +2674,7 @@ impl BetaSite { served: published.to_vec(), manifest: None, signature: None, + status: None, }; site.sign(BetaManifest { version: version.into(), @@ -2742,7 +2745,10 @@ fn serve_beta(site: Arc>) -> (Endpoints, Arc (200, vec![], site.served), - "/assets/manifest" => found(site.manifest), + "/assets/manifest" => match site.status { + Some(status) => (status, vec![], Vec::new()), + None => found(site.manifest), + }, "/releases/download/beta/beta-manifest.sig" => found(site.signature), _ => (404, vec![], Vec::new()), } @@ -3034,3 +3040,77 @@ fn the_worker_reads_the_beta_only_for_a_beta_build_and_again_before_downloading( Some(Notice::Available(next.into())) ); } + +#[test] +fn a_manifest_file_that_is_not_served_or_too_large_drops_the_beta() { + let good = BetaSite::signed(BETA, b"MZ fake beta installer"); + let mut large = good.clone(); + large.manifest = Some(vec![b'a'; MAX_MANIFEST_BYTES as usize + 1]); + let mut missing = good.clone(); + missing.status = Some(404); + let mut failing = good.clone(); + failing.status = Some(500); + for (name, site) in [("too large", large), ("404", missing), ("500", failing)] { + let (_, fetched) = fetch_site(site); + assert!( + matches!(fetched, Err(UpdateError::BadManifest(_))), + "{name}: {fetched:?}" + ); + let mut s = beta_state(Saved { + beta: Some(beta(BETA)), + ..Saved::default() + }); + s.finish_beta(fetched); + assert_eq!(s.saved.beta, None, "{name}"); + } +} + +#[test] +fn the_download_checks_the_signed_manifest_again() { + let payload = b"MZ fake beta installer"; + let site = Arc::new(Mutex::new(BetaSite::signed(BETA, payload))); + let (endpoints, _) = serve_beta(site.clone()); + let release = fetch_beta(&endpoints).unwrap(); + // Between the check and the download, the manifest stops verifying. + site.lock().manifest.as_mut().unwrap()[40] ^= 1; + let error = download_case(&endpoints, &release).unwrap_err(); + assert!( + matches!(&error, UpdateError::UnsafeDownload(why) if why.contains("signature")), + "{error}" + ); + let mut unsigned = BetaSite::signed(BETA, payload); + unsigned.signature = None; + *site.lock() = unsigned; + assert!(matches!( + download_case(&endpoints, &release), + Err(UpdateError::BadManifest(_)) + )); +} + +#[test] +fn an_older_signed_beta_than_one_verified_before_is_refused() { + let older = "0.3.5-beta.20261106.22"; + let next = "0.3.5-beta.20261108.27"; + let mut s = beta_state(Saved::default()); + s.finish_beta(Ok(beta(BETA))); + s.finish_check(1000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(1000), Some(Notice::Available(BETA.into()))); + // A replayed older signed Beta, still newer than this build: no offer, + // also after a restart. + s.finish_beta(Ok(beta(older))); + s.finish_check(2000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(2000), None); + assert_eq!(s.saved.beta_highest.as_deref(), Some(BETA)); + let mut s = beta_state(s.saved.clone()); + s.finish_beta(Ok(beta(older))); + s.finish_check(3000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(3000), None); + // The same Beta again, or a newer one, is offered. + s.finish_beta(Ok(beta(BETA))); + s.finish_check(4000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(4000), Some(Notice::Available(BETA.into()))); + s.finish_beta(Ok(beta(next))); + s.finish_check(5000, false, Ok(release("0.3.4"))); + assert_eq!(s.notice(5000), Some(Notice::Available(next.into()))); + assert_eq!(s.saved.beta_highest.as_deref(), Some(next)); +} diff --git a/scripts/sign_beta_manifest.py b/scripts/sign_beta_manifest.py index e0b79a8f..2a650815 100755 --- a/scripts/sign_beta_manifest.py +++ b/scripts/sign_beta_manifest.py @@ -1,11 +1,16 @@ #!/usr/bin/env python3 """Sign the Beta channel's manifest, so Beta installs offer the newest Beta. - sign_beta_manifest.py [--repo R] [--public-key PEM] < private-key.pem - Read the beta release, write beta-manifest.txt from it, sign that with - the Ed25519 private key read from stdin, upload both files to the - release, then verify them as a Beta install does. - sign_beta_manifest.py --verify-only [--repo R] [--public-key PEM] + sign_beta_manifest.py --sha256 HEX [--version V] < private-key.pem + Read the beta release, check that it names the installer HEX (and V), + write beta-manifest.txt from it, sign that with the Ed25519 private key + read from stdin, check the signature with the pinned public key, upload + both files to the release, then verify them as a Beta install does. + HEX is the "Installer SHA-256" in the Publish job's summary of the Beta + builds run that published the build: the hash the build computed, + which GitHub's release can't vouch for by itself. Signs only for + sidebandstudio/EternalMonitor. + sign_beta_manifest.py --verify-only [--repo R] [--public-key PEM] [--sha256 HEX] [--version V] Only verify the published manifest and signature. A Beta install offers a Beta only when beta-manifest.sig is a valid Ed25519 @@ -124,10 +129,14 @@ def verified(manifest, signature, public_key, run=subprocess.run): return result.returncode == 0 and os.path.getsize(signature) == 64 -def verify_published(github, public_key=PUBLIC_KEY, run=subprocess.run): +def verify_published(github, public_key=PUBLIC_KEY, run=subprocess.run, sha256=None, version=None): """Checks the published manifest and signature as a Beta install does, and - returns the version they vouch for.""" - version, sha256, size = read_release(github.release()) + returns the version they vouch for. With sha256, the release must name that + installer (and version).""" + found = read_release(github.release()) + if sha256 is not None: + expect(found, sha256, version) + version, sha256, size = found with tempfile.TemporaryDirectory() as tmp: try: github.download([MANIFEST, SIGNATURE], tmp) @@ -144,20 +153,32 @@ def verify_published(github, public_key=PUBLIC_KEY, run=subprocess.run): return version -def sign_and_publish(github, private_key, public_key=PUBLIC_KEY, run=subprocess.run): - """Writes, signs and uploads the manifest for the release as it is now, then - verifies what was published. Returns the version.""" - version, sha256, size = read_release(github.release()) +def expect(found, sha256, version=None): + """Refuses a release that does not name the installer the nightly run built.""" + found_version, found_sha256, _ = found + if found_sha256 != sha256.lower(): + raise SigningError(f"the release names installer {found_sha256}, not {sha256.lower()} from the nightly run") + if version is not None and found_version != version: + raise SigningError(f"the release names {found_version}, not {version}") + + +def sign_and_publish(github, private_key, sha256, version=None, run=subprocess.run): + """Writes, signs and uploads the manifest for the release as it is now, once it + names the installer sha256 (and version), then verifies what was published + against the pinned key. Returns the version.""" + found = read_release(github.release()) + expect(found, sha256, version) + version, sha256, size = found with tempfile.TemporaryDirectory() as tmp: manifest, signature = os.path.join(tmp, MANIFEST), os.path.join(tmp, SIGNATURE) with open(manifest, "w", encoding="utf-8", newline="\n") as f: f.write(manifest_text(version, sha256, size)) sign(manifest, signature, private_key, run) # A wrong key must not replace a good signature. - if not verified(manifest, signature, public_key, run): - raise SigningError("the private key does not match the public key") + if not verified(manifest, signature, PUBLIC_KEY, run): + raise SigningError("the private key does not match the pinned public key") github.upload([manifest, signature]) - published = verify_published(github, public_key, run) + published = verify_published(github, PUBLIC_KEY, run) if published != version: raise SigningError(f"the release moved to {published} while signing {version}; run again") return version @@ -166,22 +187,34 @@ def sign_and_publish(github, private_key, public_key=PUBLIC_KEY, run=subprocess. def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--repo", default=os.environ.get("GH_REPO", REPO)) - parser.add_argument("--public-key", help="PEM file to verify with instead of the pinned key") + parser.add_argument("--public-key", help="--verify-only: PEM file to verify with instead of the pinned key") + parser.add_argument("--sha256", help="the installer's SHA-256 from the nightly run (required to sign)") + parser.add_argument("--version", help="the Beta version the nightly run published") parser.add_argument("--verify-only", action="store_true") args = parser.parse_args(argv) - public_key = PUBLIC_KEY - if args.public_key: - with open(args.public_key, "rb") as f: - public_key = f.read() - github = GitHub(args.repo) + if args.sha256 is not None and not re.fullmatch(r"[0-9A-Fa-f]{64}", args.sha256): + parser.error("--sha256 takes the installer's 64-digit hex SHA-256") + if not args.verify_only: + if args.sha256 is None: + parser.error("signing needs --sha256, from the nightly run's Publish summary") + if args.public_key: + parser.error("--public-key only goes with --verify-only; signing checks the pinned key") + if args.repo != REPO: + parser.error(f"signing is only for {REPO}, not {args.repo}") try: if args.verify_only: - print(f"Verified the signed manifest for {verify_published(github, public_key)}") + public_key = PUBLIC_KEY + if args.public_key: + with open(args.public_key, "rb") as f: + public_key = f.read() + version = verify_published(GitHub(args.repo), public_key, sha256=args.sha256, version=args.version) + print(f"Verified the signed manifest for {version}") else: private_key = sys.stdin.buffer.read() if b"PRIVATE KEY" not in private_key: raise SigningError("pipe the Ed25519 private key PEM into stdin") - print(f"Signed and verified the manifest for {sign_and_publish(github, private_key, public_key)}") + version = sign_and_publish(GitHub(args.repo), private_key, args.sha256, args.version) + print(f"Signed and verified the manifest for {version}") except SigningError as error: print(f"error: {error}", file=sys.stderr) return 1 diff --git a/scripts/test_sign_beta_manifest.py b/scripts/test_sign_beta_manifest.py index 58f8f4d7..380e0e98 100644 --- a/scripts/test_sign_beta_manifest.py +++ b/scripts/test_sign_beta_manifest.py @@ -9,6 +9,7 @@ import sys import tempfile import unittest +from unittest import mock sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) import nightly # noqa: E402 @@ -120,15 +121,22 @@ def test_the_release_parts_must_agree(self): signer.read_release(data) +def pinned(public): + """Signing checks the pinned key; tests pin a throwaway one.""" + return mock.patch.object(signer, "PUBLIC_KEY", public) + + @unittest.skipUnless(ed25519_available(), "needs OpenSSL 3 with Ed25519") class SigningTest(unittest.TestCase): def test_signs_uploads_and_verifies(self): private, public = new_key() github, run = FakeGitHub(), Recorder() - self.assertEqual(signer.sign_and_publish(github, private, public, run), VERSION) + with pinned(public): + self.assertEqual(signer.sign_and_publish(github, private, HASH.upper(), VERSION, run), VERSION) self.assertEqual(github.assets[signer.MANIFEST], signer.manifest_text(VERSION, HASH, SIZE).encode()) self.assertEqual(len(github.assets[signer.SIGNATURE]), 64) self.assertEqual(signer.verify_published(github, public), VERSION) + self.assertEqual(signer.verify_published(github, public, sha256=HASH, version=VERSION), VERSION) # The key went to openssl's stdin only: never an argument, never uploaded. with_key = [args for args, given in run.calls if given == private] self.assertEqual(len(with_key), 1) @@ -138,29 +146,45 @@ def test_signs_uploads_and_verifies(self): for data in github.assets.values(): self.assertNotIn(b"PRIVATE KEY", data) + def test_a_release_that_is_not_the_nightly_runs_build_is_not_signed(self): + private, public = new_key() + run = Recorder() + for sha256, version in [("0" * 64, None), ("0" * 64, VERSION), (HASH, "0.3.5-beta.20261108.27")]: + github = FakeGitHub() + with pinned(public), self.assertRaises(signer.SigningError, msg=(sha256, version)): + signer.sign_and_publish(github, private, sha256, version, run) + self.assertEqual(github.uploads, 0) + self.assertEqual(run.calls, [], "refused before anything is signed") + github = FakeGitHub() + with pinned(public): + signer.sign_and_publish(github, private, HASH) + with self.assertRaises(signer.SigningError): + signer.verify_published(github, public, sha256="0" * 64) + def test_a_key_that_is_not_the_pinned_one_uploads_nothing(self): private, _ = new_key() _, other = new_key() github = FakeGitHub() - with self.assertRaises(signer.SigningError): - signer.sign_and_publish(github, private, other) + with pinned(other), self.assertRaises(signer.SigningError): + signer.sign_and_publish(github, private, HASH) self.assertEqual(github.uploads, 0) # Against the production key, a throwaway key fails too. with self.assertRaises(signer.SigningError): - signer.sign_and_publish(github, private) + signer.sign_and_publish(github, private, HASH) self.assertEqual(github.uploads, 0) def test_a_release_whose_parts_disagree_is_not_signed(self): private, public = new_key() github = FakeGitHub(release(digest=f"sha256:{'0' * 64}")) - with self.assertRaises(signer.SigningError): - signer.sign_and_publish(github, private, public) + with pinned(public), self.assertRaises(signer.SigningError): + signer.sign_and_publish(github, private, HASH) self.assertEqual(github.uploads, 0) def test_verify_only_refuses_a_bad_or_missing_manifest(self): private, public = new_key() github = FakeGitHub() - signer.sign_and_publish(github, private, public) + with pinned(public): + signer.sign_and_publish(github, private, HASH) _, other = new_key() with self.assertRaises(signer.SigningError): signer.verify_published(github, other) @@ -196,17 +220,50 @@ def test_openssl_signs_what_the_host_verifies(self): class CommandTest(unittest.TestCase): - def test_signing_needs_a_private_key_on_stdin(self): + def main(self, argv, stdin=b"-----BEGIN PRIVATE KEY-----\n"): + """main(argv) with stdin; the exit status, or the usage error's. Nothing + reaches GitHub: every case here stops before it would.""" class Stdin: - buffer = __import__("io").BytesIO(b"not a key") + buffer = __import__("io").BytesIO(stdin) saved = sys.stdin sys.stdin = Stdin() try: - self.assertEqual(signer.main(["--repo", "example/none"]), 1) + with mock.patch.object(signer, "GitHub", side_effect=AssertionError("no GitHub")), \ + mock.patch("sys.stderr"): + return signer.main(argv) + except SystemExit as exit: + return exit.code finally: sys.stdin = saved + def test_signing_needs_the_nightly_runs_hash(self): + self.assertEqual(self.main([]), 2) + self.assertEqual(self.main(["--sha256", "abc"]), 2) + self.assertEqual(self.main(["--verify-only", "--sha256", "abc"]), 2) + + def test_signing_is_only_for_the_real_repo_and_the_pinned_key(self): + self.assertEqual(self.main(["--sha256", HASH, "--repo", "someone/fork"]), 2) + with mock.patch.dict(os.environ, {"GH_REPO": "someone/fork"}): + self.assertEqual(self.main(["--sha256", HASH]), 2) + self.assertEqual(self.main(["--sha256", HASH, "--public-key", "other.pem"]), 2) + + def test_signing_needs_a_private_key_on_stdin(self): + self.assertEqual(self.main(["--sha256", HASH], stdin=b"not a key"), 1) + + +class WorkflowTest(unittest.TestCase): + def test_the_nightly_run_shows_the_hash_and_version_to_sign(self): + text = (ROOT / ".github/workflows/nightly.yml").read_text(encoding="utf-8") + publish = text[text.index("\n publish:"):] + self.assertIn("SHA256: ${{ needs.build.outputs.sha256 }}", publish) + self.assertIn("VERSION: ${{ needs.plan.outputs.version }}", publish) + summary = publish[publish.index("- name: Summarize the build to sign"):] + summary = summary[:summary.index("\n - name:", 1)] + self.assertIn('echo "| Installer SHA-256 | \\`$SHA256\\` |"', summary) + self.assertIn('echo "| Version | $VERSION |"', summary) + self.assertIn('} >> "$GITHUB_STEP_SUMMARY"', summary) + if __name__ == "__main__": unittest.main() From 88bd2725b44fb70bdbb453f2e9c094583318a726 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 17:32:41 -0500 Subject: [PATCH 069/226] Check an expected Beta version on its own when verifying --- scripts/sign_beta_manifest.py | 11 ++++++----- scripts/test_sign_beta_manifest.py | 3 +++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/sign_beta_manifest.py b/scripts/sign_beta_manifest.py index 2a650815..b71757ca 100755 --- a/scripts/sign_beta_manifest.py +++ b/scripts/sign_beta_manifest.py @@ -131,11 +131,10 @@ def verified(manifest, signature, public_key, run=subprocess.run): def verify_published(github, public_key=PUBLIC_KEY, run=subprocess.run, sha256=None, version=None): """Checks the published manifest and signature as a Beta install does, and - returns the version they vouch for. With sha256, the release must name that - installer (and version).""" + returns the version they vouch for. With sha256 or version, the release must + name that installer or version; each is checked on its own.""" found = read_release(github.release()) - if sha256 is not None: - expect(found, sha256, version) + expect(found, sha256, version) version, sha256, size = found with tempfile.TemporaryDirectory() as tmp: try: @@ -156,7 +155,7 @@ def verify_published(github, public_key=PUBLIC_KEY, run=subprocess.run, sha256=N def expect(found, sha256, version=None): """Refuses a release that does not name the installer the nightly run built.""" found_version, found_sha256, _ = found - if found_sha256 != sha256.lower(): + if sha256 is not None and found_sha256 != sha256.lower(): raise SigningError(f"the release names installer {found_sha256}, not {sha256.lower()} from the nightly run") if version is not None and found_version != version: raise SigningError(f"the release names {found_version}, not {version}") @@ -166,6 +165,8 @@ def sign_and_publish(github, private_key, sha256, version=None, run=subprocess.r """Writes, signs and uploads the manifest for the release as it is now, once it names the installer sha256 (and version), then verifies what was published against the pinned key. Returns the version.""" + if not sha256: + raise SigningError("signing needs the installer SHA-256 from the nightly run") found = read_release(github.release()) expect(found, sha256, version) version, sha256, size = found diff --git a/scripts/test_sign_beta_manifest.py b/scripts/test_sign_beta_manifest.py index 380e0e98..a8e5e46b 100644 --- a/scripts/test_sign_beta_manifest.py +++ b/scripts/test_sign_beta_manifest.py @@ -137,6 +137,9 @@ def test_signs_uploads_and_verifies(self): self.assertEqual(len(github.assets[signer.SIGNATURE]), 64) self.assertEqual(signer.verify_published(github, public), VERSION) self.assertEqual(signer.verify_published(github, public, sha256=HASH, version=VERSION), VERSION) + self.assertEqual(signer.verify_published(github, public, version=VERSION), VERSION) + with self.assertRaises(signer.SigningError): + signer.verify_published(github, public, version="0.3.5-beta.20990101.999") # The key went to openssl's stdin only: never an argument, never uploaded. with_key = [args for args, given in run.calls if given == private] self.assertEqual(len(with_key), 1) From d897b3f9bd09a9e4304d58ec6877f72d6af619fd Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:21:28 -0500 Subject: [PATCH 070/226] Don't take a PC that slept for a wedged pipeline The watchdog compares stage heartbeats with the host clock, and that clock counts the time a PC is asleep. At the supervisor's first look after waking, before capture has run again, every heartbeat looked as old as the sleep, and the pipeline restarted under the viewer. 7 of the 10 "capture loop silent" values in a day of field logs were 184 s to 12,886 s: PCs waking up. The supervisor looks every 250 ms. A gap of more than 2 s between two looks means nothing in the process ran in between, so the watchdog restamps the heartbeats it watches and gives the stages a fresh window. A real wedge still shows 3 s later. A generation start does not count as a gap: joining the old one can take seconds. --- host/src/capture/mod.rs | 36 +++++---- host/src/capture/second_screen.rs | 6 +- host/src/supervisor.rs | 125 +++++++++++++++++++++++++++++- 3 files changed, 145 insertions(+), 22 deletions(-) diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 1473f9a9..373f5ef4 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -1029,7 +1029,7 @@ mod tests { fn a_viewer_arriving_after_idle_is_not_taken_for_a_wedge() { use crate::supervisor::{wedge_reason_at, Machine}; use std::sync::atomic::Ordering; - let machine = Machine::new(1); + let mut machine = Machine::new(1); let start_ms = 1_000_000; let arrival_ms = start_ms + 17_458; @@ -1041,14 +1041,14 @@ mod tests { .hb_capture_loop_ms .store(arrival_ms, Ordering::Relaxed); connect_viewer(&stale); - let reason = wedge_reason_at(&stale, &machine, arrival_ms).unwrap(); + let reason = wedge_reason_at(&stale, &mut machine, arrival_ms).unwrap(); assert!(reason.starts_with("no captured frame"), "{reason}"); // The first keepalive refreshes the frame stamp before the encoder // has output anything. stale .hb_capture_frame_ms .store(arrival_ms, Ordering::Relaxed); - let reason = wedge_reason_at(&stale, &machine, arrival_ms).unwrap(); + let reason = wedge_reason_at(&stale, &mut machine, arrival_ms).unwrap(); assert!(reason.starts_with("encoder silent"), "{reason}"); // With idle heartbeats the arrival is healthy at every step. @@ -1060,16 +1060,16 @@ mod tests { idle_heartbeat(&idle, now); } connect_viewer(&idle); - assert_eq!(wedge_reason_at(&idle, &machine, arrival_ms), None); + assert_eq!(wedge_reason_at(&idle, &mut machine, arrival_ms), None); idle.hb_capture_frame_ms .store(arrival_ms + 10, Ordering::Relaxed); - assert_eq!(wedge_reason_at(&idle, &machine, arrival_ms + 10), None); + assert_eq!(wedge_reason_at(&idle, &mut machine, arrival_ms + 10), None); } #[test] fn a_viewer_arriving_during_access_lost_recovery_is_not_taken_for_a_wedge() { use crate::supervisor::{wedge_reason_at, Machine}; - let machine = Machine::new(1); + let mut machine = Machine::new(1); let shared = SharedControl::new(0, 1); let start_ms = 1_000_000; startup_heartbeat(&shared, start_ms); @@ -1081,7 +1081,7 @@ mod tests { } // A viewer registers 30 ms after a poll, before the next one. connect_viewer(&shared); - assert_eq!(wedge_reason_at(&shared, &machine, now + 30), None); + assert_eq!(wedge_reason_at(&shared, &mut machine, now + 30), None); // The next poll resends the last picture to it. assert!(recovery_poll(&shared, now + 50, Duration::from_secs(6))); } @@ -1327,7 +1327,7 @@ mod tests { #[test] fn slow_virtual_attach_keeps_watchdog_alive_without_hiding_a_stall() { let shared = SharedControl::new(9876, 15_000_000); - let machine = crate::supervisor::Machine::new(1); + let mut machine = crate::supervisor::Machine::new(1); let start = Instant::now(); let elapsed = Cell::new(Duration::ZERO); let before = vec!["primary".to_string(), "physical-secondary".to_string()]; @@ -1348,7 +1348,7 @@ mod tests { assert!( crate::supervisor::wedge_reason_at( &shared, - &machine, + &mut machine, 1_000 + elapsed.get().as_millis() as u64 ) .is_none(), @@ -1360,14 +1360,18 @@ mod tests { panic!("virtual display did not attach") }; assert_eq!(output.device_name, "virtual"); + // The driver stalls now. The supervisor looks every 250 ms; a longer + // gap between looks is a sleeping PC, not a stall. + let stalled = 1_000 + elapsed.get().as_millis() as u64; + let mut now = stalled; + while now < stalled + 3_000 { + now += 250; + assert!(crate::supervisor::wedge_reason_at(&shared, &mut machine, now).is_none()); + } assert!( - crate::supervisor::wedge_reason_at( - &shared, - &machine, - 1_000 + elapsed.get().as_millis() as u64 + 3_001 - ) - .unwrap() - .contains("capture loop silent"), + crate::supervisor::wedge_reason_at(&shared, &mut machine, now + 250) + .unwrap() + .contains("capture loop silent"), "the normal watchdog must still detect a stalled driver" ); } diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 536e8a49..2ef127b0 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -2621,7 +2621,7 @@ mod tests { fn the_watchdog_leaves_a_second_screen_proving_itself_alone() { let shared = second_screen_selected(); connect_viewer(&shared); - let machine = crate::supervisor::Machine::new(1); + let mut machine = crate::supervisor::Machine::new(1); let now_ms = 100_000; shared.hb_capture_loop_ms.store(now_ms, Ordering::Relaxed); shared @@ -2633,12 +2633,12 @@ mod tests { note_attached(&shared, "virtual".into()); start_proving(&shared, 7, Instant::now()); assert_eq!( - crate::supervisor::wedge_reason_at(&shared, &machine, now_ms), + crate::supervisor::wedge_reason_at(&shared, &mut machine, now_ms), None ); // After its first image a stalled second screen is a wedge again. note_image(&shared, 7); - assert!(crate::supervisor::wedge_reason_at(&shared, &machine, now_ms).is_some()); + assert!(crate::supervisor::wedge_reason_at(&shared, &mut machine, now_ms).is_some()); } /// Review round 2, defect 1: when no tick ended the proof before the diff --git a/host/src/supervisor.rs b/host/src/supervisor.rs index 186ffa29..0b484545 100644 --- a/host/src/supervisor.rs +++ b/host/src/supervisor.rs @@ -71,6 +71,13 @@ const BACKOFF_MAX: Duration = Duration::from_secs(30); const STORM_WINDOW: Duration = Duration::from_secs(60); const STORM_LIMIT: usize = 6; +/// The supervisor looks at the stage heartbeats every 250 ms. A longer gap +/// between two looks means this process did not run in between: the PC +/// slept or hibernated (Windows' monotonic clock counts the time asleep), or +/// the whole host was starved. Heartbeats stamped before such a gap say +/// nothing about whether a stage is wedged. +const WATCH_GAP_MS: u64 = 2_000; + #[derive(Debug, Clone, PartialEq, Eq)] pub enum SupervisorState { Running { @@ -115,6 +122,8 @@ pub struct Machine { pub state: SupervisorState, restarts: Vec, attempt: u32, + /// When the watchdog last looked at the live generation's heartbeats. + watched_ms: Option, } impl Machine { @@ -125,6 +134,7 @@ impl Machine { }, restarts: Vec::new(), attempt: 0, + watched_ms: None, } } @@ -266,6 +276,8 @@ impl Machine { pub fn note_started(&mut self, generation: u64) { self.state = SupervisorState::Running { generation }; + // Joining the old generation can take seconds; that is no sleep. + self.watched_ms = None; } /// A successful stretch of running clears the failure budget. @@ -361,7 +373,7 @@ pub fn run( let ticked = machine.on_tick(now); if ticked != Effect::None { ticked - } else if let Some(reason) = wedge_reason(&shared, &machine) { + } else if let Some(reason) = wedge_reason(&shared, &mut machine) { machine.on_wedged(&reason, now) } else { Effect::None @@ -448,16 +460,42 @@ fn stop_when_stopped(state: &SupervisorState) -> Option { /// Watchdog rules over the heartbeat atomics. Only meaningful while Running /// and after the pipeline has had a moment to warm up. -fn wedge_reason(shared: &SharedControl, machine: &Machine) -> Option { +fn wedge_reason(shared: &SharedControl, machine: &mut Machine) -> Option { wedge_reason_at(shared, machine, crate::clock::host_now_us() / 1000) } pub(crate) fn wedge_reason_at( shared: &SharedControl, - machine: &Machine, + machine: &mut Machine, now_ms: u64, ) -> Option { if !matches!(machine.state, SupervisorState::Running { .. }) { + machine.watched_ms = None; + return None; + } + // After a sleep the first look can come before any stage has run again, + // and every heartbeat looks as old as the sleep: 7 of 10 "capture loop + // silent" restarts in the field were PCs waking after 3 min to 3.5 h. + // Give the stages a fresh window instead; a real wedge still shows 3 s + // later. + let last_look = machine.watched_ms.replace(now_ms); + if let Some(gap_ms) = last_look + .map(|last| now_ms.saturating_sub(last)) + .filter(|&gap| gap > WATCH_GAP_MS) + { + info!( + gap_ms, + "The host did not run for a while (the PC slept?) — restarting the pipeline watchdog's clock" + ); + for beat in [ + &shared.hb_capture_loop_ms, + &shared.hb_capture_frame_ms, + &shared.hb_encode_frame_ms, + ] { + if beat.load(Ordering::Relaxed) > 0 { + beat.store(now_ms, Ordering::Relaxed); + } + } return None; } let loop_ms = shared.hb_capture_loop_ms.load(Ordering::Relaxed); @@ -856,6 +894,87 @@ mod tests { assert_eq!(machine.on_command(&SupervisorCommand::Start), Effect::Start); } + /// The PC sleeps for two hours mid-stream. Windows' clock counts the time + /// asleep, so at the supervisor's first look after waking, before capture + /// has run again, every heartbeat looks two hours old. The field logged + /// "capture loop silent for 184184ms" up to 12886316ms this way. + #[test] + fn a_pc_that_slept_is_not_a_wedged_pipeline() { + let shared = SharedControl::new(0, 1); + let mut machine = Machine::new(1); + let beat = |now_ms: u64| { + shared.hb_capture_loop_ms.store(now_ms, Ordering::Relaxed); + shared.hb_capture_frame_ms.store(now_ms, Ordering::Relaxed); + shared.hb_encode_frame_ms.store(now_ms, Ordering::Relaxed); + }; + let mut now_ms = 10_000; + beat(now_ms); + for _ in 0..8 { + now_ms += 250; + beat(now_ms); + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + } + // Asleep: no thread runs, the clock moves on two hours. + now_ms += 2 * 3600 * 1000; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + now_ms += 250; + beat(now_ms); + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + + // A real wedge after waking is still caught when it was before: the + // capture loop stops 3 s later. + let stuck = now_ms; + while now_ms < stuck + 3_000 { + now_ms += 250; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + } + now_ms += 250; + assert_eq!( + wedge_reason_at(&shared, &mut machine, now_ms), + Some("capture loop silent for 3250ms".into()) + ); + + // A wedge that began just before the sleep shows 3 s after waking. + let mut machine = Machine::new(2); + beat(now_ms); + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + now_ms += 250; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + now_ms += 3600 * 1000; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + let woke = now_ms; + while now_ms < woke + 3_000 { + now_ms += 250; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + } + now_ms += 250; + assert!(wedge_reason_at(&shared, &mut machine, now_ms).is_some()); + } + + /// A restart joins the old generation for up to 5 s between two looks; + /// the new generation's watchdog starts from its own first look. + #[test] + fn a_restart_is_not_taken_for_a_sleep() { + let shared = SharedControl::new(0, 1); + let mut machine = Machine::new(1); + let mut now_ms = 10_000; + shared.hb_capture_loop_ms.store(now_ms, Ordering::Relaxed); + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + assert_eq!( + machine.on_wedged("capture loop silent", Instant::now()), + Effect::StopPipeline + ); + now_ms += 100; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + // The join took 5 s, then the new generation started. + now_ms += 5_000; + machine.note_started(2); + shared + .hb_capture_loop_ms + .store(now_ms - 3_500, Ordering::Relaxed); + assert!(wedge_reason_at(&shared, &mut machine, now_ms).is_some()); + } + #[test] fn wedge_detection_restarts() { let mut machine = Machine::new(1); From f35b3a5acb83d74850ba436939b6ef69e1ad1de9 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 14:21:28 -0500 Subject: [PATCH 071/226] Recover capture when displays come and go, and tell the iPad No display (NoDisplay, 12 events on 8 installs in a day): when enumeration listed no output, capture opened the detection adapter's first output by the ordinal it had at startup. That ordinal goes stale after a dock, lid, sleep or driver change; the call never captured anything, and its error ("there is no adapter with the specified ordinal") hid what was wrong. Capture now opens the adapter and output objects from the enumeration that listed them, and reports "no display is attached to the desktop" when none is. While no display is on, the retry wait looks for one every 250 ms instead of sleeping up to 5 s, and the iPad shows "Your PC has no active display. Turn on a screen to continue." through the heartbeat notice (new NOTICE_NO_DISPLAY = 2; English only, awaiting translation). Older iPads ignore the value. A lost duplication on a display that moved: outputs now carry their adapter's LUID. After ACCESS_LOST, when the target resolves to another output or adapter than the one being duplicated (a display unplugged or back, a driver restart, a dock), the stage opens the target afresh instead of re-creating the duplication on the old output. Before, a display that came back stayed replaced by the primary. Access lost (297 events on 51 installs): it is routine with the lock screen, UAC prompts and mode changes, so it is logged at info, and a loss that is not back within a second warns once. On a 0.3.4 PC whose desktop was taken again every 1 to 4 s for about an hour, the retry ladder was held for 5 s, so every loss waited the full 2 s step and the iPad got 3 to 17 fps. A duplication that worked for 250 ms is now re-created at once, so quick repeated losses cost tens of milliseconds, and re-creations stay under four a second. The same PC's capture loop also exited seven times with ACCESS_LOST (0x887A0026), each a pipeline restart. Only AcquireNextFrame's ACCESS_LOST reached the handler; ReleaseFrame and the staging Map returned theirs with `?` and ended the loop. They now go to the same handler as a lost acquire, and the stage waits out any ACCESS_LOST that still leaves the loop as a display change instead of failing. A static second screen also keeps sending its cached picture after the duplication is re-created at the same size: keepalives waited for a new image that an unchanged screen never brings, and the watchdog restarted the pipeline 5 s later. Blocked retries: the retry pace never started over within an app run, so a PC locked for the fourth time waited up to 5 s after it was unlocked. The pace restarts once capture has worked again. --- host/src/capture/dxgi.rs | 265 +++++++++++------- host/src/capture/mod.rs | 226 ++++++++++++++- host/src/capture/recovery.rs | 256 +++++++++++++++-- host/src/capture/second_screen.rs | 1 + host/src/capture/second_screen/fake.rs | 1 + host/src/capture/second_screen/memory.rs | 1 + host/src/pipeline.rs | 15 +- .../App/ConnectionManager.swift | 10 + ios/EternalMonitor/App/DisplayView.swift | 25 ++ ios/EternalMonitor/App/UIPreview.swift | 6 +- ios/EternalMonitor/Network/WireProtocol.swift | 2 + .../Resources/Localizable.xcstrings | 12 + .../LocalizationTests.swift | 1 + .../StreamRecoveryTests.swift | 24 ++ .../DisplayStateTests.swift | 15 + proto/src/v2/control.rs | 5 +- 16 files changed, 714 insertions(+), 151 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 1139b91f..e824b62a 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -18,8 +18,8 @@ use super::recovery::{ use super::{ client_connected, frame_budget_for, frame_buffer, frame_has_image, heartbeat, idle_heartbeat, keepalive_allowed, keepalive_due, read_back_image, reconcile_virtual_display, recovery_poll, - resolve_target, FrameSlot, FrameUse, IdleGate, OutputInfo, RawFrame, ACQUIRE_TIMEOUT_MS, - FPS_WINDOW, FRAME_PADDING, + resolve_target, FrameSlot, FrameUse, IdleGate, OutputInfo, Picture, RawFrame, + ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, }; use crate::control::{CaptureTarget, SharedControl}; use crate::gpu_preference::Outcome; @@ -34,9 +34,10 @@ use windows::Win32::Graphics::Direct3D11::{ }; use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_SAMPLE_DESC}; use windows::Win32::Graphics::Dxgi::{ - CreateDXGIFactory1, IDXGIFactory1, IDXGIOutput, IDXGIOutput1, IDXGIOutputDuplication, - IDXGIResource, DXGI_ADAPTER_DESC1, DXGI_ERROR_ACCESS_LOST, DXGI_ERROR_UNSUPPORTED, - DXGI_ERROR_WAIT_TIMEOUT, DXGI_OUTDUPL_FRAME_INFO, DXGI_OUTDUPL_POINTER_SHAPE_INFO, + CreateDXGIFactory1, IDXGIAdapter1, IDXGIFactory1, IDXGIOutput, IDXGIOutput1, + IDXGIOutputDuplication, IDXGIResource, DXGI_ADAPTER_DESC1, DXGI_ERROR_ACCESS_LOST, + DXGI_ERROR_NOT_FOUND, DXGI_ERROR_UNSUPPORTED, DXGI_ERROR_WAIT_TIMEOUT, DXGI_OUTDUPL_FRAME_INFO, + DXGI_OUTDUPL_POINTER_SHAPE_INFO, }; /// Parse a null-terminated UTF-16 fixed array (DXGI `DeviceName` / `Description`) to a String. @@ -60,10 +61,30 @@ pub fn enumerate_outputs() -> Vec { return Vec::new(); } }; - enumerate_factory_outputs(&factory) + list_outputs(&factory) + .0 + .into_iter() + .map(|listed| listed.info) + .collect() } -fn enumerate_factory_outputs(factory: &IDXGIFactory1) -> Vec { +/// Whether a fresh enumeration lists any display attached to the desktop. +pub fn display_attached() -> bool { + unsafe { CreateDXGIFactory1::() } + .is_ok_and(|factory| !list_outputs(&factory).0.is_empty()) +} + +/// A desktop-attached output with the adapter and output objects it was +/// listed from. Capture opens these objects rather than looking the +/// output up again by its ordinals. +struct Listed { + info: OutputInfo, + adapter: IDXGIAdapter1, + output: IDXGIOutput, +} + +/// Every desktop-attached output, and how many adapters were listed. +fn list_outputs(factory: &IDXGIFactory1) -> (Vec, u32) { let mut outputs = Vec::new(); let mut adapter_index = 0u32; @@ -72,9 +93,9 @@ fn enumerate_factory_outputs(factory: &IDXGIFactory1) -> Vec { Ok(a) => a, Err(_) => break, }; - let adapter_name = match unsafe { adapter.GetDesc1() } { - Ok(desc) => utf16_to_string(&desc.Description), - Err(_) => String::new(), + let (adapter_name, adapter_luid) = match unsafe { adapter.GetDesc1() } { + Ok(desc) => (utf16_to_string(&desc.Description), adapter_id(desc)), + Err(_) => (String::new(), (0, 0)), }; let mut output_index = 0u32; @@ -86,16 +107,21 @@ fn enumerate_factory_outputs(factory: &IDXGIFactory1) -> Vec { if let Ok(desc) = unsafe { output.GetDesc() } { if desc.AttachedToDesktop.as_bool() { let r = desc.DesktopCoordinates; - outputs.push(OutputInfo { - adapter_index, - output_index, - device_name: utf16_to_string(&desc.DeviceName), - width: (r.right - r.left).max(0) as u32, - height: (r.bottom - r.top).max(0) as u32, - left: r.left, - top: r.top, - is_primary: r.left == 0 && r.top == 0, - adapter_name: adapter_name.clone(), + outputs.push(Listed { + info: OutputInfo { + adapter_index, + output_index, + device_name: utf16_to_string(&desc.DeviceName), + width: (r.right - r.left).max(0) as u32, + height: (r.bottom - r.top).max(0) as u32, + left: r.left, + top: r.top, + is_primary: r.left == 0 && r.top == 0, + adapter_name: adapter_name.clone(), + adapter_id: adapter_luid, + }, + adapter: adapter.clone(), + output, }); } } @@ -103,7 +129,7 @@ fn enumerate_factory_outputs(factory: &IDXGIFactory1) -> Vec { } adapter_index += 1; } - outputs + (outputs, adapter_index) } /// Enumerate adapters, select the output to duplicate, create duplication, @@ -111,13 +137,12 @@ fn enumerate_factory_outputs(factory: &IDXGIFactory1) -> Vec { pub fn run_capture_loop( slot: &FrameSlot, shared: SharedControl, - adapter_index: u32, generation: u64, frame_number: &mut u64, ) -> Result<(), Box> { // The capture adapter FOLLOWS the chosen output (which may differ from the // encoder-detection adapter on a multi-GPU system); the encoder stays vendor-based - // and independent. `adapter_index` is only the last-resort fallback below. + // and independent. let requested_target = shared.capture_target.lock().clone(); // Turn the virtual display on/off to match the request before enumerating, so the // managed virtual output only exists while we're capturing it (and only while an iPad @@ -130,7 +155,8 @@ pub fn run_capture_loop( // DXGI factories snapshot adapters at creation. Enable the driver first, // then enumerate and open the chosen output from the same fresh factory. let factory: IDXGIFactory1 = unsafe { CreateDXGIFactory1()? }; - let outputs = enumerate_factory_outputs(&factory); + let (listed, adapters) = list_outputs(&factory); + let outputs: Vec = listed.iter().map(|listed| listed.info.clone()).collect(); for o in &outputs { info!( device = %o.device_name, @@ -147,67 +173,36 @@ pub fn run_capture_loop( if !outputs.is_empty() && NO_OUTPUTS.swap(false, Ordering::Relaxed) { info!(outputs = outputs.len(), "A display output is back"); } - // A missing adapter or output (DXGI_ERROR_NOT_FOUND) is a display change - // the stage waits out, not a failure. - let (adapter, output, adapter_name, capture_label) = match resolve_target(&outputs, &target) { - Some(idx) => { - let chosen = &outputs[idx]; - let adapter = match unsafe { factory.EnumAdapters1(chosen.adapter_index) } { - Ok(a) => a, - Err(error) => { - warn!( - adapter_index = chosen.adapter_index, - error = %error, - "EnumAdapters1 failed for the chosen output's adapter — falling back to adapter 0" - ); - unsafe { factory.EnumAdapters1(0) }.map_err(opening(OpenStep::Adapter))? - } - }; - let adapter_name = match unsafe { adapter.GetDesc1() } { - Ok(desc) => utf16_to_string(&desc.Description), - Err(_) => chosen.adapter_name.clone(), - }; - let output: IDXGIOutput = unsafe { adapter.EnumOutputs(chosen.output_index) } - .map_err(opening(OpenStep::ListedOutput))?; - let label = format!( - "{} ({}x{})", - chosen.device_name, chosen.width, chosen.height - ); + // No output listed is a display change the stage waits out, not a failure. + let Some(index) = resolve_target(&outputs, &target) else { + if !NO_OUTPUTS.swap(true, Ordering::Relaxed) { info!( - device = %chosen.device_name, - adapter = %adapter_name, - width = chosen.width, - height = chosen.height, - "Capture: selected display output" + adapters, + "No display is attached to the desktop, as while a laptop's lid is closed or the screens are off" ); - (adapter, output, adapter_name, label) - } - None => { - // No outputs enumerated: a closed laptop lid or a screen that is - // off. Try the detection adapter's first output, as before. - if !NO_OUTPUTS.swap(true, Ordering::Relaxed) { - info!("No display outputs enumerated, as while a laptop's lid is closed — trying the detection adapter's first output"); - } - let adapter = match unsafe { factory.EnumAdapters1(adapter_index) } { - Ok(a) => a, - Err(error) => { - warn!( - requested = adapter_index, - error = %error, - "EnumAdapters1 failed for requested adapter index, falling back to adapter 0" - ); - unsafe { factory.EnumAdapters1(0) }.map_err(opening(OpenStep::Adapter))? - } - }; - let adapter_name = match unsafe { adapter.GetDesc1() } { - Ok(desc) => utf16_to_string(&desc.Description), - Err(_) => String::new(), - }; - let output: IDXGIOutput = - unsafe { adapter.EnumOutputs(0) }.map_err(opening(OpenStep::OnlyOutput))?; - (adapter, output, adapter_name, "primary (auto)".to_string()) } + return Err(Box::new(no_output_listed(adapters))); }; + let Listed { + info: chosen, + adapter, + output, + } = listed + .into_iter() + .nth(index) + .expect("resolved from this list"); + let adapter_name = chosen.adapter_name.clone(); + let capture_label = format!( + "{} ({}x{})", + chosen.device_name, chosen.width, chosen.height + ); + info!( + device = %chosen.device_name, + adapter = %adapter_name, + width = chosen.width, + height = chosen.height, + "Capture: selected display output" + ); info!(adapter = %adapter_name, "Capture using adapter"); // The mirror's card, by LUID, for the graphics card checks. @@ -314,8 +309,11 @@ pub fn run_capture_loop( let mut recovery = LossRecovery::default(); let mut resumed = false; let mut idle = IdleGate::new(); - // Set once this duplication has delivered a real desktop image. - let mut seen_image = false; + let mut picture = Picture::default(); + // ACCESS_LOST from a call after the acquire (ReleaseFrame, the staging + // Map). It is handled as if the next acquire had returned it: before, + // it ended the capture loop and restarted the pipeline. + let mut lost_after_acquire: Option = None; loop { if !shared.running.load(Ordering::SeqCst) @@ -361,8 +359,11 @@ pub fn run_capture_loop( let dupl = duplication .as_ref() .expect("the ACCESS_LOST handler re-creates the duplication before continuing"); - let hr = unsafe { - dupl.AcquireNextFrame(acquire_timeout_ms, &mut frame_info, &mut desktop_resource) + let hr = match lost_after_acquire.take() { + Some(lost) => Err(lost), + None => unsafe { + dupl.AcquireNextFrame(acquire_timeout_ms, &mut frame_info, &mut desktop_resource) + }, }; let acquire_us = acquire_start.elapsed().as_micros(); if hr.is_ok() @@ -438,13 +439,14 @@ pub fn run_capture_loop( // --- Copy desktop texture to staging and read pixels into a // recycled buffer (the readback is the ONLY full-frame copy) --- - if !frame_has_image(frame_info.LastPresentTime, seen_image) { - unsafe { - dupl.ReleaseFrame()?; + if !frame_has_image(frame_info.LastPresentTime, picture.seen_image) { + if let Err(error) = unsafe { dupl.ReleaseFrame() } { + lost_after_acquire = Some(access_lost_or(error)?); } continue; } - let frame_data = read_back_image(&shared, generation, &mut seen_image, || { + let seen_image = &mut picture.seen_image; + let frame_data = read_back_image(&shared, generation, seen_image, || { windows::core::Result::Ok(if let Some(ref resource) = desktop_resource { match resource.cast::() { Ok(desktop_texture) => { @@ -525,9 +527,17 @@ pub fn run_capture_loop( } None }) - })?; + }); + let frame_data = match frame_data { + Ok(frame_data) => frame_data, + Err(error) => { + lost_after_acquire = Some(access_lost_or(error)?); + continue; + } + }; if let Some(data) = frame_data { + picture.cached(); if idle.on_frame(client_connected(&shared), now) == FrameUse::Publish { PIPELINE_STATS.lock().record_capture(tex_width, tex_height); heartbeat(&shared.hb_capture_frame_ms); @@ -568,9 +578,9 @@ pub fn run_capture_loop( && keepalive_due( last_send.elapsed(), shared.force_next_idr.load(Ordering::SeqCst), - seen_image, + picture.real(), ) - && keepalive_allowed(seen_image, second_screen) + && keepalive_allowed(picture.real(), second_screen) { resend_last_frame( slot, @@ -598,8 +608,22 @@ pub fn run_capture_loop( // Normal during UAC prompts, the lock screen, mode changes and // full-screen transitions. A duplication re-created too early is // lost again at once, so the retries back off instead of spinning. + // Routine while it is back within a second; see `LOSS_QUIET`. if !recovery.in_episode() { - warn!("Desktop duplication access lost — reinitializing"); + info!("Desktop duplication access lost — reinitializing"); + } + if super::target_moved(&enumerate_outputs(), &target, &chosen) { + info!( + device = %chosen.device_name, + "The display layout changed — opening the display to capture again" + ); + return Err(Box::new(CaptureUnavailable { + block: CaptureBlock::DisplayChanging, + detail: format!( + "the display layout changed while capturing {}", + chosen.device_name + ), + })); } // A second screen that shows no picture before this is // re-created failed for a passing reason. @@ -638,9 +662,10 @@ pub fn run_capture_loop( { return Ok(()); } + warn_if_overdue(&mut recovery); let now_ms = crate::clock::host_now_us() / 1000; if recovery_poll(&shared, now_ms, last_send.elapsed()) - && keepalive_allowed(seen_image, second_screen) + && keepalive_allowed(picture.real(), second_screen) { resend_last_frame( slot, @@ -662,9 +687,10 @@ pub fn run_capture_loop( if !crate::supervisor::generation_is_current(generation) { return; } + warn_if_overdue(&mut recovery); let now_ms = crate::clock::host_now_us() / 1000; if recovery_poll(&shared, now_ms, last_send.elapsed()) - && keepalive_allowed(seen_image, second_screen) + && keepalive_allowed(picture.real(), second_screen) { resend_last_frame( slot, @@ -695,7 +721,7 @@ pub fn run_capture_loop( match retried { Ok(d) => { duplication = Some(d); - seen_image = false; + picture.recreated(); super::second_screen::note_recovery( &shared, generation, @@ -736,6 +762,7 @@ pub fn run_capture_loop( frame_bytes = row_bytes * tex_height as usize; spare = None; last_frame = Arc::new(frame_buffer(frame_bytes)); + picture.resized(); staging_texture = unsafe { let mut tex = None; device.CreateTexture2D( @@ -867,11 +894,47 @@ fn blocked_or_error(error: windows::core::Error) -> Box impl FnOnce(windows::core::Error) -> Box { - move |error| unavailable_or_error(CaptureBlock::opening(step, error.code().0), error) +/// Enumeration listed no display attached to the desktop. The detection +/// adapter's first output, found by its ordinal from startup, was tried +/// next: it never captured anything in the field, and its error text +/// (DXGI_ERROR_NOT_FOUND, "there is no adapter with the specified ordinal") +/// hid what was wrong. +fn no_output_listed(adapters: u32) -> CaptureUnavailable { + let (step, detail) = if adapters == 0 { + ( + OpenStep::Adapter, + "no graphics adapter is listed".to_string(), + ) + } else { + ( + OpenStep::OnlyOutput, + format!("no display is attached to the desktop ({adapters} graphics adapters listed)"), + ) + }; + CaptureUnavailable { + block: CaptureBlock::opening(step, DXGI_ERROR_NOT_FOUND.0) + .expect("DXGI_ERROR_NOT_FOUND is a blocked state"), + detail, + } +} + +/// `error` when it is DXGI_ERROR_ACCESS_LOST, for the loop's handler; +/// anything else ends the capture loop, as before. +fn access_lost_or( + error: windows::core::Error, +) -> Result> { + if error.code() == DXGI_ERROR_ACCESS_LOST { + Ok(error) + } else { + Err(error.into()) + } +} + +/// Logs, once per loss, that the duplication is not back within a second. +fn warn_if_overdue(recovery: &mut LossRecovery) { + if recovery.overdue(Instant::now()) { + warn!("Desktop duplication not back within a second — still retrying"); + } } fn unavailable_or_error( diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 373f5ef4..99937aca 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -222,6 +222,9 @@ pub struct OutputInfo { pub top: i32, pub is_primary: bool, pub adapter_name: String, + /// The adapter's LUID. Ordinals change when an adapter comes or goes; + /// the LUID stays until the adapter is restarted. + pub adapter_id: crate::gpu_preference::AdapterId, } /// Enumerate every desktop-attached display output. Windows-only in substance; @@ -270,6 +273,24 @@ pub(crate) fn resolve_target( } } +/// After the duplication was lost: whether `target` now resolves to another +/// output than `current`, the one being duplicated, by device name and +/// adapter LUID. A display unplugged or back, a dock or a driver restart +/// also surface as a lost duplication. Re-creating it on the same output +/// kept capturing the primary display after the chosen one came back, or an +/// output of an adapter that had gone; the stage opens the target afresh. +#[cfg(any(windows, test))] +pub(crate) fn target_moved( + outputs: &[OutputInfo], + target: &crate::control::CaptureTarget, + current: &OutputInfo, +) -> bool { + resolve_target(outputs, target).is_none_or(|index| { + let output = &outputs[index]; + output.device_name != current.device_name || output.adapter_id != current.adapter_id + }) +} + /// How long to wait for the virtual display to attach after enabling its driver. Overridable /// via `ETERNAL_VDD_TIMEOUT_SECS`, or within 30–60 s by the update manifest. The iPad mirrors /// meanwhile, so a longer wait costs the user nothing; it was 10 s while the iPad gave up 10 s @@ -787,6 +808,44 @@ pub(crate) fn reconcile_virtual_display( } } +/// What a static display's capture has to show. Sending the cached picture +/// again keeps the viewer's stream alive while the desktop changes nothing. +#[cfg(any(windows, test))] +#[derive(Debug, Default)] +pub(crate) struct Picture { + /// The current duplication has delivered an image: the first frames of a + /// new one carry none (see [`frame_has_image`]). + pub seen_image: bool, + /// The cached picture is a real image, not the black start frame. + real: bool, +} + +#[cfg(any(windows, test))] +impl Picture { + /// An image was read back into the cache. + pub fn cached(&mut self) { + self.real = true; + } + + /// The lost duplication was re-created. The cached picture is still the + /// screen's: a static second screen shows nothing new after a lock screen + /// or a UAC prompt, and while keepalives waited for a new image the + /// watchdog restarted the pipeline under the viewer ("no captured frame + /// for 5039ms with a client connected"). + pub fn recreated(&mut self) { + self.seen_image = false; + } + + /// The resolution changed: the cache is a black frame of the new size. + pub fn resized(&mut self) { + self.real = false; + } + + pub fn real(&self) -> bool { + self.real + } +} + /// Whether an idle keepalive may resend the last frame. Before the second /// screen has delivered an image, that frame is the black placeholder: /// sending it would replace the iPad's mirror with black, so the switch @@ -838,7 +897,6 @@ pub(crate) fn read_back_image( pub fn run_capture_stage( slot: FrameSlot, shared: SharedControl, - adapter_index: u32, generation: u64, ) -> Result<(), Box> { struct CloseOnExit(FrameSlot); @@ -851,13 +909,7 @@ pub fn run_capture_stage( let result = recovery::run_until_unblocked( &shared, |frame_number| { - let result = run_platform_loop( - &guard.0, - shared.clone(), - adapter_index, - generation, - frame_number, - ); + let result = run_platform_loop(&guard.0, shared.clone(), generation, frame_number); if let Err(error) = &result { if recovery::passing_error(error.as_ref()) { second_screen::note_interruption(&shared, generation); @@ -868,7 +920,7 @@ pub fn run_capture_stage( } result }, - |delay| wait_while_blocked(&shared, generation, delay), + |delay| wait_while_blocked(&shared, generation, delay, display_attached), ); if let Err(ref e) = result { error!(error = %e, "Capture loop exited with error"); @@ -876,11 +928,34 @@ pub fn run_capture_stage( result } +/// While no display is on, how often the wait looks for one to come back. +const DISPLAY_LOOK: Duration = Duration::from_millis(250); + +/// Whether Windows lists a display attached to the desktop again. +#[cfg(windows)] +fn display_attached() -> bool { + dxgi::display_attached() +} + +#[cfg(not(windows))] +fn display_attached() -> bool { + false +} + /// Sleeps between blocked-capture retries. The watchdog has nothing to fix /// while Windows withholds the desktop, so the stage keeps its heartbeats. -fn wait_while_blocked(shared: &SharedControl, generation: u64, delay: Duration) -> bool { +/// While no display is on, `display_back` is asked every [`DISPLAY_LOOK`] +/// and a display that came back ends the wait: the retries settle 5 s +/// apart, and the picture came back up to 5 s after the screen did. +fn wait_while_blocked( + shared: &SharedControl, + generation: u64, + delay: Duration, + mut display_back: impl FnMut() -> bool, +) -> bool { use std::sync::atomic::Ordering; let deadline = Instant::now() + delay; + let mut next_look = Instant::now() + DISPLAY_LOOK; loop { if !shared.running.load(Ordering::SeqCst) || !crate::supervisor::generation_is_current(generation) @@ -892,6 +967,14 @@ fn wait_while_blocked(shared: &SharedControl, generation: u64, delay: Duration) if now >= deadline { return true; } + if now >= next_look { + next_look = now + DISPLAY_LOOK; + if *shared.capture_block.lock() == Some(recovery::CaptureBlock::NoDisplay) + && display_back() + { + return true; + } + } std::thread::sleep((deadline - now).min(Duration::from_millis(50))); } } @@ -934,14 +1017,13 @@ pub(crate) fn synthetic_requested() -> bool { fn run_platform_loop( slot: &FrameSlot, shared: SharedControl, - adapter_index: u32, generation: u64, frame_number: &mut u64, ) -> Result<(), Box> { if synthetic_requested() { synthetic::run_capture_loop(slot, shared, generation, frame_number) } else { - dxgi::run_capture_loop(slot, shared, adapter_index, generation, frame_number) + dxgi::run_capture_loop(slot, shared, generation, frame_number) } } @@ -949,7 +1031,6 @@ fn run_platform_loop( fn run_platform_loop( slot: &FrameSlot, shared: SharedControl, - _adapter_index: u32, generation: u64, frame_number: &mut u64, ) -> Result<(), Box> { @@ -1436,9 +1517,128 @@ mod tests { top, is_primary: left == 0 && top == 0, adapter_name: "Test GPU".to_string(), + adapter_id: (0, 1), } } + /// A lost duplication after a display was unplugged or plugged back in, + /// or after its adapter restarted, is opened afresh from the stage. + #[test] + fn a_lost_duplication_reopens_when_the_target_moved() { + let (one, two) = (r"\\.\DISPLAY1", r"\\.\DISPLAY2"); + let both = vec![out(one, 0, 0), out(two, 1920, 0)]; + let chosen = CaptureTarget::Output(two.to_string()); + // A UAC prompt or a mode change: same output, same adapter. + assert!(!target_moved(&both, &chosen, &both[1])); + assert!(!target_moved(&both, &CaptureTarget::PrimaryAuto, &both[0])); + // The chosen display was unplugged: capture falls back to the primary. + let unplugged = vec![out(one, 0, 0)]; + assert!(target_moved(&unplugged, &chosen, &both[1])); + // It is back while the primary is captured: return to it. + assert!(target_moved(&both, &chosen, &unplugged[0])); + // Same name on an adapter that restarted, or moved to the other card + // of a dock or hybrid laptop. + let mut restarted = both.clone(); + restarted[1].adapter_id = (0, 2); + assert!(target_moved(&restarted, &chosen, &both[1])); + // Nothing listed at all: no display is on. + assert!(target_moved(&[], &CaptureTarget::PrimaryAuto, &both[0])); + } + + /// Field, 0.3.4 and the 0.3.5 beta: a static second screen lost its + /// duplication, had it back in 3 ms, and 5 s later the watchdog restarted + /// the pipeline under the viewer ("no captured frame for 5039ms with a + /// client connected"). A re-created duplication delivers no image while + /// nothing on the screen changes, and keepalives waited for one. + #[test] + fn a_static_second_screen_keeps_its_picture_after_a_lost_duplication() { + use std::sync::atomic::Ordering::Relaxed; + let shared = SharedControl::new(0, 1); + connect_viewer(&shared); + let mut machine = crate::supervisor::Machine::new(1); + let mut picture = Picture::default(); + // Before its first image the second screen sends nothing. + assert!(!keepalive_allowed(picture.real(), true)); + picture.seen_image = true; + picture.cached(); + // Lost, and re-created at the same size. + picture.recreated(); + assert!(!frame_has_image(0, picture.seen_image)); + let start = 100_000; + let mut sent = start; + shared.hb_capture_frame_ms.store(start, Relaxed); + shared.hb_encode_frame_ms.store(start, Relaxed); + let mut resends = 0; + // Ten seconds of acquire timeouts on the unchanged screen. + for now in (start..start + 10_000).step_by(50) { + shared.hb_capture_loop_ms.store(now, Relaxed); + let since_send = Duration::from_millis(now - sent); + if keepalive_due(since_send, false, picture.real()) + && keepalive_allowed(picture.real(), true) + { + sent = now; + resends += 1; + shared.hb_capture_frame_ms.store(now, Relaxed); + shared.hb_encode_frame_ms.store(now, Relaxed); + } + assert_eq!( + crate::supervisor::wedge_reason_at(&shared, &mut machine, now), + None + ); + } + assert!(resends >= 12, "{resends} keepalives in 10 s"); + // A new resolution leaves only black to resend: the second screen + // waits for the new duplication's first image again. + picture.recreated(); + picture.resized(); + assert!(!keepalive_allowed(picture.real(), true)); + assert!(keepalive_allowed(picture.real(), false), "the mirror"); + picture.seen_image = true; + picture.cached(); + assert!(keepalive_allowed(picture.real(), true)); + } + + /// The screens came back on 2 s into a 5 s retry wait. The picture comes + /// back at the next look, not when the wait ends. + #[test] + fn a_display_coming_back_ends_the_no_display_wait() { + use std::sync::atomic::Ordering::SeqCst; + let shared = SharedControl::new(0, 1); + shared.running.store(true, SeqCst); + let generation = crate::supervisor::CURRENT_GENERATION.load(SeqCst); + *shared.capture_block.lock() = Some(recovery::CaptureBlock::NoDisplay); + let started = Instant::now(); + let looks = Cell::new(0); + let back = || { + looks.set(looks.get() + 1); + started.elapsed() >= Duration::from_millis(600) + }; + assert!(wait_while_blocked( + &shared, + generation, + Duration::from_secs(5), + back + )); + let waited = started.elapsed(); + assert!(waited < Duration::from_millis(2_500), "{waited:?}"); + assert!(looks.get() >= 2, "{} looks", looks.get()); + // Other blocked states clear by the user's hand; nothing is polled. + *shared.capture_block.lock() = Some(recovery::CaptureBlock::SecureDesktop); + let looks = Cell::new(0); + let started = Instant::now(); + assert!(wait_while_blocked( + &shared, + generation, + Duration::from_millis(600), + || { + looks.set(looks.get() + 1); + true + } + )); + assert!(started.elapsed() >= Duration::from_millis(600)); + assert_eq!(looks.get(), 0); + } + #[test] fn primary_auto_picks_origin_output() { let outputs = vec![out(r"\\.\DISPLAY2", 2560, 0), out(r"\\.\DISPLAY1", 0, 0)]; diff --git a/host/src/capture/recovery.rs b/host/src/capture/recovery.rs index 61f8cf03..4f5d173c 100644 --- a/host/src/capture/recovery.rs +++ b/host/src/capture/recovery.rs @@ -209,7 +209,17 @@ pub enum LossStep { /// has worked this long. A UAC prompt, the lock screen or a full-screen game /// can take the desktop again right after it returns; starting the ladder /// over at each return re-created the duplication at full speed in a loop. -pub const LOSS_LADDER_RESET: Duration = Duration::from_secs(5); +/// It was 5 s, and on a PC whose desktop was taken again every 1 to 4 s the +/// ladder never came down from 2 s: each loss cost 2 s without a picture, +/// and the iPad got 3 to 17 fps for an hour. A duplication that worked for +/// a quarter of a second is re-created at once, so a steady stream of quick +/// losses costs tens of milliseconds each, and re-creations still stay +/// under four a second. +pub const LOSS_LADDER_RESET: Duration = Duration::from_millis(250); + +/// A loss that is back within this long is routine (the lock screen, a UAC +/// prompt, a mode change) and is logged at info; a longer one is a warning. +pub const LOSS_QUIET: Duration = Duration::from_secs(1); /// One loss episode lasts from the first ACCESS_LOST until a frame or an idle /// timeout proves the new duplication works. @@ -223,6 +233,8 @@ pub struct LossRecovery { step: usize, /// When the last episode ended, while `step` is kept. back_at: Option, + /// This episode outlasted [`LOSS_QUIET`] and said so. + overdue: bool, } impl LossRecovery { @@ -230,6 +242,18 @@ impl LossRecovery { self.since.is_some() } + /// True once per episode, when it has lasted [`LOSS_QUIET`]. + pub fn overdue(&mut self, now: Instant) -> bool { + let Some(since) = self.since else { + return false; + }; + if self.overdue || now.duration_since(since) < LOSS_QUIET { + return false; + } + self.overdue = true; + true + } + pub fn lost(&mut self, now: Instant) -> LossStep { if let Some(back_at) = self.back_at.take() { if now.duration_since(back_at) >= LOSS_LADDER_RESET { @@ -262,6 +286,7 @@ impl LossRecovery { return None; }; self.back_at = Some(now); + self.overdue = false; let losses = std::mem::take(&mut self.losses); Some((losses, now.duration_since(since))) } @@ -292,6 +317,14 @@ impl BlockedRetry { self.attempts += 1; (Duration::from_millis(BLOCKED_RETRY_MS[index]), changed) } + + /// Capture worked since the last blocked attempt. The next block is + /// retried at the quick pace again but is not logged again when it is + /// the same state: the pace climbed over a whole app run, and a PC + /// locked for the fourth time waited up to 5 s after it was unlocked. + pub fn resumed(&mut self) { + self.attempts = 0; + } } /// Runs `capture` until it ends for a reason other than a blocked desktop. @@ -341,10 +374,21 @@ pub fn run_until_unblocked( continue; } stalled = 0; + #[cfg(windows)] + let error = access_lost_is_blocked(error); let Some(unavailable) = error.downcast_ref::() else { return Err(error); }; - *shared.capture_block.lock() = Some(unavailable.block); + // The first working frame clears the block (see + // [`note_capture_resumed`]). + if shared + .capture_block + .lock() + .replace(unavailable.block) + .is_none() + { + retry.resumed(); + } let (delay, changed) = retry.blocked(unavailable.block); if changed { warn!( @@ -359,14 +403,32 @@ pub fn run_until_unblocked( } } +/// DXGI_ERROR_ACCESS_LOST that left the capture loop instead of reaching its +/// handler: a display change the stage waits out, never a pipeline failure. +/// The loop hands ACCESS_LOST from every call it knows to its handler; this +/// is for any call it does not. +#[cfg(windows)] +fn access_lost_is_blocked( + error: Box, +) -> Box { + match error.downcast_ref::() { + Some(lost) if lost.code().0 as u32 == 0x887A_0026 => Box::new(CaptureUnavailable { + block: CaptureBlock::DisplayChanging, + detail: format!("desktop duplication lost outside its handler: {lost}"), + }), + _ => error, + } +} + /// The heartbeat's notice while capture is blocked. A secure screen says the -/// PC is locked, so the iPad doesn't show a frozen picture without a word; it -/// clears with the block when capture works again. The other blocks keep the -/// held picture as before. +/// PC is locked, and no display attached says so, so the iPad doesn't show a +/// frozen picture without a word; it clears with the block when capture +/// works again. The other blocks keep the held picture as before. pub fn heartbeat_notice(block: Option) -> u8 { - use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_PC_LOCKED}; + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_NO_DISPLAY, NOTICE_PC_LOCKED}; match block { Some(CaptureBlock::SecureDesktop) => NOTICE_PC_LOCKED, + Some(CaptureBlock::NoDisplay) => NOTICE_NO_DISPLAY, _ => NOTICE_NONE, } } @@ -492,16 +554,21 @@ mod tests { } #[test] - fn only_a_secure_screen_tells_the_ipad_the_pc_is_locked() { - use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_PC_LOCKED}; + fn only_a_secure_screen_or_no_display_has_a_notice_for_the_ipad() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_NO_DISPLAY, NOTICE_PC_LOCKED}; assert_eq!( heartbeat_notice(Some(CaptureBlock::SecureDesktop)), NOTICE_PC_LOCKED ); + // A closed lid or screens that are off: the field's PCs showed a + // frozen picture without a word until a display came back. + assert_eq!( + heartbeat_notice(Some(CaptureBlock::NoDisplay)), + NOTICE_NO_DISPLAY + ); assert_eq!(heartbeat_notice(None), NOTICE_NONE); for block in [ CaptureBlock::DisplayChanging, - CaptureBlock::NoDisplay, CaptureBlock::UnsupportedOnThisGpu, CaptureBlock::TooManyCaptureApps, CaptureBlock::SessionDisconnected, @@ -635,7 +702,7 @@ mod tests { } #[test] - fn a_working_frame_ends_the_episode_and_five_healthy_seconds_reset_the_ladder() { + fn a_working_frame_ends_the_episode_and_a_healthy_quarter_second_resets_the_ladder() { let ms = Duration::from_millis; let mut recovery = LossRecovery::default(); let start = Instant::now(); @@ -645,7 +712,8 @@ mod tests { let ended = recovery.healthy(start + ms(300)); assert_eq!(ended, Some((2, ms(300)))); assert!(!recovery.in_episode()); - // Frames keep coming for 5 s: the next loss is retried at once. + // Frames keep coming for a quarter of a second: the next loss is + // retried at once. let mut now = start + ms(300); while now < start + ms(300) + LOSS_LADDER_RESET { now += ms(16); @@ -661,8 +729,8 @@ mod tests { /// A UAC prompt or a full-screen game takes the desktop again right after /// each return. The first retry is as quick as before; the returns that - /// follow within 5 s keep climbing the ladder instead of starting over - /// at 0 ms each time. + /// follow within a quarter of a second keep climbing the ladder instead of + /// starting over at 0 ms each time. #[test] fn a_duplication_lost_again_soon_after_it_returned_keeps_its_backoff() { let ms = Duration::from_millis; @@ -670,30 +738,33 @@ mod tests { let start = Instant::now(); let mut now = start; let mut delays = Vec::new(); - for _ in 0..7 { + for _ in 0..10 { let LossStep::Retry(delay) = recovery.lost(now) else { panic!("gave up") }; delays.push(delay.as_millis() as u64); - // The new duplication works for a second, then is lost again. + // The new duplication works for 200 ms, then is lost again. now += delay + ms(1); assert!( recovery.healthy(now).is_some(), "each loss is its own episode" ); - now += ms(1_000); + now += ms(200); assert_eq!(recovery.healthy(now), None); } - assert_eq!(delays, [0, 100, 250, 500, 1000, 2000, 2000]); - // The last return was a second ago. Four healthy seconds are not - // enough; five are. - let back = now - ms(1_000); - assert_eq!(recovery.healthy(back + ms(4_000)), None); - assert_eq!(recovery.lost(back + ms(4_000)), LossStep::Retry(ms(2000))); - recovery.healthy(back + ms(6_000)); - assert_eq!(recovery.healthy(back + ms(11_000)), None); assert_eq!( - recovery.lost(back + ms(11_000)), + delays, + [0, 100, 250, 500, 1000, 2000, 2000, 2000, 2000, 2000] + ); + // The last return was 200 ms ago. 240 healthy ms are not enough; + // 250 are. + let back = now - ms(200); + assert_eq!(recovery.healthy(back + ms(240)), None); + assert_eq!(recovery.lost(back + ms(240)), LossStep::Retry(ms(2000))); + recovery.healthy(back + ms(2_300)); + assert_eq!(recovery.healthy(back + ms(2_550)), None); + assert_eq!( + recovery.lost(back + ms(2_550)), LossStep::Retry(Duration::ZERO) ); // Each return still ends its episode: the give-up window measures @@ -701,6 +772,107 @@ mod tests { assert!(now.duration_since(start) > LOSS_GIVE_UP); } + /// The field's looping PC (0.3.4): the desktop was taken again 1 to 4 s + /// after each return, for about an hour. With the ladder held for 5 s + /// every retry waited 2 s, and the iPad got 3 to 17 fps. A duplication + /// that worked for a quarter of a second is re-created at once. + #[test] + fn a_desktop_taken_every_few_seconds_is_retried_at_once() { + let ms = Duration::from_millis; + // How long each re-created duplication worked, from that PC's log, + // and two quicker losses. + let lives = [3006, 1789, 650, 1227, 2008, 300, 2260, 1005].map(ms); + let mut recovery = LossRecovery::default(); + let start = Instant::now(); + let mut now = start; + let mut without_picture = Duration::ZERO; + let mut losses = 0; + while now < start + Duration::from_secs(60) { + let LossStep::Retry(delay) = recovery.lost(now) else { + panic!("gave up") + }; + losses += 1; + assert_eq!(delay, Duration::ZERO, "loss {losses} waited"); + without_picture += delay; + now += delay; + assert!(recovery.healthy(now).is_some()); + now += lives[losses % lives.len()]; + } + assert!( + without_picture < ms(100), + "{without_picture:?} without a picture in a minute" + ); + // Still never a spin: at most four re-creations a second. + assert!(losses <= 240, "{losses} re-creations in a minute"); + } + + /// ACCESS_LOST from a call the capture loop does not hand to its handler + /// (on 0.3.4, ReleaseFrame after an acquire: "Capture loop exited with + /// error" seven times in an hour, each a pipeline restart). The stage + /// waits it out like any display change, however often it comes. + #[cfg(windows)] + #[test] + fn access_lost_that_leaves_the_capture_loop_never_fails_the_stage() { + let shared = SharedControl::new(0, 1); + let mut attempts = 0; + let mut blocks = Vec::new(); + let result = run_until_unblocked( + &shared, + |_| { + attempts += 1; + if attempts <= 7 { + Err(windows::core::Error::from_hresult(windows::core::HRESULT( + 0x887A_0026_u32 as i32, + )) + .into()) + } else { + Ok(()) + } + }, + |_| { + blocks.push(*shared.capture_block.lock()); + true + }, + ); + assert!(result.is_ok(), "{result:?}"); + assert_eq!(attempts, 8); + assert_eq!(blocks, [Some(CaptureBlock::DisplayChanging); 7]); + // Other Windows errors still reach the supervisor. + let failed = run_until_unblocked( + &shared, + |_| { + Err(windows::core::Error::from_hresult(windows::core::HRESULT( + 0x887A_0005_u32 as i32, + )) + .into()) + }, + |_| panic!("a removed device was retried in the stage"), + ); + assert!(failed.is_err()); + } + + #[test] + fn only_a_loss_not_back_within_a_second_is_a_warning() { + let ms = Duration::from_millis; + let mut recovery = LossRecovery::default(); + let start = Instant::now(); + assert!(!recovery.overdue(start), "no episode yet"); + recovery.lost(start); + assert!(!recovery.overdue(start + ms(999))); + recovery.lost(start + ms(500)); + assert!(recovery.overdue(start + LOSS_QUIET)); + assert!(!recovery.overdue(start + ms(5_000)), "said once"); + recovery.healthy(start + ms(5_000)); + // The next episode is judged on its own. + let next = start + ms(9_000); + recovery.lost(next); + assert!(recovery.healthy(next + ms(300)).is_some()); + assert!(!recovery.overdue(next + ms(2_000))); + recovery.lost(next + ms(3_000)); + assert!(!recovery.overdue(next + ms(3_999))); + assert!(recovery.overdue(next + ms(4_000))); + } + /// Review r1 of #141: "No display is on" told a desktop PC to open the /// laptop lid while a display was unplugged or a driver updated. Only /// an enumeration that found no output at all is a display that's off. @@ -1036,6 +1208,40 @@ mod tests { assert!(result.is_ok(), "a stopped generation exits cleanly"); } + /// The lock screen, four times in one app run. Each unlock is retried + /// as quickly as the first, not 5 s later. + #[test] + fn a_block_after_capture_worked_again_is_retried_at_the_quick_pace() { + let shared = SharedControl::new(0, 1); + let attempts = RefCell::new(0); + let waits = RefCell::new(Vec::new()); + let result = run_until_unblocked( + &shared, + |_| { + *attempts.borrow_mut() += 1; + let attempt = *attempts.borrow(); + if attempt > 12 { + return Ok(()); + } + // Three blocked attempts, then the user unlocks and capture + // works until the next lock. + if attempt % 3 == 0 { + note_capture_resumed(&shared); + } + Err(blocked(CaptureBlock::SecureDesktop)) + }, + |delay| { + waits.borrow_mut().push(delay.as_millis()); + true + }, + ); + assert!(result.is_ok()); + assert_eq!( + *waits.borrow(), + [500, 1000, 500, 1000, 2000, 500, 1000, 2000, 500, 1000, 2000, 500] + ); + } + #[test] fn retry_pace_resets_when_the_reason_changes() { let mut retry = BlockedRetry::default(); diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 2ef127b0..deafc8a5 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -1816,6 +1816,7 @@ mod tests { top: 0, is_primary: primary, adapter_name: "Test GPU".to_string(), + adapter_id: (0, 1), } } diff --git a/host/src/capture/second_screen/fake.rs b/host/src/capture/second_screen/fake.rs index 4ab29bcd..d065ba2c 100644 --- a/host/src/capture/second_screen/fake.rs +++ b/host/src/capture/second_screen/fake.rs @@ -156,6 +156,7 @@ impl Driver for FakeDisplay { top: 0, is_primary: primary, adapter_name: APP_GPU.to_string(), + adapter_id: APP_CARD, }; let mut outputs = vec![output("primary", true)]; if attached { diff --git a/host/src/capture/second_screen/memory.rs b/host/src/capture/second_screen/memory.rs index 9ed2916b..4d9b77b7 100644 --- a/host/src/capture/second_screen/memory.rs +++ b/host/src/capture/second_screen/memory.rs @@ -1112,6 +1112,7 @@ mod tests { top: 0, is_primary: true, adapter_name: "Test GPU".to_string(), + adapter_id: (0, 1), }] } diff --git a/host/src/pipeline.rs b/host/src/pipeline.rs index 2633a12e..594c0a9a 100644 --- a/host/src/pipeline.rs +++ b/host/src/pipeline.rs @@ -60,19 +60,14 @@ pub async fn run_pipeline_supervised( let capture_reporter = reporter.clone(); let capture_shared = shared.clone(); - let adapter_index = gpu_info.adapter_index; let capture_thread = std::thread::Builder::new() .name(format!("capture-g{generation}")) .spawn(move || { - let outcome = match capture::run_capture_stage( - frame_producer, - capture_shared, - adapter_index, - generation, - ) { - Ok(()) => StageOutcome::Completed, - Err(e) => StageOutcome::Failed(e.to_string()), - }; + let outcome = + match capture::run_capture_stage(frame_producer, capture_shared, generation) { + Ok(()) => StageOutcome::Completed, + Err(e) => StageOutcome::Failed(e.to_string()), + }; capture_reporter.stage_exited(Stage::Capture, outcome); }); if let Err(e) = capture_thread { diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 543ab12f..edaa516d 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -114,6 +114,9 @@ final class ConnectionManager: ObservableObject { /// The PC is locked or shows a permission prompt, so it can't capture and /// the picture is held. Its heartbeats say so until capture works again. @Published var pcLocked = false + /// The PC has no display turned on, so there is nothing to capture and + /// the picture is held. Its heartbeats say so until one comes back. + @Published var pcNoDisplay = false /// Nonzero while an automatic reconnect cycle is running. @Published private(set) var reconnectAttempt = 0 @@ -441,6 +444,7 @@ final class ConnectionManager: ObservableObject { versionMismatch = nil signalLost = false pcLocked = false + pcNoDisplay = false degradedSinceUs = nil prevCounters = FrameAssembler.Counters() diagnostics.removeAll() @@ -808,6 +812,11 @@ final class ConnectionManager: ObservableObject { self.pcLocked = locked self.record(.info, "ctrl", locked ? "PC locked: holding the picture" : "PC unlocked") } + let noDisplay = heartbeat.notice == Heartbeat.noticeNoDisplay + if noDisplay != self.pcNoDisplay { + self.pcNoDisplay = noDisplay + self.record(.info, "ctrl", noDisplay ? "PC has no active display: holding the picture" : "PC display back") + } } } channel.onStreamConfig = { [weak self, weak assembler] config in @@ -1466,6 +1475,7 @@ final class ConnectionManager: ObservableObject { signalLost = false pictureRestoring = false pcLocked = false + pcNoDisplay = false degradedSinceUs = nil prevCounters = FrameAssembler.Counters() hostInfo = nil diff --git a/ios/EternalMonitor/App/DisplayView.swift b/ios/EternalMonitor/App/DisplayView.swift index 9ef35f46..bb557073 100644 --- a/ios/EternalMonitor/App/DisplayView.swift +++ b/ios/EternalMonitor/App/DisplayView.swift @@ -55,6 +55,10 @@ struct DisplayView: View { pcLockedPill .transition(.opacity) .allowsHitTesting(false) + } else if connectionManager.pcNoDisplay { + noDisplayPill + .transition(.opacity) + .allowsHitTesting(false) } else if connectionManager.pictureRestoring { restoringPill .transition(.opacity) @@ -118,6 +122,7 @@ struct DisplayView: View { .animation(.easeInOut(duration: 0.25), value: connectionManager.signalLost) .animation(.easeInOut(duration: 0.25), value: connectionManager.pictureRestoring) .animation(.easeInOut(duration: 0.25), value: connectionManager.pcLocked) + .animation(.easeInOut(duration: 0.25), value: connectionManager.pcNoDisplay) .animation(reduceMotion ? nil : .easeInOut(duration: 0.35), value: appeared) .onAppear { appeared = true @@ -435,6 +440,26 @@ struct DisplayView: View { .accessibilityIdentifier("display.pcLocked") } + /// Over the held picture while the PC has no display turned on, until + /// one comes back. + private var noDisplayPill: some View { + HStack(spacing: 10) { + Image(systemName: "display") + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(Theme.text) + .accessibilityHidden(true) + Text(L("Your PC has no active display. Turn on a screen to continue.")) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + } + .padding(.horizontal, 18) + .frame(minHeight: 44) + .background(glass(Capsule())) + .accessibilityElement(children: .ignore) + .accessibilityLabel(L("Your PC has no active display. Turn on a screen to continue.")) + .accessibilityIdentifier("display.pcNoDisplay") + } + /// The PC is still connected but a lost frame left the picture waiting /// for a fresh one; usually gone in well under a second. private var restoringPill: some View { diff --git a/ios/EternalMonitor/App/UIPreview.swift b/ios/EternalMonitor/App/UIPreview.swift index 2d35d2df..47cad88f 100644 --- a/ios/EternalMonitor/App/UIPreview.swift +++ b/ios/EternalMonitor/App/UIPreview.swift @@ -12,7 +12,7 @@ import SwiftUI /// Launch with `-allowUSB NO` so the USB listener stays off as well. enum UIPreview { static let mode = ProcessInfo.processInfo.environment["EM_UI_PREVIEW"] ?? "" - static var showsBackdrop: Bool { mode == "display" || mode == "quality" || mode == "restoring" || mode == "locked" } + static var showsBackdrop: Bool { mode == "display" || mode == "quality" || mode == "restoring" || mode == "locked" || mode == "no-display" } static var opensQuality: Bool { mode == "quality" } static var opensSettings: Bool { mode == "settings" || mode == "settings-about" } /// Scrolls Settings to Support and About. @@ -50,6 +50,10 @@ enum UIPreview { manager.transportMode = "WiFi" manager.state = .connected manager.pcLocked = true + case "no-display": + manager.transportMode = "WiFi" + manager.state = .connected + manager.pcNoDisplay = true case "connecting": manager.state = .connecting case "error": diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index f7a15526..32fa987f 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -318,6 +318,8 @@ struct Heartbeat: Equatable { static let noticeNone: UInt8 = 0 /// The PC is locked or shows a permission prompt, so it can't capture. static let noticePCLocked: UInt8 = 1 + /// No display is on at the PC (a closed lid, screens off or unplugged). + static let noticeNoDisplay: UInt8 = 2 } enum ByeReason: UInt8 { diff --git a/ios/EternalMonitor/Resources/Localizable.xcstrings b/ios/EternalMonitor/Resources/Localizable.xcstrings index 9d27ffc2..4866bd5e 100644 --- a/ios/EternalMonitor/Resources/Localizable.xcstrings +++ b/ios/EternalMonitor/Resources/Localizable.xcstrings @@ -9043,6 +9043,18 @@ } } }, + "Your PC has no active display. Turn on a screen to continue." : { + "comment" : "Over the held picture while no display is on at the PC: a closed laptop lid, or screens that are off or unplugged", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Your PC has no active display. Turn on a screen to continue." + } + } + } + }, "Your PC is locked. Unlock it to continue." : { "comment" : "Over the held picture while the PC shows its lock screen or a permission prompt", "extractionState" : "manual", diff --git a/ios/EternalMonitorTests/LocalizationTests.swift b/ios/EternalMonitorTests/LocalizationTests.swift index 5a33593f..4147cfd5 100644 --- a/ios/EternalMonitorTests/LocalizationTests.swift +++ b/ios/EternalMonitorTests/LocalizationTests.swift @@ -24,6 +24,7 @@ final class LocalizationTests: XCTestCase { /// then every language shows them in English. private let awaitingTranslation: Set = [ "Your PC is locked. Unlock it to continue.", + "Your PC has no active display. Turn on a screen to continue.", ] /// Every language has exactly English's strings, none empty, each with diff --git a/ios/EternalMonitorTests/StreamRecoveryTests.swift b/ios/EternalMonitorTests/StreamRecoveryTests.swift index 622b7bbc..c0317ecc 100644 --- a/ios/EternalMonitorTests/StreamRecoveryTests.swift +++ b/ios/EternalMonitorTests/StreamRecoveryTests.swift @@ -109,6 +109,30 @@ final class StreamRecoveryTests: XCTestCase { XCTAssertFalse(manager.pcLocked, "the next session starts without it") } + /// A closed lid or screens that are off leave the PC nothing to + /// capture. Its heartbeats say so, and the iPad says so over the held + /// picture, still connected, until a display comes back. + func testAPCWithNoDisplayIsShownOverTheHeldPictureUntilOneComesBack() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + XCTAssertFalse(manager.pcNoDisplay) + host.sendHeartbeat(notice: Heartbeat.noticeNoDisplay) + try await until("the PC says it has no display") { manager.pcNoDisplay } + XCTAssertEqual(manager.state, .connected) + XCTAssertFalse(manager.pcLocked) + // Unlocking a PC that still has no display doesn't say it is locked. + host.sendHeartbeat(notice: Heartbeat.noticePCLocked) + try await until("locked instead") { manager.pcLocked && !manager.pcNoDisplay } + host.sendHeartbeat(notice: Heartbeat.noticeNoDisplay) + try await until("no display again") { manager.pcNoDisplay && !manager.pcLocked } + host.sendHeartbeat() + try await until("a display is back") { !manager.pcNoDisplay } + host.sendHeartbeat(notice: Heartbeat.noticeNoDisplay) + try await until("no display again") { manager.pcNoDisplay } + manager.cancel() + XCTAssertFalse(manager.pcNoDisplay, "the next session starts without it") + } + func testContiguousNumberingRevealsALostReopenKeyframe() async throws { // The PC reopened its encoder and its new keyframe (seq 2) was lost; // the new encoder's first P-frame continues the codec's count. A PC diff --git a/ios/EternalMonitorUITests/DisplayStateTests.swift b/ios/EternalMonitorUITests/DisplayStateTests.swift index 1fc818c6..66019d3f 100644 --- a/ios/EternalMonitorUITests/DisplayStateTests.swift +++ b/ios/EternalMonitorUITests/DisplayStateTests.swift @@ -41,4 +41,19 @@ final class DisplayStateTests: UITestCase { screenshot.lifetime = .keepAlways add(screenshot) } + + func testAPCWithNoDisplaySaysSoOverTheHeldPicture() { + let app = XCUIApplication() + app.launchArguments = ["-didSeeOnboarding", "YES", "-allowUSB", "NO"] + app.launchEnvironment["EM_UI_PREVIEW"] = "no-display" + launchCleanly(app) + let note = app.otherElements["display.pcNoDisplay"] + XCTAssertTrue(note.waitForExistence(timeout: 10), "a PC with no display on must not look like a frozen picture") + XCTAssertEqual(note.label, "Your PC has no active display. Turn on a screen to continue.") + XCTAssertFalse(app.otherElements["display.pcLocked"].exists) + let screenshot = XCTAttachment(screenshot: app.screenshot()) + screenshot.name = "pc-no-display" + screenshot.lifetime = .keepAlways + add(screenshot) + } } diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index 72386841..b498ab03 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -176,6 +176,9 @@ pub const NOTICE_NONE: u8 = 0; /// The PC is locked or shows a permission prompt, so Windows lets nothing /// capture its screen and the picture stops until the user returns. pub const NOTICE_PC_LOCKED: u8 = 1; +/// No display is attached to the PC's desktop (a closed lid, screens off or +/// unplugged), so there is nothing to capture until one comes back. +pub const NOTICE_NO_DISPLAY: u8 = 2; /// Host → client, every heartbeat interval while a session is active. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1129,7 +1132,7 @@ mod tests { // A notice is one byte more. Such a body, cut back to the old length // as an older iPad reads it, is the plain heartbeat; an unknown value // reaches the iPad as it is, for it to ignore. - for notice in [NOTICE_PC_LOCKED, 0xC8] { + for notice in [NOTICE_PC_LOCKED, NOTICE_NO_DISPLAY, 0xC8] { let datagram = encode_control(7, 1, &heartbeat(notice)); assert_eq!(datagram.len(), plain.len() + 1); assert_eq!(datagram[..plain.len()][8..], plain[8..]); From 5ec65523246d1a71664d706887d2bac052718613 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 17:57:33 -0500 Subject: [PATCH 072/226] Keep the pointer in the video on the second screen The virtual display has no hardware pointer, so Windows draws the pointer into its image: on the reference PC every pointer move on the second screen was a new image, and Desktop Duplication kept reporting the pointer visible at one stale position. An iPad drawing the pointer there would show that stale pointer too. On the second screen the host now keeps the pointer in the video as before and tells the iPad it is hidden; Mirror sends it apart. --- host/src/capture/dxgi.rs | 15 +++++++++++++-- host/src/capture/mod.rs | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 07cb4bc1..ab3e6f9f 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -319,6 +319,15 @@ pub fn run_capture_loop( // Whether `last_frame` has the pointer drawn in: it has unless the // client draws the pointer itself (`SharedControl::client_draws_pointer`). let mut last_frame_has_pointer = true; + if second_screen { + // A position from a mirror before this one would stay drawn. + shared.pointer.set_position(super::pointer::Position { + visible: false, + x: 0, + y: 0, + host_time_us: crate::clock::host_now_us(), + }); + } loop { if !shared.running.load(Ordering::SeqCst) @@ -353,7 +362,7 @@ pub fn run_capture_loop( // other client, or none, one with it. The staging texture still // holds the last desktop image, so the cached frame is redrawn from // it when the connected client's answer differs from the cache's. - let draws_pointer = shared.client_draws_pointer(); + let draws_pointer = pointer_apart(shared.client_draws_pointer(), second_screen); if seen_image && last_frame_has_pointer == draws_pointer { let mut pixel_buf = frame_buffer(frame_bytes); copy_staging( @@ -441,7 +450,8 @@ pub fn run_capture_loop( cursor_x = frame_info.PointerPosition.Position.x; cursor_y = frame_info.PointerPosition.Position.y; shared.pointer.set_position(super::pointer::Position { - visible: cursor_visible, + // The second screen's picture has the pointer in it. + visible: cursor_visible && !second_screen, x: cursor_x, y: cursor_y, host_time_us: crate::clock::host_now_us(), @@ -449,6 +459,7 @@ pub fn run_capture_loop( } if frame_info.PointerShapeBufferSize > 0 && cursor.update_shape(dupl, frame_info.PointerShapeBufferSize) + && !second_screen { if let Some(image) = super::pointer::convert_shape( cursor.shape_type, diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 77cec87d..4fccce95 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -108,6 +108,19 @@ pub(crate) fn frame_has_image(last_present_time: i64, seen_image: bool) -> bool last_present_time != 0 || seen_image } +/// Whether the pointer goes apart from the video for this capture. On the +/// second screen it never does: the virtual display has no hardware +/// pointer, so Windows draws the pointer into its image, and Desktop +/// Duplication kept reporting a stale position there (measured on the +/// reference PC: every pointer move was a new image, and the reported +/// position never moved). An iPad that draws the pointer would show that +/// stale one too. There the pointer stays in the video, as before, and the +/// iPad is told it is hidden. +#[cfg(any(windows, test))] +pub(crate) fn pointer_apart(client_draws_pointer: bool, second_screen: bool) -> bool { + client_draws_pointer && !second_screen +} + /// What the capture loop does with an acquired frame. #[cfg(any(windows, test))] #[derive(Debug, PartialEq, Eq)] @@ -1048,6 +1061,17 @@ mod tests { ); } + #[test] + fn the_second_screen_keeps_the_pointer_in_the_video() { + assert!(pointer_apart(true, false), "Mirror, an iPad that draws it"); + assert!( + !pointer_apart(true, true), + "second screen: Windows draws it" + ); + assert!(!pointer_apart(false, false), "an older iPad"); + assert!(!pointer_apart(false, true)); + } + pub(super) fn connect_viewer(shared: &SharedControl) { use crate::transport::link::PeerId; use crate::transport::session::ConfigSource; From f8b110b376ef928793567b62c544853e7bcb788f Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:03:04 -0500 Subject: [PATCH 073/226] Make a replacement window device off the window's thread, time-boxed Listing adapters and making a device, where a driver call can stall, ran on the window's thread, so one stuck call froze the window for good. Now they run on their own thread (winit::Answer), which the window polls each frame. One that hasn't answered in 10 s counts as a failed try for the backoff and give-up, and no other starts while it is out; a late device is still taken. The window's thread keeps only the surface and its swapchain. New test faults, debug and fault-injection builds only: device-lost-later-fail and device-lost-later-hang fail or hang every replacement, and device-lost-early-fail loses the device at the first frame. --- host/src/gui/renderer.rs | 4 + host/src/startup/mod.rs | 38 +++-- host/tests/window_device_loss.rs | 85 ++++++++++- vendor/README.md | 16 ++- vendor/egui-wgpu/src/winit.rs | 239 ++++++++++++++++++++++++++----- 5 files changed, 335 insertions(+), 47 deletions(-) diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 7f4f3aaa..65a8a8da 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -652,6 +652,10 @@ fn select( if replacing { lost_before_drawing(); } + #[cfg(any(debug_assertions, feature = "fault-injection"))] + if replacing { + crate::startup::simulate_replacement_failure()?; + } let usable: Vec<&wgpu::Adapter> = adapters .iter() .filter(|adapter| surface.is_none_or(|surface| adapter.is_surface_supported(surface))) diff --git a/host/src/startup/mod.rs b/host/src/startup/mod.rs index 24984341..b687a1b2 100644 --- a/host/src/startup/mod.rs +++ b/host/src/startup/mod.rs @@ -35,7 +35,9 @@ static SHOWN_AT: OnceLock = OnceLock::new(); /// fail while the window renderer starts; `early-error` to fail right after the failure reporter /// starts; `panic-later` or, on Windows, `fail-fast-later` to fail 15 s after /// the window works; `device-lost-later` to lose the window's graphics -/// device 15 s after the window works; or `surface-error` to report a surface +/// device 15 s after the window works, with `-fail` or `-hang` to fail or +/// hang every replacement device; `device-lost-early-fail` to lose it at the +/// first frame and fail every replacement; or `surface-error` to report a surface /// error on every frame of the first two seconds. A `-quiet` suffix skips the /// message box. /// Only debug builds and builds with the `fault-injection` feature read it; @@ -585,21 +587,41 @@ pub(crate) fn simulate_window_failure() { } } -/// [`SIMULATE_ENV`] asks for `device-lost-later`. +/// [`SIMULATE_ENV`] asks for `device-lost-later`, `device-lost-later-fail` or +/// `device-lost-later-hang` (every replacement device fails or never comes), +/// or `device-lost-early-fail` (at the first frame, before the window works). #[cfg(any(debug_assertions, feature = "fault-injection"))] pub(crate) fn device_loss_requested() -> bool { - simulated() == "device-lost-later" + simulated().starts_with("device-lost-") } -/// True once, 15 s after the window works, when [`device_loss_requested`]: -/// the window's graphics device is to be lost then. +/// Fail or hang the choice of a replacement device, when [`SIMULATE_ENV`] +/// asks. It runs on egui-wgpu's replacement thread. +#[cfg(any(debug_assertions, feature = "fault-injection"))] +pub(crate) fn simulate_replacement_failure() -> Result<(), String> { + match simulated().as_str() { + "device-lost-later-fail" | "device-lost-early-fail" => { + Err(format!("Simulated replacement failure from {SIMULATE_ENV}")) + } + "device-lost-later-hang" => loop { + std::thread::park(); + }, + _ => Ok(()), + } +} + +/// True once when [`device_loss_requested`]: at the first frame for +/// `device-lost-early-*`, else 15 s after the window works. The window's +/// graphics device is to be lost then. #[cfg(any(debug_assertions, feature = "fault-injection"))] pub(crate) fn simulate_device_loss_now() -> bool { static DONE: AtomicBool = AtomicBool::new(false); + let early = simulated().starts_with("device-lost-early"); device_loss_requested() - && SHOWN_AT - .get() - .is_some_and(|shown| shown.elapsed() > Duration::from_secs(15)) + && (early + || SHOWN_AT + .get() + .is_some_and(|shown| shown.elapsed() > Duration::from_secs(15))) && !DONE.swap(true, Ordering::SeqCst) } diff --git a/host/tests/window_device_loss.rs b/host/tests/window_device_loss.rs index 67d7b84c..13949d4c 100644 --- a/host/tests/window_device_loss.rs +++ b/host/tests/window_device_loss.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant}; use eframe::egui; -use eframe::egui_wgpu::winit::Replacement; +use eframe::egui_wgpu::winit::{Answer, Asked, Replacement, REPLACEMENT_TIMEOUT}; use eframe::egui_wgpu::{Renderer, RendererOptions, ScreenDescriptor}; use eframe::wgpu; @@ -150,3 +150,86 @@ fn a_device_that_drew_for_a_while_starts_the_count_again() { replacement.failed(now); assert_eq!(replacement.wait(now), Some(Duration::from_millis(500))); } + +#[test] +fn a_stuck_replacement_times_out_without_blocking_and_its_late_device_is_taken() { + let (release, released) = std::sync::mpsc::channel::<()>(); + let start = Instant::now(); + // As a driver call that stalls while the replacement device is made. + let mut answer = Answer::ask("test replacement", start, REPLACEMENT_TIMEOUT, move || { + let _ = released.recv(); + "device" + }) + .unwrap(); + let polling = Instant::now(); + assert!(matches!(answer.poll(start), Asked::Waiting)); + assert!(matches!( + answer.poll(start + REPLACEMENT_TIMEOUT), + Asked::TimedOut + )); + // One timeout per period, each a failed try for the backoff. + assert!(matches!( + answer.poll(start + REPLACEMENT_TIMEOUT), + Asked::Waiting + )); + assert!(matches!( + answer.poll(start + REPLACEMENT_TIMEOUT * 2), + Asked::TimedOut + )); + assert!( + polling.elapsed() < SECOND, + "polling never waits for the worker" + ); + + release.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + match answer.poll(start) { + Asked::Answered(device) => break assert_eq!(device, "device"), + Asked::Waiting => std::thread::sleep(Duration::from_millis(10)), + other => panic!("{other:?}"), + } + assert!(Instant::now() < deadline); + } +} + +#[test] +fn a_replacement_thread_that_dies_ends_the_wait() { + let start = Instant::now(); + let mut answer = Answer::<()>::ask("test replacement", start, REPLACEMENT_TIMEOUT, || { + panic!("simulated driver panic") + }) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !matches!(answer.poll(start), Asked::Ended) { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +fn a_replacement_that_never_answers_ends_in_giving_up() { + // The painter counts each timeout of a stuck replacement as a failed + // try, and starts no other while it is out. + let (_never, never) = std::sync::mpsc::channel::<()>(); + let mut now = Instant::now(); + let mut replacement = Replacement::default(); + replacement.lost(now); + let mut answer = Answer::ask("test replacement", now, REPLACEMENT_TIMEOUT, move || { + let _ = never.recv(); + }) + .unwrap(); + let mut timeouts = 0; + for _ in 0..100 { + if replacement.wait(now).is_none() { + break; + } + now += REPLACEMENT_TIMEOUT; + if let Asked::TimedOut = answer.poll(now) { + timeouts += 1; + replacement.failed(now); + } + assert!(timeouts <= Replacement::MAX_FAILURES); + } + assert_eq!(timeouts, Replacement::MAX_FAILURES); +} diff --git a/vendor/README.md b/vendor/README.md index c01a3b66..d596a453 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -30,11 +30,21 @@ The patch: its own devices and is not affected. The new device's first frame uploads the font atlas again. Other textures are the app's to load again (`host/src/gui/renderer.rs` counts devices for that). +- A replacement never blocks the window's thread. The window makes the new + surface (it belongs to the window, and Metal wants it on the main thread). + Listing adapters and making the device, where a driver can stall, run on + their own thread (`winit::Answer`), which the window polls each frame. One + that hasn't answered in 10 s counts as a failed try, and no other starts + while it is out; a late device is still taken. Configuring the new surface, + which makes the swapchain, stays on the window's thread, because DXGI sends + the window messages while it does that. - `configure_surface` replaces the device, instead of panicking, when the adapter supports no surface any more. -- `Renderer::upload_failed` and `winit::SKIPPED_FRAMES` let the app see - skipped frames: the host counts them, so a backend that only skips frames - is never recorded as one that drew. +- `Renderer::upload_failed`, `winit::SKIPPED_FRAMES` and `winit::DEVICES_LOST` + let the app see skipped frames and dropped devices. The host counts them, so + a backend that only skips frames is never recorded as one that drew, and a + device lost before the window worked is recorded as its backend's failure + even if no replacement is ever made. - A D3D12 device is only replaced once nothing holds the lost one, so the host drops the copy eframe keeps in `Frame` (`release_render_state`). diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs index 5bace98d..d9ec857d 100644 --- a/vendor/egui-wgpu/src/winit.rs +++ b/vendor/egui-wgpu/src/winit.rs @@ -20,6 +20,98 @@ use web_time::Instant; /// failures. pub static SKIPPED_FRAMES: AtomicU64 = AtomicU64::new(0); +/// EternalMonitor patch: devices the painter dropped because they were lost, whether or +/// not a replacement was ever made. +pub static DEVICES_LOST: AtomicU64 = AtomicU64::new(0); + +/// EternalMonitor patch: how long a replacement device may take before the try counts as +/// failed. A driver call can stall for much longer, so the window never waits on it. +pub const REPLACEMENT_TIMEOUT: Duration = Duration::from_secs(10); + +/// EternalMonitor patch: how often the window looks for a replacement device being made. +const REPLACEMENT_POLL: Duration = Duration::from_millis(100); + +/// EternalMonitor patch: what [`Answer::poll`] found. +#[derive(Debug)] +pub enum Asked { + Answered(T), + Waiting, + /// No answer by the deadline. The next deadline is a timeout later; a late answer + /// still arrives. + TimedOut, + /// The thread ended without an answer, as by a panic. + Ended, +} + +/// EternalMonitor patch: work done on its own thread, which the window polls without ever +/// waiting for it. +pub struct Answer { + receiver: std::sync::mpsc::Receiver, + deadline: Instant, + timeout: Duration, +} + +impl Answer { + pub fn ask( + name: &str, + now: Instant, + timeout: Duration, + work: impl FnOnce() -> T + Send + 'static, + ) -> std::io::Result { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + std::thread::Builder::new() + .name(name.to_owned()) + .spawn(move || { + let _ = sender.send(work()); + })?; + Ok(Self { + receiver, + deadline: now + timeout, + timeout, + }) + } + + /// Never blocks. + pub fn poll(&mut self, now: Instant) -> Asked { + match self.receiver.try_recv() { + Ok(answer) => Asked::Answered(answer), + Err(std::sync::mpsc::TryRecvError::Disconnected) => Asked::Ended, + Err(std::sync::mpsc::TryRecvError::Empty) if now >= self.deadline => { + self.deadline = now + self.timeout; + Asked::TimedOut + } + Err(std::sync::mpsc::TryRecvError::Empty) => Asked::Waiting, + } + } +} + +/// EternalMonitor patch: run `future` on this thread. wgpu's native futures are ready at +/// once; this parks until woken otherwise. +fn block_on(future: F) -> F::Output { + struct Unpark(std::thread::Thread); + impl std::task::Wake for Unpark { + fn wake(self: Arc) { + self.0.unpark(); + } + } + let waker = std::task::Waker::from(Arc::new(Unpark(std::thread::current()))); + let mut cx = std::task::Context::from_waker(&waker); + let mut future = core::pin::pin!(future); + loop { + if let core::task::Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + std::thread::park(); + } +} + +/// EternalMonitor patch: a replacement device being made on another thread, with the +/// surface it was made for. +type Replacing = Answer<( + wgpu::Surface<'static>, + Result, +)>; + /// EternalMonitor patch: when the painter tries to replace a lost device, and when it /// stops trying. /// @@ -132,6 +224,9 @@ pub struct Painter { restore_fonts: bool, /// When to try replacing a lost device, and when to stop. replacement: Replacement, + /// The replacement being made, if any. Never more than one: a stuck one is waited + /// for, not joined by another. + replacing: Option, } impl Painter { @@ -177,6 +272,7 @@ impl Painter { rebuilding: false, restore_fonts: false, replacement: Replacement::default(), + replacing: None, } } @@ -299,37 +395,13 @@ impl Painter { if let Some(window) = window { let size = window.inner_size(); if !self.surfaces.contains_key(&viewport_id) { - // EternalMonitor patch: a lost device is replaced when `replacement` says, - // not on every repaint, and not at all once it gave up. + // EternalMonitor patch: a lost device is replaced off this thread. if self.rebuilding { - match self.replacement.wait(Instant::now()) { - None => return Ok(()), - Some(wait) if !wait.is_zero() => { - self.context.request_repaint_after(wait); - return Ok(()); - } - Some(_) => {} - } - } - let added = match self.instance.create_surface(window) { - Ok(surface) => self.add_surface(surface, viewport_id, size).await, - Err(err) => Err(err.into()), - }; - match added { - // EternalMonitor patch: while a lost device is replaced, the window - // skips frames until a new one can be made. - Err(err) if self.rebuilding => { - if self.replacement.failed(Instant::now()) { - Self::log_gave_up(&err); - } else { - log::warn!("Could not replace the lost wgpu device yet: {err}"); - if let Some(wait) = self.replacement.wait(Instant::now()) { - self.context.request_repaint_after(wait); - } - } - } - added => added?, + self.replace_device(viewport_id, window); + return Ok(()); } + let surface = self.instance.create_surface(window)?; + self.add_surface(surface, viewport_id, size).await?; } } else { log::warn!("No window - clearing all surfaces"); @@ -378,11 +450,6 @@ impl Painter { RenderState::create(&self.config, &self.instance, Some(&surface), self.options) .await?; self.watch_device(&render_state.device); - if self.rebuilding { - log::info!("Replaced the lost wgpu device"); - self.rebuilding = false; - self.restore_fonts = true; - } self.render_state = Some(render_state); } self.install_surface(surface, viewport_id, size.width, size.height, false); @@ -420,6 +487,7 @@ impl Painter { self.screen_capture_state = None; self.rebuilding = true; self.restore_fonts = false; + DEVICES_LOST.fetch_add(1, Ordering::SeqCst); let now = Instant::now(); if self.replacement.lost(now) { Self::log_gave_up(&"the new device was lost too"); @@ -431,6 +499,107 @@ impl Painter { } } + /// EternalMonitor patch: work towards a new device for a lost one, without blocking. + /// + /// The surface is made here, on the window's thread: it is the window's (Metal wants + /// it on the main thread) and calls no driver for a device. Listing adapters and + /// making the device, where a driver can stall, run on another thread, which this + /// polls. One that hasn't answered in [`REPLACEMENT_TIMEOUT`] counts as a failed try, + /// and no other starts while it is out; its late device is still taken. Configuring + /// the new surface, which makes the swapchain, stays here too: DXGI sends the window + /// messages while it does. + fn replace_device(&mut self, viewport_id: ViewportId, window: Arc) { + let now = Instant::now(); + if let Some(replacing) = &mut self.replacing { + match replacing.poll(now) { + Asked::Waiting => {} + Asked::TimedOut => self.replacement_failed(now, &"no new device within 10 s"), + Asked::Ended => { + self.replacing = None; + self.replacement_failed(now, &"the thread making it ended"); + } + Asked::Answered((surface, Ok(render_state))) => { + self.replacing = None; + self.watch_device(&render_state.device); + self.render_state = Some(render_state); + let size = window.inner_size(); + self.install_surface(surface, viewport_id, size.width, size.height, false); + self.rebuilding = false; + self.restore_fonts = true; + log::info!("Replaced the lost wgpu device"); + return; + } + Asked::Answered((_, Err(err))) => { + self.replacing = None; + self.replacement_failed(now, &err); + } + } + if let Some(wait) = self.next_check(now) { + self.context.request_repaint_after(wait); + } + return; + } + match self.replacement.wait(now) { + None => return, + Some(wait) if !wait.is_zero() => { + self.context.request_repaint_after(wait); + return; + } + Some(_) => {} + } + let surface = match self.instance.create_surface(window) { + Ok(surface) => surface, + Err(err) => { + self.replacement_failed(now, &err); + return; + } + }; + let (config, instance, options) = + (self.config.clone(), self.instance.clone(), self.options); + let asked = Answer::ask( + "wgpu device replacement", + now, + REPLACEMENT_TIMEOUT, + move || { + let made = block_on(RenderState::create( + &config, + &instance, + Some(&surface), + options, + )); + (surface, made) + }, + ); + match asked { + Ok(replacing) => { + self.replacing = Some(replacing); + self.context.request_repaint_after(REPLACEMENT_POLL); + } + Err(err) => self.replacement_failed(now, &err), + } + } + + /// EternalMonitor patch: a try at a new device failed at `now`. + fn replacement_failed(&mut self, now: Instant, why: &dyn core::fmt::Display) { + if self.replacement.failed(now) { + Self::log_gave_up(why); + // A stuck thread may still answer; nothing waits for it. + self.replacing = None; + } else { + log::warn!("Could not replace the lost wgpu device yet: {why}"); + } + } + + /// EternalMonitor patch: when the window should look again at a device being replaced, + /// or `None` once the painter gave up. + fn next_check(&self, now: Instant) -> Option { + if self.replacing.is_some() { + Some(REPLACEMENT_POLL) + } else { + self.replacement.wait(now) + } + } + /// EternalMonitor patch: said once, when [`Replacement`] gives up. fn log_gave_up(last: &dyn core::fmt::Display) { log::error!( @@ -713,7 +882,7 @@ impl Painter { // the last frame draws nothing again, so it goes. if self.rebuilding { SKIPPED_FRAMES.fetch_add(1, Ordering::SeqCst); - if let Some(wait) = self.replacement.wait(Instant::now()) { + if let Some(wait) = self.next_check(Instant::now()) { self.context.request_repaint_after(wait); } return vsync_sec; From 34692a84d2f21dec670dbc050519ea0b813ba279 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:03:05 -0500 Subject: [PATCH 074/226] Record a device lost before the window works when it is dropped The failure was recorded only when a replacement chose its adapter. If every replacement failed before that, as when the window's surface can't be made, the painter gave up and a normal quit then cleared the start marker without recording anything. The painter now counts the devices it drops (DEVICES_LOST), and the window records the backend it started on as failed, once, at the next frame after a drop. --- host/src/gui/renderer.rs | 75 +++++++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 13 deletions(-) diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 65a8a8da..6a874005 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -125,6 +125,8 @@ static STATUS: Mutex> = Mutex::new(None); /// Adapters chosen for the window this launch. After the first, each /// replaces a lost device (`vendor/egui-wgpu`). static SELECTIONS: AtomicU64 = AtomicU64::new(0); +/// The painter's count of devices dropped as lost, when last looked at. +static LOSSES_SEEN: AtomicU64 = AtomicU64::new(0); /// How this launch draws its window, for Settings > Troubleshooting. #[derive(Clone, Debug, Default, PartialEq)] @@ -438,6 +440,11 @@ fn save_marker(directory: &Path, owner: ProcessId, marker: &Marker) -> bool { /// window has kept drawing without a surface error. pub(super) fn frame_started(ctx: &egui::Context) { crate::startup::simulate_surface_error(&FAILED_ACQUIRES); + // Whether or not a replacement is ever made. + let lost = eframe::egui_wgpu::winit::DEVICES_LOST.load(Ordering::SeqCst); + if LOSSES_SEEN.swap(lost, Ordering::SeqCst) != lost { + lost_before_drawing(); + } let visible = ctx.input(|input| input.viewport().minimized != Some(true)); let Ok(mut tracker) = PRESENT.lock() else { return; @@ -468,17 +475,25 @@ fn failures() -> u64 { + eframe::egui_wgpu::winit::SKIPPED_FRAMES.load(Ordering::SeqCst) } -/// The window's device was lost before the window was confirmed working: the -/// backend it started on failed to draw. Record that, and end this launch's -/// marker, so the replacement drawing later is not counted for it. +/// The painter dropped a lost device. Before the window was confirmed +/// working, that means the backend it started on failed to draw. fn lost_before_drawing() { - if PRESENT.lock().is_ok_and(|tracker| tracker.confirmed) { + let confirmed = PRESENT.lock().is_ok_and(|tracker| tracker.confirmed); + if let (Some(directory), Ok(mut started)) = (STATE_DIR.get(), STARTED.lock()) { + lose_started(directory, &mut started, confirmed, ProcessId::current()); + } +} + +/// Record the failure of this launch's `started` backend, once, unless the +/// window was `confirmed`. The marker ends with it, so neither a replacement +/// that draws later nor a normal quit counts that backend as drawing. +fn lose_started(directory: &Path, started: &mut Option, confirmed: bool, owner: ProcessId) { + if confirmed { return; } - let marker = STARTED.lock().ok().and_then(|mut started| started.take()); - if let (Some(directory), Some(marker)) = (STATE_DIR.get(), marker) { + if let Some(marker) = started.take() { warn!(gpu = %marker.gpu, backend = %marker.backend, "Window device lost before the window worked"); - lose_launch(directory, &marker, ProcessId::current()); + lose_launch(directory, &marker, owner); } } @@ -645,13 +660,10 @@ fn select( rules: &[RendererRule], overruled: Option<&'static str>, ) -> Result { - // A replacement for a lost device chooses as the start did, and starts no - // marker. The window may have worked, so a later end is not a failed - // start; if it hadn't yet, the backend it started on failed. + // A replacement for a lost device (on egui-wgpu's replacement thread) + // chooses as the start did, and starts no marker: the loss itself was + // recorded when the device was dropped (`frame_started`). let replacing = SELECTIONS.fetch_add(1, Ordering::SeqCst) > 0; - if replacing { - lost_before_drawing(); - } #[cfg(any(debug_assertions, feature = "fault-injection"))] if replacing { crate::startup::simulate_replacement_failure()?; @@ -2152,6 +2164,43 @@ mod tests { fs::remove_dir_all(directory).unwrap(); } + #[test] + fn a_device_lost_before_the_window_works_is_recorded_once_even_if_never_replaced() { + let directory = temp_dir("lost-unreplaced"); + let owner = ProcessId::current(); + let gpu = "10de:2c02 32.0.15.9186"; + let mut started = Some(marker(gpu, "vulkan")); + assert!(save_marker(&directory, owner, started.as_ref().unwrap())); + // The painter drops the device, then every replacement fails making + // its surface, so no adapter is ever chosen, until it gives up. The + // host sees the drop on the next frames. + for _ in 0..3 { + lose_started(&directory, &mut started, false, owner); + } + // Then a normal quit (`window_closed`). + if let Some(marker) = started.as_ref() { + assert!(finish_launch(&directory, marker, owner, false)); + } + let history = read_history(&directory); + assert_eq!(history.len(), 1); + assert_eq!(history[0].backend, "vulkan"); + assert_eq!(history[0].failures, 1); + assert_eq!(history[0].fault.as_deref(), Some(DEVICE_LOST)); + assert!(!marker_path(&directory, owner).exists()); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn a_device_lost_after_the_window_works_is_not_a_failed_start() { + let directory = temp_dir("lost-confirmed"); + let owner = ProcessId::current(); + let mut started = Some(marker("10de:2c02 32.0.15.9186", "dx12")); + lose_started(&directory, &mut started, true, owner); + assert!(started.is_some()); + assert!(read_history(&directory).is_empty()); + fs::remove_dir_all(directory).unwrap(); + } + #[test] fn a_minimized_window_is_not_confirmed() { let start = Instant::now(); From 03f97547fd71861d8d5939d782b64250d82873fa Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:12:47 -0500 Subject: [PATCH 075/226] Capture the GPU detection test's log lines whatever ran first The candidate's log-capturing tests call prepare_log_capture first, so a stall line another test's thread logged before any subscriber still reaches the capture. The detection test asserts that line too. --- host/src/gpu.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/host/src/gpu.rs b/host/src/gpu.rs index f19fadcf..7c0dda08 100644 --- a/host/src/gpu.rs +++ b/host/src/gpu.rs @@ -407,6 +407,7 @@ mod tests { .with_ansi(false) .with_writer(move || Sink(Arc::clone(&writer))) .finish(); + crate::logging::prepare_log_capture(); let started = std::time::Instant::now(); let info = tracing::subscriber::with_default(subscriber, || { detect_within(lanes, limits, found, detect) From eba142fe27279e7336dfcd9eb382a685bc8df5a9 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:23:15 -0500 Subject: [PATCH 076/226] Give a waking PC's displays 10 s before the watchdog judges capture After a gap in the watchdog's looks (a PC that slept), the heartbeats were restamped and a capture loop still silent 3 s later was restarted. Waking displays and drivers can hold capture inside a call for longer than that while nothing is wrong. The stages now get 10 s after such a gap; a capture loop still silent when it ends is restarted then, and a wedge once capture runs again is caught 3 s later, as before. --- host/src/supervisor.rs | 50 +++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/host/src/supervisor.rs b/host/src/supervisor.rs index 0b484545..9f4268b2 100644 --- a/host/src/supervisor.rs +++ b/host/src/supervisor.rs @@ -78,6 +78,11 @@ const STORM_LIMIT: usize = 6; /// nothing about whether a stage is wedged. const WATCH_GAP_MS: u64 = 2_000; +/// After such a gap the stages get this long before the watchdog judges +/// them. Waking displays and drivers can hold capture for seconds while +/// nothing is wrong; a stage still silent after it is restarted then. +const WAKE_GRACE_MS: u64 = 10_000; + #[derive(Debug, Clone, PartialEq, Eq)] pub enum SupervisorState { Running { @@ -124,6 +129,8 @@ pub struct Machine { attempt: u32, /// When the watchdog last looked at the live generation's heartbeats. watched_ms: Option, + /// When the watchdog last found such a gap, during [`WAKE_GRACE_MS`]. + woke_ms: Option, } impl Machine { @@ -135,6 +142,7 @@ impl Machine { restarts: Vec::new(), attempt: 0, watched_ms: None, + woke_ms: None, } } @@ -278,6 +286,7 @@ impl Machine { self.state = SupervisorState::Running { generation }; // Joining the old generation can take seconds; that is no sleep. self.watched_ms = None; + self.woke_ms = None; } /// A successful stretch of running clears the failure budget. @@ -471,13 +480,14 @@ pub(crate) fn wedge_reason_at( ) -> Option { if !matches!(machine.state, SupervisorState::Running { .. }) { machine.watched_ms = None; + machine.woke_ms = None; return None; } // After a sleep the first look can come before any stage has run again, // and every heartbeat looks as old as the sleep: 7 of 10 "capture loop // silent" restarts in the field were PCs waking after 3 min to 3.5 h. - // Give the stages a fresh window instead; a real wedge still shows 3 s - // later. + // Give the stages a fresh window instead, with a grace while the + // displays wake; a real wedge still shows when the grace ends. let last_look = machine.watched_ms.replace(now_ms); if let Some(gap_ms) = last_look .map(|last| now_ms.saturating_sub(last)) @@ -496,8 +506,15 @@ pub(crate) fn wedge_reason_at( beat.store(now_ms, Ordering::Relaxed); } } + machine.woke_ms = Some(now_ms); return None; } + if let Some(woke_ms) = machine.woke_ms { + if now_ms.saturating_sub(woke_ms) < WAKE_GRACE_MS { + return None; + } + machine.woke_ms = None; + } let loop_ms = shared.hb_capture_loop_ms.load(Ordering::Relaxed); let frame_ms = shared.hb_capture_frame_ms.load(Ordering::Relaxed); let encode_ms = shared.hb_encode_frame_ms.load(Ordering::Relaxed); @@ -917,12 +934,21 @@ mod tests { // Asleep: no thread runs, the clock moves on two hours. now_ms += 2 * 3600 * 1000; assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); - now_ms += 250; - beat(now_ms); - assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + // The displays wake slowly: capture is stuck in a driver call for + // 6 s, then runs again. No restart. + let woke = now_ms; + while now_ms < woke + 6_000 { + now_ms += 250; + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + } + while now_ms < woke + 12_000 { + now_ms += 250; + beat(now_ms); + assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); + } - // A real wedge after waking is still caught when it was before: the - // capture loop stops 3 s later. + // A real wedge once awake is caught as before: 3 s after the capture + // loop stops. let stuck = now_ms; while now_ms < stuck + 3_000 { now_ms += 250; @@ -934,7 +960,8 @@ mod tests { Some("capture loop silent for 3250ms".into()) ); - // A wedge that began just before the sleep shows 3 s after waking. + // A capture loop that never runs again after waking is restarted + // when the grace ends. let mut machine = Machine::new(2); beat(now_ms); assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); @@ -943,12 +970,15 @@ mod tests { now_ms += 3600 * 1000; assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); let woke = now_ms; - while now_ms < woke + 3_000 { + while now_ms < woke + WAKE_GRACE_MS - 250 { now_ms += 250; assert_eq!(wedge_reason_at(&shared, &mut machine, now_ms), None); } now_ms += 250; - assert!(wedge_reason_at(&shared, &mut machine, now_ms).is_some()); + assert_eq!( + wedge_reason_at(&shared, &mut machine, now_ms), + Some("capture loop silent for 10000ms".into()) + ); } /// A restart joins the old generation for up to 5 s between two looks; From d34cf2ca619fd77d468680bdac7eefb647d532b4 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:23:15 -0500 Subject: [PATCH 077/226] Check a lost output cheaply, reopen a moved display at once, cap a held picture Review r1 of #205. After ACCESS_LOST the handler listed every adapter and output from a new DXGI factory, with no time limit, at every loss: up to four times a second during mode changes and wake-ups. It now asks the duplicated output for its own description first, and lists the outputs only when that output left the desktop or was renamed, while capture runs on the primary for a missing display, or after a re-creation failed. Both calls run under a new stall limit, list_outputs (1 s warning, 4 s limit, as for device creation), and keep the picture alive while they wait. A call that does not return in time counts as no move. Each listing logs how long it took. That check looked the target up through resolve_target, which warns that the chosen display is missing: one warning per loss while it was. It has its own lookup now, which logs nothing. A moved display was reopened through the blocked pace, 500 ms later. The loop now ends with Reopen, and the stage opens the display again at once; a second Reopen within a second waits at the blocked pace, so a layout that keeps changing cannot reopen it in a loop. Resending the cached picture after a re-created duplication hid a virtual display that sends no frames any more: the picture would stay frozen for good. A second screen now resends it for at most 15 s from the first re-creation without a new image. Then the stage reopens it without the cached picture, and the watchdog restarts the pipeline 5 s later if it is still dark, as before the cached picture was kept. The mirror keeps resending. The no-display wait test allows 4 s instead of 2.5 s on a loaded test machine; the wait it checks is 5 s. --- host/src/capture/dxgi.rs | 193 +++++++++++++++++++++++++++-------- host/src/capture/mod.rs | 150 +++++++++++++++++++++++++-- host/src/capture/recovery.rs | 80 +++++++++++++++ host/src/stall.rs | 12 ++- 4 files changed, 386 insertions(+), 49 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index e824b62a..0f3affe9 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -13,13 +13,13 @@ use windows::core::Interface; use super::recovery::{ note_capture_resumed, note_gpu_outcome, CaptureBlock, CaptureUnavailable, LossRecovery, - LossStep, OpenStep, LOSS_GIVE_UP, + LossStep, OpenStep, Reopen, LOSS_GIVE_UP, }; use super::{ client_connected, frame_budget_for, frame_buffer, frame_has_image, heartbeat, idle_heartbeat, keepalive_allowed, keepalive_due, read_back_image, reconcile_virtual_display, recovery_poll, resolve_target, FrameSlot, FrameUse, IdleGate, OutputInfo, Picture, RawFrame, - ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, + ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, PICTURE_HOLD, }; use crate::control::{CaptureTarget, SharedControl}; use crate::gpu_preference::Outcome; @@ -323,6 +323,19 @@ pub fn run_capture_loop( break; } heartbeat(&shared.hb_capture_loop_ms); + if picture.held_too_long(Instant::now(), second_screen) { + info!( + device = %chosen.device_name, + held_s = PICTURE_HOLD.as_secs(), + "The second screen showed no new image since its duplication came back — opening it again" + ); + return Err(Box::new(Reopen { + detail: format!( + "no new image from {} since its duplication came back", + chosen.device_name + ), + })); + } let wait = idle.wait_before_acquire(client_connected(&shared), Instant::now()); if !wait.is_zero() { @@ -612,18 +625,25 @@ pub fn run_capture_loop( if !recovery.in_episode() { info!("Desktop duplication access lost — reinitializing"); } - if super::target_moved(&enumerate_outputs(), &target, &chosen) { - info!( - device = %chosen.device_name, - "The display layout changed — opening the display to capture again" - ); - return Err(Box::new(CaptureUnavailable { - block: CaptureBlock::DisplayChanging, - detail: format!( - "the display layout changed while capturing {}", - chosen.device_name - ), - })); + let allowed = keepalive_allowed(picture.real(), second_screen); + let dims = (tex_width, tex_height); + let call = super::second_screen::driver_call(&shared, generation); + let moved = target_moved(&output1, &chosen, &target, false, || { + if crate::supervisor::generation_is_current(generation) { + keep_picture_alive( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + dims, + allowed, + ); + } + }); + drop(call); + if moved { + return Err(moved_away(&chosen)); } // A second screen that shows no picture before this is // re-created failed for a passing reason. @@ -663,19 +683,15 @@ pub fn run_capture_loop( return Ok(()); } warn_if_overdue(&mut recovery); - let now_ms = crate::clock::host_now_us() / 1000; - if recovery_poll(&shared, now_ms, last_send.elapsed()) - && keepalive_allowed(picture.real(), second_screen) - { - resend_last_frame( - slot, - &shared, - frame_number, - &mut last_send, - &last_frame, - (tex_width, tex_height), - ); - } + keep_picture_alive( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + (tex_width, tex_height), + keepalive_allowed(picture.real(), second_screen), + ); let now = Instant::now(); if now >= deadline { break; @@ -688,19 +704,15 @@ pub fn run_capture_loop( return; } warn_if_overdue(&mut recovery); - let now_ms = crate::clock::host_now_us() / 1000; - if recovery_poll(&shared, now_ms, last_send.elapsed()) - && keepalive_allowed(picture.real(), second_screen) - { - resend_last_frame( - slot, - &shared, - frame_number, - &mut last_send, - &last_frame, - (tex_width, tex_height), - ); - } + keep_picture_alive( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + (tex_width, tex_height), + keepalive_allowed(picture.real(), second_screen), + ); }; let call = super::second_screen::driver_call(&shared, generation); let retried = duplicate_output(&output1, &device, poll); @@ -721,7 +733,7 @@ pub fn run_capture_loop( match retried { Ok(d) => { duplication = Some(d); - picture.recreated(); + picture.recreated(Instant::now()); super::second_screen::note_recovery( &shared, generation, @@ -740,6 +752,27 @@ pub fn run_capture_loop( if let Some(block) = CaptureBlock::from_hresult(last_code.0) { *shared.capture_block.lock() = Some(block); } + // The output may be gone: look at the whole layout. + let allowed = keepalive_allowed(picture.real(), second_screen); + let dims = (tex_width, tex_height); + let call = super::second_screen::driver_call(&shared, generation); + let moved = target_moved(&output1, &chosen, &target, true, || { + if crate::supervisor::generation_is_current(generation) { + keep_picture_alive( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + dims, + allowed, + ); + } + }); + drop(call); + if moved { + return Err(moved_away(&chosen)); + } } } } @@ -930,6 +963,84 @@ fn access_lost_or( } } +/// After ACCESS_LOST: whether the stage should open the target afresh, +/// because it now resolves to another output or adapter than `chosen` (see +/// [`super::target_moved`]). The duplicated output's own description is +/// asked first, and the outputs are listed (a new factory and every adapter) +/// only when it is off the desktop or renamed, while capture runs on the +/// primary for a missing display, or when `list` asks after a failed +/// re-creation. Both run under the stall limits while `progress` keeps the +/// picture alive; one that does not return in time counts as no move, and +/// the re-creation goes on. +fn target_moved( + output: &IDXGIOutput1, + chosen: &OutputInfo, + target: &CaptureTarget, + list: bool, + mut progress: impl FnMut(), +) -> bool { + let fallback = matches!(target, CaptureTarget::Output(name) if *name != chosen.device_name); + if !list && !fallback { + let (output, name) = (output.clone(), chosen.device_name.clone()); + let same = crate::stall::run( + Call::ListOutputs, + move || { + unsafe { output.GetDesc() }.is_ok_and(|desc| { + desc.AttachedToDesktop.as_bool() && utf16_to_string(&desc.DeviceName) == name + }) + }, + &mut progress, + ); + if same.unwrap_or(true) { + return false; + } + } + let (target, chosen) = (target.clone(), chosen.clone()); + let started = Instant::now(); + let moved = crate::stall::run( + Call::ListOutputs, + move || super::target_moved(&enumerate_outputs(), &target, &chosen), + progress, + ) + .unwrap_or(false); + info!( + took_ms = started.elapsed().as_millis() as u64, + moved, "Listed the display outputs after a lost duplication" + ); + moved +} + +/// [`Reopen`] for a target that moved off `chosen`. +fn moved_away(chosen: &OutputInfo) -> Box { + info!( + device = %chosen.device_name, + "The display layout changed — opening the display to capture again" + ); + Box::new(Reopen { + detail: format!( + "the display layout changed while capturing {}", + chosen.device_name + ), + }) +} + +/// While the duplication is away: resends the last picture when the viewer +/// would otherwise go without one (see [`recovery_poll`]), if `allowed`. +fn keep_picture_alive( + slot: &FrameSlot, + shared: &SharedControl, + frame_number: &mut u64, + last_send: &mut Instant, + last_frame: &Arc>, + dims: (u32, u32), + allowed: bool, +) { + let now_ms = crate::clock::host_now_us() / 1000; + if recovery_poll(shared, now_ms, last_send.elapsed()) && allowed { + resend_last_frame(slot, shared, frame_number, last_send, last_frame, dims); + } +} + /// Logs, once per loss, that the duplication is not back within a second. fn warn_if_overdue(recovery: &mut LossRecovery) { if recovery.overdue(Instant::now()) { diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 99937aca..0c0f2994 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -279,14 +279,25 @@ pub(crate) fn resolve_target( /// also surface as a lost duplication. Re-creating it on the same output /// kept capturing the primary display after the chosen one came back, or an /// output of an adapter that had gone; the stage opens the target afresh. +/// +/// It looks the target up as [`resolve_target`] does, without its warning: +/// it runs at every loss while the chosen display is missing. #[cfg(any(windows, test))] pub(crate) fn target_moved( outputs: &[OutputInfo], target: &crate::control::CaptureTarget, current: &OutputInfo, ) -> bool { - resolve_target(outputs, target).is_none_or(|index| { - let output = &outputs[index]; + use crate::control::CaptureTarget; + let primary = || outputs.iter().find(|o| o.is_primary).or(outputs.first()); + let wanted = match target { + CaptureTarget::Output(name) => outputs + .iter() + .find(|o| &o.device_name == name) + .or_else(primary), + CaptureTarget::PrimaryAuto | CaptureTarget::VirtualExtended => primary(), + }; + wanted.is_none_or(|output| { output.device_name != current.device_name || output.adapter_id != current.adapter_id }) } @@ -818,27 +829,53 @@ pub(crate) struct Picture { pub seen_image: bool, /// The cached picture is a real image, not the black start frame. real: bool, + /// When a duplication was re-created with a real picture cached and no + /// new image since. + held_since: Option, } +/// How long a second screen resends its cached picture after its +/// duplication was re-created without a new image. A virtual display that +/// stopped sending frames looks the same as one that shows nothing new, and +/// the resends hid it from the watchdog; after this the stage opens it +/// again without the cached picture, and the watchdog judges what follows. +#[cfg(any(windows, test))] +pub(crate) const PICTURE_HOLD: Duration = Duration::from_secs(15); + #[cfg(any(windows, test))] impl Picture { /// An image was read back into the cache. pub fn cached(&mut self) { self.real = true; + self.held_since = None; } /// The lost duplication was re-created. The cached picture is still the /// screen's: a static second screen shows nothing new after a lock screen /// or a UAC prompt, and while keepalives waited for a new image the /// watchdog restarted the pipeline under the viewer ("no captured frame - /// for 5039ms with a client connected"). - pub fn recreated(&mut self) { + /// for 5039ms with a client connected"). It is resent for at most + /// [`PICTURE_HOLD`] from the first re-creation without a new image. + pub fn recreated(&mut self, now: Instant) { self.seen_image = false; + if self.real { + self.held_since.get_or_insert(now); + } } /// The resolution changed: the cache is a black frame of the new size. pub fn resized(&mut self) { self.real = false; + self.held_since = None; + } + + /// A second screen has resent its cached picture for [`PICTURE_HOLD`] + /// without a new image: the stage opens it again. + pub fn held_too_long(&self, now: Instant, second_screen: bool) -> bool { + second_screen + && self + .held_since + .is_some_and(|since| now.duration_since(since) >= PICTURE_HOLD) } pub fn real(&self) -> bool { @@ -1545,6 +1582,40 @@ mod tests { assert!(target_moved(&[], &CaptureTarget::PrimaryAuto, &both[0])); } + /// Review r1 of #205: while the chosen display is missing, capture runs + /// on the primary and checks at every loss whether it is back. The + /// check logged resolve_target's warning each time. + #[test] + fn checking_for_a_missing_display_at_each_loss_logs_nothing() { + use std::sync::{Arc, Mutex}; + struct Sink(Arc>>); + impl std::io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let out_log = Arc::new(Mutex::new(Vec::::new())); + let writer = Arc::clone(&out_log); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || Sink(Arc::clone(&writer))) + .finish(); + let primary = out(r"\\.\DISPLAY1", 0, 0); + let chosen = CaptureTarget::Output(r"\\.\DISPLAY2".to_string()); + let unplugged = vec![primary.clone()]; + tracing::subscriber::with_default(subscriber, || { + for _ in 0..20 { + assert!(!target_moved(&unplugged, &chosen, &primary)); + } + }); + let log = String::from_utf8(out_log.lock().unwrap().clone()).unwrap(); + assert!(log.is_empty(), "{log}"); + } + /// Field, 0.3.4 and the 0.3.5 beta: a static second screen lost its /// duplication, had it back in 3 ms, and 5 s later the watchdog restarted /// the pipeline under the viewer ("no captured frame for 5039ms with a @@ -1562,7 +1633,8 @@ mod tests { picture.seen_image = true; picture.cached(); // Lost, and re-created at the same size. - picture.recreated(); + let lost = Instant::now(); + picture.recreated(lost); assert!(!frame_has_image(0, picture.seen_image)); let start = 100_000; let mut sent = start; @@ -1585,11 +1657,16 @@ mod tests { crate::supervisor::wedge_reason_at(&shared, &mut machine, now), None ); + let elapsed = Duration::from_millis(now - start); + assert!(!picture.held_too_long(lost + elapsed, true)); } assert!(resends >= 12, "{resends} keepalives in 10 s"); + // A new image ends the hold. + picture.cached(); + assert!(!picture.held_too_long(lost + PICTURE_HOLD, true)); // A new resolution leaves only black to resend: the second screen // waits for the new duplication's first image again. - picture.recreated(); + picture.recreated(lost); picture.resized(); assert!(!keepalive_allowed(picture.real(), true)); assert!(keepalive_allowed(picture.real(), false), "the mirror"); @@ -1598,6 +1675,64 @@ mod tests { assert!(keepalive_allowed(picture.real(), true)); } + /// Review r1 of #205: resending the cached picture hid a virtual display + /// that sends no frames after its duplication was re-created, and the + /// picture would stay frozen for good. After [`PICTURE_HOLD`] the stage + /// opens it again without the cached picture, and the watchdog restarts + /// the pipeline 5 s later if it is still dark, as before. The mirror + /// keeps resending: it is never dark. + #[test] + fn a_second_screen_dark_after_a_lost_duplication_is_reopened_then_judged() { + use std::sync::atomic::Ordering::Relaxed; + let shared = SharedControl::new(0, 1); + connect_viewer(&shared); + let mut machine = crate::supervisor::Machine::new(1); + let mut picture = Picture::default(); + picture.seen_image = true; + picture.cached(); + let lost = Instant::now(); + picture.recreated(lost); + // Lost again before any new image: the hold runs from the first. + picture.recreated(lost + Duration::from_secs(5)); + let start = 100_000; + let mut sent = start; + shared.hb_capture_frame_ms.store(start, Relaxed); + shared.hb_encode_frame_ms.store(start, Relaxed); + let mut reopened_at = None; + let mut wedge = None; + for now in (start..start + 30_000).step_by(50) { + shared.hb_capture_loop_ms.store(now, Relaxed); + let at = lost + Duration::from_millis(now - start); + assert!(!picture.held_too_long(at, false), "the mirror"); + if reopened_at.is_none() && picture.held_too_long(at, true) { + // The stage opens it again: a fresh loop, nothing cached. + reopened_at = Some(now); + picture = Picture::default(); + } + let since_send = Duration::from_millis(now - sent); + if keepalive_due(since_send, false, picture.real()) + && keepalive_allowed(picture.real(), true) + { + sent = now; + shared.hb_capture_frame_ms.store(now, Relaxed); + shared.hb_encode_frame_ms.store(now, Relaxed); + } + if let Some(reason) = crate::supervisor::wedge_reason_at(&shared, &mut machine, now) { + wedge = Some((now, reason)); + break; + } + } + let reopened_at = reopened_at.expect("reopened"); + assert_eq!(reopened_at - start, PICTURE_HOLD.as_millis() as u64); + let (at, reason) = wedge.expect("the watchdog judged the dark screen"); + assert!(reason.starts_with("no captured frame"), "{reason}"); + assert!( + at - reopened_at <= 5_500, + "{} ms after reopening", + at - reopened_at + ); + } + /// The screens came back on 2 s into a 5 s retry wait. The picture comes /// back at the next look, not when the wait ends. #[test] @@ -1620,7 +1755,8 @@ mod tests { back )); let waited = started.elapsed(); - assert!(waited < Duration::from_millis(2_500), "{waited:?}"); + // Well before the 5 s wait ends, with room for a loaded test machine. + assert!(waited < Duration::from_millis(4_000), "{waited:?}"); assert!(looks.get() >= 2, "{} looks", looks.get()); // Other blocked states clear by the user's hand; nothing is polled. *shared.capture_block.lock() = Some(recovery::CaptureBlock::SecureDesktop); diff --git a/host/src/capture/recovery.rs b/host/src/capture/recovery.rs index 4f5d173c..f7b7ea0a 100644 --- a/host/src/capture/recovery.rs +++ b/host/src/capture/recovery.rs @@ -112,6 +112,9 @@ pub fn passing_error(error: &(dyn std::error::Error + Send + Sync + 'static)) -> if let Some(unavailable) = error.downcast_ref::() { return unavailable.block != CaptureBlock::UnsupportedOnThisGpu; } + if error.downcast_ref::().is_some() { + return true; + } if error.downcast_ref::().is_some() { return true; } @@ -190,6 +193,27 @@ impl std::fmt::Display for CaptureUnavailable { impl std::error::Error for CaptureUnavailable {} +/// The capture loop ended to open its display afresh: after a lost +/// duplication the target is now another output or adapter, or a re-created +/// duplication showed no new image for too long. The stage opens it again at +/// once, not at the blocked pace. +#[derive(Debug)] +pub struct Reopen { + pub detail: String, +} + +impl std::fmt::Display for Reopen { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "capture reopens: {}", self.detail) + } +} + +impl std::error::Error for Reopen {} + +/// A [`Reopen`] within this long of the last one waits at the blocked pace: +/// a layout that keeps changing must not reopen the display in a loop. +const REOPEN_SPACING: Duration = Duration::from_secs(1); + /// Delays between duplication re-creations after ACCESS_LOST. The first retry /// is immediate, as before; a duplication that is lost again right after it /// was re-created waits longer each time instead of spinning. @@ -345,11 +369,27 @@ pub fn run_until_unblocked( let mut retry = BlockedRetry::default(); let mut stalled = 0; let mut frame_number = 0; + let mut last_reopen: Option = None; loop { let error = match capture(&mut frame_number) { Ok(()) => return Ok(()), Err(error) => error, }; + let error: Box = match error.downcast::() { + Ok(reopen) => { + let now = Instant::now(); + if last_reopen.is_none_or(|at| now.duration_since(at) >= REOPEN_SPACING) { + last_reopen = Some(now); + info!(detail = %reopen.detail, "Opening the display to capture again"); + continue; + } + Box::new(CaptureUnavailable { + block: CaptureBlock::DisplayChanging, + detail: reopen.detail, + }) + } + Err(error) => error, + }; // A driver call that did not return in time, or was refused while an // earlier one is stuck. Restarting the pipeline cannot free the // driver, and refusals would spend the restart budget until the @@ -1242,6 +1282,46 @@ mod tests { ); } + /// Review r1 of #205: a display that moved was reopened after the + /// blocked pace's 500 ms. It is reopened at once; a second move right + /// after waits at that pace instead of reopening in a loop. + #[test] + fn a_moved_display_is_reopened_at_once_but_not_in_a_loop() { + let shared = SharedControl::new(0, 1); + let mut attempts = 0; + let mut waits = Vec::new(); + let result = run_until_unblocked( + &shared, + |_| { + attempts += 1; + if attempts <= 2 { + Err(Reopen { + detail: "the display layout changed".into(), + } + .into()) + } else { + Ok(()) + } + }, + |delay| { + waits.push(delay.as_millis()); + true + }, + ); + assert!(result.is_ok()); + assert_eq!(attempts, 3); + assert_eq!(waits, [500]); + assert_eq!( + *shared.capture_block.lock(), + Some(CaptureBlock::DisplayChanging) + ); + // Not a failure of the second screen. + let reopen: Box = Box::new(Reopen { + detail: String::new(), + }); + assert!(passing_error(reopen.as_ref())); + } + #[test] fn retry_pace_resets_when_the_reason_changes() { let mut retry = BlockedRetry::default(); diff --git a/host/src/stall.rs b/host/src/stall.rs index 9d8b5a20..9a9a61cd 100644 --- a/host/src/stall.rs +++ b/host/src/stall.rs @@ -40,9 +40,12 @@ pub enum Call { /// Reading the graphics cards and the screens' wiring for the second /// screen's decisions (`capture::second_screen::facts`). GpuFacts, + /// After a lost duplication: asking the duplicated output whether it is + /// still on the desktop, and listing the outputs when it is not. + ListOutputs, } -const CALLS: usize = 6; +const CALLS: usize = 7; /// When a call is slow enough to log, and when it is given up on. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -70,6 +73,7 @@ impl Call { Self::VddEnable => "vdd_enable", Self::VddDisable => "vdd_disable", Self::GpuFacts => "gpu_facts", + Self::ListOutputs => "list_outputs", } } @@ -99,6 +103,10 @@ impl Call { // adapters, as device creation begins, then asks DXCore and reads // the display configuration; it keeps device creation's limits. Self::GpuFacts => Limits::ms(1_000, 4_000), + // Not timed in the field. Listing creates a DXGI factory and asks + // every adapter for its outputs, which is how device creation + // begins; it keeps device creation's limits. + Self::ListOutputs => Limits::ms(1_000, 4_000), } } } @@ -781,6 +789,7 @@ mod tests { Call::VddEnable, Call::VddDisable, Call::GpuFacts, + Call::ListOutputs, ] .map(Call::name) .into(); @@ -798,6 +807,7 @@ mod tests { Call::EncoderOpen, Call::VddEnable, Call::GpuFacts, + Call::ListOutputs, ] { let limits = call.limits(); assert!(limits.limit >= Duration::from_secs(3), "{call:?}"); From 163ade5c901c35252189150cd47cac4090a81831 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:35:12 -0500 Subject: [PATCH 078/226] End the swscale frame when a conversion fails sws_frame_start in FFmpeg 7.1 takes a reference to the source before it references the output, and sws_scale_frame returns at once if that second step fails, leaving the source held until the next conversion. For a borrowed capture buffer that reference must not outlive the call, so a failed sws_scale_frame now also calls sws_frame_end. --- host/src/encoder/input.rs | 58 +++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/host/src/encoder/input.rs b/host/src/encoder/input.rs index 105e86d6..33a1be4a 100644 --- a/host/src/encoder/input.rs +++ b/host/src/encoder/input.rs @@ -196,15 +196,33 @@ impl Scaler { } self.check_output(output)?; // SAFETY: both frames own their buffers and have the sizes checked. - let result = unsafe { - ffmpeg_next::ffi::sws_scale_frame(self.context, output.as_mut_ptr(), input.as_ptr()) - }; + let result = unsafe { self.scale_frame(output.as_mut_ptr(), input.as_ptr()) }; if result < 0 { return Err(ffmpeg_next::Error::from(result)); } Ok(()) } + /// `sws_scale_frame`, after which swscale holds no reference to either + /// frame. It drops them itself once it gets as far as converting. In + /// FFmpeg 7.1 `sws_frame_start` can fail after taking the source, which + /// would leave it held until the next conversion, so an error ends the + /// frame here too. + /// + /// # Safety + /// Both frames must be valid, and `output` reference-counted. + unsafe fn scale_frame( + &mut self, + output: *mut ffmpeg_next::ffi::AVFrame, + source: *const ffmpeg_next::ffi::AVFrame, + ) -> libc::c_int { + let result = unsafe { ffmpeg_next::ffi::sws_scale_frame(self.context, output, source) }; + if result < 0 { + unsafe { ffmpeg_next::ffi::sws_frame_end(self.context) }; + } + result + } + fn check_output(&self, output: &ffmpeg_next::frame::Video) -> Result<(), ffmpeg_next::Error> { if output.format() != self.output || output.width() != self.width @@ -240,8 +258,7 @@ pub(super) fn convert_packed_bgra( output: &mut ffmpeg_next::frame::Video, ) -> Result<(), ffmpeg_next::Error> { use ffmpeg_next::ffi::{ - av_buffer_create, av_frame_alloc, av_frame_free, sws_scale_frame, AVPixelFormat, - AV_BUFFER_FLAG_READONLY, + av_buffer_create, av_frame_alloc, av_frame_free, AVPixelFormat, AV_BUFFER_FLAG_READONLY, }; let (width, height) = (scaler.width, scaler.height); let picture_bytes = width as usize * 4; @@ -256,9 +273,9 @@ pub(super) fn convert_packed_bgra( return scaler.run(&copied, output); } // sws_scale_frame takes reference-counted frames, so `rows` is wrapped - // in a buffer that never frees it. sws_scale_frame drops its references - // before it returns, and the frame is freed below, so nothing refers to - // `rows` after this call. + // in a buffer that never frees it. `scale_frame` leaves swscale holding + // no reference, whether it converted or failed, and the frame is freed + // below, so nothing refers to `rows` after this call. unsafe extern "C" fn borrowed(_: *mut libc::c_void, _: *mut u8) {} let mut source = unsafe { av_frame_alloc() }; if source.is_null() { @@ -286,7 +303,7 @@ pub(super) fn convert_packed_bgra( let result = if (*source).buf[0].is_null() { -libc::ENOMEM } else { - sws_scale_frame(scaler.context, output.as_mut_ptr(), source) + scaler.scale_frame(output.as_mut_ptr(), source) }; av_frame_free(&mut source); result @@ -638,6 +655,29 @@ mod tests { assert_eq!(set, scaler_threads() as i64); } + /// A conversion swscale refuses still leaves it holding no reference to + /// the source, so a borrowed capture buffer is free when the call ends. + /// This error comes after swscale holds both frames. The one that needed + /// `scale_frame`'s own `sws_frame_end`, in FFmpeg 7.1's `sws_frame_start` + /// after it took the source, only happens when memory runs out. + #[test] + fn a_failed_conversion_keeps_no_reference_to_the_source() { + ffmpeg_next::init().unwrap(); + let (width, height) = (96, 64); + let source = ffmpeg_next::frame::Video::new(Pixel::BGRA, width, height); + for threads in [1, 4] { + let mut scaler = Scaler::new(Pixel::YUV420P, width, height, threads).unwrap(); + // An output larger than the scaler's makes swscale fail once it + // holds both frames. + let mut output = ffmpeg_next::frame::Video::new(Pixel::YUV420P, width, height * 2); + let result = unsafe { scaler.scale_frame(output.as_mut_ptr(), source.as_ptr()) }; + assert!(result < 0, "threads={threads}"); + let held = + unsafe { ffmpeg_next::ffi::av_buffer_get_ref_count((*source.as_ptr()).buf[0]) }; + assert_eq!(held, 1, "threads={threads}"); + } + } + #[test] fn capture_padding_is_ffmpegs() { assert_eq!( From b3de1f7a1154e37a07656ac929db59c97cfa8af2 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:35:35 -0500 Subject: [PATCH 079/226] Skip the startup Disable only when no driver task is queued or running Review r1, P1: an Enable task the previous host gave up on can still be queued or running in Task Scheduler while the device reads disabled. Startup recorded the display as off and ran nothing; the Enable then added the monitor, and every later Disable, the quit's included, returned "already off", leaving a phantom monitor. The startup cleanup now decides on its own turn, after handing the turn back to startup. It reads Task Scheduler first: if either driver task has an instance queued or running, or Task Scheduler can't say, the state stays unknown and the Disable runs as before, behind that instance. Only with nothing pending does it read the device, and only problem 22 counts as off. The task's run step is now shared with every other request. The COM query runs on the startup thread, not the window's. Test: an Enable left running by the previous host ends after startup, and the quit's Disable still turns the display off. A native, ignored test checks the Task Scheduler query on Windows. --- host/Cargo.toml | 1 + host/src/vdd.rs | 259 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 223 insertions(+), 37 deletions(-) diff --git a/host/Cargo.toml b/host/Cargo.toml index d8ece94d..86f719b6 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -105,6 +105,7 @@ features = [ "Win32_System_Registry", "Win32_System_StationsAndDesktops", "Win32_System_SystemInformation", + "Win32_System_TaskScheduler", "Win32_System_IO", "Win32_NetworkManagement_IpHelper", "Win32_NetworkManagement_Ndis", diff --git a/host/src/vdd.rs b/host/src/vdd.rs index 2ed43730..f4b63895 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -214,15 +214,29 @@ impl DriverTasks { self.gate.lock().health } - /// Before any request, with the device as Configuration Manager reports - /// it: one Windows reports disabled (problem 22) is off already, so no - /// Disable needs to run. True when so. - fn found_off(&self, device: crate::vdd_device::DeviceState) -> bool { - let off = device == crate::vdd_device::DeviceState::Problem(22); - if off { - self.gate.lock().known_enabled = Some(false); - } - off + /// The startup cleanup: a Disable on its own turn, unless `already_off` + /// says, on that turn, that the display is off with no task left to + /// change it. Then it counts as a Disable that completed, so later + /// requests know the display is off. + fn disable_at_startup( + &self, + on_turn: impl FnOnce(), + already_off: impl FnOnce() -> bool, + run: impl FnOnce(bool, &mut dyn FnMut()) -> TaskOutcome, + ) -> bool { + self.set_enabled_on_turn( + false, + Purpose::Cleanup, + || {}, + on_turn, + |enabled, progress| { + if already_off() { + TaskOutcome::Done + } else { + run(enabled, progress) + } + }, + ) } #[cfg(test)] @@ -472,20 +486,7 @@ fn request_on_turn( #[cfg(windows)] { TASKS.set_enabled_on_turn(enabled, purpose, progress, on_turn, |enabled, progress| { - let mut outcome = TaskOutcome::Failed(None); - // Read here, after the gate decided and before the task runs. - let display_off = TASKS.gate.lock().known_enabled == Some(false); - crate::startup::display::change(enabled, progress, |progress| { - outcome = crate::display_guard::around( - enabled, - crate::startup::ProcessId::current(), - log_screens, - display_off, - || run_task(if enabled { TASK_ENABLE } else { TASK_DISABLE }, progress), - |outcome| *outcome == TaskOutcome::Done, - ); - outcome == TaskOutcome::Done - }); + let outcome = run_on_turn(enabled, progress); *ran = Some(outcome); outcome }) @@ -498,6 +499,26 @@ fn request_on_turn( } } +/// Run the driver task for `enabled`, once the gate let a request run. +#[cfg(windows)] +fn run_on_turn(enabled: bool, progress: &mut dyn FnMut()) -> TaskOutcome { + let mut outcome = TaskOutcome::Failed(None); + // Read here, after the gate decided and before the task runs. + let display_off = TASKS.gate.lock().known_enabled == Some(false); + crate::startup::display::change(enabled, progress, |progress| { + outcome = crate::display_guard::around( + enabled, + crate::startup::ProcessId::current(), + log_screens, + display_off, + || run_task(if enabled { TASK_ENABLE } else { TASK_DISABLE }, progress), + |outcome| *outcome == TaskOutcome::Done, + ); + outcome == TaskOutcome::Done + }); + outcome +} + /// Disable the virtual display device (best effort — failures are non-fatal). /// For startup, quit and panics: a hung or refused earlier task does not stop /// it, and it waits for an Enable still running, within the task's bounds. @@ -512,14 +533,11 @@ pub fn disable() -> bool { /// A hung task no longer keeps a Mirror stream black for up to 30 s. /// /// A display Windows already reports disabled, as at almost every start, -/// runs no task. That Disable was the task that hung most often in the -/// field, and every later request waited behind it and failed. +/// runs no task, unless an Enable or Disable task is still queued or running +/// from before: an Enable the previous host gave up on can still turn the +/// display on. That Disable was the task that hung most often in the field, +/// and every later request waited behind it and failed. pub fn disable_in_background() { - #[cfg(windows)] - if TASKS.found_off(crate::vdd_device::query()) { - info!("Virtual display already off — not running the Disable task"); - return; - } let (held, turn) = std::sync::mpsc::channel(); let spawned = std::thread::Builder::new() .name("vdd-startup-off".into()) @@ -527,7 +545,20 @@ pub fn disable_in_background() { let on_turn = move || { let _ = held.send(()); }; - request_on_turn(false, Purpose::Cleanup, || {}, on_turn, &mut None) + #[cfg(windows)] + TASKS.disable_at_startup( + on_turn, + || { + let off = off_with_nothing_pending(task_pending(), crate::vdd_device::query); + if off { + info!("Virtual display already off — not running the Disable task"); + } + off + }, + run_on_turn, + ); + #[cfg(not(windows))] + request_on_turn(false, Purpose::Cleanup, || {}, on_turn, &mut None); }); match spawned { Ok(_) => { @@ -539,6 +570,65 @@ pub fn disable_in_background() { } } +/// Whether the display is off and stays off: Task Scheduler reports no +/// driver task queued or running (`pending`, `None` when it could not say), +/// and only then does the device read disabled (problem 22). In that order, +/// so an Enable that ends meanwhile shows in the device. +#[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. +fn off_with_nothing_pending( + pending: Option, + device: impl FnOnce() -> crate::vdd_device::DeviceState, +) -> bool { + pending == Some(false) && device() == crate::vdd_device::DeviceState::Problem(22) +} + +/// Whether either driver task has an instance queued or running, as Task +/// Scheduler reports it; a missing task has none. `None` when it can't say. +#[cfg(windows)] +fn task_pending() -> Option { + tasks_pending(&[TASK_ENABLE, TASK_DISABLE]) +} + +#[cfg(windows)] +fn tasks_pending(names: &[&str]) -> Option { + use windows::core::{BSTR, HRESULT, VARIANT}; + use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_MULTITHREADED, + }; + use windows::Win32::System::TaskScheduler::{ + ITaskService, TaskScheduler, TASK_STATE_QUEUED, TASK_STATE_RUNNING, + }; + struct Apartment; + impl Drop for Apartment { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } + } + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) } + .ok() + .ok()?; + let _apartment = Apartment; + let service: ITaskService = + unsafe { CoCreateInstance(&TaskScheduler, None, CLSCTX_INPROC_SERVER) }.ok()?; + let local = VARIANT::default(); + unsafe { service.Connect(&local, &local, &local, &local) }.ok()?; + let folder = unsafe { service.GetFolder(&BSTR::from("\\")) }.ok()?; + let mut pending = false; + for &name in names { + match unsafe { folder.GetTask(&BSTR::from(name)) } { + Ok(task) => { + let state = unsafe { task.State() }.ok()?; + pending |= state == TASK_STATE_QUEUED || state == TASK_STATE_RUNNING; + } + // 0x80070002: the task does not exist. + Err(error) if error.code() == HRESULT(0x8007_0002_u32 as i32) => {} + Err(_) => return None, + } + } + Some(pending) +} + /// Disable the virtual display after `host` ended abnormally, if it owned the /// display and no other host of this install runs. For the failure reporter. #[cfg(windows)] @@ -597,8 +687,13 @@ mod tests { use crate::vdd_device::DeviceState; use std::cell::Cell; let tasks = DriverTasks::new(); - assert!(tasks.found_off(DeviceState::Problem(22))); - // Neither the startup cleanup nor a quit before any stream runs one. + let off = || off_with_nothing_pending(Some(false), || DeviceState::Problem(22)); + assert!(tasks.disable_at_startup( + || {}, + off, + |_, _| { panic!("ran the Disable task for a display that is off") } + )); + // Nor does a quit before any stream. assert!(tasks.set_enabled( false, Purpose::Cleanup, @@ -628,12 +723,10 @@ mod tests { DeviceState::Missing, ] { let tasks = DriverTasks::new(); - assert!(!tasks.found_off(device), "{device:?}"); let ran = Cell::new(false); - tasks.set_enabled( - false, - Purpose::Cleanup, + tasks.disable_at_startup( || {}, + || off_with_nothing_pending(Some(false), || device), |_, _| { ran.set(true); TaskOutcome::Done @@ -643,6 +736,98 @@ mod tests { } } + /// Review r1: an Enable task the previous host gave up on can still be + /// queued or running while the device reads disabled. Recording "off" + /// then let it add the monitor, and every later Disable, the quit's + /// too, found the display "already off": a phantom monitor stayed. + #[test] + fn an_enable_left_running_by_the_previous_host_is_turned_off_at_startup() { + use crate::vdd_device::DeviceState; + use std::cell::Cell; + let tasks = DriverTasks::new(); + // Task Scheduler still runs the Enable, and Windows has not added the + // display yet. + let enable_running = Cell::new(true); + let display_on = Cell::new(false); + let device = || { + if display_on.get() { + DeviceState::Ready + } else { + DeviceState::Problem(22) + } + }; + let enable_ends = || { + if enable_running.replace(false) { + display_on.set(true); + } + }; + let queried_device = Cell::new(false); + let disabled = tasks.disable_at_startup( + || {}, + || { + off_with_nothing_pending(Some(enable_running.get()), || { + queried_device.set(true); + device() + }) + }, + // The startup Disable runs vdd_task.ps1 with -After Enable, which + // waits for the Enable to end before it disables. + |enabled, _| { + assert!(!enabled); + enable_ends(); + display_on.set(false); + TaskOutcome::Done + }, + ); + assert!(disabled); + // A surviving Enable ends whenever it ends; then the host quits. + enable_ends(); + tasks.set_enabled( + false, + Purpose::Cleanup, + || {}, + |_, _| { + display_on.set(false); + TaskOutcome::Done + }, + ); + assert!(!display_on.get(), "a phantom monitor stayed after quit"); + assert!( + !queried_device.get(), + "decided on the device while a task was pending" + ); + + // When Task Scheduler can't say, the Disable runs too. + let tasks = DriverTasks::new(); + let ran = Cell::new(false); + tasks.disable_at_startup( + || {}, + || off_with_nothing_pending(None, || DeviceState::Problem(22)), + |_, _| { + ran.set(true); + TaskOutcome::Done + }, + ); + assert!(ran.get(), "no Disable when the task state was unknown"); + } + + /// Native only (`--ignored` on Windows): Task Scheduler answers for the + /// driver tasks, a missing task has nothing pending, and a task started + /// beforehand and named in `EM_TEST_PENDING_TASK` reads as pending. + #[cfg(windows)] + #[test] + #[ignore] + fn task_scheduler_reports_pending_driver_tasks() { + assert_eq!( + tasks_pending(&["EternalMonitor test task that does not exist"]), + Some(false) + ); + assert!(task_pending().is_some(), "Task Scheduler did not answer"); + if let Ok(running) = std::env::var("EM_TEST_PENDING_TASK") { + assert_eq!(tasks_pending(&[running.as_str()]), Some(true), "{running}"); + } + } + #[test] fn a_hung_task_is_not_waited_out_again_until_the_cooldown_ends() { let mut gate = TaskGate::new(); From 26330a7de3e774942833d057efaeccb4ce75a937 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:35:35 -0500 Subject: [PATCH 080/226] Leave a UTF-32 settings file untouched Review r1, P2: UTF-32's little-endian byte order mark (FF FE 00 00) starts like UTF-16's. The file decoded as UTF-16 full of NULs, did not parse, and a driver Setup installed with a card to name had the whole file replaced. Both UTF-32 byte order marks are now refused before UTF-16 is tried. Decoded text holding a NUL, which XML never allows (UTF-32 without a byte order mark reads as UTF-8 that way), is refused too. Such files are left byte for byte, whoever installed the driver, with or without a card to name. The UTF-16 decoder now uses as_chunks, which clippy asks for. --- host/src/vdd_settings.rs | 70 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/host/src/vdd_settings.rs b/host/src/vdd_settings.rs index 02487971..1a6f275c 100644 --- a/host/src/vdd_settings.rs +++ b/host/src/vdd_settings.rs @@ -668,8 +668,8 @@ pub fn pin_result( /// back as UTF-8, the encoding the driver's own file has and every file this /// host writes. A UTF-16 document's `encoding="utf-16"` declaration becomes /// `utf-8` with it, so the declaration always names what is on disk. UTF-8 -/// keeps any byte order mark. Any other encoding is left alone, and the -/// error says so. +/// keeps any byte order mark. Any other encoding, UTF-32 included, is left +/// alone, and the error says so. fn read_settings(path: &Path) -> io::Result { decode_settings(&std::fs::read(path)?).ok_or_else(|| { io::Error::new( @@ -680,6 +680,17 @@ fn read_settings(path: &Path) -> io::Result { } fn decode_settings(bytes: &[u8]) -> Option { + let text = decode_text(bytes)?; + // XML never holds NUL: text with one was read in the wrong encoding, + // such as UTF-32 without a byte order mark, which reads as UTF-8. + (!text.contains('\0')).then_some(text) +} + +fn decode_text(bytes: &[u8]) -> Option { + // UTF-32's byte order marks; the little-endian one starts like UTF-16's. + if bytes.starts_with(&[0xFF, 0xFE, 0, 0]) || bytes.starts_with(&[0, 0, 0xFE, 0xFF]) { + return None; + } let (units, big_endian): (&[u8], bool) = match bytes { [0xFF, 0xFE, rest @ ..] => (rest, false), [0xFE, 0xFF, rest @ ..] => (rest, true), @@ -688,11 +699,11 @@ fn decode_settings(bytes: &[u8]) -> Option { [0, b'<', 0, b'?', ..] => (bytes, true), _ => return String::from_utf8(bytes.to_vec()).ok(), }; - if units.len() % 2 != 0 { + let (pairs, odd) = units.as_chunks::<2>(); + if !odd.is_empty() { return None; } - let units = units.chunks_exact(2).map(|pair| { - let pair = [pair[0], pair[1]]; + let units = pairs.iter().map(|&pair| { if big_endian { u16::from_be_bytes(pair) } else { @@ -1692,6 +1703,55 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + /// Review r1: UTF-32's little-endian byte order mark starts like + /// UTF-16's. Read as UTF-16 it held NULs, did not parse, and a driver + /// Setup installed with a card to name had the whole file replaced. + #[test] + fn a_utf32_settings_file_is_left_byte_for_byte() { + let dir = std::env::temp_dir().join(format!("vdd-encoding-32-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let (path, record) = ( + dir.join("vdd_settings.xml"), + dir.join("app").join("vdd-pin.txt"), + ); + const INTEL: &str = "Intel(R) HD Graphics 630"; + let utf32 = |big_endian: bool, bom: bool| -> Vec { + bom.then_some('\u{feff}') + .into_iter() + .chain(VENDOR.chars()) + .flat_map(|c| { + let c = c as u32; + if big_endian { + c.to_be_bytes() + } else { + c.to_le_bytes() + } + }) + .collect() + }; + for (name, bytes) in [ + ("utf-32 le, bom", utf32(false, true)), + ("utf-32 be, bom", utf32(true, true)), + ("utf-32 le, no bom", utf32(false, false)), + ("utf-32 be, no bom", utf32(true, false)), + ] { + for owned in [false, true] { + std::fs::write(&path, &bytes).unwrap(); + let written = write_with_pin(&path, &record, owned, &default_modes(), Some(INTEL)); + assert!(written.is_err(), "{name} owned={owned}"); + assert!( + apply_pin(&path, &record, owned, Some(INTEL)).is_err(), + "{name}" + ); + assert!(pin_gpu_in(&path, INTEL).is_err(), "{name}"); + assert_eq!(std::fs::read(&path).unwrap(), bytes, "{name} owned={owned}"); + assert!(!record.exists(), "{name} owned={owned}"); + } + } + std::fs::remove_dir_all(&dir).unwrap(); + } + /// A file in another encoding, or broken UTF-16, may hold anything: /// it stays byte for byte, and the error says why. #[test] From e79e6dcb017773bdb28485eddfe6fee09859b677 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:35:51 -0500 Subject: [PATCH 081/226] Convert on one thread when swscale can't start its threads A threaded swscale context can fail to initialise where a single-threaded one doesn't, and that failure made the encoder fail to open. The scaler now logs it once and opens on one thread, as before threading. --- host/src/encoder/input.rs | 62 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/host/src/encoder/input.rs b/host/src/encoder/input.rs index 33a1be4a..ffc1cb8c 100644 --- a/host/src/encoder/input.rs +++ b/host/src/encoder/input.rs @@ -1,5 +1,5 @@ use ffmpeg_next::format::Pixel; -use tracing::info; +use tracing::{info, warn}; use crate::capture::FRAME_PADDING; @@ -104,7 +104,26 @@ pub(super) fn conversion_context( if !matches!(format, Pixel::YUV420P | Pixel::NV12) { return Ok(None); } - Scaler::new(format, width, height, scaler_threads()).map(Some) + with_one_thread_fallback(scaler_threads(), |threads| { + Scaler::new(format, width, height, threads) + }) + .map(Some) +} + +/// Opens a scaler with `open` on `threads` threads, or on one when that +/// fails: swscale can fail to start its threads, and the single-threaded +/// conversion, which needs none, must still open wherever it did before. +fn with_one_thread_fallback( + threads: usize, + mut open: impl FnMut(usize) -> Result, +) -> Result { + match open(threads) { + Err(error) if threads > 1 => { + warn!(%error, threads, "swscale could not start its threads; converting on one"); + open(1) + } + opened => opened, + } } /// Threads one conversion is split across: half the logical processors, at @@ -678,6 +697,45 @@ mod tests { } } + /// A scaler whose threads fail to start opens on one thread instead; a + /// failure on one thread is the encoder's, as before threading. + #[test] + fn a_scaler_whose_threads_fail_converts_on_one() { + ffmpeg_next::init().unwrap(); + let refused = || ffmpeg_next::Error::Other { + errno: libc::ENOMEM, + }; + let mut tried = Vec::new(); + let scaler = with_one_thread_fallback(4, |threads| { + tried.push(threads); + if threads > 1 { + return Err(refused()); + } + Scaler::new(Pixel::YUV420P, 96, 64, threads) + }) + .unwrap(); + assert_eq!(tried, [4, 1]); + assert_eq!(scaler.threads(), 1); + + for threads in [4, 1] { + let mut tried = Vec::new(); + let opened = with_one_thread_fallback(threads, |threads| { + tried.push(threads); + Err(refused()) + }); + assert!(opened.is_err()); + assert_eq!(tried, if threads > 1 { vec![4, 1] } else { vec![1] }); + } + + let mut tried = Vec::new(); + let scaler = with_one_thread_fallback(4, |threads| { + tried.push(threads); + Scaler::new(Pixel::YUV420P, 96, 64, threads) + }) + .unwrap(); + assert_eq!((tried, scaler.threads()), (vec![4], 4)); + } + #[test] fn capture_padding_is_ffmpegs() { assert_eq!( From 69ec7ca53c7f896fc87ecb8445ff0d0661eaa0bb Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:37:00 -0500 Subject: [PATCH 082/226] Keep swscale on one thread where Accelerate converts On macOS, YUV420P input is converted by Accelerate, and swscale is only its fallback. Its context now opens on one thread there, so the threaded one keeps no workers waiting. conversion_context takes the thread count. --- host/src/encoder/input.rs | 19 +++++++----- host/src/encoder/mod.rs | 65 +++++++++++++++++++++++++++++---------- 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/host/src/encoder/input.rs b/host/src/encoder/input.rs index ffc1cb8c..43960c40 100644 --- a/host/src/encoder/input.rs +++ b/host/src/encoder/input.rs @@ -96,15 +96,18 @@ pub(super) fn retries_as_yuv(mode: EncoderInput, format: Pixel) -> bool { mode == EncoderInput::Automatic && matches!(format, Pixel::BGRA | Pixel::BGRZ) } +/// swscale converting BGRA to `format` on `threads` threads, or nothing +/// when the encoder takes BGRA itself. pub(super) fn conversion_context( format: Pixel, width: u32, height: u32, + threads: usize, ) -> Result, ffmpeg_next::Error> { if !matches!(format, Pixel::YUV420P | Pixel::NV12) { return Ok(None); } - with_one_thread_fallback(scaler_threads(), |threads| { + with_one_thread_fallback(threads, |threads| { Scaler::new(format, width, height, threads) }) .map(Some) @@ -129,7 +132,7 @@ fn with_one_thread_fallback( /// Threads one conversion is split across: half the logical processors, at /// most four. A 2732x2048 frame takes about 14 ms on one thread of a desktop /// CPU, most of a 60 fps frame's 16.7 ms. -fn scaler_threads() -> usize { +pub(super) fn scaler_threads() -> usize { threads_for(std::thread::available_parallelism().map_or(1, |count| count.get())) } @@ -626,7 +629,9 @@ mod tests { let label = format!("{width}x{height} {format:?} threads={threads}"); assert_same_picture(&before, &after, &label); } - let mut scaler = conversion_context(format, width, height).unwrap().unwrap(); + let mut scaler = conversion_context(format, width, height, scaler_threads()) + .unwrap() + .unwrap(); let mut after = ffmpeg_next::frame::Video::new(format, width, height); // A short buffer or another size is refused, never read past. assert_eq!( @@ -657,7 +662,7 @@ mod tests { for (processors, threads) in [(1, 1), (2, 1), (4, 2), (6, 3), (8, 4), (16, 4), (64, 4)] { assert_eq!(threads_for(processors), threads, "{processors} processors"); } - let scaler = conversion_context(Pixel::YUV420P, 2732, 2048) + let scaler = conversion_context(Pixel::YUV420P, 2732, 2048, scaler_threads()) .unwrap() .unwrap(); assert_eq!(scaler.threads(), scaler_threads()); @@ -880,7 +885,7 @@ mod tests { let mut planar = ffmpeg_next::frame::Video::new(Pixel::YUV420P, width, height); let mut nv12 = ffmpeg_next::frame::Video::new(Pixel::NV12, width, height); for output in [&mut planar, &mut nv12] { - conversion_context(output.format(), width, height) + conversion_context(output.format(), width, height, 1) .unwrap() .unwrap() .run(&bgra, output) @@ -910,10 +915,10 @@ mod tests { ); } } - assert!(conversion_context(Pixel::BGRA, width, height) + assert!(conversion_context(Pixel::BGRA, width, height, 1) .unwrap() .is_none()); - assert!(conversion_context(Pixel::BGRZ, width, height) + assert!(conversion_context(Pixel::BGRZ, width, height, 1) .unwrap() .is_none()); } diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index 37c84b25..f49646d0 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -814,7 +814,32 @@ impl EncoderState { let mut h264 = H264BitstreamState::default(); h264.refresh_parameter_sets_from_extradata(encoder_extradata(&encoder)); - let scaler = input::conversion_context(pixel_format, width, height)?; + #[cfg(target_os = "macos")] + let native_converter = if pixel_format == ffmpeg_next::format::Pixel::YUV420P { + match vimage::Converter::new() { + Ok(converter) => { + info!("Accelerate BGRA->YUV420P converter ready"); + Some(converter) + } + Err(error) => { + warn!(%error, "Using swscale color conversion"); + None + } + } + } else { + None + }; + // Where Accelerate converts, swscale is only its fallback, so it + // keeps no threads waiting. + #[cfg(target_os = "macos")] + let threads = if native_converter.is_some() { + 1 + } else { + input::scaler_threads() + }; + #[cfg(not(target_os = "macos"))] + let threads = input::scaler_threads(); + let scaler = input::conversion_context(pixel_format, width, height, threads)?; if let Some(scaler) = &scaler { info!( output = ?pixel_format, @@ -825,20 +850,7 @@ impl EncoderState { Ok(Self { #[cfg(target_os = "macos")] - native_converter: if pixel_format == ffmpeg_next::format::Pixel::YUV420P { - match vimage::Converter::new() { - Ok(converter) => { - info!("Accelerate BGRA->YUV420P converter ready"); - Some(converter) - } - Err(error) => { - warn!(%error, "Using swscale color conversion"); - None - } - } - } else { - None - }, + native_converter, encoder, scaler, frame: ffmpeg_next::frame::Video::new(pixel_format, width, height), @@ -2355,6 +2367,21 @@ mod tests { .unwrap_or_else(|error| panic!("{width}x{height}: {error}")) } + /// swscale gets threads only where it converts every frame: with + /// Accelerate converting on macOS it is a fallback and runs on one. + #[test] + fn swscale_gets_threads_only_where_it_converts_every_frame() { + let state = libx264_at(96, 64); + let scaler = state.scaler.as_ref().expect("x264 takes YUV420P"); + #[cfg(target_os = "macos")] + assert_eq!( + (state.native_converter.is_some(), scaler.threads()), + (true, 1) + ); + #[cfg(not(target_os = "macos"))] + assert_eq!(scaler.threads(), input::scaler_threads()); + } + /// A synthetic capture, in a buffer as capture hands it over. fn synthetic_capture(width: u32, height: u32, frame_number: u64) -> Vec { let frame_bytes = width as usize * 4 * height as usize; @@ -2496,7 +2523,13 @@ mod tests { _ => Pixel::YUV420P, }; state.frame = ffmpeg_next::frame::Video::new(format, even_width, even_height); - state.scaler = input::conversion_context(format, even_width, even_height).unwrap(); + state.scaler = input::conversion_context( + format, + even_width, + even_height, + input::scaler_threads(), + ) + .unwrap(); assert_eq!(state.scaler.is_some(), format != Pixel::BGRA); state.convert(&alone, even_bytes).unwrap(); let expected = picture(&state.frame); From 513a4268b780678593e05f392aa08a8b0abf6b3d Mon Sep 17 00:00:00 2001 From: Ali Younes <159104116+whoisaldo@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:40:31 -0500 Subject: [PATCH 083/226] Answer the iPad while the PC isn't streaming, and say why (#196) While the PC is restarting the stream, stopped by the user, or stopped after repeated failures, it now answers the iPad's HELLO with Restarting, Stopped or Failed (with a short reason) instead of staying silent. A small standby responder serves the stream port and USB tunnels between pipeline runs, and the bound socket is handed over without closing, so no HELLO is lost across a restart. Nothing starts streaming by itself after Stop. New iPads advertise a feature bit for these answers, pad their HELLO2 so every answer fits, wait with the right card and connect as soon as the PC streams again. A standby or stopping-stream answer is never larger than the HELLO it answers. The standby follows changes to the chosen network and to Listen on. Old iPads keep today's behaviour. Supersedes #85. --- host/src/capture/recovery.rs | 14 + host/src/control.rs | 6 + host/src/supervisor.rs | 111 +++- host/src/transport/mod.rs | 104 ++-- host/src/transport/session.rs | 545 +++++++++++++++++- host/src/transport/standby.rs | 243 ++++++++ host/src/transport/usb.rs | 21 +- host/tests/hello_standby.rs | 533 +++++++++++++++++ .../App/ConnectionManager.swift | 34 +- .../App/ConnectionProblem.swift | 56 +- .../Network/ControlChannel.swift | 37 +- ios/EternalMonitor/Network/WireProtocol.swift | 36 +- .../ConnectionProblemTests.swift | 28 + ios/EternalMonitorTests/FakeUDPHost.swift | 26 +- .../PCNotStreamingTests.swift | 170 ++++++ .../WireProtocolGoldenTests.swift | 49 +- proto/src/v2/control.rs | 247 +++++++- proto/testdata/v2_vectors.txt | 4 + proto/tests/golden.rs | 43 ++ 19 files changed, 2217 insertions(+), 90 deletions(-) create mode 100644 host/src/transport/standby.rs create mode 100644 host/tests/hello_standby.rs create mode 100644 ios/EternalMonitorTests/PCNotStreamingTests.swift diff --git a/host/src/capture/recovery.rs b/host/src/capture/recovery.rs index 1cc4232b..04e5a008 100644 --- a/host/src/capture/recovery.rs +++ b/host/src/capture/recovery.rs @@ -38,6 +38,20 @@ pub enum CaptureBlock { } impl CaptureBlock { + /// How an iPad that asks after the storm brake is told this block. + pub fn failed_reason(self) -> eternal_wire::v2::control::FailedReason { + use eternal_wire::v2::control::FailedReason; + match self { + Self::SecureDesktop => FailedReason::SecureDesktop, + Self::DisplayChanging => FailedReason::DisplayChanging, + Self::NoDisplay => FailedReason::NoDisplay, + Self::UnsupportedOnThisGpu => FailedReason::UnsupportedOnThisGpu, + Self::TooManyCaptureApps => FailedReason::TooManyCaptureApps, + Self::SessionDisconnected => FailedReason::SessionDisconnected, + Self::DriverNotResponding => FailedReason::DriverNotResponding, + } + } + /// Maps a DXGI/Win32 HRESULT from DuplicateOutput or AcquireNextFrame. /// `None` means a real failure that the supervisor should handle. pub fn from_hresult(code: i32) -> Option { diff --git a/host/src/control.rs b/host/src/control.rs index e7cd0ce8..e549c480 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -42,6 +42,11 @@ pub struct SharedControl { /// Set while Windows blocks screen capture (lock screen, UAC, a display /// it will not duplicate); the GUI explains it instead of a frozen picture. pub capture_block: Arc>>, + /// The stream port between two owners (a generation's transport and the + /// standby responder), bound and with its queue, so a HELLO2 sent while + /// it changes hands waits for the next one. The supervisor closes it + /// when it shuts down. + pub stream_socket: Arc>>, /// With `UnsupportedOnThisGpu`: the card the captured screen is wired to /// and the switch that would move EternalMonitor onto it. pub gpu_diagnosis: Arc>>, @@ -182,6 +187,7 @@ impl SharedControl { vdd_status: Arc::new(Mutex::new(VddStatus::Inactive)), vdd_problem: Arc::new(Mutex::new(None)), capture_block: Arc::new(Mutex::new(None)), + stream_socket: Arc::new(Mutex::new(None)), gpu_diagnosis: Arc::new(Mutex::new(None)), screen_watch: Arc::new(Mutex::new(crate::gpu_preference::ScreenWatch::new())), gpu_switch: Arc::new(Mutex::new(None)), diff --git a/host/src/supervisor.rs b/host/src/supervisor.rs index 186ffa29..bf2bd173 100644 --- a/host/src/supervisor.rs +++ b/host/src/supervisor.rs @@ -16,7 +16,8 @@ use tracing::{error, info, warn}; use crate::control::{SharedControl, SupervisorCommand}; use crate::gpu::GpuInfo; use crate::stats::PIPELINE_STATS; -use crate::transport::session::{StageFailure, StreamStop}; +use crate::transport::session::{NotStreaming, StageFailure, StreamStop}; +use crate::transport::standby::Standby; /// The current pipeline generation. Stage health reports carry the generation /// they belong to; anything from an older generation is ignored (a "poisoned" @@ -71,6 +72,21 @@ const BACKOFF_MAX: Duration = Duration::from_secs(30); const STORM_WINDOW: Duration = Duration::from_secs(60); const STORM_LIMIT: usize = 6; +/// Lets integration tests trip the storm brake in milliseconds instead of +/// half a minute: the first backoff, in ms. +#[cfg(feature = "test-hooks")] +pub static TEST_BACKOFF_BASE_MS: AtomicU64 = AtomicU64::new(BACKOFF_BASE.as_millis() as u64); + +#[cfg(feature = "test-hooks")] +fn backoff_base() -> Duration { + Duration::from_millis(TEST_BACKOFF_BASE_MS.load(Ordering::SeqCst)) +} + +#[cfg(not(feature = "test-hooks"))] +fn backoff_base() -> Duration { + BACKOFF_BASE +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum SupervisorState { Running { @@ -233,7 +249,7 @@ impl Machine { } self.restarts.push(now); self.attempt += 1; - let backoff = BACKOFF_BASE + let backoff = backoff_base() .saturating_mul(1u32 << (self.attempt - 1).min(6)) .min(BACKOFF_MAX); warn!( @@ -332,6 +348,8 @@ pub fn run( } else { machine.wait_for_port(Instant::now() + crate::startup::port::RECHECK); } + let mut standby = None; + settle(&machine, &shared, &mut standby, listen_port); let mut healthy_since = Instant::now(); let mut transport_watch = crate::transport::beat::Watch::default(); @@ -373,16 +391,34 @@ pub fn run( match effect { Effect::None => {} Effect::StopPipeline => { + // Before the join: a HELLO2 the stopping transport still + // takes hears why. + shared + .session + .lock() + .set_not_streaming(not_streaming(&machine.state, &shared)); stop_pipeline(&shared, &machine.state); bounded_join(&mut pipeline, generation); } Effect::Restart | Effect::Start => { + // A HELLO2 meanwhile is told to ask again in a moment. + shared + .session + .lock() + .set_not_streaming(Some(NotStreaming::Restarting)); shared.stop(); bounded_join(&mut pipeline, generation); + // The check below binds the port: the standby lets go first. + if let Some(standby) = standby.take() { + standby.stop(); + } // Before anything of the new generation starts: a held port - // leaves capture and the encoder closed while it waits. - if !crate::startup::port::check(listen_port) { + // leaves capture and the encoder closed while it waits. One + // the last owner left bound is this host's own. + let ours = shared.stream_socket.lock().is_some(); + if !ours && !crate::startup::port::check(listen_port) { machine.wait_for_port(now + crate::startup::port::RECHECK); + settle(&machine, &shared, &mut standby, listen_port); continue; } crate::audio::restart_stats(&PIPELINE_STATS, |stats| { @@ -391,6 +427,7 @@ pub fn run( }); generation = CURRENT_GENERATION.fetch_add(1, Ordering::SeqCst) + 1; machine.note_started(generation); + settle(&machine, &shared, &mut standby, listen_port); healthy_since = Instant::now(); pipeline = Some(spawn_generation( generation, @@ -404,17 +441,83 @@ pub fn run( Effect::Shutdown => { shared.stop(); bounded_join(&mut pipeline, generation); + // The port closes as the stream ends, as it always did: a + // copy that takes over after this one quits binds it. + shared.stream_socket.lock().take(); break; } } + settle(&machine, &shared, &mut standby, listen_port); // Stopped by the user (or given up) while waiting: no notice. if !matches!(machine.state, SupervisorState::WaitingForPort { .. }) { crate::startup::port::forget(); } } + if let Some(standby) = standby.take() { + standby.stop(); + } + shared.stream_socket.lock().take(); info!("Supervisor loop ended"); } +/// Makes what iPads hear match `machine`: the session's answer to a HELLO2, +/// and the standby responder on the port while no generation streams. +fn settle( + machine: &Machine, + shared: &SharedControl, + standby: &mut Option, + listen_port: u16, +) { + if !standby_answers(&machine.state) { + if let Some(standby) = standby.take() { + standby.stop(); + } + } + // Cleared only once the standby has stopped: it never starts a session. + shared + .session + .lock() + .set_not_streaming(not_streaming(&machine.state, shared)); + if standby_answers(&machine.state) && standby.is_none() { + *standby = Standby::start(listen_port, shared.clone()); + } +} + +/// Whether the standby responder holds the port in `state`. Not while a +/// generation runs, and not while another program holds the port. +fn standby_answers(state: &SupervisorState) -> bool { + matches!( + state, + SupervisorState::BackingOff { .. } + | SupervisorState::Stopped + | SupervisorState::Failed { .. } + ) +} + +/// What a HELLO2 hears in `state`. Nothing starts streaming by itself after +/// Stop or the storm brake: only Start does. +fn not_streaming(state: &SupervisorState, shared: &SharedControl) -> Option { + use eternal_wire::v2::control::FailedReason; + match state { + SupervisorState::Running { .. } => None, + SupervisorState::BackingOff { .. } | SupervisorState::WaitingForPort { .. } => { + Some(NotStreaming::Restarting) + } + SupervisorState::Stopped | SupervisorState::ShutDown => Some(NotStreaming::Stopped), + SupervisorState::Failed { stage, .. } => { + let block = *shared.capture_block.lock(); + Some(NotStreaming::Failed(block.map_or( + match stage { + Stage::Capture | Stage::Audio => FailedReason::Capture, + Stage::Encoder => FailedReason::Encoder, + Stage::Transport => FailedReason::Transport, + }, + crate::capture::recovery::CaptureBlock::failed_reason, + ))) + } + } +} + /// Stops the pipeline that the machine stopped in `state`, first ending the /// session as [`stop_when_stopped`] says: its goodbye goes out with the /// transport, which runs until it sees `running` false. diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index f38ba27c..c6050db6 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -7,6 +7,7 @@ pub mod link; pub mod pacer; pub mod retransmit; pub mod session; +pub mod standby; pub mod usb; pub mod usbmuxd; @@ -200,42 +201,11 @@ pub async fn start_sender( ); } let bind_addr = SocketAddr::from((bind_ip, listen_port)); - let socket = { - let raw = socket2::Socket::new(socket2::Domain::IPV4, socket2::Type::DGRAM, None)?; - // A 4 MiB send buffer lets the kernel absorb whatever the pacer's - // hard budget doesn't smooth (best effort — some platforms clamp it). - let _ = raw.set_send_buffer_size(4 * 1024 * 1024); - let _ = raw.set_recv_buffer_size(4 * 1024 * 1024); - let _ = raw.set_tos(0xB8); - #[cfg(windows)] - { - use std::os::windows::io::AsRawSocket; - use windows::Win32::Networking::WinSock::{ - WSAGetLastError, WSAIoctl, SIO_UDP_CONNRESET, SOCKET, - }; - let disabled = 0u32; - let mut returned = 0; - let result = unsafe { - WSAIoctl( - SOCKET(raw.as_raw_socket() as usize), - SIO_UDP_CONNRESET, - Some((&disabled as *const u32).cast()), - std::mem::size_of_val(&disabled) as u32, - None, - 0, - &mut returned, - None, - None, - ) - }; - if result != 0 { - warn!(error = ?unsafe { WSAGetLastError() }, "Could not disable UDP connection-reset notifications"); - } - } - raw.set_nonblocking(true)?; - raw.bind(&bind_addr.into())?; - UdpSocket::from_std(raw.into())? + let socket = match take_stream_socket(&shared, bind_addr) { + Some(socket) => socket, + None => bind_stream_socket(bind_addr)?, }; + let socket = UdpSocket::from_std(socket)?; // Never 0 — the receiver treats epoch 0 as invalid. (Wrap after 2^32 runs is // harmless: a fresh session id accompanies any host relaunch.) @@ -428,6 +398,7 @@ pub async fn start_sender( src, &header, message, + datagram.len(), &config, now(), ); @@ -602,10 +573,71 @@ pub async fn start_sender( if let Some(goodbye) = goodbye { say_goodbye(&links, goodbye).await; } + park_stream_socket(&shared, links.into_udp_socket()); info!("NAL channel closed, transport sender shutting down"); Ok(()) } +/// The UDP socket iPads send HELLO2 to and the stream is sent from, bound +/// to `bind_addr`. The standby responder binds it the same way. +fn bind_stream_socket(bind_addr: SocketAddr) -> std::io::Result { + let raw = socket2::Socket::new(socket2::Domain::IPV4, socket2::Type::DGRAM, None)?; + // A 4 MiB send buffer lets the kernel absorb whatever the pacer's + // hard budget doesn't smooth (best effort — some platforms clamp it). + let _ = raw.set_send_buffer_size(4 * 1024 * 1024); + let _ = raw.set_recv_buffer_size(4 * 1024 * 1024); + let _ = raw.set_tos(0xB8); + #[cfg(windows)] + { + use std::os::windows::io::AsRawSocket; + use windows::Win32::Networking::WinSock::{ + WSAGetLastError, WSAIoctl, SIO_UDP_CONNRESET, SOCKET, + }; + let disabled = 0u32; + let mut returned = 0; + let result = unsafe { + WSAIoctl( + SOCKET(raw.as_raw_socket() as usize), + SIO_UDP_CONNRESET, + Some((&disabled as *const u32).cast()), + std::mem::size_of_val(&disabled) as u32, + None, + 0, + &mut returned, + None, + None, + ) + }; + if result != 0 { + warn!(error = ?unsafe { WSAGetLastError() }, "Could not disable UDP connection-reset notifications"); + } + } + raw.set_nonblocking(true)?; + raw.bind(&bind_addr.into())?; + Ok(raw.into()) +} + +/// The stream socket the last owner of the port left, if it is bound to +/// `bind_addr`. One bound elsewhere (the chosen network changed) closes. +fn take_stream_socket( + shared: &SharedControl, + bind_addr: SocketAddr, +) -> Option { + shared + .stream_socket + .lock() + .take() + .filter(|socket| socket.local_addr().is_ok_and(|addr| addr == bind_addr)) +} + +/// Leaves the stream socket, still bound, for the next owner of the port. +fn park_stream_socket(shared: &SharedControl, socket: UdpSocket) { + match socket.into_std() { + Ok(socket) => *shared.stream_socket.lock() = Some(socket), + Err(error) => debug!(%error, "Closed the stream port between owners"), + } +} + /// Marks the transport ended however `start_sender` stops, so the watch /// never reads a finished loop's last mark as a stuck one. struct BeatEnds<'a>(&'a beat::Beat); @@ -760,7 +792,7 @@ async fn repair_while_pacing( #[cfg(feature = "test-hooks")] probe::taken(peer); let actions = shared.session.lock().handle_datagram(PeerId::udp(peer), &header, - message, config, now()); + message, len, config, now()); act(actions, state, links, shared, config, supervisor_tx).await; continue; } diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 50be83ab..b667ed13 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -13,8 +13,9 @@ use super::link::{LinkId, PeerId}; use std::time::{Duration, Instant}; use eternal_wire::v2::control::{ - ByeReason, ControlHeader, ControlMessage, HelloAck, HelloStatus, InputEvent, KeyframeRequest, - Nack, ReceiverReport, StreamConfig, FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_NACK, + ByeReason, ControlHeader, ControlMessage, FailedReason, HelloAck, HelloStatus, InputEvent, + KeyframeRequest, Nack, ReceiverReport, StreamConfig, FEATURE_STANDBY_STATUS, + FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_NACK, }; use tracing::{info, warn}; @@ -366,6 +367,28 @@ impl SeqWindow { } } +/// Why no stream runs, for an iPad that asks meanwhile. The supervisor sets +/// it whenever no pipeline generation streams (see +/// [`Session::set_not_streaming`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NotStreaming { + /// Someone clicked Stop: only Start on the PC streams again. + Stopped, + /// The supervisor is waiting out the pause before a restart. + Restarting, + /// The storm brake gave up after repeated errors, until Start. + Failed(FailedReason), +} + +/// Why a HELLO2 can't start a session ([`Session::closed_to_clients`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Closed { + /// Turned away with Busy, logged with this line. + Busy(&'static str), + /// Told why, once it passes the pairing check. + NotStreaming(NotStreaming), +} + /// Provides the current stream parameters for HELLO_ACK/heartbeats. pub trait ConfigSource { fn pairing(&self) -> Option<&parking_lot::Mutex> { @@ -401,6 +424,8 @@ pub struct Session { /// has resumed since: new clients are turned away, as nothing would /// serve them. stream_stopped: bool, + /// No stream runs: HELLO2s are told why instead of starting a session. + not_streaming: Option, /// The goodbye for a stream an update or a stop ended, sent by the next /// tick or by the transport on its way out. goodbye: Option<(PeerId, Vec)>, @@ -425,6 +450,7 @@ impl Session { last_end: None, updating: false, stream_stopped: false, + not_streaming: None, goodbye: None, stopped: None, ended_for_liveness: None, @@ -510,6 +536,17 @@ impl Session { self.goodbye.take() } + /// Whether a stream runs, and why not. While it is set, a HELLO2 that + /// passes the version, update and pairing checks is told why, and no + /// session starts. A session the stream keeps across a restart stays. + pub fn set_not_streaming(&mut self, state: Option) { + self.not_streaming = state; + } + + pub fn not_streaming(&self) -> Option { + self.not_streaming + } + /// The update handoff failed; clients may connect again. pub fn end_update(&mut self) { self.updating = false; @@ -526,21 +563,18 @@ impl Session { self.stopped.take().or(active) } - /// Why a HELLO2 is turned away before anything else is looked at, if it - /// is: an update is about to close the host, or the stream stopped and - /// its transport is on its way out. The iPad shows Busy; a retry once - /// the stream runs again works. - fn closed_to_clients(&self) -> Option<(HelloStatus, &'static str)> { + /// Why a HELLO2 can't start a session now, if it can't: an update is + /// about to close the host, no stream runs (the supervisor says why), or + /// the stream stopped and its transport is on its way out (the app + /// quits). The first and last are turned away with Busy before anything + /// else is looked at; a retry once the stream runs again works. + fn closed_to_clients(&self) -> Option { if self.updating { - Some(( - HelloStatus::Busy, - "Client turned away while an update starts", - )) + Some(Closed::Busy("Client turned away while an update starts")) + } else if let Some(state) = self.not_streaming { + Some(Closed::NotStreaming(state)) } else if self.stream_stopped { - Some(( - HelloStatus::Busy, - "Client turned away while the stream stops", - )) + Some(Closed::Busy("Client turned away while the stream stops")) } else { None } @@ -631,18 +665,22 @@ impl Session { /// Feed one control datagram off a link, with its header. Input whose /// msg_seq this session took already, or that is too old, is dropped. + /// + /// `datagram_len` is the datagram's size: an answer saying the PC isn't + /// streaming is never larger than the HELLO2 it answers. pub fn handle_datagram( &mut self, source: PeerId, header: &ControlHeader, message: ControlMessage, + datagram_len: usize, config: &impl ConfigSource, now: Instant, ) -> Actions { self.handle( source, header.session_id, - Some(header.msg_seq), + Some((header.msg_seq, datagram_len)), message, config, now, @@ -670,7 +708,8 @@ impl Session { &mut self, source: PeerId, header_session_id: u32, - msg_seq: Option, + // The datagram's msg_seq and size, when it came off a link. + datagram: Option<(u32, usize)>, message: ControlMessage, config: &impl ConfigSource, now: Instant, @@ -695,13 +734,26 @@ impl Session { } } match message { - ControlMessage::Hello2(hello) => self.handle_hello(source, hello, config, now), + ControlMessage::Hello2(hello) => { + let len = datagram.map(|(_, len)| len); + let mut actions = self.handle_hello(source, hello, len, config, now); + // While no stream runs, no answer to a HELLO2 is larger than + // it: refusals included, after the checks that chose them. + if self.not_streaming.is_some() { + if let Some(len) = len { + actions.replies.retain(|(_, reply)| reply.len() <= len); + } + } + actions + } ControlMessage::Identify(request) => self.answer_identify(source, request, config, now), ControlMessage::Bye(reason) => self.handle_bye(source, reason, now), ControlMessage::KeyframeRequest(request) => self.handle_keyframe(source, request, now), ControlMessage::ReceiverReport(report) => self.handle_report(source, report, now), ControlMessage::Ping(ping) => self.handle_ping(source, ping, now), - ControlMessage::InputEvent(event) => self.handle_input(source, event, msg_seq, now), + ControlMessage::InputEvent(event) => { + self.handle_input(source, event, datagram.map(|(seq, _)| seq), now) + } ControlMessage::Nack(nack) => self.handle_nack(source, nack, now), // Host-outbound types arriving inbound: ignore. _ => Actions::default(), @@ -775,6 +827,7 @@ impl Session { &mut self, source: PeerId, hello: eternal_wire::v2::control::Hello2, + datagram_len: Option, config: &impl ConfigSource, now: Instant, ) -> Actions { @@ -787,9 +840,10 @@ impl Session { } // Don't start a stream nothing will serve. - if let Some((status, why)) = self.closed_to_clients() { + let closed = self.closed_to_clients(); + if let Some(Closed::Busy(why)) = closed { info!(peer = %source, "{why}"); - return self.refuse(source, status, &hello, config, now); + return self.refuse(source, HelloStatus::Busy, &hello, config, now); } // Keep credentials locked through authorization, rotation and ACK creation: @@ -803,6 +857,17 @@ impl Session { { // Exact replay of an accepted handshake. In particular, losing the // first code-based ACK must not require the now-rotated code. + if let Some(Closed::NotStreaming(state)) = closed { + drop(pairing); + return self.answer_not_streaming( + source, + state, + &hello, + datagram_len, + config, + now, + ); + } actions.replies.push((source, session.ack.clone())); return actions; } @@ -821,6 +886,12 @@ impl Session { new_pairing: false, }, }; + // Only an iPad that may connect learns why the PC isn't streaming. + // A right pairing code isn't spent: it still works after Start. + if let Some(Closed::NotStreaming(state)) = closed { + drop(pairing); + return self.answer_not_streaming(source, state, &hello, datagram_len, config, now); + } // The media target: the client's advertised listen port at its source IP // (normally identical to the source port; 0 = malformed, use the source). @@ -1269,6 +1340,47 @@ impl Session { /// Answers `hello` with a refusal. The first refusal of a nonce ends /// that attempt; a retransmit of it gets the answer again. + /// Tells an iPad the PC isn't streaming, in the words it understands. + /// 0.3.4 and older drop an ACK whose status they don't know. A restart + /// is short and their HELLO2 retries outlast it, so they hear nothing, + /// as before. A stopped or failed PC waits for someone at the PC, so it + /// turns them down with Error: "Your PC turned down the connection", + /// which also ends their reconnect cycle. + fn answer_not_streaming( + &mut self, + source: PeerId, + state: NotStreaming, + hello: &eternal_wire::v2::control::Hello2, + datagram_len: Option, + config: &impl ConfigSource, + now: Instant, + ) -> Actions { + let (status, reason) = if hello.feature_caps & FEATURE_STANDBY_STATUS == 0 { + match state { + NotStreaming::Restarting => return Actions::default(), + NotStreaming::Stopped | NotStreaming::Failed(_) => (HelloStatus::Error, ""), + } + } else { + match state { + NotStreaming::Stopped => (HelloStatus::Stopped, ""), + NotStreaming::Restarting => (HelloStatus::Restarting, ""), + NotStreaming::Failed(reason) => (HelloStatus::Failed, reason.token()), + } + }; + let mut actions = self.refuse_with_reason(source, status, reason, hello, config, now); + // Never larger than the HELLO2: iPads that understand these answers + // pad theirs so every answer fits. A stopping stream answers here + // too, until its transport ends, so the bound is here as well as in + // the standby responder. + actions + .replies + .retain(|(_, reply)| datagram_len.is_none_or(|len| reply.len() <= len)); + if actions.ended.is_some() && !actions.replies.is_empty() { + info!(peer = %source, ?status, reason, "Told an iPad the PC isn't streaming"); + } + actions + } + fn refuse( &mut self, source: PeerId, @@ -1277,7 +1389,19 @@ impl Session { config: &impl ConfigSource, now: Instant, ) -> Actions { - let ack = self.make_ack(status, hello.client_nonce, 0, config); + self.refuse_with_reason(source, status, "", hello, config, now) + } + + fn refuse_with_reason( + &mut self, + source: PeerId, + status: HelloStatus, + reason: &str, + hello: &eternal_wire::v2::control::Hello2, + config: &impl ConfigSource, + now: Instant, + ) -> Actions { + let ack = self.make_reply(status, hello.client_nonce, 0, config, [0; 16], reason); self.refused .retain(|(_, at)| now.saturating_duration_since(*at) < REFUSED_FOR); let first = !self @@ -1337,23 +1461,25 @@ impl Session { } } - fn make_ack( + fn make_ack_with_token( &mut self, status: HelloStatus, client_nonce: u32, session_id: u32, config: &impl ConfigSource, + auth_token: [u8; 16], ) -> Vec { - self.make_ack_with_token(status, client_nonce, session_id, config, [0; 16]) + self.make_reply(status, client_nonce, session_id, config, auth_token, "") } - fn make_ack_with_token( + fn make_reply( &mut self, status: HelloStatus, client_nonce: u32, session_id: u32, config: &impl ConfigSource, auth_token: [u8; 16], + reason: &str, ) -> Vec { let msg_seq = self.next_msg_seq(); let ack = ControlMessage::HelloAck(HelloAck { @@ -1368,6 +1494,7 @@ impl Session { host_name: config.host_name(), auth_token, host_caps: HOSTCAP_NACK | config.host_caps(), + reason: reason.to_string(), }); eternal_wire::v2::control::encode_control(session_id, msg_seq, &ack) } @@ -2347,7 +2474,7 @@ mod tests { msg_seq, }; let message = ControlMessage::InputEvent(event); - let actions = session.handle_datagram(peer, &header, message, &TestConfig, now); + let actions = session.handle_datagram(peer, &header, message, 64, &TestConfig, now); actions .input .map(|(_, event)| event) @@ -3941,4 +4068,370 @@ mod tests { assert_eq!(late, 0, "{ppm} ppm"); } } + + fn paired_hello(nonce: u32, feature_caps: u16) -> Hello2 { + let mut hello = pair_hello(nonce); + hello.auth_token = [7; 16]; + hello.feature_caps = feature_caps; + hello + } + + fn ask_while( + session: &mut Session, + config: &PairConfig, + state: NotStreaming, + hello: Hello2, + ) -> Actions { + session.set_not_streaming(Some(state)); + pair_send( + session, + config, + addr([192, 0, 2, 1], 50000), + hello, + Instant::now(), + ) + } + + /// A paired iPad that asks while no stream runs hears why, and nothing + /// starts: no session, no media target, no pairing token. + #[test] + fn a_new_ipad_hears_why_the_pc_is_not_streaming() { + let config = pair_config(true); + let mut session = Session::new(10); + for (state, status, reason) in [ + (NotStreaming::Stopped, HelloStatus::Stopped, ""), + (NotStreaming::Restarting, HelloStatus::Restarting, ""), + ( + NotStreaming::Failed(FailedReason::DisplayChanging), + HelloStatus::Failed, + "display_changing", + ), + ] { + let actions = ask_while( + &mut session, + &config, + state, + paired_hello(1, FEATURE_STANDBY_STATUS), + ); + let ack = pair_ack(&actions); + assert_eq!((ack.status, ack.reason.as_str()), (status, reason)); + assert_eq!((ack.session_id, ack.auth_token), (0, [0; 16])); + assert!(actions.new_target.is_none() && !actions.force_idr); + assert!(!session.is_active()); + } + // Streaming again: the same HELLO2 connects. + session.set_not_streaming(None); + let actions = pair_send( + &mut session, + &config, + addr([192, 0, 2, 1], 50000), + paired_hello(1, FEATURE_STANDBY_STATUS), + Instant::now(), + ); + assert_eq!(pair_ack(&actions).status, HelloStatus::Ok); + assert!(session.is_active()); + } + + /// 0.3.4 and older drop statuses past RateLimited. While the PC restarts + /// they hear nothing and keep retrying, as before; a stopped or failed + /// PC turns them down with Error, a card they have. + #[test] + fn an_old_ipad_gets_only_statuses_it_knows() { + let config = pair_config(true); + let mut session = Session::new(10); + let restarting = ask_while( + &mut session, + &config, + NotStreaming::Restarting, + paired_hello(1, 0), + ); + assert!(restarting.replies.is_empty() && restarting.ended.is_none()); + for state in [ + NotStreaming::Stopped, + NotStreaming::Failed(FailedReason::Encoder), + ] { + let ack = pair_ack(&ask_while(&mut session, &config, state, paired_hello(2, 0))); + assert_eq!(ack.status, HelloStatus::Error); + assert!(ack.reason.is_empty()); + } + assert!(!session.is_active()); + } + + /// Only an iPad that may connect learns the stream state. Wrong codes + /// count toward the rate limit here too, and the right code is not + /// spent, so it still works once the stream starts. + #[test] + fn pairing_still_decides_who_hears_it() { + let config = pair_config(true); + let mut session = Session::new(10); + let mut unpaired = pair_hello(1); + unpaired.feature_caps = FEATURE_STANDBY_STATUS; + for state in [ + NotStreaming::Stopped, + NotStreaming::Restarting, + NotStreaming::Failed(FailedReason::Capture), + ] { + let ack = pair_ack(&ask_while(&mut session, &config, state, unpaired.clone())); + assert_eq!(ack.status, HelloStatus::Unauthorized); + assert!(ack.reason.is_empty()); + } + let mut statuses = Vec::new(); + for attempt in 1..=5 { + let mut guess = unpaired.clone(); + guess.client_nonce = 10 + attempt; + guess.pairing_code = 1_000_000 + attempt; + statuses.push( + pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Stopped, + guess, + )) + .status, + ); + } + assert_eq!(statuses.last(), Some(&HelloStatus::RateLimited)); + + let config = pair_config(true); + let code = config.0.lock().code(); + let mut right = unpaired; + right.client_nonce = 20; + right.pairing_code = code; + let ack = pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Stopped, + right.clone(), + )); + assert_eq!( + (ack.status, ack.auth_token), + (HelloStatus::Stopped, [0; 16]) + ); + assert_eq!(config.0.lock().code(), code, "the code stays valid"); + session.set_not_streaming(None); + let accepted = pair_send( + &mut session, + &config, + addr([192, 0, 2, 1], 50000), + right, + Instant::now(), + ); + assert_eq!(pair_ack(&accepted).status, HelloStatus::Ok); + } + + /// Stop ends the session and closes the PC to clients; the supervisor + /// then says why, and a HELLO2 hears it instead of Busy. A quit, which + /// no supervisor state follows, still turns it away with Busy. + #[test] + fn a_stop_is_answered_with_why_not_busy() { + let config = pair_config(true); + let mut session = Session::new(10); + let now = Instant::now(); + session.shut_down(now, StreamStop::Quit); + let busy = pair_send( + &mut session, + &config, + addr([192, 0, 2, 1], 50000), + paired_hello(1, FEATURE_STANDBY_STATUS), + now, + ); + assert_eq!(pair_ack(&busy).status, HelloStatus::Busy); + session.shut_down(now, StreamStop::User); + let ack = pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Stopped, + paired_hello(2, FEATURE_STANDBY_STATUS), + )); + assert_eq!(ack.status, HelloStatus::Stopped); + } + + #[test] + fn the_version_and_update_refusals_come_first() { + let config = pair_config(true); + let mut session = Session::new(10); + let mut old = paired_hello(1, FEATURE_STANDBY_STATUS); + old.proto_min = 1; + old.proto_max = 1; + let ack = pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Stopped, + old, + )); + assert_eq!(ack.status, HelloStatus::VersionUnsupported); + assert!(session.begin_update(false)); + let ack = pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Stopped, + paired_hello(2, FEATURE_STANDBY_STATUS), + )); + assert_eq!(ack.status, HelloStatus::Busy); + } + + /// A session kept across a restart stays, and its iPad's repeated + /// HELLO2 hears that the PC is restarting instead of the old OK. + #[test] + fn a_session_kept_across_a_restart_stays() { + let config = pair_config(true); + let mut session = Session::new(10); + let hello = paired_hello(1, FEATURE_STANDBY_STATUS); + let peer = addr([192, 0, 2, 1], 50000); + let accepted = pair_send(&mut session, &config, peer, hello.clone(), Instant::now()); + assert_eq!(pair_ack(&accepted).status, HelloStatus::Ok); + let ack = pair_ack(&ask_while( + &mut session, + &config, + NotStreaming::Restarting, + hello, + )); + assert_eq!(ack.status, HelloStatus::Restarting); + assert!(session.is_active()); + } + + /// The stream that is stopping still takes HELLO2s until its transport + /// ends. Its not-streaming answers are never larger than the HELLO2 + /// either, as the standby's aren't. + #[test] + fn a_not_streaming_answer_is_never_larger_than_the_hello() { + let config = pair_config(false); + let mut session = Session::new(10); + let peer = addr([192, 0, 2, 1], 50000); + let mut small = pair_hello(1); + small.device_name = String::new(); + small.device_id = 0; + let unpadded = + eternal_wire::v2::control::encode_control(0, 1, &ControlMessage::Hello2(small.clone())); + // The same small HELLO2 claiming the bit, without its padding. + let mut claimed = unpadded.clone(); + claimed[26..28].copy_from_slice(&FEATURE_STANDBY_STATUS.to_le_bytes()); + let mut padded_hello = small; + padded_hello.client_nonce = 2; + padded_hello.feature_caps = FEATURE_STANDBY_STATUS; + let padded = + eternal_wire::v2::control::encode_control(0, 2, &ControlMessage::Hello2(padded_hello)); + session.set_not_streaming(Some(NotStreaming::Failed(FailedReason::Encoder))); + let mut answer = |datagram: &[u8]| { + let (header, message) = eternal_wire::v2::control::parse_control(datagram).unwrap(); + session.handle_datagram( + peer, + &header, + message, + datagram.len(), + &config, + Instant::now(), + ) + }; + for small in [&unpadded, &claimed] { + assert_eq!(small.len(), 38); + let actions = answer(small); + assert!( + actions.replies.is_empty(), + "{} bytes answered", + actions.replies[0].1.len() + ); + } + let actions = answer(&padded); + let (_, reply) = &actions.replies[0]; + assert!(reply.len() <= padded.len()); + let (_, ControlMessage::HelloAck(ack)) = + eternal_wire::v2::control::parse_control(reply).unwrap() + else { + panic!("expected an ACK"); + }; + assert_eq!( + (ack.status, ack.reason.as_str()), + (HelloStatus::Failed, "encoder") + ); + } + + /// While no stream runs, every answer to a HELLO2 is bounded by its + /// size, the version and pairing refusals too. A padded HELLO2 still + /// hears them; while streaming, nothing changes. + #[test] + fn every_hello_answer_is_bounded_while_not_streaming() { + use eternal_wire::v2::control::{encode_control, parse_control}; + let peer = addr([192, 0, 2, 1], 50000); + let send = |session: &mut Session, config: &PairConfig, hello: &Hello2| { + let datagram = encode_control(0, 1, &ControlMessage::Hello2(hello.clone())); + let (header, message) = parse_control(&datagram).unwrap(); + let actions = session.handle_datagram( + peer, + &header, + message, + datagram.len(), + config, + Instant::now(), + ); + for (_, reply) in &actions.replies { + assert!( + reply.len() <= datagram.len(), + "{} > {}", + reply.len(), + datagram.len() + ); + } + actions + .replies + .first() + .map(|(_, reply)| match parse_control(reply) { + Ok((_, ControlMessage::HelloAck(ack))) => ack.status, + other => panic!("not an ACK: {other:?}"), + }) + }; + let small = |nonce: u32, caps: u16| { + let mut hello = pair_hello(nonce); + hello.device_name = String::new(); + hello.device_id = 0; + hello.feature_caps = caps; + hello + }; + + let config = pair_config(true); + let mut session = Session::new(10); + session.set_not_streaming(Some(NotStreaming::Stopped)); + // The version check. + let mut old_version = small(1, 0); + old_version.proto_min = 1; + old_version.proto_max = 1; + assert_eq!(send(&mut session, &config, &old_version), None); + old_version.feature_caps = FEATURE_STANDBY_STATUS; + assert_eq!( + send(&mut session, &config, &old_version), + Some(HelloStatus::VersionUnsupported) + ); + // The pairing checks: no code, then wrong codes until rate limited. + assert_eq!(send(&mut session, &config, &small(2, 0)), None); + for attempt in 0..6 { + let mut guess = small(10 + attempt, 0); + guess.pairing_code = 1_000_000 + attempt; + assert_eq!( + send(&mut session, &config, &guess), + None, + "attempt {attempt}" + ); + } + let mut padded = small(20, FEATURE_STANDBY_STATUS); + padded.pairing_code = 1_000_020; + assert_eq!( + send(&mut session, &config, &padded), + Some(HelloStatus::RateLimited) + ); + + // Streaming: the refusals are as they always were. + let config = pair_config(true); + let mut session = Session::new(10); + let datagram = encode_control(0, 1, &ControlMessage::Hello2(small(30, 0))); + let (header, message) = parse_control(&datagram).unwrap(); + let actions = session.handle_datagram( + peer, + &header, + message, + datagram.len(), + &config, + Instant::now(), + ); + assert_eq!(pair_ack(&actions).status, HelloStatus::Unauthorized); + } } diff --git a/host/src/transport/standby.rs b/host/src/transport/standby.rs new file mode 100644 index 00000000..e7f36077 --- /dev/null +++ b/host/src/transport/standby.rs @@ -0,0 +1,243 @@ +//! Answers iPads while no pipeline generation streams. +//! +//! The stream port used to be open only inside a running pipeline, so a +//! HELLO2 sent while the stream waited to restart, after Stop, or after the +//! storm brake gave up went unanswered, and the iPad said the PC didn't +//! answer. Between generations the supervisor now runs this responder on +//! the same port and the USB tunnels. It answers each HELLO2 through the +//! session, which says why nothing streams ([`Session::set_not_streaming`]) +//! after the same version, update and pairing checks as always. It never +//! starts a session or a stream, and lets go of the port before the +//! supervisor checks it for the next generation. +//! +//! [`Session::set_not_streaming`]: super::session::Session::set_not_streaming + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use eternal_wire::v2::control::{parse_control, ControlMessage}; +use eternal_wire::v2::{classify, Classified}; +use tokio::net::UdpSocket; +use tokio::sync::oneshot; +use tracing::{debug, info, warn}; + +use super::link::PeerId; +use super::usb::{LinkEvent, TransportLinks}; +use crate::control::SharedControl; + +/// How often the responder tries again to bind a port it couldn't: the +/// generation that just ended may hold it a moment longer. +const BIND_RETRY: Duration = Duration::from_millis(250); +/// How often the responder looks for a goodbye left for a stopped session. +const GOODBYE_CHECK: Duration = Duration::from_millis(250); +/// How often the responder checks the chosen network's address, so it +/// follows the network as the stream does, a little sooner. +const NETWORK_CHECK: Duration = Duration::from_secs(3); +/// How long [`Standby::stop`] waits for the port to close. +const STOP_TIMEOUT: Duration = Duration::from_secs(2); + +/// The running responder. [`Standby::stop`] closes the port. +pub struct Standby { + stop: oneshot::Sender<()>, + thread: std::thread::JoinHandle<()>, +} + +impl Standby { + /// Starts answering on `listen_port`, on its own thread. + pub fn start(listen_port: u16, shared: SharedControl) -> Option { + let (stop, stopped) = oneshot::channel(); + std::thread::Builder::new() + .name("standby".into()) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build(); + match runtime { + Ok(runtime) => runtime.block_on(run(listen_port, shared, stopped)), + Err(error) => warn!(%error, "Could not answer iPads while the stream is off"), + } + }) + .inspect_err(|error| warn!(%error, "Could not answer iPads while the stream is off")) + .ok() + .map(|thread| Self { stop, thread }) + } + + /// Stops answering, and returns once the port and the USB tunnels are + /// free for the next generation. + pub fn stop(self) { + let _ = self.stop.send(()); + let deadline = Instant::now() + STOP_TIMEOUT; + while !self.thread.is_finished() { + if Instant::now() >= deadline { + warn!("The standby responder didn't let go of the stream port in time"); + return; + } + std::thread::sleep(Duration::from_millis(5)); + } + let _ = self.thread.join(); + } +} + +async fn run(listen_port: u16, shared: SharedControl, mut stopped: oneshot::Receiver<()>) { + loop { + let (mut links, bound) = tokio::select! { + opened = open(listen_port, &shared) => opened, + _ = &mut stopped => return, + }; + debug!(%bound, "Answering iPads while no stream runs"); + if !serve(&mut links, bound, &shared, &mut stopped).await { + super::park_stream_socket(&shared, links.into_udp_socket()); + debug!("Stopped answering iPads while no stream runs"); + return; + } + // The chosen network went away, came back or changed its address: + // listen where iPads can reach the PC now, as the stream would. + info!(was = %bound, "The chosen network changed: answering iPads on its new address"); + } +} + +/// Answers until the supervisor stops the responder (false), or until the +/// chosen network's address is no longer `bound` (true). +async fn serve( + links: &mut TransportLinks, + bound: SocketAddr, + shared: &SharedControl, + stopped: &mut oneshot::Receiver<()>, +) -> bool { + let mut goodbyes = tokio::time::interval(GOODBYE_CHECK); + let mut network_check = tokio::time::interval(NETWORK_CHECK); + network_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + biased; + _ = &mut *stopped => return false, + _ = network_check.tick() => { + // Every network listens on 0.0.0.0, so a change to or from + // it shows here too. + let listen_on = shared.listen_on.lock().clone(); + if crate::network::listen_ip_now(&listen_on) != bound.ip() { + return true; + } + } + // A Stop during a restart pause ends a session no transport + // can tell: say it here. + _ = goodbyes.tick() => { + let goodbye = shared.session.lock().take_goodbye(); + if let Some(goodbye) = goodbye { + super::say_goodbye(links, goodbye).await; + } + } + event = links.receive() => match event { + Ok(LinkEvent::Datagram(peer, datagram)) => { + answer(links, shared, bound, peer, &datagram).await; + } + // A tunnel that closes now is gone for the next generation + // too; a session it carried ends on its liveness there. + Ok(LinkEvent::Closed(_)) => {} + Err(error) => { + debug!(%error, "Standby receive failed"); + tokio::time::sleep(BIND_RETRY).await; + } + }, + } + } +} + +/// Takes the stream port from the generation that just ended, or binds it +/// as the transport does, retrying while that generation still holds it. +/// Returns the links and the address they listen on. +async fn open(listen_port: u16, shared: &SharedControl) -> (TransportLinks, SocketAddr) { + let mut logged = false; + loop { + let listen_on = shared.listen_on.lock().clone(); + let ip = crate::network::listen_ip_now(&listen_on); + let bind_addr = SocketAddr::from((ip, listen_port)); + let socket = match super::take_stream_socket(shared, bind_addr) { + Some(socket) => Ok(socket), + None => super::bind_stream_socket(bind_addr), + }; + match socket.and_then(UdpSocket::from_std) { + Ok(socket) => { + if logged { + info!(%bind_addr, "Answering iPads while the stream is off"); + } + // The Address row and QR code follow, as for the stream. + let shown = crate::gui::shown_address(ip, listen_port); + crate::stats::PIPELINE_STATS.lock().set_listen_addr(shown); + let links = TransportLinks::new(socket, Arc::clone(&shared.force_next_idr)); + return (links, bind_addr); + } + Err(error) if !logged => { + logged = true; + warn!(%bind_addr, %error, "Can't answer iPads while the stream is off yet"); + } + Err(_) => {} + } + tokio::time::sleep(BIND_RETRY).await; + } +} + +/// Answers a HELLO2. Everything else waits for the next generation: a +/// session kept across a restart picks up there. +async fn answer( + links: &TransportLinks, + shared: &SharedControl, + bound: SocketAddr, + peer: PeerId, + datagram: &[u8], +) { + match classify(datagram) { + Classified::Control(_) => {} + Classified::LegacyHello => { + shared.session.lock().note_legacy_hello(peer); + return; + } + _ => return, + } + let Ok((header, message @ ControlMessage::Hello2(_))) = parse_control(datagram) else { + return; + }; + let config = super::SharedConfigSource { + shared, + stream_epoch: 0, + pen_available: false, + listen: bound, + }; + let actions = { + let mut session = shared.session.lock(); + // The supervisor clears it only after stopping this responder; a + // HELLO2 here never starts a session. + if session.not_streaming().is_none() { + return; + } + session.handle_datagram( + peer, + &header, + message, + datagram.len(), + &config, + super::now(), + ) + }; + if let Some(end) = &actions.ended { + crate::diagnostics::log_session_end(end, shared); + } + for (destination, reply) in &actions.replies { + // An answer is never larger than the HELLO2 it answers. iPads that + // understand these answers pad their HELLO2 so every answer fits + // (`STANDBY_HELLO2_LEN`); an older one may get today's silence. + if reply.len() > datagram.len() { + debug!( + peer = %destination, + reply = reply.len(), + request = datagram.len(), + "Left a HELLO2 unanswered: the answer is larger than it" + ); + continue; + } + if let Err(error) = links.send_to(reply, *destination).await { + debug!(peer = %destination, %error, "Could not answer a HELLO2 while the stream is off"); + } + } +} diff --git a/host/src/transport/usb.rs b/host/src/transport/usb.rs index 424a2810..b3a3e94c 100644 --- a/host/src/transport/usb.rs +++ b/host/src/transport/usb.rs @@ -235,7 +235,16 @@ pub struct TransportLinks { pub udp: UdpLink, usb: HashMap, events: mpsc::Receiver, - token: u64, + _subscription: HubToken, +} + +/// The hub's events go to this owner until it is dropped. +struct HubToken(u64); + +impl Drop for HubToken { + fn drop(&mut self) { + UsbHub::get().unsubscribe(self.0); + } } impl TransportLinks { @@ -245,7 +254,7 @@ impl TransportLinks { udp: UdpLink::new(socket), usb: HashMap::new(), events: subscription.events, - token: subscription.token, + _subscription: HubToken(subscription.token), }; for (id, generation, link) in subscription.open { info!(?id, generation, "USB tunnel kept across pipeline restart"); @@ -356,11 +365,11 @@ impl TransportLinks { } } } -} -impl Drop for TransportLinks { - fn drop(&mut self) { - UsbHub::get().unsubscribe(self.token); + /// Lets go of the USB tunnels and returns the UDP socket, for the next + /// owner of the stream port. + pub fn into_udp_socket(self) -> UdpSocket { + self.udp.socket } } diff --git a/host/tests/hello_standby.rs b/host/tests/hello_standby.rs new file mode 100644 index 00000000..c21e56a4 --- /dev/null +++ b/host/tests/hello_standby.rs @@ -0,0 +1,533 @@ +//! The PC answers an iPad's HELLO2 while it isn't streaming: in the pause +//! before an automatic restart, across a restart, after Stop, and after the +//! storm brake. Drives the REAL supervisor with synthetic capture and +//! libx264, talked to by fake iPads over loopback UDP on ephemeral ports. + +use std::collections::BTreeMap; +use std::net::UdpSocket; +use std::sync::atomic::Ordering; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +use eternal_host::control::{SharedControl, SupervisorCommand}; +use eternal_host::gpu::GpuInfo; +use eternal_host::pipeline; +use eternal_host::stats::PIPELINE_STATS; +use eternal_host::transport::session::NotStreaming; +use eternal_wire::v2::control::{ + encode_control, parse_control, ControlMessage, Hello2, HelloAck, HelloStatus, CAP_DECODE_H264, + FEATURE_STANDBY_STATUS, +}; + +/// Tests share process-global env (ETERNAL_*) and pipeline state; run them +/// one at a time. +static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// An iPad on 0.3.5 or later, and one on 0.3.4 or older. +const NEW: u16 = FEATURE_STANDBY_STATUS; +const OLD: u16 = 0; + +struct Host { + shared: SharedControl, + port: u16, + commands: mpsc::Sender, + supervisor: Option>, + _env: std::sync::MutexGuard<'static, ()>, +} + +impl Host { + fn start(gpu: GpuInfo, pairing: bool) -> Self { + Self::start_with_backoff(gpu, pairing, 500) + } + + /// With the first automatic restart `backoff_ms` after a failure. + fn start_with_backoff(gpu: GpuInfo, pairing: bool, backoff_ms: u64) -> Self { + Self::start_with(gpu, backoff_ms, |shared| { + shared.pairing.lock().required = pairing + }) + } + + /// With `configure` applied before the supervisor starts. + fn start_with(gpu: GpuInfo, backoff_ms: u64, configure: impl FnOnce(&SharedControl)) -> Self { + let env = ENV_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + eternal_host::supervisor::TEST_BACKOFF_BASE_MS.store(backoff_ms, Ordering::SeqCst); + eternal_host::clock::init(); + // Never claim a physically connected iPad. + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_SIZE", "640x360"); + for name in [ + "ETERNAL_DROP", + "ETERNAL_REORDER", + "ETERNAL_JITTER_MS", + "ETERNAL_FAULT_ENCODER_AFTER", + ] { + std::env::remove_var(name); + } + let _ = ffmpeg_next::init(); + let port = UdpSocket::bind("0.0.0.0:0") + .unwrap() + .local_addr() + .unwrap() + .port(); + let shared = SharedControl::new(port, pipeline::DEFAULT_BITRATE_BPS); + configure(&shared); + let (commands, command_rx) = mpsc::channel(); + let supervisor = { + let (shared, commands) = (shared.clone(), commands.clone()); + std::thread::spawn(move || { + eternal_host::supervisor::run(port, shared, gpu, commands, command_rx); + }) + }; + Self { + shared, + port, + commands, + supervisor: Some(supervisor), + _env: env, + } + } + + fn send(&self, command: SupervisorCommand) { + self.commands.send(command).unwrap(); + } + + fn streaming(&self) -> bool { + PIPELINE_STATS.lock().pipeline_running + } + + /// Waits for the supervisor to stop the stream after Stop. + fn wait_until_stopped(&self) { + let deadline = Instant::now() + Duration::from_secs(20); + while self.shared.session.lock().not_streaming() != Some(NotStreaming::Stopped) { + assert!(Instant::now() < deadline, "the stream never stopped"); + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +impl Drop for Host { + fn drop(&mut self) { + eternal_host::supervisor::TEST_BACKOFF_BASE_MS.store(500, Ordering::SeqCst); + #[cfg(debug_assertions)] + eternal_host::network::set_test_adapters(None); + self.shared.stop(); + let _ = self.commands.send(SupervisorCommand::Shutdown); + if let Some(supervisor) = self.supervisor.take() { + let _ = supervisor.join(); + } + } +} + +/// A GPU whose encoder FFmpeg doesn't have: every generation fails as it +/// starts, as a broken driver does. +fn broken_gpu() -> GpuInfo { + GpuInfo { + encoder_name: "no_such_encoder".into(), + ..GpuInfo::software_fallback() + } +} + +struct FakeIpad { + socket: UdpSocket, + host: String, + device_id: u64, + feature_caps: u16, + auth_token: [u8; 16], + /// The bytes of the last HELLO2 sent and of the last ACK received. + sizes: std::cell::Cell<(usize, usize)>, +} + +impl FakeIpad { + fn new(host: &Host, device_id: u64, feature_caps: u16) -> Self { + Self::at(std::net::Ipv4Addr::LOCALHOST, host, device_id, feature_caps) + } + + /// An iPad that reaches the PC at `ip`. + fn at(ip: std::net::Ipv4Addr, host: &Host, device_id: u64, feature_caps: u16) -> Self { + let socket = UdpSocket::bind((ip, 0)).unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let _ = socket2::SockRef::from(&socket).set_recv_buffer_size(4 * 1024 * 1024); + Self { + socket, + host: format!("{ip}:{}", host.port), + device_id, + feature_caps, + auth_token: [0; 16], + sizes: Default::default(), + } + } + + fn hello(&self, nonce: u32) { + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: nonce, + listen_port: self.socket.local_addr().unwrap().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: self.feature_caps, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 640, + screen_pt_h: 360, + refresh_hz: 60, + // A long name, so a 0.3.4 iPad's HELLO2 is larger than an + // answer carrying any PC name a test machine has. + device_name: "standby fake iPad named like a real one".to_string(), + device_id: self.device_id, + preferred_fps: 0, + auth_token: self.auth_token, + pairing_code: 0, + }); + self.send(&encode_control(0, nonce, &hello)); + } + + fn send(&self, datagram: &[u8]) { + self.socket.send_to(datagram, &self.host).unwrap(); + self.sizes.set((datagram.len(), 0)); + } + + /// The next HELLO_ACK for `nonce` within `wait`, skipping video, + /// heartbeats and older answers. + fn ack(&self, nonce: u32, wait: Duration) -> Option { + let deadline = Instant::now() + wait; + let mut buffer = [0; 2048]; + while Instant::now() < deadline { + if let Ok(len) = self.socket.recv(&mut buffer) { + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&buffer[..len]) { + if ack.client_nonce == nonce { + self.sizes.set((self.sizes.get().0, len)); + return Some(ack); + } + } + } + } + None + } + + /// Asks once, as an iPad's first HELLO2 does. The answer must be prompt. + fn ask(&self, nonce: u32) -> Option { + self.hello(nonce); + self.ack(nonce, Duration::from_secs(2)) + } + + /// Keeps asking, as a waiting iPad does, until the PC answers at all. + fn ask_until_answered(&self, nonce: u32, within: Duration) -> Option { + let deadline = Instant::now() + within; + while Instant::now() < deadline { + self.hello(nonce); + if let Some(ack) = self.ack(nonce, Duration::from_millis(500)) { + return Some(ack); + } + } + None + } + + /// Keeps asking, as an iPad does, until the PC streams to it. + fn connect(&self, nonce: u32) -> HelloAck { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + assert!(Instant::now() < deadline, "never connected"); + self.hello(nonce); + if let Some(ack) = self.ack(nonce, Duration::from_millis(250)) { + if ack.status == HelloStatus::Ok { + return ack; + } + } + } + } +} + +/// The smallest HELLO2 there is: no name, no device id or token, no +/// feature bits: 38 bytes. +fn minimal_hello(nonce: u32) -> Vec { + encode_control( + 0, + nonce, + &ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: nonce, + listen_port: 0, + decoder_caps: CAP_DECODE_H264, + feature_caps: 0, + screen_px_w: 1, + screen_px_h: 1, + screen_pt_w: 1, + screen_pt_h: 1, + refresh_hz: 60, + device_name: String::new(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }), + ) +} + +/// While no stream runs, the PC never answers with more bytes than it was +/// sent. A small HELLO2 from an unpaired sender hears nothing; an iPad that +/// asks for standby answers pads its HELLO2 and hears every answer. +#[test] +fn a_stopped_pc_never_answers_with_more_than_it_was_sent() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + + let small_sender = FakeIpad::new(&host, 1, OLD); + let small = minimal_hello(1); + assert_eq!(small.len(), 38); + small_sender.send(&small); + assert!( + small_sender.ack(1, Duration::from_secs(2)).is_none(), + "an answer larger than a 38-byte HELLO2 went out" + ); + + let unpaired = FakeIpad::new(&host, 2, NEW); + let ack = unpaired.ask(2).expect("a padded HELLO2 is answered"); + assert_eq!(ack.status, HelloStatus::Unauthorized); + let (sent, received) = unpaired.sizes.get(); + assert!(received <= sent, "{received} > {sent}"); + + let mut paired = FakeIpad::new(&host, 3, NEW); + paired.auth_token = host.shared.pairing.lock().token(); + let ack = paired.ask(3).expect("a padded HELLO2 is answered"); + assert_eq!(ack.status, HelloStatus::Stopped); + let (sent, received) = paired.sizes.get(); + assert!(received <= sent, "{received} > {sent}"); + + // A paired 0.3.4 iPad: an answer that fits goes out, one that doesn't + // is today's silence. + let mut old = FakeIpad::new(&host, 4, OLD); + old.auth_token = paired.auth_token; + if let Some(ack) = old.ask(4) { + assert_eq!(ack.status, HelloStatus::Error); + let (sent, received) = old.sizes.get(); + assert!(received <= sent, "{received} > {sent}"); + } +} + +/// After Stop, a new iPad hears that the PC isn't streaming, an old one is +/// turned down at once instead of timing out, and nothing starts by itself. +/// After Start, the waiting iPad connects. +#[test] +fn a_stopped_pc_says_so_and_streams_again_only_after_start() { + let host = Host::start(GpuInfo::software_fallback(), false); + let ipad = FakeIpad::new(&host, 1, NEW); + assert_eq!(ipad.connect(1).status, HelloStatus::Ok); + + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let ack = ipad.ask(2).expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Stopped); + assert_eq!(ack.session_id, 0); + let old = FakeIpad::new(&host, 2, OLD); + let ack = old.ask(3).expect("a stopped PC must answer an old iPad"); + assert_eq!(ack.status, HelloStatus::Error); + + // Asking again and again never starts the stream: only Start does. + for nonce in 4..10 { + let ack = ipad.ask(nonce).expect("still answered"); + assert_eq!(ack.status, HelloStatus::Stopped); + std::thread::sleep(Duration::from_millis(200)); + } + assert!(!host.streaming(), "a HELLO2 started the stream after Stop"); + + host.send(SupervisorCommand::Start); + let ack = ipad.connect(10); + assert_ne!(ack.session_id, 0); +} + +/// An unpaired iPad learns nothing new from a stopped PC: it is asked for +/// the pairing code, as when it streams. A paired one hears it's stopped. +#[test] +fn a_stopped_pc_still_asks_an_unpaired_ipad_to_pair() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let unpaired = FakeIpad::new(&host, 1, NEW); + let ack = unpaired.ask(1).expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Unauthorized); + assert!(ack.reason.is_empty()); + let mut paired = FakeIpad::new(&host, 2, NEW); + paired.auth_token = host.shared.pairing.lock().token(); + let ack = paired.ask(2).expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Stopped); + assert_eq!(ack.auth_token, [0; 16], "no token while stopped"); +} + +/// While the supervisor waits out the pause before an automatic restart, a +/// new iPad hears "restarting", and an old iPad is never turned down: it +/// hears nothing and keeps retrying, as before. +#[test] +fn a_pc_restarting_after_a_failure_says_so() { + let host = Host::start(broken_gpu(), false); + let ipad = FakeIpad::new(&host, 1, NEW); + let old = FakeIpad::new(&host, 2, OLD); + let deadline = Instant::now() + Duration::from_secs(20); + let mut restarting = 0; + let mut old_answers = Vec::new(); + for nonce in 1.. { + assert!(Instant::now() < deadline, "never heard Restarting"); + if let Some(ack) = ipad.ask(nonce) { + restarting += usize::from(ack.status == HelloStatus::Restarting); + } + old.hello(nonce); + if let Some(ack) = old.ack(nonce, Duration::from_millis(50)) { + old_answers.push(ack.status); + } + if restarting >= 3 { + break; + } + } + assert!( + old_answers.iter().all(|status| *status == HelloStatus::Ok), + "an old iPad must not be turned down mid-restart: {old_answers:?}" + ); +} + +/// Every HELLO2 sent across a Restart is answered: the port stays bound +/// while it changes hands, so none falls into the gap. +#[test] +fn no_hello_is_lost_across_a_restart() { + let host = Host::start(GpuInfo::software_fallback(), false); + let ipad = FakeIpad::new(&host, 1, NEW); + assert_eq!(ipad.connect(1).status, HelloStatus::Ok); + + const SENT: u32 = 300; + let mut answers = BTreeMap::new(); + let mut buffer = [0; 2048]; + let mut collect = |ipad: &FakeIpad, answers: &mut BTreeMap| { + while let Ok(len) = ipad.socket.recv(&mut buffer) { + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&buffer[..len]) { + answers.insert(ack.client_nonce, ack.status); + } + } + }; + for nonce in 100..100 + SENT { + if nonce == 150 { + host.send(SupervisorCommand::Restart); + } + ipad.hello(nonce); + std::thread::sleep(Duration::from_millis(5)); + if nonce % 10 == 0 { + collect(&ipad, &mut answers); + } + } + let deadline = Instant::now() + Duration::from_secs(10); + while answers.len() < SENT as usize && Instant::now() < deadline { + collect(&ipad, &mut answers); + } + let missing: Vec<_> = (100..100 + SENT) + .filter(|nonce| !answers.contains_key(nonce)) + .collect(); + assert!(missing.is_empty(), "unanswered HELLO2 nonces: {missing:?}"); + assert!( + answers + .values() + .all(|status| matches!(status, HelloStatus::Ok | HelloStatus::Restarting)), + "{answers:?}" + ); +} + +/// After repeated failures the storm brake stops the stream until someone +/// clicks Start. A new iPad hears why, an old one is turned down, and the +/// stream doesn't start by itself. +#[test] +fn the_storm_brake_says_why_and_waits_for_start() { + let host = Host::start_with_backoff(broken_gpu(), false, 1); + let ipad = FakeIpad::new(&host, 1, NEW); + let deadline = Instant::now() + Duration::from_secs(30); + let failed = (1..) + .find_map(|nonce| { + assert!(Instant::now() < deadline, "never heard Failed"); + ipad.ask(nonce) + .filter(|ack| ack.status == HelloStatus::Failed) + }) + .unwrap(); + assert_eq!(failed.reason, "encoder"); + assert_eq!(failed.session_id, 0); + let (sent, received) = ipad.sizes.get(); + assert!(received <= sent, "{received} > {sent}"); + let old = FakeIpad::new(&host, 2, OLD); + assert_eq!( + old.ask(1) + .expect("a failed PC must answer an old iPad") + .status, + HelloStatus::Error + ); + std::thread::sleep(Duration::from_secs(1)); + assert!(!host.streaming()); + let ack = ipad.ask(1000).expect("still answered"); + assert_eq!(ack.status, HelloStatus::Failed); +} + +/// A stopped PC listens where iPads can reach it, as the stream does: on +/// this PC only while the chosen network is down, on the network's address +/// once it is back, on this PC again when it goes, and on every network +/// once Listen on says so. It never starts the stream to do so. +#[cfg(debug_assertions)] +#[test] +fn a_stopped_pc_follows_the_chosen_network() { + use eternal_host::network::{default_route, set_test_adapters, Adapter, Kind, ListenOn}; + use std::net::IpAddr; + // Binding the network's address needs a real one. + let Some(IpAddr::V4(ip)) = default_route().filter(|ip| !ip.is_loopback()) else { + eprintln!("SKIP: no IPv4 default route to stand in for the chosen network"); + return; + }; + let wifi = move |up| { + vec![Adapter { + name: "Wi-Fi".into(), + kind: Kind::Wifi, + up, + gateway: true, + ipv4: vec![ip], + }] + }; + let host = Host::start_with(GpuInfo::software_fallback(), 500, |shared| { + shared.pairing.lock().required = false; + *shared.listen_on.lock() = ListenOn::Adapter("Wi-Fi".into()); + set_test_adapters(Some(wifi(false))); + }); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let here = FakeIpad::new(&host, 1, NEW); + let lan = FakeIpad::at(ip, &host, 2, NEW); + let ack = here + .ask(1) + .expect("answered on this PC while the network is down"); + assert_eq!(ack.status, HelloStatus::Stopped); + + set_test_adapters(Some(wifi(true))); + let ack = lan + .ask_until_answered(2, Duration::from_secs(20)) + .expect("never answered on the network's address once it came back"); + assert_eq!(ack.status, HelloStatus::Stopped); + + set_test_adapters(Some(wifi(false))); + let ack = here + .ask_until_answered(3, Duration::from_secs(20)) + .expect("never answered on this PC once the network went"); + assert_eq!(ack.status, HelloStatus::Stopped); + + // Listen on goes back to every network while stopped: the network's + // address answers again, though the adapter is still down. + *host.shared.listen_on.lock() = ListenOn::All; + let ack = lan + .ask_until_answered(4, Duration::from_secs(20)) + .expect("never answered on every network after Listen on changed to All"); + assert_eq!(ack.status, HelloStatus::Stopped); + assert!( + !host.streaming(), + "following the network started the stream" + ); + assert_eq!( + host.shared.session.lock().not_streaming(), + Some(NotStreaming::Stopped) + ); +} diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 543ab12f..7a920dab 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -95,12 +95,15 @@ final class ConnectionManager: ObservableObject { @Published var pairingPrompt: PairingSheetModel? /// HELLO2 feature bits. Every build says it checks for its own updates, - /// so the PC tells only older iPad apps to update. + /// so the PC tells only older iPad apps to update, and that it + /// understands why a PC isn't streaming. nonisolated static func featureCaps(wantsInput: Bool, wantsAudio: Bool, isUSB: Bool) -> UInt16 { (wantsInput ? Hello2.featureWantsInput : 0) | (wantsAudio ? Hello2.featureWantsAudio : 0) - | (isUSB ? 0 : Hello2.featureSupportsNack) | Hello2.featureChecksUpdates + | (isUSB ? 0 : Hello2.featureSupportsNack) | Hello2.featureChecksUpdates | Hello2.featureStandbyStatus } private var awaitingPairing = false + /// The PC answered that it isn't streaming; the attempt waits for it. + private var waitingForStream = false private var explicitPairingToken: Data? private let pairingStore: PairingStore @Published private(set) var diagnostics: [DiagnosticEntry] = [] @@ -445,6 +448,7 @@ final class ConnectionManager: ObservableObject { prevCounters = FrameAssembler.Counters() diagnostics.removeAll() didExtendTimeout = false + waitingForStream = false attemptStartedAt = ProcessInfo.processInfo.systemUptime sessionAcceptedAt = nil firstFrameAt = nil @@ -677,6 +681,11 @@ final class ConnectionManager: ObservableObject { at: isUSB ? nil : "\(normalizedHost):\(port)") self.livenessTimeoutUs = UInt64(info.livenessTimeoutMs) * 1000 self.hostInfo = info + if self.waitingForStream { + self.waitingForStream = false + if case .pcNotStreaming = self.problem { self.problem = nil } + self.armConnectTimeout(seconds: self.timing.connectTimeoutSeconds) + } let seen = self.now() self.keepAppleDevicesSighting(info.hostCaps & HelloAck.hostCapUSB == 0 ? AppleDevicesSighting(target: pcTarget, at: seen, @@ -744,6 +753,20 @@ final class ConnectionManager: ObservableObject { self.disconnect(.pcRefused) } } + channel.onPCNotStreaming = { [weak self] status, reason in + Task { @MainActor in + guard let self, self.currentAttempt.withLock({ $0 == connectionID }), + let wait = ConnectionProblem.StreamWait(status, reason: reason) else { return } + if self.problem != .pcNotStreaming(wait) { + self.record(.info, "ctrl", "PC isn't streaming (\(status)\(reason.isEmpty ? "" : " \(reason)")); waiting for it") + } + // No reconnect attempt is spent while the PC says why it + // isn't streaming. If it stops answering, the timeout says so. + self.waitingForStream = true + self.problem = .pcNotStreaming(wait) + self.armConnectTimeout(seconds: self.timing.connectTimeoutSeconds) + } + } channel.onHandshakeTimeout = { [weak self, weak receiver] in Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } @@ -1216,7 +1239,9 @@ final class ConnectionManager: ObservableObject { self.record(level, "timeout", debug.timeoutSummary(handshakeDone: handshakeDone) + self.attemptTimings) // Only an attempt the PC answered counts. if handshakeDone { self.noPictureStreak.endedWithoutPicture() } - let problem = self.noPictureStreak.shown(.timeout( + // A PC that said it isn't streaming and then went quiet (it + // closed, or slept) didn't answer: its answers were no picture. + let problem: ConnectionProblem = self.waitingForStream ? .noReply(usb: isUSB) : self.noPictureStreak.shown(.timeout( listenerReady: debug.listenerReadyPort != nil, helloAttempts: debug.helloAttempts, handshakeDone: handshakeDone, datagramsReceived: debug.datagramsReceived, isUSB: isUSB, host: debug.host), at: isUSB ? nil : "\(debug.host):\(debug.port)") @@ -1270,6 +1295,8 @@ final class ConnectionManager: ObservableObject { timeoutTask?.cancel() endReconnectCycle() disconnect() + // The user stopped waiting for the PC to stream. + if case .pcNotStreaming = problem { problem = nil } } /// Try the last connection again: the cable when that is what failed, @@ -1427,6 +1454,7 @@ final class ConnectionManager: ObservableObject { recordSummary(end) } awaitingPairing = false + waitingForStream = false pairingPrompt = nil currentAttempt.withLock { $0 = nil } timeoutTask?.cancel() diff --git a/ios/EternalMonitor/App/ConnectionProblem.swift b/ios/EternalMonitor/App/ConnectionProblem.swift index 20c5df8f..ace092a3 100644 --- a/ios/EternalMonitor/App/ConnectionProblem.swift +++ b/ios/EternalMonitor/App/ConnectionProblem.swift @@ -41,6 +41,46 @@ enum ConnectionProblem: Equatable { case pcBusy /// The PC stopped the stream to install an update and restarts on it. case pcUpdating(UpdateWait) + /// The PC answered but isn't streaming. This iPad keeps asking and + /// connects as soon as it is. + case pcNotStreaming(StreamWait) + + enum StreamWait: Equatable { + /// Someone clicked Stop on the PC. + case stopped + case restarting + /// The PC stopped after repeated errors, for the reason it gave (a + /// token; one this iPad doesn't know is no reason). + case failed(String) + + /// What the PC's answer means, if it is one of these. + init?(_ status: HelloStatus, reason: String) { + switch status { + case .stopped: self = .stopped + case .restarting: self = .restarting + case .failed: self = .failed(reason) + default: return nil + } + } + + /// The PC's reason for `.failed`, in words, if this iPad knows it. + /// The PC's Stream page says the same. + static func failedReason(_ token: String) -> String? { + switch token { + case "capture": return "Capturing your screen kept failing" + case "encoder": return "Encoding the video kept failing" + case "transport": return "Sending the video kept failing" + case "secure_desktop": return "Your PC is showing the lock screen or a permission prompt" + case "display_changing": return "Windows is changing the display" + case "no_display": return "No display is on" + case "unsupported_gpu": return "Windows won't share this screen from the graphics card EternalMonitor uses" + case "too_many_capture_apps": return "Another app is recording or sharing your screen" + case "session_disconnected": return "This Windows session is disconnected" + case "driver_not_responding": return "The graphics driver isn't responding" + default: return nil + } + } + } /// What this iPad does while the PC updates. enum UpdateWait: Equatable { @@ -89,6 +129,9 @@ enum ConnectionProblem: Equatable { case .pcBusy: return "Your PC is already streaming to another device" case .pcUpdating(.reconnecting), .pcUpdating(.manual): return "Your PC is updating EternalMonitor" case .pcUpdating(.notBack): return "Your PC hasn't come back from its update yet" + case .pcNotStreaming(.stopped): return "Your PC isn't streaming" + case .pcNotStreaming(.restarting): return "Your PC is restarting the stream…" + case .pcNotStreaming(.failed): return "Your PC stopped streaming after repeated errors" } } @@ -148,6 +191,13 @@ enum ConnectionProblem: Equatable { return "Check that EternalMonitor opened again on the PC, then try again. Windows may still be waiting for you to approve the update there." case .pcBusy: return "Disconnect the other iPad first, then try again." + case .pcNotStreaming(.stopped): + return "Click Start streaming in EternalMonitor on the PC. This iPad connects as soon as it starts." + case .pcNotStreaming(.restarting): + return "This iPad connects as soon as it's back." + case .pcNotStreaming(.failed(let token)): + let start = "Click Start streaming in EternalMonitor on the PC to try again. This iPad connects as soon as it starts." + return StreamWait.failedReason(token).map { "\($0). \(start)" } ?? start } } @@ -166,7 +216,8 @@ enum ConnectionProblem: Equatable { var canReport: Bool { switch self { case .reconnecting, .pcUpdating(.reconnecting), .pcUpdating(.manual), - .lost(.cableUnplugged), .lost(.usbTurnedOff), .pcBusy, .pairingCodeAsAddress: + .lost(.cableUnplugged), .lost(.usbTurnedOff), .pcBusy, .pairingCodeAsAddress, + .pcNotStreaming(.stopped), .pcNotStreaming(.restarting): return false default: return true @@ -176,7 +227,8 @@ enum ConnectionProblem: Equatable { var actions: [Action] { switch self { case .localNetworkBlocked: return [.openSettings, .retry] - case .reconnecting, .lost(.cableUnplugged), .lost(.usbTurnedOff), .receiverNotReady, .pairingCodeAsAddress: return [] + case .reconnecting, .lost(.cableUnplugged), .lost(.usbTurnedOff), .receiverNotReady, .pairingCodeAsAddress, + .pcNotStreaming: return [] default: return [.retry] } } diff --git a/ios/EternalMonitor/Network/ControlChannel.swift b/ios/EternalMonitor/Network/ControlChannel.swift index 2c2f2d58..5f540e64 100644 --- a/ios/EternalMonitor/Network/ControlChannel.swift +++ b/ios/EternalMonitor/Network/ControlChannel.swift @@ -22,6 +22,9 @@ final class ControlChannel { var onSessionEstablished: ((SessionInfo) -> Void)? var onRejected: ((HelloStatus) -> Void)? + /// The PC answered that it isn't streaming, with its reason token. The + /// same HELLO2 keeps going out slowly and connects once the PC streams. + var onPCNotStreaming: ((HelloStatus, String) -> Void)? /// The last refusal's version mismatch, if any. Read inside `onRejected`. private(set) var lastMismatch: VersionMismatch? /// HELLO2 went unanswered for the whole retry budget. @@ -47,6 +50,8 @@ final class ControlChannel { private var helloBytes = Data() private var helloTemplate: Hello2? private var helloAttempts = 0 + /// The PC said it isn't streaming: keep asking, without a retry budget. + private var waitingForStream = false private var helloTimer: DispatchSourceTimer? private var reportTimer: DispatchSourceTimer? private var pingTimer: DispatchSourceTimer? @@ -71,6 +76,10 @@ final class ControlChannel { private static let maxHelloAttempts = 8 private static let helloRetryMs = 250 + /// How often to ask a PC that answered it isn't streaming. A restart + /// takes a second or so; Start takes someone at the PC. + static let restartingRetryMs = 1000 + static let stoppedRetryMs = 2000 private static let pingIntervalMs = 2000 private static let pingBurstCount = 5 private static let pingBurstSpacingMs = 200 @@ -123,6 +132,7 @@ final class ControlChannel { hostCaps = 0 msgSeq = 0 helloAttempts = 0 + waitingForStream = false clientNonce = UInt32.random(in: 1...UInt32.max) lastHelloNonce = clientNonce let hello = Hello2( @@ -160,14 +170,15 @@ final class ControlChannel { hello.pairingCode = code helloTemplate = hello helloAttempts = 0 + waitingForStream = false helloBytes = Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .hello2(hello)) startHelloTimer() } } - private func startHelloTimer() { + private func startHelloTimer(everyMs: Int = ControlChannel.helloRetryMs, firstAfterMs: Int = 0) { let timer = DispatchSource.makeTimerSource(queue: queue) - timer.schedule(deadline: .now(), repeating: .milliseconds(Self.helloRetryMs)) + timer.schedule(deadline: .now() + .milliseconds(firstAfterMs), repeating: .milliseconds(everyMs)) timer.setEventHandler { [weak self] in self?.helloTick() } helloTimer?.cancel() helloTimer = timer @@ -180,6 +191,10 @@ final class ControlChannel { helloTimer = nil return } + if waitingForStream { + send(helloBytes) + return + } helloAttempts += 1 if helloAttempts > Self.maxHelloAttempts { helloTimer?.cancel() @@ -238,6 +253,23 @@ final class ControlChannel { helloTimer?.cancel() helloTimer = nil + if ack.status.pcNotStreaming { + // The nonce outlives the wait, so a late or reordered copy can + // come after the OK. The session is up: it means nothing now. + guard sessionId == 0 else { + onDiagnostic?("Ignored a late HELLO_ACK (\(ack.status)) for a session already established") + return + } + // Same nonce: the PC answers this HELLO2 with a session once it streams. + if !waitingForStream { onDiagnostic?("PC answered that it isn't streaming (\(ack.status))") } + waitingForStream = true + let everyMs = ack.status == .restarting ? Self.restartingRetryMs : Self.stoppedRetryMs + startHelloTimer(everyMs: everyMs, firstAfterMs: everyMs) + onPCNotStreaming?(ack.status, ack.reason) + return + } + waitingForStream = false + guard ack.status == .ok else { clientNonce = 0 lastMismatch = helloTemplate.flatMap { @@ -370,6 +402,7 @@ final class ControlChannel { helloTemplate = nil helloBytes = Data() clientNonce = 0 + waitingForStream = false sessionId = 0 hostCaps = 0 estimator.reset() diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index f7a15526..88cce1da 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -255,6 +255,13 @@ struct Hello2: Equatable { static let featureWantsInput: UInt16 = 1 << 0 static let featureWantsAudio: UInt16 = 1 << 1 static let featureSupportsNack: UInt16 = 1 << 2 + /// This app understands the stopped, restarting and failed statuses. A PC + /// sends them only to an iPad that sets it. + static let featureStandbyStatus: UInt16 = 1 << 3 + /// A HELLO2 that sets `featureStandbyStatus` is padded with zeros to at + /// least this many bytes: the longest HELLO_ACK there is. A PC answers + /// while it isn't streaming only with no more bytes than it was sent. + static let standbyLength = 164 /// This app checks for its own updates, so the PC need not tell the user. 0.3.1 and older never set it. static let featureChecksUpdates: UInt16 = 1 << 4 static let maxNameLength = 64 @@ -284,6 +291,15 @@ enum HelloStatus: UInt8 { case error = 3 case unauthorized = 4 case rateLimited = 5 + /// The PC answered but isn't streaming: someone clicked Stop on it, + case stopped = 6 + /// it is restarting its stream, + case restarting = 7 + /// or it stopped after repeated errors (`HelloAck.reason` says why). + case failed = 8 + + /// The PC is there and may stream again: keep asking. + var pcNotStreaming: Bool { self == .stopped || self == .restarting || self == .failed } } struct HelloAck: Equatable { @@ -306,6 +322,10 @@ struct HelloAck: Equatable { var hostName: String var authToken = Data(repeating: 0, count: 16) var hostCaps: UInt16 = 0 + /// Why the PC isn't streaming, for `.failed`: a token such as + /// "display_changing". Empty otherwise. + var reason = "" + static let maxReasonLength = 32 } struct Heartbeat: Equatable { @@ -498,6 +518,10 @@ extension Wire { body.append(h.authToken) body.appendLE(h.pairingCode) } + let padded = Hello2.standbyLength - ControlHeader.size + if h.featureCaps & Hello2.featureStandbyStatus != 0, body.count < padded { + body.append(Data(count: padded - body.count)) + } case .helloAck(let a): let name = Data(a.hostName.utf8).prefixOnCharBoundary(a.hostName, max: Hello2.maxNameLength) body.append(a.status.rawValue) @@ -511,10 +535,15 @@ extension Wire { body.append(UInt8(name.count)) body.append(name) precondition(a.authToken.count == 16) - if a.hostCaps != 0 || a.authToken.contains(where: { $0 != 0 }) { + let reason = Data(a.reason.utf8).prefixOnCharBoundary(a.reason, max: HelloAck.maxReasonLength) + if a.hostCaps != 0 || a.authToken.contains(where: { $0 != 0 }) || !reason.isEmpty { body.append(a.authToken) body.appendLE(a.hostCaps) } + if !reason.isEmpty { + body.append(UInt8(reason.count)) + body.append(reason) + } case .heartbeat(let hb): body.appendLE(hb.hostTimeUs) hb.streamConfig.encode(into: &body) @@ -745,6 +774,11 @@ extension Wire { ack.authToken = Data(token) ack.hostCaps = caps } + if r.remaining > 0 { + guard let length = r.readU8(), Int(length) <= HelloAck.maxReasonLength, let bytes = r.take(Int(length)), + let reason = String(bytes: bytes, encoding: .utf8) else { return nil } + ack.reason = reason + } return .helloAck(ack) } diff --git a/ios/EternalMonitorTests/ConnectionProblemTests.swift b/ios/EternalMonitorTests/ConnectionProblemTests.swift index dc73d20a..e29aff0a 100644 --- a/ios/EternalMonitorTests/ConnectionProblemTests.swift +++ b/ios/EternalMonitorTests/ConnectionProblemTests.swift @@ -14,6 +14,34 @@ final class ConnectionProblemTests: XCTestCase { XCTAssertEqual(ConnectionProblem.timeout(listenerReady: true, helloAttempts: 1, handshakeDone: true, datagramsReceived: 0, isUSB: true), .noPicture) } + /// The PC's not-streaming answers, in plain words. None offers Try + /// again: the iPad is already asking again by itself. + func testNotStreamingAnswersSayWhatToDo() { + typealias Wait = ConnectionProblem.StreamWait + XCTAssertEqual(Wait(.stopped, reason: ""), .stopped) + XCTAssertEqual(Wait(.restarting, reason: "ignored"), .restarting) + XCTAssertEqual(Wait(.failed, reason: "encoder"), .failed("encoder")) + for status: HelloStatus in [.ok, .busy, .versionUnsupported, .error, .unauthorized, .rateLimited] { + XCTAssertNil(Wait(status, reason: ""), "\(status)") + } + XCTAssertEqual(ConnectionProblem.pcNotStreaming(.restarting).detail, "This iPad connects as soon as it's back.") + XCTAssertEqual(ConnectionProblem.pcNotStreaming(.failed("encoder")).detail, + "Encoding the video kept failing. Click Start streaming in EternalMonitor on the PC to try again. " + + "This iPad connects as soon as it starts.") + // Every token the PC sends has words here. + for token in ["capture", "encoder", "transport", "secure_desktop", "display_changing", "no_display", + "unsupported_gpu", "too_many_capture_apps", "session_disconnected", "driver_not_responding"] { + XCTAssertNotNil(Wait.failedReason(token), token) + } + for wait: Wait in [.stopped, .restarting, .failed("encoder")] { + XCTAssertEqual(ConnectionProblem.pcNotStreaming(wait).actions, []) + } + // Only a PC that ran into errors offers Report this problem. + XCTAssertFalse(ConnectionProblem.pcNotStreaming(.stopped).canReport) + XCTAssertFalse(ConnectionProblem.pcNotStreaming(.restarting).canReport) + XCTAssertTrue(ConnectionProblem.pcNotStreaming(.failed("")).canReport) + } + /// PCs on 0.3.1 and 0.3.2 answer but send no picture in Second screen /// mode. The card says to mirror, naming the choice as both versions do. func testNoPictureSaysToMirrorTheScreen() { diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index e2ba4be5..ee97164a 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -14,6 +14,9 @@ final class FakeUDPHost: @unchecked Sendable { private var answered = Set() /// The nonce of the HELLO2 answered last. private(set) var lastNonce: UInt32 = 0 + /// The HELLO2 answered last, and its size in bytes. + private(set) var lastHello: Hello2? + private(set) var lastHelloSize = 0 /// The pairing token the HELLO2 answered last carried. private(set) var lastAuthToken = Data() /// A HELLO2 came before the identify request was answered. @@ -53,18 +56,22 @@ final class FakeUDPHost: @unchecked Sendable { /// Answers the next HELLO2 attempt, with session 41 unless `status` /// refuses it. False after thirty quiet seconds, a backstop that a - /// loaded machine's slow start does not reach. + /// loaded machine's slow start does not reach. An attempt told the PC + /// isn't streaming keeps its nonce and can be answered again. func acceptSession(status: HelloStatus = .ok, acceptedVersion: UInt8 = 2, hostCaps: UInt16 = 0, - hostName: String = "Test PC", authToken: Data = Data(17...32)) -> Bool { + hostName: String = "Test PC", authToken: Data = Data(17...32), reason: String = "") -> Bool { for _ in 0..<100 { guard let (data, from, length) = receive() else { continue } guard case .hello2(let hello) = Wire.parseControl(data)?.message, - answered.insert(hello.clientNonce).inserted else { continue } + !answered.contains(hello.clientNonce) else { continue } + if !status.pcNotStreaming { answered.insert(hello.clientNonce) } client = (from, length) lastNonce = hello.clientNonce + lastHello = hello + lastHelloSize = data.count lastAuthToken = hello.authToken send(Self.ack(nonce: hello.clientNonce, status: status, acceptedVersion: acceptedVersion, - hostName: hostName, authToken: authToken, hostCaps: hostCaps)) + hostName: hostName, authToken: authToken, hostCaps: hostCaps, reason: reason)) return true } return false @@ -73,11 +80,13 @@ final class FakeUDPHost: @unchecked Sendable { /// The HELLO_ACK for the attempt with `nonce` from the PC `hostName` /// with pairing token `authToken`: session 41, unless `status` refuses it. static func ack(nonce: UInt32, status: HelloStatus = .ok, acceptedVersion: UInt8 = 2, - hostName: String = "Test PC", authToken: Data = Data(17...32), hostCaps: UInt16 = 0) -> Data { + hostName: String = "Test PC", authToken: Data = Data(17...32), hostCaps: UInt16 = 0, + reason: String = "") -> Data { Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .helloAck(HelloAck( status: status, acceptedVersion: acceptedVersion, clientNonce: nonce, sessionId: status == .ok ? 41 : 0, heartbeatIntervalMs: 1000, reportIntervalMs: 500, livenessTimeoutMs: 3000, - streamConfig: StreamConfig(), hostName: hostName, authToken: authToken, hostCaps: hostCaps))) + streamConfig: StreamConfig(), hostName: hostName, authToken: authToken, hostCaps: hostCaps, + reason: reason))) } private func send(_ datagram: Data) { @@ -91,6 +100,11 @@ final class FakeUDPHost: @unchecked Sendable { } } + /// A HELLO_ACK for `nonce`, sent again late, as Wi-Fi can reorder it. + func sendAck(nonce: UInt32, status: HelloStatus) { + send(Self.ack(nonce: nonce, status: status)) + } + /// Answers the next identify request as the PC holding `token` would, /// receiving at this fake's port on `addresses` (loopback unless given), /// or with `proof` instead. False after thirty quiet seconds. diff --git a/ios/EternalMonitorTests/PCNotStreamingTests.swift b/ios/EternalMonitorTests/PCNotStreamingTests.swift new file mode 100644 index 00000000..f0ff7b04 --- /dev/null +++ b/ios/EternalMonitorTests/PCNotStreamingTests.swift @@ -0,0 +1,170 @@ +import XCTest +@testable import EternalMonitor + +/// A PC that answers but isn't streaming (Stop clicked, restarting, or +/// stopped after repeated errors) says so. The iPad shows why, keeps asking +/// slowly without spending reconnect attempts, and connects as soon as the +/// PC streams again. +@MainActor +final class PCNotStreamingTests: XCTestCase { + private func until(_ description: String, seconds: Double = 30, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + /// Longer than the slow HELLO2 retry, so a waiting attempt never times + /// out between two answers, even on a loaded machine. + private let timing = ConnectionManager.Timing( + connectTimeoutSeconds: 5, reconnectGraceSeconds: 0.1, reconnectBackoffSeconds: { _ in 0.05 }) + + func testAStoppedPCIsWaitedForAndConnectsOnceItStarts() async throws { + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let answered = await Task.detached { host.acceptSession(status: .stopped) }.value + XCTAssertTrue(answered, "the iPad never sent HELLO2") + let advertised = (host.lastHello?.featureCaps ?? 0) & Hello2.featureStandbyStatus + XCTAssertNotEqual(advertised, 0, "the PC must know this iPad understands the answer") + XCTAssertGreaterThanOrEqual(host.lastHelloSize, Hello2.standbyLength, "padded so any answer fits") + let firstNonce = host.lastNonce + try await until("told the PC isn't streaming") { manager.problem == .pcNotStreaming(.stopped) } + XCTAssertEqual(manager.problem?.title, "Your PC isn't streaming") + XCTAssertEqual(manager.problem?.detail, + "Click Start streaming in EternalMonitor on the PC. This iPad connects as soon as it starts.") + XCTAssertEqual(manager.state, .connecting) + + // It keeps asking, every two seconds rather than four times a + // second, with the same HELLO2, past the connect timeout. + var answeredAt = [Date()] + for _ in 0..<3 { + let again = await Task.detached { host.acceptSession(status: .stopped) }.value + XCTAssertTrue(again, "the iPad stopped asking") + answeredAt.append(Date()) + XCTAssertEqual(host.lastNonce, firstNonce) + } + for (earlier, later) in zip(answeredAt.dropLast(), answeredAt.dropFirst()) { + XCTAssertGreaterThan(later.timeIntervalSince(earlier), 1.5, "asking too often") + } + XCTAssertEqual(manager.state, .connecting) + XCTAssertEqual(manager.problem, .pcNotStreaming(.stopped)) + + // Start on the PC: the same attempt connects. + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted) + XCTAssertEqual(host.lastNonce, firstNonce) + try await until("handshake") { manager.hostInfo != nil } + XCTAssertNil(manager.problem) + } + + func testAFailedPCSaysWhyInWords() async throws { + for (token, detail) in [ + ("display_changing", "Windows is changing the display. Click Start streaming in EternalMonitor on the PC " + + "to try again. This iPad connects as soon as it starts."), + ("a_reason_from_a_newer_pc", "Click Start streaming in EternalMonitor on the PC to try again. " + + "This iPad connects as soon as it starts.") + ] { + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + manager.connect(host: "127.0.0.1", port: host.port) + let answered = await Task.detached { host.acceptSession(status: .failed, reason: token) }.value + XCTAssertTrue(answered) + try await until("told why") { manager.problem == .pcNotStreaming(.failed(token)) } + XCTAssertEqual(manager.problem?.title, "Your PC stopped streaming after repeated errors") + XCTAssertEqual(manager.problem?.detail, detail) + XCTAssertEqual(manager.problem?.actions, []) + XCTAssertEqual(manager.state, .connecting) + manager.cancel() + } + } + + /// The PC was stopped, then closed: once it stops answering, the iPad + /// says it didn't answer, not that no picture arrived. + func testAPCThatStopsAnsweringTimesOutAsNoReply() async throws { + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let answered = await Task.detached { host.acceptSession(status: .stopped) }.value + XCTAssertTrue(answered) + try await until("timed out") { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .noReply(usb: false)) + } + + /// A restarting PC doesn't run the reconnect cycle out: the cycle's + /// attempt waits, and connects when the stream is back. + func testARestartingPCDoesNotSpendReconnectAttempts() async throws { + let previous = UserDefaults.standard.object(forKey: "autoReconnect") + UserDefaults.standard.set(true, forKey: "autoReconnect") + defer { UserDefaults.standard.set(previous, forKey: "autoReconnect") } + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let first = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(first) + host.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming") { manager.state == .connected } + + // The PC's stream fails and it restarts. + host.sendBye(.error) + try await until("reconnecting") { manager.reconnectAttempt == 1 } + var answeredAt: [Date] = [] + for _ in 0..<4 { + let answered = await Task.detached { host.acceptSession(status: .restarting) }.value + XCTAssertTrue(answered, "the iPad stopped asking") + answeredAt.append(Date()) + } + for (earlier, later) in zip(answeredAt.dropLast(), answeredAt.dropFirst()) { + XCTAssertGreaterThan(later.timeIntervalSince(earlier), 0.7, "asking too often") + } + XCTAssertEqual(manager.problem, .pcNotStreaming(.restarting)) + XCTAssertEqual(manager.problem?.title, "Your PC is restarting the stream…") + XCTAssertEqual(manager.reconnectAttempt, 1, "waiting must not spend attempts") + + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted) + try await until("handshake") { manager.hostInfo != nil } + host.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming again") { manager.state == .connected } + XCTAssertNil(manager.problem) + XCTAssertEqual(manager.reconnectAttempt, 0) + } + + /// The attempt keeps its nonce while it waits, so a Restarting answer + /// can arrive after the OK (Wi-Fi reordered it, or a late copy). Once the + /// session is up, it changes nothing: no card, no new timeout. + func testARestartingAnswerAfterTheSessionStartedIsIgnored() async throws { + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let waiting = await Task.detached { host.acceptSession(status: .restarting) }.value + XCTAssertTrue(waiting) + try await until("waiting") { manager.problem == .pcNotStreaming(.restarting) } + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted) + try await until("handshake") { manager.hostInfo != nil } + XCTAssertNil(manager.problem) + + host.sendAck(nonce: host.lastNonce, status: .restarting) + try await Task.sleep(nanoseconds: 500_000_000) + XCTAssertNil(manager.problem, "a late Restarting answer brought the card back") + host.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming") { manager.state == .connected } + XCTAssertNil(manager.problem) + } + + func testCancelWhileWaitingClearsTheMessage() async throws { + let host = FakeUDPHost() + let manager = ConnectionManager(timing: timing) + manager.connect(host: "127.0.0.1", port: host.port) + let answered = await Task.detached { host.acceptSession(status: .stopped) }.value + XCTAssertTrue(answered) + try await until("waiting") { manager.problem == .pcNotStreaming(.stopped) } + manager.cancel() + XCTAssertEqual(manager.state, .disconnected) + XCTAssertNil(manager.problem) + } +} diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index 9bb9df14..888bd53d 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,7 +35,7 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 32, + Self.vectors.count, 36, "golden vector count drifted — update both test suites together" ) } @@ -79,6 +79,53 @@ final class WireProtocolGoldenTests: XCTestCase { } } + /// 0.3.5: the PC answers while it isn't streaming, a Failed answer with + /// its reason token. + func testNotStreamingVectors() throws { + for (name, status, reason): (String, HelloStatus, String) in [ + ("hello_ack_stopped", .stopped, ""), ("hello_ack_restarting", .restarting, ""), + ("hello_ack_failed", .failed, "display_changing") + ] { + let data = try vector(name) + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .helloAck(let ack) = message else { return XCTFail("wrong type") } + XCTAssertEqual(ack.status, status) + XCTAssertTrue(ack.status.pcNotStreaming) + XCTAssertEqual(ack.reason, reason) + XCTAssertEqual(ack.sessionId, 0) + XCTAssertEqual(ack.hostCaps, HelloAck.hostCapNack) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + } + } + + /// An iPad that asks for standby answers pads its HELLO2 with zeros, as + /// the PC encodes it, so no answer is larger than the question. + func testAStandbyHelloIsPaddedToFitAnyAnswer() throws { + let data = try vector("hello2_standby_padded") + XCTAssertEqual(data.count, Hello2.standbyLength) + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .hello2(let hello) = message else { return XCTFail("wrong type") } + XCTAssertNotEqual(hello.featureCaps & Hello2.featureStandbyStatus, 0) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + // Without the bit, nothing is added. + var plain = hello + plain.featureCaps &= ~Hello2.featureStandbyStatus + XCTAssertLessThan(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .hello2(plain)).count, data.count) + } + + /// How 0.3.4 and older treat a status they don't know: the whole ACK is + /// dropped, as if it were lost. The PC sends 6-8 only to iPads that set + /// `featureStandbyStatus`. + func testAnUnknownStatusIsDroppedLikeALostAck() throws { + var data = try vector("hello_ack_busy") + data[16] = 9 + XCTAssertNil(Wire.parseControl(data)) + XCTAssertEqual(Hello2.featureStandbyStatus, 1 << 3) + for status: HelloStatus in [.ok, .busy, .versionUnsupported, .error, .unauthorized, .rateLimited] { + XCTAssertFalse(status.pcNotStreaming, "\(status)") + } + } + func testNativePenVectorsAndTruncatedTilt() throws { for (name, phase, buttons, pressure): (String, UInt8, UInt8, UInt16) in [ ("input_pen_down", 0, 1, 750), ("input_pen_hover", 1, 0, 0), ("input_pen_cancel", 3, 1, 0) diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index 72386841..f9528fe2 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -37,6 +37,10 @@ pub const CAP_DECODE_HEVC_10BIT: u16 = 1 << 2; pub const FEATURE_WANTS_INPUT: u16 = 1 << 0; pub const FEATURE_WANTS_AUDIO: u16 = 1 << 1; pub const FEATURE_SUPPORTS_NACK: u16 = 1 << 2; +/// The iPad understands [`HelloStatus::Stopped`], [`HelloStatus::Restarting`] +/// and [`HelloStatus::Failed`] (0.3.5 and later). Older iPads drop an ACK +/// whose status they don't know, so the host sends them only statuses 0-5. +pub const FEATURE_STANDBY_STATUS: u16 = 1 << 3; /// The iPad app checks for its own updates (0.3.2 and later), so a HELLO2 /// without it comes from 0.3.1 or older. Older hosts ignore it. pub const FEATURE_CHECKS_UPDATES: u16 = 1 << 4; @@ -136,6 +140,13 @@ pub enum HelloStatus { Error = 3, Unauthorized = 4, RateLimited = 5, + /// The PC answered but isn't streaming: someone clicked Stop on it. + Stopped = 6, + /// The PC is restarting its stream; ask again shortly. + Restarting = 7, + /// The PC stopped streaming after repeated errors, until someone clicks + /// Start. [`HelloAck::reason`] carries a [`FailedReason`] token. + Failed = 8, } impl HelloStatus { @@ -147,6 +158,9 @@ impl HelloStatus { 3 => Self::Error, 4 => Self::Unauthorized, 5 => Self::RateLimited, + 6 => Self::Stopped, + 7 => Self::Restarting, + 8 => Self::Failed, _ => return None, }) } @@ -167,6 +181,76 @@ pub struct HelloAck { pub host_name: String, pub auth_token: [u8; 16], pub host_caps: u16, + /// Why the PC isn't streaming, for [`HelloStatus::Failed`]: a + /// [`FailedReason`] token, empty otherwise. Appended after `host_caps` + /// (0.3.5), at most [`MAX_REASON_LEN`] bytes; older parsers ignore it. + pub reason: String, +} + +/// Longest [`HelloAck::reason`] on the wire, in bytes. +pub const MAX_REASON_LEN: usize = 32; + +/// The longest HELLO_ACK datagram: the header, the fixed fields and stream +/// config, the longest host name, the token and host caps, and the longest +/// reason. +pub const MAX_HELLO_ACK_LEN: usize = + CONTROL_HEADER_SIZE + 16 + STREAM_CONFIG_SIZE + 1 + MAX_NAME_LEN + 16 + 2 + 1 + MAX_REASON_LEN; + +/// A HELLO2 that sets [`FEATURE_STANDBY_STATUS`] is padded with zeros to at +/// least this size, so any answer to it is no larger than the question. A +/// PC answers while it isn't streaming only within that bound. Older +/// parsers ignore the padding, as any trailing body bytes. +pub const STANDBY_HELLO2_LEN: usize = MAX_HELLO_ACK_LEN; + +/// Why a PC stopped streaming after repeated errors, as [`HelloAck::reason`] +/// carries it. A token, not text: the iPad says it in its own words, and an +/// iPad that doesn't know a token shows the general advice. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FailedReason { + Capture, + Encoder, + Transport, + SecureDesktop, + DisplayChanging, + NoDisplay, + UnsupportedOnThisGpu, + TooManyCaptureApps, + SessionDisconnected, + DriverNotResponding, +} + +impl FailedReason { + pub const ALL: [FailedReason; 10] = [ + Self::Capture, + Self::Encoder, + Self::Transport, + Self::SecureDesktop, + Self::DisplayChanging, + Self::NoDisplay, + Self::UnsupportedOnThisGpu, + Self::TooManyCaptureApps, + Self::SessionDisconnected, + Self::DriverNotResponding, + ]; + + pub fn token(self) -> &'static str { + match self { + Self::Capture => "capture", + Self::Encoder => "encoder", + Self::Transport => "transport", + Self::SecureDesktop => "secure_desktop", + Self::DisplayChanging => "display_changing", + Self::NoDisplay => "no_display", + Self::UnsupportedOnThisGpu => "unsupported_gpu", + Self::TooManyCaptureApps => "too_many_capture_apps", + Self::SessionDisconnected => "session_disconnected", + Self::DriverNotResponding => "driver_not_responding", + } + } + + pub fn from_token(token: &str) -> Option { + Self::ALL.into_iter().find(|reason| reason.token() == token) + } } /// [`Heartbeat::notice`] values. A notice is appended to the heartbeat only @@ -446,6 +530,9 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - body.extend_from_slice(&h.auth_token); body.extend_from_slice(&h.pairing_code.to_le_bytes()); } + if h.feature_caps & FEATURE_STANDBY_STATUS != 0 { + body.resize(body.len().max(STANDBY_HELLO2_LEN - CONTROL_HEADER_SIZE), 0); + } } ControlMessage::HelloAck(a) => { let name = truncated_name(&a.host_name); @@ -461,10 +548,15 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - body.extend_from_slice(&cfg); body.push(name.len() as u8); body.extend_from_slice(name.as_bytes()); - if a.host_caps != 0 || a.auth_token != [0; 16] { + let reason = truncated(&a.reason, MAX_REASON_LEN); + if a.host_caps != 0 || a.auth_token != [0; 16] || !reason.is_empty() { body.extend_from_slice(&a.auth_token); body.extend_from_slice(&a.host_caps.to_le_bytes()); } + if !reason.is_empty() { + body.push(reason.len() as u8); + body.extend_from_slice(reason.as_bytes()); + } } ControlMessage::Heartbeat(hb) => { body.extend_from_slice(&hb.host_time_us.to_le_bytes()); @@ -654,6 +746,16 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), } else { ([0; 16], 0) }; + let reason = if r.remaining() > 0 { + let len = usize::from(r.u8()?); + if len > MAX_REASON_LEN { + return Err(WireError::InvalidField("reason_len")); + } + String::from_utf8(r.take(len)?.to_vec()) + .map_err(|_| WireError::InvalidField("reason_utf8"))? + } else { + String::new() + }; if status == HelloStatus::Ok && session_id == 0 { return Err(WireError::InvalidField("session_id")); } @@ -669,6 +771,7 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), host_name, auth_token, host_caps, + reason, }) } PacketType::Heartbeat => ControlMessage::Heartbeat(Heartbeat { @@ -800,11 +903,15 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), } fn truncated_name(name: &str) -> &str { - if name.len() <= MAX_NAME_LEN { + truncated(name, MAX_NAME_LEN) +} + +fn truncated(name: &str, max: usize) -> &str { + if name.len() <= max { return name; } // Truncate on a char boundary at or below the cap. - let mut end = MAX_NAME_LEN; + let mut end = max; while end > 0 && !name.is_char_boundary(end) { end -= 1; } @@ -1018,6 +1125,7 @@ mod tests { host_name: "ALI-PC".to_string(), auth_token: [0; 16], host_caps: 0, + reason: String::new(), }), ControlMessage::Heartbeat(Heartbeat { host_time_us: 987_654_321, @@ -1175,6 +1283,7 @@ mod tests { host_name: String::new(), auth_token: [0; 16], host_caps: 0, + reason: String::new(), }); // encode_control would happily serialize it; the parser must reject. let datagram = encode_control(0, 1, &ack); @@ -1266,4 +1375,136 @@ mod tests { let _ = super::super::classify(&bytes); } } + + fn standby_ack(status: HelloStatus, reason: &str) -> HelloAck { + HelloAck { + status, + accepted_version: 2, + client_nonce: 9, + session_id: 0, + heartbeat_interval_ms: 1000, + report_interval_ms: 500, + liveness_timeout_ms: 3000, + stream_config: StreamConfig::default(), + host_name: "PC".to_string(), + auth_token: [0; 16], + host_caps: 0, + reason: reason.to_string(), + } + } + + #[test] + fn not_streaming_statuses_and_their_reason_round_trip() { + for status in [ + HelloStatus::Stopped, + HelloStatus::Restarting, + HelloStatus::Failed, + ] { + for reason in ["", "display_changing"] { + let ack = standby_ack(status, reason); + let bytes = encode_control(0, 1, &ControlMessage::HelloAck(ack.clone())); + assert_eq!( + parse_control(&bytes).unwrap().1, + ControlMessage::HelloAck(ack) + ); + } + } + // A reason past the cap is cut, so it always parses. + let long = "x".repeat(100); + let bytes = encode_control( + 0, + 1, + &ControlMessage::HelloAck(standby_ack(HelloStatus::Failed, &long)), + ); + let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&bytes) else { + panic!("a long reason must still parse"); + }; + assert_eq!(ack.reason.len(), MAX_REASON_LEN); + } + + /// 0.3.4 and older know statuses 0-5 only and drop the whole ACK for + /// any other, like a lost datagram: the host must not send them 6-8. + #[test] + fn a_status_past_the_known_ones_is_rejected() { + let mut bytes = encode_control( + 0, + 1, + &ControlMessage::HelloAck(standby_ack(HelloStatus::Error, "")), + ); + bytes[CONTROL_HEADER_SIZE] = 9; + assert_eq!( + parse_control(&bytes), + Err(WireError::InvalidField("status")) + ); + } + + #[test] + fn failed_reason_tokens_are_short_distinct_and_round_trip() { + let mut seen = std::collections::HashSet::new(); + for reason in FailedReason::ALL { + let token = reason.token(); + assert!(token.len() <= MAX_REASON_LEN && token.is_ascii(), "{token}"); + assert!(seen.insert(token), "{token} twice"); + assert_eq!(FailedReason::from_token(token), Some(reason)); + } + assert_eq!(FailedReason::from_token("something new"), None); + } + + fn standby_hello2(feature_caps: u16, device_name: &str) -> Vec { + encode_control( + 0, + 1, + &ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 7, + listen_port: 0, + decoder_caps: CAP_DECODE_H264, + feature_caps, + screen_px_w: 1, + screen_px_h: 1, + screen_pt_w: 1, + screen_pt_h: 1, + refresh_hz: 60, + device_name: device_name.into(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }), + ) + } + + /// Any answer fits in a HELLO2 that asks for standby answers: the + /// longest ACK there is, with a 64-byte host name, a token and the + /// longest reason, is no larger. The padding parses as the HELLO2 it was. + #[test] + fn a_hello2_that_asks_for_standby_answers_fits_any_ack() { + let mut longest = standby_ack(HelloStatus::Failed, &"r".repeat(MAX_REASON_LEN)); + longest.host_name = "h".repeat(MAX_NAME_LEN); + longest.auth_token = [9; 16]; + longest.host_caps = 0xFFFF; + let ack = encode_control(0, 1, &ControlMessage::HelloAck(longest)); + assert_eq!(ack.len(), MAX_HELLO_ACK_LEN); + for name in ["", "iPad", &"n".repeat(MAX_NAME_LEN)] { + let padded = standby_hello2(FEATURE_STANDBY_STATUS, name); + assert!( + padded.len() >= ack.len(), + "{} < {}", + padded.len(), + ack.len() + ); + let Ok((_, ControlMessage::Hello2(hello))) = parse_control(&padded) else { + panic!("a padded HELLO2 must parse"); + }; + assert_eq!(hello.device_name, name); + assert_eq!((hello.device_id, hello.auth_token), (0, [0; 16])); + } + assert_eq!( + standby_hello2(FEATURE_STANDBY_STATUS, "").len(), + STANDBY_HELLO2_LEN + ); + // Without the bit, as from 0.3.4, nothing is added. + assert_eq!(standby_hello2(0, "").len(), CONTROL_HEADER_SIZE + 22); + } } diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 0e02c1bc..4c62e3c4 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -33,5 +33,9 @@ receiver_report_v030 454d021500003200785634123200000003000000881300007e1300000a0 hello_ack_paired 454d02110000390000000000030000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f100100 hello_ack_contiguous_seq 454d02110000390000000000050000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f102100 hello2_v030 454d02100000410000000000040000000202efbeadde94260300010074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e20100 +hello2_standby_padded 454d02100000940000000000040000000202efbeadde94260300090074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e201000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 identify 454d021a00004400000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 identity 454d021b00004400000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf942602c000020ac633640a000000000000000018fb81d76bbb96e91101119775ad9766b0c332b2c49441e6f07ec33be1c6d94e +hello_ack_stopped 454d02110000390000000000050000000602efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d5043000000000000000000000000000000000100 +hello_ack_restarting 454d02110000390000000000050000000702efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d5043000000000000000000000000000000000100 +hello_ack_failed 454d021100004a0000000000050000000802efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d504300000000000000000000000000000000010010646973706c61795f6368616e67696e67 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index 4ab055fb..cbe81264 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -111,6 +111,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { host_name: "ALI-PC".into(), auth_token: [0; 16], host_caps: 0, + reason: String::new(), }), ), )); @@ -218,6 +219,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { host_name: "ALI-PC".to_string(), auth_token: [0; 16], host_caps: 0, + reason: String::new(), }), ), ( @@ -236,6 +238,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { host_name: "ALI-PC".to_string(), auth_token: [0; 16], host_caps: 0, + reason: String::new(), }), ), ( @@ -413,6 +416,13 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { hello.pairing_code = 123_456; vectors.push(( "hello2_v030", + encode_control(0, 4, &ControlMessage::Hello2(hello.clone())), + )); + // 0.3.5: an iPad that asks for standby answers pads its HELLO2, so an + // answer is never larger than the question. + hello.feature_caps |= FEATURE_STANDBY_STATUS; + vectors.push(( + "hello2_standby_padded", encode_control(0, 4, &ControlMessage::Hello2(hello)), )); let nonce = std::array::from_fn(|index| 0xA0 + index as u8); @@ -444,6 +454,39 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { }), ), )); + // 0.3.5: answers while the PC isn't streaming, a Failed one with its + // reason token appended. + for (name, status, reason) in [ + ("hello_ack_stopped", HelloStatus::Stopped, ""), + ("hello_ack_restarting", HelloStatus::Restarting, ""), + ( + "hello_ack_failed", + HelloStatus::Failed, + FailedReason::DisplayChanging.token(), + ), + ] { + vectors.push(( + name, + encode_control( + 0, + 5, + &ControlMessage::HelloAck(HelloAck { + status, + accepted_version: 2, + client_nonce: 0xDEAD_BEEF, + session_id: 0, + heartbeat_interval_ms: 1000, + report_interval_ms: 500, + liveness_timeout_ms: 3000, + stream_config: StreamConfig::default(), + host_name: "ALI-PC".into(), + auth_token: [0; 16], + host_caps: HOSTCAP_NACK, + reason: reason.into(), + }), + ), + )); + } vectors } From 59b0d190a077219df3460c3b752b252698df5868 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:43:11 -0500 Subject: [PATCH 084/226] Check the hard-limit release itself, not the quit's releases_past_the_hard_limit_still_release_everything waited with finish(), which now queues a release of everything first. That reset passed the test even with the overflow's releases lost. It now waits with wait_idle(), which releases nothing. With the overflow marker disabled, the last thing Windows gets is a press. --- host/src/input/injector.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs index e804355e..ca324cc9 100644 --- a/host/src/input/injector.rs +++ b/host/src/input/injector.rs @@ -1062,7 +1062,7 @@ mod tests { harness.send(KIND_KEY, PHASE_BEGAN, 20_000); harness.injector.as_ref().unwrap().reset(); harness.go.send(()).unwrap(); - assert!(harness.thread.finish(Duration::from_secs(5))); + assert!(harness.thread.wait_idle(Duration::from_secs(5))); let calls = harness.calls.lock().clone(); assert_eq!( calls.last(), From 9598ae9dafdb8b42ba7b9d22d8571095e6ac60b2 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:51:01 -0500 Subject: [PATCH 085/226] Leave the virtual display on when making a window device panics The panic hook turns the virtual display off for a panic on any background thread, and the window's device replacement thread is one, so a panic there could end a Second screen stream although the window only counts a failed try. The thread is now named (REPLACEMENT_THREAD) and the hook only logs a panic on it. The hook's body moved to on_panic so a test can run it. --- host/src/startup/mod.rs | 85 ++++++++++++++++++++++++++++++----- vendor/README.md | 5 ++- vendor/egui-wgpu/src/winit.rs | 27 ++++++----- 3 files changed, 91 insertions(+), 26 deletions(-) diff --git a/host/src/startup/mod.rs b/host/src/startup/mod.rs index b687a1b2..8d3bb84c 100644 --- a/host/src/startup/mod.rs +++ b/host/src/startup/mod.rs @@ -66,18 +66,9 @@ fn crash_dir() -> Option { pub fn install_failure_hooks() { let default_hook = std::panic::take_hook(); std::panic::set_hook(Box::new(move |info| { - if std::thread::current().name() == Some("main") { - in_report_order( - || save(&panic_detail(info)), - || { - crate::vdd::disable(); - }, - show, - ); - } else { - tracing::error!(panic = %info, backtrace = %std::backtrace::Backtrace::force_capture(), "Host panic"); + on_panic(info, || { crate::vdd::disable(); - } + }); default_hook(info); })); #[cfg(windows)] @@ -116,6 +107,25 @@ fn prepare_crash_reporting() { reporter::start(&directory, owner, &launch_id); } +/// What [`install_failure_hooks`] does for a panic, by its thread, with +/// `disable_virtual_display` turning the virtual display off. A panic on +/// egui-wgpu's device replacement thread ends only that try: the window tries +/// again or gives up, and the stream, maybe on the virtual display, goes on. +fn on_panic(info: &std::panic::PanicHookInfo<'_>, disable_virtual_display: impl Fn()) { + match std::thread::current().name() { + Some("main") => { + in_report_order(|| save(&panic_detail(info)), &disable_virtual_display, show) + } + Some(eframe::egui_wgpu::winit::REPLACEMENT_THREAD) => { + tracing::warn!(panic = %info, "Making a new device for the window panicked"); + } + _ => { + tracing::error!(panic = %info, backtrace = %std::backtrace::Backtrace::force_capture(), "Host panic"); + disable_virtual_display(); + } + } +} + /// Save the report, clean up, then ask for the message: the dialog never /// holds the cleanup, and killing a stuck app cannot skip it. fn in_report_order( @@ -898,6 +908,59 @@ mod tests { assert!(detail.contains("mod.rs:"), "{detail}"); } + #[test] + fn a_panic_making_a_window_device_leaves_the_virtual_display_on() { + use eframe::egui_wgpu::winit::{ + Answer, Asked, Replacement, REPLACEMENT_THREAD, REPLACEMENT_TIMEOUT, + }; + use std::sync::atomic::AtomicUsize; + use std::sync::Arc; + const PIPELINE: &str = "test pipeline thread"; + static TURNED_OFF: AtomicUsize = AtomicUsize::new(0); + // The host's hook, for this test's threads; the hook is process-wide, + // so other tests' panics go to the old one. + let previous: Arc) + Send + Sync> = + std::panic::take_hook().into(); + let fallback = previous.clone(); + std::panic::set_hook(Box::new(move |info| match std::thread::current().name() { + Some(REPLACEMENT_THREAD | PIPELINE) => on_panic(info, || { + TURNED_OFF.fetch_add(1, Ordering::SeqCst); + }), + _ => fallback(info), + })); + + // The painter's replacement thread panics, as in a driver. + let start = Instant::now(); + let mut replacing = + Answer::<()>::ask(REPLACEMENT_THREAD, start, REPLACEMENT_TIMEOUT, || { + panic!("simulated panic making a device") + }) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !matches!(replacing.poll(start), Asked::Ended) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let ended = matches!(replacing.poll(start), Asked::Ended); + let after_replacement = TURNED_OFF.load(Ordering::SeqCst); + // Any other background thread's panic still turns it off. + let _ = std::thread::Builder::new() + .name(PIPELINE.into()) + .spawn(|| panic!("simulated pipeline panic")) + .unwrap() + .join(); + let after_pipeline = TURNED_OFF.load(Ordering::SeqCst); + std::panic::set_hook(Box::new(move |info| previous(info))); + assert!(ended); + assert_eq!(after_replacement, 0); + assert_eq!(after_pipeline, 1); + + // The painter counts the ended try as failed and tries again later. + let mut replacement = Replacement::default(); + replacement.lost(start); + replacement.failed(start); + assert_eq!(replacement.wait(start), Some(Duration::from_millis(500))); + } + #[test] fn a_gui_panic_turns_the_virtual_display_off_before_the_message() { let steps = std::cell::RefCell::new(Vec::new()); diff --git a/vendor/README.md b/vendor/README.md index d596a453..586f3a21 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -35,7 +35,10 @@ The patch: Listing adapters and making the device, where a driver can stall, run on their own thread (`winit::Answer`), which the window polls each frame. One that hasn't answered in 10 s counts as a failed try, and no other starts - while it is out; a late device is still taken. Configuring the new surface, + while it is out; a late device is still taken. The thread is named + `winit::REPLACEMENT_THREAD`, so the host's panic hook treats a panic there + as one failed try: it doesn't turn the virtual display off as it does for + other background threads. Configuring the new surface, which makes the swapchain, stays on the window's thread, because DXGI sends the window messages while it does that. - `configure_surface` replaces the device, instead of panicking, when the diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs index d9ec857d..9a560a3d 100644 --- a/vendor/egui-wgpu/src/winit.rs +++ b/vendor/egui-wgpu/src/winit.rs @@ -28,6 +28,10 @@ pub static DEVICES_LOST: AtomicU64 = AtomicU64::new(0); /// failed. A driver call can stall for much longer, so the window never waits on it. pub const REPLACEMENT_TIMEOUT: Duration = Duration::from_secs(10); +/// EternalMonitor patch: the name of the thread that makes a replacement device. A panic +/// there ends only that try, so an app's panic hook can tell it from others. +pub const REPLACEMENT_THREAD: &str = "wgpu device replacement"; + /// EternalMonitor patch: how often the window looks for a replacement device being made. const REPLACEMENT_POLL: Duration = Duration::from_millis(100); @@ -556,20 +560,15 @@ impl Painter { }; let (config, instance, options) = (self.config.clone(), self.instance.clone(), self.options); - let asked = Answer::ask( - "wgpu device replacement", - now, - REPLACEMENT_TIMEOUT, - move || { - let made = block_on(RenderState::create( - &config, - &instance, - Some(&surface), - options, - )); - (surface, made) - }, - ); + let asked = Answer::ask(REPLACEMENT_THREAD, now, REPLACEMENT_TIMEOUT, move || { + let made = block_on(RenderState::create( + &config, + &instance, + Some(&surface), + options, + )); + (surface, made) + }); match asked { Ok(replacing) => { self.replacing = Some(replacing); From 13a9cab6276b3708bc2ea5df41ef428899fda62d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:51:01 -0500 Subject: [PATCH 086/226] Give the polling test room for a loaded machine It checks that polling a stuck replacement never waits for it; a slow scheduler alone must not fail it. --- host/tests/window_device_loss.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/host/tests/window_device_loss.rs b/host/tests/window_device_loss.rs index 13949d4c..000d6ae0 100644 --- a/host/tests/window_device_loss.rs +++ b/host/tests/window_device_loss.rs @@ -176,8 +176,9 @@ fn a_stuck_replacement_times_out_without_blocking_and_its_late_device_is_taken() answer.poll(start + REPLACEMENT_TIMEOUT * 2), Asked::TimedOut )); + // The worker stays stuck throughout; a blocking poll would hang here. assert!( - polling.elapsed() < SECOND, + polling.elapsed() < SECOND * 3, "polling never waits for the worker" ); From ddbffdd668c92d0c278abf4b7e05ea7c9ad4f107 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:51:31 -0500 Subject: [PATCH 087/226] Let go of the kept picture under the mailbox lock OwedKeyframe::next dropped its kept picture only after take_within had returned the newer frame and released the mailbox. An encoder thread descheduled in between held both, capture could publish a third frame, and the synthetic source's next render found no free buffer. take_within now takes the kept picture and drops it under the mailbox lock before the newer frame leaves the mailbox. A test-only hook runs in the window after a take; the regression test publishes the third buffer there and checks that capture still has one free. --- host/src/capture/mod.rs | 109 +++++++++++++++++++++++++++++++--------- host/src/encoder/mod.rs | 9 +--- 2 files changed, 87 insertions(+), 31 deletions(-) diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index ee831d4f..cb21df05 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -150,6 +150,10 @@ pub struct FrameSlot { struct SlotInner { state: Mutex, condvar: Condvar, + /// Runs after [`FrameSlot::take_within`] took a frame and let go of the + /// mailbox, where a taker can be descheduled. + #[cfg(test)] + after_take: Mutex>>, } #[derive(Default)] @@ -164,6 +168,8 @@ impl FrameSlot { inner: Arc::new(SlotInner { state: Mutex::new(SlotState::default()), condvar: Condvar::new(), + #[cfg(test)] + after_take: Mutex::new(None), }), } } @@ -200,30 +206,45 @@ impl FrameSlot { } } - /// As [`Self::blocking_take`], but gives up after `timeout`. - pub fn take_within(&self, timeout: Duration) -> Taken { + /// As [`Self::blocking_take`], but gives up after `timeout`. A frame + /// taken replaces `kept`, a frame the caller still holds: `kept` is let + /// go under the mailbox lock before the frame leaves it, so capture, + /// which recycles a few buffers, never finds them all held at once. + pub fn take_within(&self, timeout: Duration, kept: &mut Option) -> Taken { let deadline = Instant::now() + timeout; - let mut state = self.inner.state.lock(); - loop { - if let Some(frame) = state.frame.take() { - return Taken::Frame(frame); - } - if state.closed { - return Taken::Closed; + let taken = { + let mut state = self.inner.state.lock(); + let mut timed_out = false; + loop { + if state.frame.is_some() { + *kept = None; + break Taken::Frame(state.frame.take().expect("checked above")); + } + if state.closed { + break Taken::Closed; + } + if timed_out { + break Taken::Nothing; + } + timed_out = self + .inner + .condvar + .wait_until(&mut state, deadline) + .timed_out(); } - if self - .inner - .condvar - .wait_until(&mut state, deadline) - .timed_out() - { - return match state.frame.take() { - Some(frame) => Taken::Frame(frame), - None if state.closed => Taken::Closed, - None => Taken::Nothing, - }; + }; + #[cfg(test)] + if matches!(taken, Taken::Frame(_)) { + if let Some(hook) = self.inner.after_take.lock().as_mut() { + hook(); } } + taken + } + + #[cfg(test)] + pub(crate) fn after_take(&self, hook: impl FnMut() + Send + 'static) { + *self.inner.after_take.lock() = Some(Box::new(hook)); } } @@ -1013,13 +1034,17 @@ mod tests { height: 2, }; let short = Duration::from_millis(5); - assert!(matches!(consumer.take_within(short), super::Taken::Nothing)); + assert!(matches!( + consumer.take_within(short, &mut None), + super::Taken::Nothing + )); producer.publish(frame(1)); assert!(matches!( - consumer.take_within(short), + consumer.take_within(short, &mut None), super::Taken::Frame(taken) if taken.frame_number == 1 )); - let waiting = std::thread::spawn(move || consumer.take_within(Duration::from_secs(10))); + let waiting = + std::thread::spawn(move || consumer.take_within(Duration::from_secs(10), &mut None)); std::thread::sleep(Duration::from_millis(20)); producer.publish(frame(2)); assert!(matches!( @@ -1028,7 +1053,43 @@ mod tests { )); let (producer, consumer) = super::frame_slot(); producer.close(); - assert!(matches!(consumer.take_within(short), super::Taken::Closed)); + assert!(matches!( + consumer.take_within(short, &mut None), + super::Taken::Closed + )); + } + + /// The synthetic source recycles three buffers. A taker that still held + /// a kept picture next to the frame it took, while capture published a + /// third, left capture no buffer to render its next frame into. + #[test] + fn a_frame_taken_in_place_of_a_kept_one_leaves_capture_a_free_buffer() { + let (producer, consumer) = super::frame_slot(); + let pool: Vec>> = (0..3).map(|_| Arc::new(vec![0u8; 16])).collect(); + let frame = |n: usize| super::RawFrame { + frame_number: n as u64, + timestamp: Instant::now(), + data: Arc::clone(&pool[n]), + width: 2, + height: 2, + }; + let mut kept = Some(frame(0)); + producer.publish(frame(1)); + let third = frame(2); + let capture = producer.handle(); + // The taker is descheduled right after the take: capture publishes + // its third buffer and looks for a free one, as the synthetic source + // does before it renders. + consumer.after_take(move || { + capture.publish(third.clone()); + assert!( + pool.iter().any(|buffer| Arc::strong_count(buffer) == 1), + "capture has no free buffer for its next frame" + ); + }); + let taken = consumer.take_within(Duration::from_secs(1), &mut kept); + assert!(matches!(taken, super::Taken::Frame(frame) if frame.frame_number == 1)); + assert!(kept.is_none()); } #[test] diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index d3f56416..fda575ef 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -1728,13 +1728,8 @@ impl OwedKeyframe { /// else the last picture again, until the deadline. fn next(&mut self, frames: &FrameSlot) -> Next { let wait = KEYFRAME_FLUSH_WAIT.min(self.until.saturating_duration_since(Instant::now())); - match frames.take_within(wait) { - // The newer picture replaces the kept one, which is let go before - // the newer one is converted: capture has no buffer to spare. - Taken::Frame(frame) => { - self.picture = None; - Next::Frame(frame) - } + match frames.take_within(wait, &mut self.picture) { + Taken::Frame(frame) => Next::Frame(frame), Taken::Closed => Next::Closed, Taken::Nothing => match self.picture.take() { Some(picture) if Instant::now() < self.until => { From 8171c7b1f16912ea336995dbe5ab0e6376d0fd17 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:52:29 -0500 Subject: [PATCH 088/226] Count the keyframe retry deadline from the keyframe's submission The 500 ms deadline for giving AMF its last picture again was taken after send_frame and the receive loop, so time spent in them was not counted. OwedKeyframe::after_encode now sets it from the moment the keyframe was submitted, and keeps it across the retries that follow. --- host/src/encoder/mod.rs | 50 ++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index fda575ef..c46ac46d 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -651,16 +651,8 @@ fn run_encode_loop( Err(e) => return Err(e.into()), } } - owed_keyframe = match picture { - Some(picture) if !keyframe_out => { - let mut owed = owed_keyframe - .take() - .unwrap_or_else(|| OwedKeyframe::until(Instant::now() + KEYFRAME_FLUSH_FOR)); - owed.picture = Some(picture); - Some(owed) - } - _ => None, - }; + owed_keyframe = + OwedKeyframe::after_encode(owed_keyframe.take(), picture, keyframe_out, encode_start); } info!("Capture channel closed, encoder shutting down"); @@ -1724,6 +1716,21 @@ impl OwedKeyframe { } } + /// What is owed after a frame was submitted at `submitted`: nothing once + /// the keyframe came out or the encoder holds no frame, else `picture` + /// to give again, until [`KEYFRAME_FLUSH_FOR`] after the first try. + fn after_encode( + owed: Option, + picture: Option, + keyframe_out: bool, + submitted: Instant, + ) -> Option { + let picture = picture.filter(|_| !keyframe_out)?; + let mut owed = owed.unwrap_or_else(|| Self::until(submitted + KEYFRAME_FLUSH_FOR)); + owed.picture = Some(picture); + Some(owed) + } + /// Capture's next picture if one arrives within [`KEYFRAME_FLUSH_WAIT`], /// else the last picture again, until the deadline. fn next(&mut self, frames: &FrameSlot) -> Next { @@ -4759,6 +4766,29 @@ mod tests { ); } + #[test] + fn the_deadline_counts_from_the_first_keyframe_submission() { + let kept = std::sync::Arc::new(vec![0u8; 16]); + let submitted = Instant::now() + .checked_sub(Duration::from_millis(200)) + .unwrap(); + // The keyframe went in 200 ms ago; sending and receiving took that long. + let owed = OwedKeyframe::after_encode(None, Some(raw(1, &kept)), false, submitted) + .expect("the keyframe is owed"); + assert_eq!(owed.until, submitted + KEYFRAME_FLUSH_FOR); + // A retry submitted later keeps the first deadline. + let owed = + OwedKeyframe::after_encode(Some(owed), Some(raw(1, &kept)), false, Instant::now()) + .expect("still owed"); + assert_eq!(owed.until, submitted + KEYFRAME_FLUSH_FOR); + assert!( + OwedKeyframe::after_encode(Some(owed), Some(raw(1, &kept)), true, Instant::now()) + .is_none(), + "the keyframe came out" + ); + assert!(OwedKeyframe::after_encode(None, None, false, submitted).is_none()); + } + #[test] fn the_kept_picture_is_given_again_only_until_the_deadline() { let (_producer, consumer) = crate::capture::frame_slot(); From e6172dd4ac1711ae373d770f151a3381d7e1895a Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:53:02 -0500 Subject: [PATCH 089/226] Test the keyframe retry deadline without racing the scheduler The deadline test waited 20 ms against a 60 ms deadline and then slept past it, so a slow scheduler could fail it. It is now two tests: the picture is given again under a deadline a minute away, and is let go at once under one that has already passed. --- host/src/encoder/mod.rs | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index c46ac46d..114b33c2 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -4790,22 +4790,39 @@ mod tests { } #[test] - fn the_kept_picture_is_given_again_only_until_the_deadline() { + fn the_kept_picture_is_given_again_before_the_deadline() { let (_producer, consumer) = crate::capture::frame_slot(); let kept = std::sync::Arc::new(vec![0u8; 16]); - let mut owed = OwedKeyframe::until(Instant::now() + Duration::from_millis(60)); + let mut owed = OwedKeyframe::until(Instant::now() + Duration::from_secs(60)); owed.picture = Some(raw(1, &kept)); let started = Instant::now(); let Next::Frame(frame) = owed.next(&consumer) else { panic!("the picture was not given again"); }; assert_eq!(frame.frame_number, 1); - assert!(started.elapsed() >= KEYFRAME_FLUSH_WAIT); - // The loop keeps the picture again after encoding it. - owed.picture = Some(frame); - std::thread::sleep(Duration::from_millis(80)); - assert!(matches!(owed.next(&consumer), Next::GaveUp)); assert_eq!(owed.given, 1); + assert!( + started.elapsed() >= KEYFRAME_FLUSH_WAIT, + "it did not wait for capture first" + ); + } + + #[test] + fn the_kept_picture_is_not_given_again_after_the_deadline() { + let (_producer, consumer) = crate::capture::frame_slot(); + let kept = std::sync::Arc::new(vec![0u8; 16]); + let passed = Instant::now() + .checked_sub(Duration::from_millis(1)) + .unwrap(); + let mut owed = OwedKeyframe::until(passed); + owed.picture = Some(raw(1, &kept)); + assert!(matches!(owed.next(&consumer), Next::GaveUp)); + assert_eq!(owed.given, 0); + assert_eq!( + std::sync::Arc::strong_count(&kept), + 1, + "the picture is let go" + ); } #[test] From 0948574ab9cb542eddf08d78e6bb0ddbd072060a Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:54:26 -0500 Subject: [PATCH 090/226] Test the panic hook in its own process, as installed The test installed its own hook around on_panic, which skipped the installer and the hook it chains to, and raced the other test that changes the process-wide hook. Now the test binary runs itself again with EM_TEST_PANIC_HOOK_CHILD, installs the host's panic hook there with a virtual display stand-in that notes its thread, and checks that a panic on the replacement thread left it alone while another background thread's turned it off. The polling test drops its wall-clock bound. --- host/src/startup/mod.rs | 134 +++++++++++++++++++++++-------- host/tests/window_device_loss.rs | 6 -- 2 files changed, 99 insertions(+), 41 deletions(-) diff --git a/host/src/startup/mod.rs b/host/src/startup/mod.rs index 8d3bb84c..d582e7ed 100644 --- a/host/src/startup/mod.rs +++ b/host/src/startup/mod.rs @@ -64,13 +64,9 @@ fn crash_dir() -> Option { /// threads belong to the pipeline, which the supervisor restarts, so they keep /// the plain log line. Native crashes on any thread end the process. pub fn install_failure_hooks() { - let default_hook = std::panic::take_hook(); - std::panic::set_hook(Box::new(move |info| { - on_panic(info, || { - crate::vdd::disable(); - }); - default_hook(info); - })); + install_panic_hook(|| { + crate::vdd::disable(); + }); #[cfg(windows)] prepare_crash_reporting(); if simulated() == "early-error" { @@ -107,6 +103,16 @@ fn prepare_crash_reporting() { reporter::start(&directory, owner, &launch_id); } +/// The panic hook of [`install_failure_hooks`], ahead of the one before it, +/// with `disable_virtual_display` turning the virtual display off. +fn install_panic_hook(disable_virtual_display: impl Fn() + Send + Sync + 'static) { + let default_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + on_panic(info, &disable_virtual_display); + default_hook(info); + })); +} + /// What [`install_failure_hooks`] does for a panic, by its thread, with /// `disable_virtual_display` turning the virtual display off. A panic on /// egui-wgpu's device replacement thread ends only that try: the window tries @@ -908,26 +914,39 @@ mod tests { assert!(detail.contains("mod.rs:"), "{detail}"); } + /// Set for [`panic_hook_child`]: where it notes what happened. + const PANIC_HOOK_CHILD: &str = "EM_TEST_PANIC_HOOK_CHILD"; + const PIPELINE_THREAD: &str = "test pipeline thread"; + + /// Run alone, in its own process, by + /// [`a_panic_making_a_window_device_leaves_the_virtual_display_on`]: the + /// panic hook is process-wide. Installs the host's panic hook with a + /// virtual display "turn off" that notes the thread it ran for, then + /// panics on egui-wgpu's replacement thread and on another background + /// thread. #[test] - fn a_panic_making_a_window_device_leaves_the_virtual_display_on() { - use eframe::egui_wgpu::winit::{ - Answer, Asked, Replacement, REPLACEMENT_THREAD, REPLACEMENT_TIMEOUT, + fn panic_hook_child() { + use eframe::egui_wgpu::winit::{Answer, Asked, REPLACEMENT_THREAD, REPLACEMENT_TIMEOUT}; + let Some(notes) = std::env::var_os(PANIC_HOOK_CHILD).map(PathBuf::from) else { + return; }; - use std::sync::atomic::AtomicUsize; - use std::sync::Arc; - const PIPELINE: &str = "test pipeline thread"; - static TURNED_OFF: AtomicUsize = AtomicUsize::new(0); - // The host's hook, for this test's threads; the hook is process-wide, - // so other tests' panics go to the old one. - let previous: Arc) + Send + Sync> = - std::panic::take_hook().into(); - let fallback = previous.clone(); - std::panic::set_hook(Box::new(move |info| match std::thread::current().name() { - Some(REPLACEMENT_THREAD | PIPELINE) => on_panic(info, || { - TURNED_OFF.fetch_add(1, Ordering::SeqCst); - }), - _ => fallback(info), - })); + let note = move |line: String| { + use std::io::Write; + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(¬es) + .unwrap(); + writeln!(file, "{line}").unwrap(); + }; + let off = note.clone(); + install_panic_hook(move || { + let thread = std::thread::current() + .name() + .unwrap_or("unnamed") + .to_string(); + off(format!("virtual display off: {thread}")); + }); // The painter's replacement thread panics, as in a driver. let start = Instant::now(); @@ -937,24 +956,69 @@ mod tests { }) .unwrap(); let deadline = Instant::now() + Duration::from_secs(10); - while !matches!(replacing.poll(start), Asked::Ended) && Instant::now() < deadline { + while !matches!(replacing.poll(start), Asked::Ended) { + assert!( + Instant::now() < deadline, + "the painter never saw the try end" + ); std::thread::sleep(Duration::from_millis(10)); } - let ended = matches!(replacing.poll(start), Asked::Ended); - let after_replacement = TURNED_OFF.load(Ordering::SeqCst); - // Any other background thread's panic still turns it off. + note("replacement ended".into()); + + // Any other background thread. let _ = std::thread::Builder::new() - .name(PIPELINE.into()) + .name(PIPELINE_THREAD.into()) .spawn(|| panic!("simulated pipeline panic")) .unwrap() .join(); - let after_pipeline = TURNED_OFF.load(Ordering::SeqCst); - std::panic::set_hook(Box::new(move |info| previous(info))); - assert!(ended); - assert_eq!(after_replacement, 0); - assert_eq!(after_pipeline, 1); + note("pipeline ended".into()); + } + + #[test] + fn a_panic_making_a_window_device_leaves_the_virtual_display_on() { + use eframe::egui_wgpu::winit::{Replacement, REPLACEMENT_THREAD}; + let directory = std::env::temp_dir().join(format!( + "eternal-panic-hook-{}-{:?}", + std::process::id(), + Instant::now() + )); + fs::create_dir_all(&directory).unwrap(); + let notes = directory.join("notes.txt"); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "startup::tests::panic_hook_child", + "--exact", + "--nocapture", + "--test-threads=1", + ]) + .env(PANIC_HOOK_CHILD, ¬es) + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&child.stderr); + assert!(child.status.success(), "{stderr}"); + let notes = fs::read_to_string(¬es).unwrap(); + // The worker's panic left the virtual display alone, the other + // thread's turned it off, in order. + assert_eq!( + notes.lines().collect::>(), + [ + "replacement ended", + &format!("virtual display off: {PIPELINE_THREAD}"), + "pipeline ended", + ], + "{stderr}" + ); + assert!(!notes.contains(REPLACEMENT_THREAD)); + // The hook before the host's still ran for both. + assert!( + stderr.contains("simulated panic making a device"), + "{stderr}" + ); + assert!(stderr.contains("simulated pipeline panic"), "{stderr}"); + fs::remove_dir_all(directory).unwrap(); // The painter counts the ended try as failed and tries again later. + let start = Instant::now(); let mut replacement = Replacement::default(); replacement.lost(start); replacement.failed(start); diff --git a/host/tests/window_device_loss.rs b/host/tests/window_device_loss.rs index 000d6ae0..c2b0305a 100644 --- a/host/tests/window_device_loss.rs +++ b/host/tests/window_device_loss.rs @@ -161,7 +161,6 @@ fn a_stuck_replacement_times_out_without_blocking_and_its_late_device_is_taken() "device" }) .unwrap(); - let polling = Instant::now(); assert!(matches!(answer.poll(start), Asked::Waiting)); assert!(matches!( answer.poll(start + REPLACEMENT_TIMEOUT), @@ -176,11 +175,6 @@ fn a_stuck_replacement_times_out_without_blocking_and_its_late_device_is_taken() answer.poll(start + REPLACEMENT_TIMEOUT * 2), Asked::TimedOut )); - // The worker stays stuck throughout; a blocking poll would hang here. - assert!( - polling.elapsed() < SECOND * 3, - "polling never waits for the worker" - ); release.send(()).unwrap(); let deadline = Instant::now() + Duration::from_secs(10); From 113153132c40c5275545a133cbd13b6baee38f5f Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:59:34 -0500 Subject: [PATCH 091/226] Try a BGRA open that fails in the background again at YUV420 An Automatic BGRA open that outlived the first picture's wait finished through EncoderState::opened directly, so a BGRA failure after the wait skipped open_at_input's YUV420 retry and left NVENC remembered as failed for the launch. The background open now retries at YUV420 in the background, the pipeline keeps YUV420, and the failure is handed back, to be remembered, only if YUV420 fails too. --- host/src/encoder/mod.rs | 203 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 196 insertions(+), 7 deletions(-) diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index 8028a11f..3fb4d52c 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -441,7 +441,20 @@ fn run_encode_loop( // once it returned, or a new one. let reopened = if let Some(result) = background .as_mut() - .and_then(|behind| behind.open.poll()) + .and_then(|behind| { + behind.open.poll(&mut input_mode, |args| { + EncoderState::new( + &args.encoder_name, + args.size.0, + args.size.1, + args.bitrate_bps, + args.target_fps, + &mut EncoderInput::Yuv420, + Opening::Behind, + &mut || {}, + ) + }) + }) { let behind = background.take().expect("it was just polled"); if let Err(error) = &result { @@ -959,7 +972,7 @@ impl EncoderState { formats }; info!(encoder = encoder_name, advertised_formats = ?supported, input = ?input_mode, "Encoder advertised input formats"); - open_at_input(input_mode, encoder_name, &supported, |pixel_format| { + let opened = open_at_input(input_mode, encoder_name, &supported, |pixel_format| { Self::open_at( codec, encoder_name, @@ -970,7 +983,16 @@ impl EncoderState { opening, &mut *progress, ) - }) + }); + // One that goes on in the background is tried at YUV420 there. + if let Err(error) = &opened { + if let Some(still) = error.downcast_ref::() { + if let Some(open) = still.1.lock().as_mut() { + open.yuv_retry = input::retries_as_yuv(*input_mode, open.args.pixel_format); + } + } + } + opened } #[allow(clippy::too_many_arguments)] @@ -1025,7 +1047,11 @@ impl EncoderState { Opened::Now(result) => Self::opened(result, &args), Opened::Behind(pending) => Err(Box::new(StillOpening( encoder_name.to_string(), - parking_lot::Mutex::new(Some(BackgroundOpen { pending, args })), + parking_lot::Mutex::new(Some(BackgroundOpen { + pending, + args, + yuv_retry: false, + })), ))), } } @@ -1388,6 +1414,9 @@ struct SessionArgs { struct BackgroundOpen { pending: PendingOpen, args: SessionArgs, + /// An Automatic BGRA open: if it fails, YUV420 is tried, as + /// [`open_at_input`] tries it for an open that fails while waited for. + yuv_retry: bool, } impl BackgroundOpen { @@ -1396,10 +1425,37 @@ impl BackgroundOpen { } /// The session once its open returned, or the open's error; past its - /// limit, its stall. None while it opens within its limit. - fn poll(&mut self) -> Option>> { + /// limit, its stall. None while it opens within its limit. A BGRA open + /// that failed is tried again at YUV420 with `retry_yuv`, and `input` + /// keeps YUV420 for the pipeline's later opens: the error comes back only + /// if that fails too, so an encoder that works at YUV420 is not left for + /// the launch. + fn poll( + &mut self, + input: &mut EncoderInput, + retry_yuv: impl FnOnce(&SessionArgs) -> Result>, + ) -> Option>> { let result = self.pending.poll()?; - Some(EncoderState::opened(result, &self.args)) + let opened = EncoderState::opened(result, &self.args); + let Err(error) = &opened else { + return Some(opened); + }; + if !std::mem::take(&mut self.yuv_retry) || is_open_stall(error.as_ref()) { + return Some(opened); + } + warn!( + encoder = %self.args.encoder_name, + %error, + "Encoder did not open with BGRA input; using YUV420 for this stream" + ); + *input = EncoderInput::Yuv420; + match retry_yuv(&self.args) { + Err(error) if is_still_opening(error.as_ref()) => { + *self = take_still_opening(error.as_ref()).expect("the open going on"); + None + } + retried => Some(retried), + } } } @@ -5155,6 +5211,139 @@ mod tests { assert!(!retried.contains("earlier in this launch"), "{retried}"); } + /// A background open in `lanes` of `name` at `pixel_format`, parked + /// already, that hangs until the returned sender sends, then gives `open`. + fn parked_background_open( + lanes: &'static crate::stall::Lanes, + name: &str, + pixel_format: ffmpeg_next::format::Pixel, + open: impl FnOnce() -> Result + + Send + + 'static, + ) -> (std::sync::mpsc::Sender<()>, BackgroundOpen) { + let (release, hang) = stuck_open(); + let mut pending = crate::stall::start_in( + lanes, + crate::stall::Call::EncoderOpen, + crate::stall::Wait { + limits: open_limits(name, Opening::Behind), + slice: Duration::from_millis(1), + now: Instant::now as fn() -> Instant, + }, + move || { + hang(); + open() + }, + crate::stall::no_hooks(), + ) + .unwrap(); + assert!(pending.park().is_none()); + let args = SessionArgs { + encoder_name: name.to_string(), + size: (64, 64), + bitrate_bps: 8_000_000, + target_fps: 60, + pixel_format, + legacy_pts: false, + }; + ( + release, + BackgroundOpen { + pending, + args, + yuv_retry: false, + }, + ) + } + + /// A real libx264 session at 64x64 YUV420P, opened on the call's thread + /// as a hardware encoder would be. + fn real_yuv_encoder() -> Result { + let codec = ffmpeg_next::encoder::find_by_name("libx264").expect("libx264"); + let mut video = ffmpeg_next::codec::Context::new_with_codec(codec) + .encoder() + .video()?; + video.set_width(64); + video.set_height(64); + video.set_format(ffmpeg_next::format::Pixel::YUV420P); + video.set_time_base(ffmpeg_next::Rational(1, 60)); + video.open_with(encoder_options("libx264")) + } + + #[test] + fn a_bgra_open_that_fails_in_the_background_is_tried_again_at_yuv420_there() { + use ffmpeg_next::format::Pixel; + ffmpeg_next::init().unwrap(); + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let refused = || Err(ffmpeg_next::Error::Other { errno: libc::EINVAL }); + // NVENC's Automatic BGRA open outlived the first picture's wait, and + // fails after it, as an NVENC that takes only YUV420 does. + let (release, mut open) = parked_background_open(lanes, "h264_nvenc", Pixel::BGRA, refused); + open.yuv_retry = true; + let mut input = EncoderInput::Automatic; + assert!(open + .poll(&mut input, |_| panic!("tried again before it failed")) + .is_none()); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + // YUV420 is tried at once, in the background too, and the pipeline + // keeps it; the BGRA failure is not handed back to be remembered. + let mut retried_at = None; + let mut yuv_release = None; + let polled = open.poll(&mut input, |args| { + retried_at = Some((args.encoder_name.clone(), args.size, args.bitrate_bps)); + let (release, yuv) = + parked_background_open(lanes, &args.encoder_name, Pixel::YUV420P, real_yuv_encoder); + yuv_release = Some(release); + Err(Box::new(StillOpening( + args.encoder_name.clone(), + parking_lot::Mutex::new(Some(yuv)), + ))) + }); + assert!(polled.is_none(), "the BGRA failure was handed back"); + assert_eq!( + retried_at, + Some(("h264_nvenc".to_string(), (64, 64), 8_000_000)) + ); + assert_eq!(input, EncoderInput::Yuv420); + assert_eq!(open.args.pixel_format, Pixel::YUV420P); + assert!(open + .poll(&mut input, |_| panic!("tried again while YUV420 opens")) + .is_none()); + // It opens: the stream takes that session. + yuv_release.unwrap().send(()).unwrap(); + wait_for_clear_lane(lanes); + let state = open + .poll(&mut input, |_| panic!("YUV420 is tried once")) + .expect("returned") + .expect("the YUV420 open failed"); + assert_eq!(state.opened_encoder, "h264_nvenc"); + assert_eq!(state.frame.format(), Pixel::YUV420P); + + // When YUV420 fails too, that failure is handed back, to be + // remembered for the launch. + let (release, mut open) = parked_background_open(lanes, "h264_nvenc", Pixel::BGRA, refused); + open.yuv_retry = true; + let mut input = EncoderInput::Automatic; + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + let failed = open.poll(&mut input, |_| Err("h264_nvenc open failed at YUV420".into())); + let error = failed.expect("returned").err().expect("it opened"); + assert!(error.to_string().contains("at YUV420"), "{error}"); + assert!(!is_open_stall(error.as_ref())); + assert_eq!(input, EncoderInput::Yuv420); + + // A YUV420 open, or a picked BGRA, is not tried again; nor is a + // stall, which says nothing about BGRA. + let (release, mut open) = parked_background_open(lanes, "h264_nvenc", Pixel::BGRA, refused); + let mut input = EncoderInput::Bgra; + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + let failed = open.poll(&mut input, |_| panic!("a picked BGRA was tried again")); + assert!(failed.expect("returned").is_err()); + assert_eq!(input, EncoderInput::Bgra); + } + #[test] fn the_reopen_line_says_why() { let nvenc = ("h264_nvenc", 60); From 8b39c87b993df9b960c8623f0c2bfe4742e3d805 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 18:59:34 -0500 Subject: [PATCH 092/226] Drop an HEVC open in the background once the frame wants H.264 A background session was taken without looking at the codec negotiated since it started, so an HEVC open could replace libx264 after HEVC was turned off or a client that decodes only H.264 connected. Such an open is now dropped each frame before it can be taken, and the session that streams goes on. The frame's preferred encoder comes from one function, as before. --- host/src/encoder/mod.rs | 140 ++++++++++++++++++++++++++++++++++------ 1 file changed, 121 insertions(+), 19 deletions(-) diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index 3fb4d52c..2261a40a 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -216,25 +216,17 @@ fn run_encode_loop( if !hevc_wanted { hevc_open_failed = false; } - let preferred: String = if hevc_wanted && !hevc_open_failed { - match hevc_encoder_for(&encoder_name) { - Some(hevc) if ffmpeg_next::encoder::find_by_name(hevc).is_some() => { - hevc.to_string() - } - _ => { - if !hevc_unavailable_warned { - hevc_unavailable_warned = true; - warn!( - encoder = %encoder_name, - "HEVC requested but no H.265 encoder is available — staying on H.264" - ); - } - encoder_name.clone() - } - } - } else { - encoder_name.clone() - }; + let preferred = preferred_encoder( + &encoder_name, + hevc_wanted, + hevc_open_failed, + &mut hevc_unavailable_warned, + ); + // An HEVC open going on in the background is of no use once this + // frame wants H.264: the stream goes on, and reopens from there. + if drop_obsolete_background(&mut background, &preferred) { + stall_recovery.trying = false; + } // No open given up on, or going on in the background, still holds // FFmpeg's lock. Until then, an open in that lane would only be // refused. @@ -1393,6 +1385,53 @@ fn open_limits(encoder_name: &str, opening: Opening) -> crate::stall::Limits { } } +/// The encoder a frame prefers: the H.265 encoder for `encoder_name` when +/// HEVC is wanted (the user's preference and the client's decoding) and has +/// not failed, if FFmpeg has it; else `encoder_name`. `warned` says once +/// that there is none. +fn preferred_encoder( + encoder_name: &str, + hevc_wanted: bool, + hevc_open_failed: bool, + warned: &mut bool, +) -> String { + if !hevc_wanted || hevc_open_failed { + return encoder_name.to_string(); + } + match hevc_encoder_for(encoder_name) { + Some(hevc) if ffmpeg_next::encoder::find_by_name(hevc).is_some() => hevc.to_string(), + _ => { + if !*warned { + *warned = true; + warn!( + encoder = %encoder_name, + "HEVC requested but no H.265 encoder is available — staying on H.264" + ); + } + encoder_name.to_string() + } + } +} + +/// Drops an HEVC open going on in the background once the frame prefers +/// H.264 (HEVC turned off, or a client that decodes only H.264): adopted, +/// it would replace a session the client can decode with one it may not. +/// The session that streams meanwhile goes on. True when one was dropped; +/// its result is released when it returns. +fn drop_obsolete_background(background: &mut Option, preferred: &str) -> bool { + let obsolete = background + .as_ref() + .is_some_and(|behind| is_hevc_encoder(behind.open.encoder()) && !is_hevc_encoder(preferred)); + if let Some(behind) = background.take_if(|_| obsolete) { + info!( + encoder = behind.open.encoder(), + preferred, + "The codec changed while an encoder opened in the background; that session will not be used" + ); + } + obsolete +} + /// Whether a reopen from the `current` session to `wanted` opens in the /// background: libx264 needs no lane, so it streams on while an encoder in /// the lane opens, instead of freezing for as long as the open takes. @@ -5344,6 +5383,69 @@ mod tests { assert_eq!(input, EncoderInput::Bgra); } + #[test] + fn an_hevc_open_in_the_background_is_dropped_once_the_frame_wants_h264() { + use ffmpeg_next::format::Pixel; + ffmpeg_next::init().unwrap(); + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + // An H.264 encoder whose H.265 sibling this FFmpeg build has. + let Some(h264) = ["libx264", "h264_nvenc", "h264_videotoolbox"] + .into_iter() + .find(|name| ffmpeg_next::encoder::find_by_name(hevc_variant_of(name).unwrap()).is_some()) + else { + eprintln!("no H.265 encoder in this FFmpeg build"); + return; + }; + // The encode loop's negotiation, from the user's preference and the + // client's decoding. + let preferred = |hevc_enabled: bool, client_hevc: bool| { + preferred_encoder(h264, hevc_enabled && client_hevc, false, &mut true) + }; + let hevc = preferred(true, true); + assert!(is_hevc_encoder(&hevc), "{hevc}"); + for change in ["an H.264-only client connected", "HEVC was turned off"] { + let (release, open) = parked_background_open(lanes, &hevc, Pixel::YUV420P, real_yuv_encoder); + let mut background = Some(Behind::first(open, 8_000_000)); + assert!(!drop_obsolete_background(&mut background, &preferred(true, true))); + assert!(background.is_some(), "{change}: dropped while wanted"); + let now = match change { + "an H.264-only client connected" => preferred(true, false), + _ => preferred(false, true), + }; + assert_eq!(now, h264, "{change}"); + // The HEVC session opens meanwhile, ready to be taken. + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + assert!( + drop_obsolete_background(&mut background, &now), + "{change}: the HEVC session would replace one the client decodes" + ); + assert!(background.is_none(), "{change}"); + } + // Still wanted, it is taken once it opens. + let (release, open) = parked_background_open(lanes, &hevc, Pixel::YUV420P, real_yuv_encoder); + let mut background = Some(Behind::first(open, 8_000_000)); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + assert!(!drop_obsolete_background(&mut background, &hevc)); + let state = background + .as_mut() + .unwrap() + .open + .poll(&mut EncoderInput::Yuv420, |_| panic!("tried again")) + .expect("returned") + .expect("opened"); + assert_eq!(state.opened_encoder, hevc); + // An H.264 open is kept when HEVC is turned on: the client decodes + // it, and the stream reopens to HEVC from there. + let (release, open) = parked_background_open(lanes, h264, Pixel::YUV420P, real_yuv_encoder); + let mut background = Some(Behind::first(open, 8_000_000)); + assert!(!drop_obsolete_background(&mut background, &hevc)); + assert!(background.is_some()); + release.send(()).unwrap(); + wait_for_clear_lane(lanes); + } + #[test] fn the_reopen_line_says_why() { let nvenc = ("h264_nvenc", 60); From e4496eafa3859fcd293fca5420bace49cfd75000 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 19:29:55 -0500 Subject: [PATCH 093/226] Let a stopped or restarting PC prove it is the saved PC While the PC isn't streaming, the standby responder answered only HELLO2. An iPad that found its saved PC at a new address checks it with IDENTIFY first, and got no answer until streaming started again, so it said the PC didn't answer instead of showing that the PC is stopped. The standby responder now hands IDENTIFY to the session too, which answers it exactly as while streaming: the same proof, the same limits per address and in all, no address list older than its freshness, Wi-Fi only, and an answer no larger than the request. The responder also lists the home network addresses when it binds every address, as the stream does. --- host/src/transport/session.rs | 41 ++++++- host/src/transport/standby.rs | 30 +++-- host/tests/hello_standby.rs | 224 +++++++++++++++++++++++++++++++++- 3 files changed, 282 insertions(+), 13 deletions(-) diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index b667ed13..3678a4b1 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -1078,7 +1078,8 @@ impl Session { /// An iPad checking that this is the PC it paired with, at the address /// it asked, before it reconnects there. It starts no session and works - /// while another device streams. Only Wi-Fi asks, within + /// while another device streams, and through the standby responder while + /// nothing streams. Only Wi-Fi asks, within /// [`IdentifyLimiter`]'s bounds. fn answer_identify( &mut self, @@ -2784,6 +2785,44 @@ mod tests { assert!(answered(&mut session, asker, &identify_config(true), now)); } + /// While no stream runs, the standby responder hands an IDENTIFY to the + /// session: it is answered over Wi-Fi as while streaming, never over a + /// cable, with an answer no larger than the request, and it starts + /// nothing. + #[test] + fn identify_while_not_streaming_is_answered_as_while_streaming() { + let config = identify_config(true); + let request = eternal_wire::v2::control::encode_control(0, 1, &identify(1, [3; 16])); + for state in [ + NotStreaming::Stopped, + NotStreaming::Restarting, + NotStreaming::Failed(FailedReason::Encoder), + ] { + let mut session = Session::new(100); + session.set_not_streaming(Some(state)); + let mut ask = |peer: PeerId| { + let (header, message) = eternal_wire::v2::control::parse_control(&request).unwrap(); + session.handle_datagram( + peer, + &header, + message, + request.len(), + &config, + Instant::now(), + ) + }; + assert!(ask(PeerId::usb(1)).replies.is_empty(), "{state:?}"); + let actions = ask(addr([192, 0, 2, 9], 50100)); + let identity = identity_reply(&actions); + assert_eq!(identity.nonce, [3; 16]); + assert_eq!(actions.replies[0].1.len(), request.len()); + assert_eq!(request.len(), 84); + assert!(actions.new_target.is_none()); + assert!(session.session_id().is_none()); + assert_eq!(session.not_streaming(), Some(state)); + } + } + /// A burst from one address gets 8 answers in 10 s, and the PC 32 in /// all from at most 16 addresses; each count starts over when its /// window ends. diff --git a/host/src/transport/standby.rs b/host/src/transport/standby.rs index e7f36077..15307305 100644 --- a/host/src/transport/standby.rs +++ b/host/src/transport/standby.rs @@ -6,7 +6,9 @@ //! answer. Between generations the supervisor now runs this responder on //! the same port and the USB tunnels. It answers each HELLO2 through the //! session, which says why nothing streams ([`Session::set_not_streaming`]) -//! after the same version, update and pairing checks as always. It never +//! after the same version, update and pairing checks as always. It answers +//! an IDENTIFY through the session too, so an iPad that finds the PC at a +//! new address can check it is the saved PC while nothing streams. It never //! starts a session or a stream, and lets go of the port before the //! supervisor checks it for the next generation. //! @@ -162,6 +164,11 @@ async fn open(listen_port: u16, shared: &SharedControl) -> (TransportLinks, Sock if logged { info!(%bind_addr, "Answering iPads while the stream is off"); } + if ip.is_unspecified() { + // Lists the home network addresses before anyone asks + // for them, as the stream does. + crate::network::home_addresses(); + } // The Address row and QR code follow, as for the stream. let shown = crate::gui::shown_address(ip, listen_port); crate::stats::PIPELINE_STATS.lock().set_listen_addr(shown); @@ -178,8 +185,8 @@ async fn open(listen_port: u16, shared: &SharedControl) -> (TransportLinks, Sock } } -/// Answers a HELLO2. Everything else waits for the next generation: a -/// session kept across a restart picks up there. +/// Answers a HELLO2 or an IDENTIFY. Everything else waits for the next +/// generation: a session kept across a restart picks up there. async fn answer( links: &TransportLinks, shared: &SharedControl, @@ -195,7 +202,9 @@ async fn answer( } _ => return, } - let Ok((header, message @ ControlMessage::Hello2(_))) = parse_control(datagram) else { + let Ok((header, message @ (ControlMessage::Hello2(_) | ControlMessage::Identify(_)))) = + parse_control(datagram) + else { return; }; let config = super::SharedConfigSource { @@ -206,8 +215,8 @@ async fn answer( }; let actions = { let mut session = shared.session.lock(); - // The supervisor clears it only after stopping this responder; a - // HELLO2 here never starts a session. + // The supervisor clears it only after stopping this responder; + // nothing answered here starts a session. if session.not_streaming().is_none() { return; } @@ -224,20 +233,21 @@ async fn answer( crate::diagnostics::log_session_end(end, shared); } for (destination, reply) in &actions.replies { - // An answer is never larger than the HELLO2 it answers. iPads that + // An answer is never larger than the request it answers. iPads that // understand these answers pad their HELLO2 so every answer fits - // (`STANDBY_HELLO2_LEN`); an older one may get today's silence. + // (`STANDBY_HELLO2_LEN`); an older one may get today's silence. An + // IDENTIFY is as long as its answer. if reply.len() > datagram.len() { debug!( peer = %destination, reply = reply.len(), request = datagram.len(), - "Left a HELLO2 unanswered: the answer is larger than it" + "Left a request unanswered: the answer is larger than it" ); continue; } if let Err(error) = links.send_to(reply, *destination).await { - debug!(peer = %destination, %error, "Could not answer a HELLO2 while the stream is off"); + debug!(peer = %destination, %error, "Could not answer an iPad while the stream is off"); } } } diff --git a/host/tests/hello_standby.rs b/host/tests/hello_standby.rs index c21e56a4..44148c8d 100644 --- a/host/tests/hello_standby.rs +++ b/host/tests/hello_standby.rs @@ -1,7 +1,9 @@ //! The PC answers an iPad's HELLO2 while it isn't streaming: in the pause //! before an automatic restart, across a restart, after Stop, and after the -//! storm brake. Drives the REAL supervisor with synthetic capture and -//! libx264, talked to by fake iPads over loopback UDP on ephemeral ports. +//! storm brake. A stopped PC also proves it is the saved PC to an iPad that +//! found it at a new address. Drives the REAL supervisor with synthetic +//! capture and libx264, talked to by fake iPads over loopback UDP on +//! ephemeral ports. use std::collections::BTreeMap; use std::net::UdpSocket; @@ -531,3 +533,221 @@ fn a_stopped_pc_follows_the_chosen_network() { Some(NotStreaming::Stopped) ); } + +/// A stopped PC still proves it is the saved PC to an iPad that found it +/// at a new address, within the same bounds as while it streams. The PC's +/// home network address is a test listing, so debug builds only. +#[cfg(debug_assertions)] +mod identify { + use super::*; + use eternal_host::network::{home_addresses, set_test_adapters, Adapter, Kind}; + use eternal_wire::v2::control::{identity_message, Identify, Identity, IDENTIFY_VERSION}; + + impl FakeIpad { + /// Asks the PC to prove it is the saved PC, as an iPad that found it at a + /// new address does. + fn identify(&self, nonce: [u8; 16]) { + let identify = ControlMessage::Identify(Identify { + version: IDENTIFY_VERSION, + nonce, + }); + self.send(&encode_control(0, 1, &identify)); + } + + /// The next IDENTITY for `nonce` within `wait`, skipping anything else. + fn identity(&self, nonce: [u8; 16], wait: Duration) -> Option { + let deadline = Instant::now() + wait; + let mut buffer = [0; 2048]; + while Instant::now() < deadline { + if let Ok(len) = self.socket.recv(&mut buffer) { + if let Ok((_, ControlMessage::Identity(identity))) = + parse_control(&buffer[..len]) + { + if identity.nonce == nonce { + self.sizes.set((self.sizes.get().0, len)); + return Some(identity); + } + } + } + } + None + } + } + + /// The address the test PC lists as its home network's. It listens on + /// every address, so an iPad that found it there reaches it on loopback. + const HOME: std::net::Ipv4Addr = std::net::Ipv4Addr::new(192, 168, 1, 20); + + /// Lists [`HOME`] as the PC's only home network address, and waits until a + /// fresh listing has it. + fn list_home_address() { + set_test_adapters(Some(vec![Adapter { + name: "Wi-Fi".into(), + kind: Kind::Wifi, + up: true, + gateway: true, + ipv4: vec![HOME], + }])); + let deadline = Instant::now() + Duration::from_secs(10); + while home_addresses() != [HOME] { + assert!( + Instant::now() < deadline, + "the home address was never listed" + ); + std::thread::sleep(Duration::from_millis(5)); + } + } + + /// What the iPad checks before it moves to `asked`: the proof is the + /// pairing token's over its nonce and the port and addresses the answer + /// names, and those are the port and one of the addresses it asked at. + fn ipad_accepts( + identity: &Identity, + token: [u8; 16], + nonce: [u8; 16], + asked: std::net::SocketAddrV4, + ) -> bool { + let key = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, &token); + let signed = identity_message(&nonce, identity.port, &identity.addresses); + ring::hmac::verify(&key, &signed, &identity.proof).is_ok() + && identity.port == asked.port() + && identity.addresses.contains(&asked.ip().octets()) + } + + /// A stopped PC that changed address proves it is the saved PC, so the iPad + /// moves to it and hears that it's stopped. The answer is no larger than + /// the request, and nothing starts. + #[test] + fn a_stopped_pc_proves_it_is_the_saved_pc() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let mut ipad = FakeIpad::new(&host, 1, NEW); + let token = host.shared.pairing.lock().token(); + + list_home_address(); + ipad.identify([1; 16]); + let identity = ipad + .identity([1; 16], Duration::from_secs(2)) + .expect("a stopped PC must answer IDENTIFY"); + let asked = std::net::SocketAddrV4::new(HOME, host.port); + assert!( + ipad_accepts(&identity, token, [1; 16], asked), + "{identity:?}" + ); + assert!(!ipad_accepts(&identity, [9; 16], [1; 16], asked)); + assert_eq!(ipad.sizes.get(), (84, 84)); + + assert!(!host.streaming(), "an IDENTIFY started the stream"); + assert_eq!( + host.shared.session.lock().not_streaming(), + Some(NotStreaming::Stopped) + ); + ipad.auth_token = token; + let ack = ipad.ask(2).expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Stopped); + } + + /// Something at another address that passes the iPad's IDENTIFY on to the + /// stopped PC gets the PC's real answer back, and the iPad turns it down: + /// it names the PC's own port, and no rewrite of the port or addresses + /// keeps the proof. + #[test] + fn a_forwarded_identify_fails_the_ipads_check_while_stopped() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let token = host.shared.pairing.lock().token(); + let forwarder = UdpSocket::bind("127.0.0.1:0").unwrap(); + forwarder + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let std::net::SocketAddr::V4(forwarder_addr) = forwarder.local_addr().unwrap() else { + unreachable!() + }; + let mut ipad = FakeIpad::new(&host, 1, NEW); + ipad.host = forwarder_addr.to_string(); + + list_home_address(); + ipad.identify([1; 16]); + let mut buffer = [0; 2048]; + let (len, from) = forwarder.recv_from(&mut buffer).unwrap(); + forwarder + .send_to(&buffer[..len], ("127.0.0.1", host.port)) + .unwrap(); + let (len, _) = forwarder + .recv_from(&mut buffer) + .expect("a stopped PC must answer a forwarded IDENTIFY"); + forwarder.send_to(&buffer[..len], from).unwrap(); + let identity = ipad + .identity([1; 16], Duration::from_secs(2)) + .expect("the forwarder passed the answer back"); + + // It is the PC's real answer, for the PC's own address... + let pc = std::net::SocketAddrV4::new(HOME, host.port); + assert!(ipad_accepts(&identity, token, [1; 16], pc)); + // ...so not for the address the iPad asked at. + assert!(!ipad_accepts(&identity, token, [1; 16], forwarder_addr)); + let mut rewritten = identity.clone(); + rewritten.port = forwarder_addr.port(); + rewritten.addresses = vec![forwarder_addr.ip().octets()]; + assert!(!ipad_accepts(&rewritten, token, [1; 16], forwarder_addr)); + } + + /// A stopped PC answers no more IDENTIFYs than a streaming one: a burst + /// from one address gets the per-address limit, and the PC still answers + /// HELLO2. + #[test] + fn identify_answers_stay_bounded_while_stopped() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let mut ipad = FakeIpad::new(&host, 1, NEW); + + list_home_address(); + for nonce in 0..20u8 { + ipad.identify([nonce; 16]); + } + let mut answers = 0; + let mut buffer = [0; 2048]; + let deadline = Instant::now() + Duration::from_millis(1500); + while Instant::now() < deadline { + if let Ok(len) = ipad.socket.recv(&mut buffer) { + answers += usize::from(matches!( + parse_control(&buffer[..len]), + Ok((_, ControlMessage::Identity(_))) + )); + } + } + // Per asking address and 10 s window (`IDENTIFY_PER_SOURCE`). + assert_eq!(answers, 8); + + ipad.auth_token = host.shared.pairing.lock().token(); + let ack = ipad.ask(1).expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Stopped); + } + + /// A stopped PC signs no address listing older than its 2 s freshness: the + /// IDENTIFY that finds it expired goes unanswered, the next one is answered + /// from the fresh listing it started. + #[test] + fn a_stopped_pc_signs_no_expired_listing() { + let host = Host::start(GpuInfo::software_fallback(), true); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let ipad = FakeIpad::new(&host, 1, NEW); + + list_home_address(); + std::thread::sleep(Duration::from_millis(2200)); + ipad.identify([1; 16]); + assert!( + ipad.identity([1; 16], Duration::from_secs(1)).is_none(), + "signed an expired listing" + ); + ipad.identify([2; 16]); + let identity = ipad + .identity([2; 16], Duration::from_secs(2)) + .expect("never answered from a fresh listing"); + assert_eq!(identity.addresses, [HOME.octets()]); + } +} From e09aee99c836ea4cc23b7014f565c67815caee2e Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 19:46:07 -0500 Subject: [PATCH 094/226] Count IDENTIFY answers only, and keep test address lists apart A request left unanswered because the PC had no fresh addresses to sign no longer uses up one of the asker's answers. Debug builds forget the last home network listing whenever the test adapters change, and drop a listing that was under way, so no test hears the addresses of another. The standby IDENTIFY tests ask up to 8 times, 250 ms apart, as the iPad does, and a new test checks that the first IDENTIFY after Stop is answered though the stream's listing has expired. --- host/src/network.rs | 88 ++++++++++++++++++++++++++++--- host/src/transport/session.rs | 29 ++++++++-- host/tests/hello_standby.rs | 99 ++++++++++++++++++++++++++++------- 3 files changed, 184 insertions(+), 32 deletions(-) diff --git a/host/src/network.rs b/host/src/network.rs index d4715d4a..b7ba8e54 100644 --- a/host/src/network.rs +++ b/host/src/network.rs @@ -407,6 +407,9 @@ pub struct HomeCache { struct HomeState { listed: Option<(Instant, Vec)>, listing: bool, + /// Bumped each time debug builds clear the cache: a listing begun + /// before that is dropped. + cleared: u64, } impl HomeCache { @@ -432,9 +435,12 @@ impl HomeCache { let addresses = fresh.map(|(_, addresses)| addresses.clone()); let start = addresses.is_none() && !state.listing; state.listing |= start; - (addresses.unwrap_or_default(), start) + ( + addresses.unwrap_or_default(), + start.then_some(state.cleared), + ) }; - if start { + if let Some(cleared) = start { let (list, ttl, listing) = (self.list, self.ttl, HomeListing(self.state.clone())); // A thread that can't start drops the closure, and with it the guard. let _ = std::thread::Builder::new() @@ -444,13 +450,23 @@ impl HomeCache { let addresses = list() .map(|adapters| home_addresses_of(&adapters)) .unwrap_or_default(); - if started.elapsed() < ttl { - listing.0.lock().listed = Some((started, addresses)); + let mut state = listing.0.lock(); + if started.elapsed() < ttl && state.cleared == cleared { + state.listed = Some((started, addresses)); } }); } addresses } + + /// Debug builds only: forgets the last listing, and any listing under + /// way, so the next one reads the adapters as they are now. + #[cfg(debug_assertions)] + fn clear(&self) { + let mut state = self.state.lock(); + state.listed = None; + state.cleared += 1; + } } /// Ends a listing when dropped, so a worker that panics loses only that @@ -470,11 +486,12 @@ fn listed_home() -> Option> { /// This PC's home network addresses (see [`home_addresses_of`] and /// [`HomeCache`]). pub fn home_addresses() -> Vec { - static HOME: std::sync::LazyLock = - std::sync::LazyLock::new(|| HomeCache::with(listed_home, HOME_TTL)); HOME.get() } +static HOME: std::sync::LazyLock = + std::sync::LazyLock::new(|| HomeCache::with(listed_home, HOME_TTL)); + /// What kind of interface a Unix name and flags say: network cards are /// named en*, eth* or wl*; point-to-point interfaces are tunnels. #[cfg(unix)] @@ -594,12 +611,15 @@ pub fn log_addresses() { #[cfg(debug_assertions)] static TEST_ADAPTERS: Mutex>> = Mutex::new(None); -/// Debug builds only: has the Address row's hint read `adapters` instead of -/// the PC's, until called with `None`. +/// Debug builds only: has the Address row's hint and the home network +/// addresses read `adapters` instead of the PC's, until called with `None`. +/// The last listing of home network addresses is forgotten each time, so no +/// test hears another's. #[cfg(debug_assertions)] #[doc(hidden)] pub fn set_test_adapters(adapters: Option>) { *TEST_ADAPTERS.lock() = adapters; + HOME.clear(); } fn listed() -> Option> { @@ -1227,6 +1247,58 @@ mod tests { assert_eq!(HOME_CALLS.load(Ordering::SeqCst), 1); } + static SWITCHED_GATE: Mutex<()> = Mutex::new(()); + static SWITCHED_CALLS: AtomicUsize = AtomicUsize::new(0); + static SWITCHED_IP: Mutex<[u8; 4]> = Mutex::new([192, 168, 1, 20]); + + fn switched_home() -> Option> { + let listed = *SWITCHED_IP.lock(); + SWITCHED_CALLS.fetch_add(1, Ordering::SeqCst); + let _open = SWITCHED_GATE.lock(); + Some(vec![adapter(Kind::Wifi, true, listed)]) + } + + /// Clearing forgets a fresh listing at once, and drops one that was + /// under way when the adapters changed: a test never hears the + /// addresses of the adapters it replaced. + #[cfg(debug_assertions)] + #[test] + fn a_cleared_cache_never_gives_the_old_addresses() { + let cache = HomeCache::with(switched_home, HOME_TTL); + let deadline = Instant::now() + Duration::from_secs(10); + while cache.get() != [ip([192, 168, 1, 20])] { + assert!(Instant::now() < deadline, "never listed"); + std::thread::sleep(Duration::from_millis(5)); + } + // The get after clearing starts a listing of the old adapters; the + // adapters change and the cache is cleared again while it runs. + let closed = SWITCHED_GATE.lock(); + cache.clear(); + assert!(cache.get().is_empty(), "a cleared listing was given out"); + let deadline = Instant::now() + Duration::from_secs(10); + while SWITCHED_CALLS.load(Ordering::SeqCst) < 2 { + assert!(Instant::now() < deadline, "no listing started"); + std::thread::sleep(Duration::from_millis(5)); + } + *SWITCHED_IP.lock() = [192, 168, 1, 30]; + cache.clear(); + drop(closed); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let addresses = cache.get(); + assert_ne!( + addresses, + [ip([192, 168, 1, 20])], + "the old listing came back" + ); + if addresses == [ip([192, 168, 1, 30])] { + break; + } + assert!(Instant::now() < deadline, "the new addresses never came"); + std::thread::sleep(Duration::from_millis(1)); + } + } + #[cfg(unix)] #[test] fn unix_interfaces_are_told_apart_by_flags_then_name() { diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 3678a4b1..40a48a9c 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -62,8 +62,9 @@ const IDENTIFY_OVERALL: u32 = 32; /// Asking addresses tracked in a window; more get no answer until it ends. const IDENTIFY_SOURCES: usize = 16; -/// Bounds what IDENTIFY costs: each answer is an HMAC, an address listing -/// and a datagram, and anyone on the network may ask. +/// Bounds what IDENTIFY costs: each answer is an HMAC and a datagram, and +/// anyone on the network may ask. Only answers count: a request left +/// unanswered for want of addresses to sign uses up nothing. #[derive(Debug, Default)] struct IdentifyLimiter { overall: Option<(Instant, u32)>, @@ -1095,14 +1096,15 @@ impl Session { let Some(pairing) = config.pairing() else { return actions; }; - if request.version != eternal_wire::v2::control::IDENTIFY_VERSION - || !self.identify_limiter.allow(asker.ip(), now) - { + if request.version != eternal_wire::v2::control::IDENTIFY_VERSION { return actions; } let Some((port, addresses)) = config.identity_endpoints() else { return actions; }; + if !self.identify_limiter.allow(asker.ip(), now) { + return actions; + } let addresses: Vec<[u8; 4]> = addresses .into_iter() .take(eternal_wire::v2::control::IDENTITY_MAX_ADDRESSES) @@ -2823,6 +2825,23 @@ mod tests { } } + /// Requests that get no answer, while the PC has no addresses to sign, + /// use up none of the asker's answers. + #[test] + fn unanswered_identify_uses_up_no_answers() { + let mut session = Session::new(100); + let now = Instant::now(); + let asker = addr([192, 0, 2, 9], 50100); + for _ in 0..IDENTIFY_OVERALL { + assert!(!answered(&mut session, asker, &pair_config(true), now)); + } + let config = identify_config(true); + let answers = (0..IDENTIFY_PER_SOURCE * 2) + .filter(|_| answered(&mut session, asker, &config, now)) + .count(); + assert_eq!(answers, IDENTIFY_PER_SOURCE as usize); + } + /// A burst from one address gets 8 answers in 10 s, and the PC 32 in /// all from at most 16 addresses; each count starts over when its /// window ends. diff --git a/host/tests/hello_standby.rs b/host/tests/hello_standby.rs index 44148c8d..8a7d59cb 100644 --- a/host/tests/hello_standby.rs +++ b/host/tests/hello_standby.rs @@ -572,15 +572,23 @@ mod identify { } None } + + /// Asks up to 8 times, 250 ms apart, as an iPad's check does, and + /// returns the first answer. + fn check(&self, nonce: [u8; 16]) -> Option { + (0..8).find_map(|_| { + self.identify(nonce); + self.identity(nonce, Duration::from_millis(250)) + }) + } } /// The address the test PC lists as its home network's. It listens on /// every address, so an iPad that found it there reaches it on loopback. const HOME: std::net::Ipv4Addr = std::net::Ipv4Addr::new(192, 168, 1, 20); - /// Lists [`HOME`] as the PC's only home network address, and waits until a - /// fresh listing has it. - fn list_home_address() { + /// Makes [`HOME`] the PC's only home network address. + fn set_home_adapter() { set_test_adapters(Some(vec![Adapter { name: "Wi-Fi".into(), kind: Kind::Wifi, @@ -588,6 +596,12 @@ mod identify { gateway: true, ipv4: vec![HOME], }])); + } + + /// Makes [`HOME`] the PC's only home network address, and waits until a + /// fresh listing has it. + fn list_home_address() { + set_home_adapter(); let deadline = Instant::now() + Duration::from_secs(10); while home_addresses() != [HOME] { assert!( @@ -626,9 +640,8 @@ mod identify { let token = host.shared.pairing.lock().token(); list_home_address(); - ipad.identify([1; 16]); let identity = ipad - .identity([1; 16], Duration::from_secs(2)) + .check([1; 16]) .expect("a stopped PC must answer IDENTIFY"); let asked = std::net::SocketAddrV4::new(HOME, host.port); assert!( @@ -660,28 +673,43 @@ mod identify { let token = host.shared.pairing.lock().token(); let forwarder = UdpSocket::bind("127.0.0.1:0").unwrap(); forwarder - .set_read_timeout(Some(Duration::from_secs(2))) + .set_read_timeout(Some(Duration::from_millis(20))) .unwrap(); let std::net::SocketAddr::V4(forwarder_addr) = forwarder.local_addr().unwrap() else { unreachable!() }; let mut ipad = FakeIpad::new(&host, 1, NEW); ipad.host = forwarder_addr.to_string(); + // Passes what the iPad sends on to the PC, and the PC's answers back. + let relaying = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + let relay = { + let relaying = relaying.clone(); + let pc = std::net::SocketAddr::from(([127, 0, 0, 1], host.port)); + std::thread::spawn(move || { + let mut buffer = [0; 2048]; + let mut asker = None; + while relaying.load(Ordering::SeqCst) { + let Ok((len, from)) = forwarder.recv_from(&mut buffer) else { + continue; + }; + let to = if from == pc { + asker + } else { + asker = Some(from); + Some(pc) + }; + if let Some(to) = to { + let _ = forwarder.send_to(&buffer[..len], to); + } + } + }) + }; list_home_address(); - ipad.identify([1; 16]); - let mut buffer = [0; 2048]; - let (len, from) = forwarder.recv_from(&mut buffer).unwrap(); - forwarder - .send_to(&buffer[..len], ("127.0.0.1", host.port)) - .unwrap(); - let (len, _) = forwarder - .recv_from(&mut buffer) - .expect("a stopped PC must answer a forwarded IDENTIFY"); - forwarder.send_to(&buffer[..len], from).unwrap(); - let identity = ipad - .identity([1; 16], Duration::from_secs(2)) - .expect("the forwarder passed the answer back"); + let identity = ipad.check([1; 16]); + relaying.store(false, Ordering::SeqCst); + relay.join().unwrap(); + let identity = identity.expect("a stopped PC must answer a forwarded IDENTIFY"); // It is the PC's real answer, for the PC's own address... let pc = std::net::SocketAddrV4::new(HOME, host.port); @@ -750,4 +778,37 @@ mod identify { .expect("never answered from a fresh listing"); assert_eq!(identity.addresses, [HOME.octets()]); } + + /// The standby lists the home network addresses as it starts on every + /// address, as the stream does, so the first IDENTIFY after Stop is + /// answered though the stream's listing expired long ago. + #[test] + fn the_first_identify_after_stop_is_answered() { + let host = Host::start_with(GpuInfo::software_fallback(), 500, |shared| { + shared.pairing.lock().required = true; + set_home_adapter(); + }); + let deadline = Instant::now() + Duration::from_secs(20); + while !host.streaming() { + assert!(Instant::now() < deadline, "never streamed"); + std::thread::sleep(Duration::from_millis(10)); + } + std::thread::sleep(Duration::from_millis(2200)); + host.send(SupervisorCommand::Stop); + host.wait_until_stopped(); + let mut ipad = FakeIpad::new(&host, 1, NEW); + ipad.auth_token = host.shared.pairing.lock().token(); + // Answered: the standby holds the port. Its listing runs on a + // thread of its own: give it a moment. + let ack = ipad + .ask_until_answered(1, Duration::from_secs(10)) + .expect("a stopped PC must answer"); + assert_eq!(ack.status, HelloStatus::Stopped); + std::thread::sleep(Duration::from_millis(300)); + ipad.identify([1; 16]); + assert!( + ipad.identity([1; 16], Duration::from_secs(2)).is_some(), + "the first IDENTIFY after Stop went unanswered" + ); + } } From 6d35f25be18ec1fc46b83f4029c53457cae12572 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 19:48:58 -0500 Subject: [PATCH 095/226] Skip the startup Disable without recording the display as off Review r2: the Task Scheduler probe made synchronous COM calls with no deadline while holding the driver turn, so a hung call blocked every later request, quit included. It also could not see a previous host's launcher that had not started its task yet. The probe is gone. At startup, a device Configuration Manager reports disabled (problem 22) still runs no Disable task, which keeps the field fix: no hung startup Disable for Enables to wait behind. The skip now records nothing, so the state stays unknown, as it was before this change. Every later Disable (Stop, a mode change, quitting, the failure reporter) runs the task as before, and turns off a display a stale Enable from a previous host turned on. An unknown state never holds up or skips an Enable, and Second screen runs no Disable before its Enable. Tests: the startup skip with problem 22 leaves the state unknown and an Enable runs at once; an Enable left by the previous host is turned off at quit. --- host/Cargo.toml | 1 - host/src/vdd.rs | 212 ++++++++++++++---------------------------------- 2 files changed, 59 insertions(+), 154 deletions(-) diff --git a/host/Cargo.toml b/host/Cargo.toml index 86f719b6..d8ece94d 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -105,7 +105,6 @@ features = [ "Win32_System_Registry", "Win32_System_StationsAndDesktops", "Win32_System_SystemInformation", - "Win32_System_TaskScheduler", "Win32_System_IO", "Win32_NetworkManagement_IpHelper", "Win32_NetworkManagement_Ndis", diff --git a/host/src/vdd.rs b/host/src/vdd.rs index f4b63895..4b15eab8 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -215,28 +215,17 @@ impl DriverTasks { } /// The startup cleanup: a Disable on its own turn, unless `already_off` - /// says, on that turn, that the display is off with no task left to - /// change it. Then it counts as a Disable that completed, so later - /// requests know the display is off. + /// says, on that turn, that the display is off. That skip records + /// nothing: the state stays unknown, so every later Disable (Stop, a + /// mode change, quitting, the failure reporter) still runs the task, and + /// turns off a display an Enable left from before turns on meanwhile. fn disable_at_startup( &self, on_turn: impl FnOnce(), already_off: impl FnOnce() -> bool, run: impl FnOnce(bool, &mut dyn FnMut()) -> TaskOutcome, ) -> bool { - self.set_enabled_on_turn( - false, - Purpose::Cleanup, - || {}, - on_turn, - |enabled, progress| { - if already_off() { - TaskOutcome::Done - } else { - run(enabled, progress) - } - }, - ) + self.set_enabled_unless(false, Purpose::Cleanup, || {}, on_turn, already_off, run) } #[cfg(test)] @@ -253,11 +242,25 @@ impl DriverTasks { /// As `set_enabled`, calling `on_turn` once this request holds the turn: /// every request made after that queues behind it. fn set_enabled_on_turn( + &self, + enabled: bool, + purpose: Purpose, + progress: impl FnMut(), + on_turn: impl FnOnce(), + run: impl FnOnce(bool, &mut dyn FnMut()) -> TaskOutcome, + ) -> bool { + self.set_enabled_unless(enabled, purpose, progress, on_turn, || false, run) + } + + /// As `set_enabled_on_turn`, returning true without running or recording + /// anything when `skip`, asked on the turn, says so. + fn set_enabled_unless( &self, enabled: bool, purpose: Purpose, mut progress: impl FnMut(), on_turn: impl FnOnce(), + skip: impl FnOnce() -> bool, run: impl FnOnce(bool, &mut dyn FnMut()) -> TaskOutcome, ) -> bool { use std::sync::atomic::Ordering::SeqCst; @@ -284,6 +287,9 @@ impl DriverTasks { progress(); }; on_turn(); + if skip() { + return true; + } // The gate guard ends with this statement. Logging below reads the // health through the same mutex, which deadlocked when the guard // lived through the whole match. @@ -533,10 +539,9 @@ pub fn disable() -> bool { /// A hung task no longer keeps a Mirror stream black for up to 30 s. /// /// A display Windows already reports disabled, as at almost every start, -/// runs no task, unless an Enable or Disable task is still queued or running -/// from before: an Enable the previous host gave up on can still turn the -/// display on. That Disable was the task that hung most often in the field, -/// and every later request waited behind it and failed. +/// runs no task, and its state stays unknown. That Disable was the task that +/// hung most often in the field, and every later request waited behind it +/// and failed. pub fn disable_in_background() { let (held, turn) = std::sync::mpsc::channel(); let spawned = std::thread::Builder::new() @@ -549,7 +554,7 @@ pub fn disable_in_background() { TASKS.disable_at_startup( on_turn, || { - let off = off_with_nothing_pending(task_pending(), crate::vdd_device::query); + let off = reports_disabled(crate::vdd_device::query()); if off { info!("Virtual display already off — not running the Disable task"); } @@ -570,63 +575,11 @@ pub fn disable_in_background() { } } -/// Whether the display is off and stays off: Task Scheduler reports no -/// driver task queued or running (`pending`, `None` when it could not say), -/// and only then does the device read disabled (problem 22). In that order, -/// so an Enable that ends meanwhile shows in the device. +/// Whether the device reads disabled (problem 22): the startup cleanup has +/// nothing to turn off. #[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. -fn off_with_nothing_pending( - pending: Option, - device: impl FnOnce() -> crate::vdd_device::DeviceState, -) -> bool { - pending == Some(false) && device() == crate::vdd_device::DeviceState::Problem(22) -} - -/// Whether either driver task has an instance queued or running, as Task -/// Scheduler reports it; a missing task has none. `None` when it can't say. -#[cfg(windows)] -fn task_pending() -> Option { - tasks_pending(&[TASK_ENABLE, TASK_DISABLE]) -} - -#[cfg(windows)] -fn tasks_pending(names: &[&str]) -> Option { - use windows::core::{BSTR, HRESULT, VARIANT}; - use windows::Win32::System::Com::{ - CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, - COINIT_MULTITHREADED, - }; - use windows::Win32::System::TaskScheduler::{ - ITaskService, TaskScheduler, TASK_STATE_QUEUED, TASK_STATE_RUNNING, - }; - struct Apartment; - impl Drop for Apartment { - fn drop(&mut self) { - unsafe { CoUninitialize() }; - } - } - unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) } - .ok() - .ok()?; - let _apartment = Apartment; - let service: ITaskService = - unsafe { CoCreateInstance(&TaskScheduler, None, CLSCTX_INPROC_SERVER) }.ok()?; - let local = VARIANT::default(); - unsafe { service.Connect(&local, &local, &local, &local) }.ok()?; - let folder = unsafe { service.GetFolder(&BSTR::from("\\")) }.ok()?; - let mut pending = false; - for &name in names { - match unsafe { folder.GetTask(&BSTR::from(name)) } { - Ok(task) => { - let state = unsafe { task.State() }.ok()?; - pending |= state == TASK_STATE_QUEUED || state == TASK_STATE_RUNNING; - } - // 0x80070002: the task does not exist. - Err(error) if error.code() == HRESULT(0x8007_0002_u32 as i32) => {} - Err(_) => return None, - } - } - Some(pending) +fn reports_disabled(device: crate::vdd_device::DeviceState) -> bool { + device == crate::vdd_device::DeviceState::Problem(22) } /// Disable the virtual display after `host` ended abnormally, if it owned the @@ -687,20 +640,21 @@ mod tests { use crate::vdd_device::DeviceState; use std::cell::Cell; let tasks = DriverTasks::new(); - let off = || off_with_nothing_pending(Some(false), || DeviceState::Problem(22)); + let off = || reports_disabled(DeviceState::Problem(22)); assert!(tasks.disable_at_startup( || {}, off, |_, _| { panic!("ran the Disable task for a display that is off") } )); - // Nor does a quit before any stream. - assert!(tasks.set_enabled( - false, - Purpose::Cleanup, - || {}, - |_, _| { panic!("ran the Disable task for a display that is off") } - )); - // The first Enable runs, and so does the Disable after it. + // Nothing is recorded: the state stays unknown. + { + let gate = tasks.gate.lock(); + assert_eq!(gate.known_enabled, None); + assert!(!gate.enable_ran); + assert_eq!(gate.health, VddTaskHealth::Ok); + } + // An unknown state never holds up an Enable: it runs at once, and + // the Disable after it runs too. let ran = Cell::new(Vec::new()); let run = |enabled: bool, _: &mut dyn FnMut()| { let mut calls = ran.take(); @@ -726,7 +680,7 @@ mod tests { let ran = Cell::new(false); tasks.disable_at_startup( || {}, - || off_with_nothing_pending(Some(false), || device), + || reports_disabled(device), |_, _| { ran.set(true); TaskOutcome::Done @@ -737,15 +691,16 @@ mod tests { } /// Review r1: an Enable task the previous host gave up on can still be - /// queued or running while the device reads disabled. Recording "off" - /// then let it add the monitor, and every later Disable, the quit's - /// too, found the display "already off": a phantom monitor stayed. + /// running, or not yet started, while the device reads disabled. + /// Recording "off" then let it add the monitor, and every later Disable, + /// the quit's too, found the display "already off": a phantom monitor + /// stayed. The startup skip now records nothing. #[test] - fn an_enable_left_running_by_the_previous_host_is_turned_off_at_startup() { + fn an_enable_left_running_by_the_previous_host_is_turned_off_at_quit() { use crate::vdd_device::DeviceState; use std::cell::Cell; let tasks = DriverTasks::new(); - // Task Scheduler still runs the Enable, and Windows has not added the + // The Enable is still on its way, and Windows has not added the // display yet. let enable_running = Cell::new(true); let display_on = Cell::new(false); @@ -756,76 +711,27 @@ mod tests { DeviceState::Problem(22) } }; - let enable_ends = || { - if enable_running.replace(false) { - display_on.set(true); - } - }; - let queried_device = Cell::new(false); - let disabled = tasks.disable_at_startup( + assert!(tasks.disable_at_startup( || {}, - || { - off_with_nothing_pending(Some(enable_running.get()), || { - queried_device.set(true); - device() - }) - }, - // The startup Disable runs vdd_task.ps1 with -After Enable, which - // waits for the Enable to end before it disables. - |enabled, _| { - assert!(!enabled); - enable_ends(); - display_on.set(false); - TaskOutcome::Done - }, - ); - assert!(disabled); - // A surviving Enable ends whenever it ends; then the host quits. - enable_ends(); - tasks.set_enabled( + || reports_disabled(device()), + |_, _| { panic!("ran the startup Disable for a display that reads disabled") } + )); + // The surviving Enable ends whenever it ends; then the host quits. + if enable_running.replace(false) { + display_on.set(true); + } + let disabled = tasks.set_enabled( false, Purpose::Cleanup, || {}, - |_, _| { + |enabled, _| { + assert!(!enabled); display_on.set(false); TaskOutcome::Done }, ); + assert!(disabled); assert!(!display_on.get(), "a phantom monitor stayed after quit"); - assert!( - !queried_device.get(), - "decided on the device while a task was pending" - ); - - // When Task Scheduler can't say, the Disable runs too. - let tasks = DriverTasks::new(); - let ran = Cell::new(false); - tasks.disable_at_startup( - || {}, - || off_with_nothing_pending(None, || DeviceState::Problem(22)), - |_, _| { - ran.set(true); - TaskOutcome::Done - }, - ); - assert!(ran.get(), "no Disable when the task state was unknown"); - } - - /// Native only (`--ignored` on Windows): Task Scheduler answers for the - /// driver tasks, a missing task has nothing pending, and a task started - /// beforehand and named in `EM_TEST_PENDING_TASK` reads as pending. - #[cfg(windows)] - #[test] - #[ignore] - fn task_scheduler_reports_pending_driver_tasks() { - assert_eq!( - tasks_pending(&["EternalMonitor test task that does not exist"]), - Some(false) - ); - assert!(task_pending().is_some(), "Task Scheduler did not answer"); - if let Ok(running) = std::env::var("EM_TEST_PENDING_TASK") { - assert_eq!(tasks_pending(&[running.as_str()]), Some(true), "{running}"); - } } #[test] From 805b22e94def75ffbba141e26dae360ca88ce5e3 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 19:50:16 -0500 Subject: [PATCH 096/226] Build the dark second screen test's picture with its first image seen --- host/src/capture/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 0c0f2994..615c590e 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -1687,8 +1687,10 @@ mod tests { let shared = SharedControl::new(0, 1); connect_viewer(&shared); let mut machine = crate::supervisor::Machine::new(1); - let mut picture = Picture::default(); - picture.seen_image = true; + let mut picture = Picture { + seen_image: true, + ..Picture::default() + }; picture.cached(); let lost = Instant::now(); picture.recreated(lost); From 4fe24effd6f0b0161d95bd087afd76e7e688e6fa Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 19:52:56 -0500 Subject: [PATCH 097/226] Import the second screen's pointer rule in the Windows capture loop --- host/src/capture/dxgi.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index ab3e6f9f..3f9cb5d7 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -17,8 +17,8 @@ use super::recovery::{ }; use super::{ acquired, client_connected, frame_budget_for, frame_buffer, heartbeat, idle_heartbeat, - keepalive_allowed, keepalive_due, read_back_image, reconcile_virtual_display, recovery_poll, - resolve_target, Acquired, FrameSlot, FrameUse, IdleGate, OutputInfo, RawFrame, + keepalive_allowed, keepalive_due, pointer_apart, read_back_image, reconcile_virtual_display, + recovery_poll, resolve_target, Acquired, FrameSlot, FrameUse, IdleGate, OutputInfo, RawFrame, ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, }; use crate::control::{CaptureTarget, SharedControl}; From 80bf784fe4a1be224818a744fc81fba9263e389d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 19:53:53 -0500 Subject: [PATCH 098/226] Wait for the standby's address listing itself in the tests The first-IDENTIFY test waits until a fresh listing of the home network addresses exists, read without starting one, instead of sleeping. The cleared-cache test keeps asking while it waits for the second listing, so that listing starts once the first one's worker has let go. --- host/src/network.rs | 33 ++++++++++++++++++++++++++++----- host/tests/hello_standby.rs | 18 ++++++++++++++---- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/host/src/network.rs b/host/src/network.rs index b7ba8e54..3217c20e 100644 --- a/host/src/network.rs +++ b/host/src/network.rs @@ -428,11 +428,7 @@ impl HomeCache { pub fn get(&self) -> Vec { let (addresses, start) = { let mut state = self.state.lock(); - let fresh = state - .listed - .as_ref() - .filter(|(started, _)| started.elapsed() < self.ttl); - let addresses = fresh.map(|(_, addresses)| addresses.clone()); + let addresses = self.fresh(&state); let start = addresses.is_none() && !state.listing; state.listing |= start; ( @@ -459,6 +455,22 @@ impl HomeCache { addresses } + /// The last listing's addresses while it is fresh. + fn fresh(&self, state: &HomeState) -> Option> { + state + .listed + .as_ref() + .filter(|(started, _)| started.elapsed() < self.ttl) + .map(|(_, addresses)| addresses.clone()) + } + + /// Debug builds only: the last listing's addresses while it is fresh, + /// without starting a listing. + #[cfg(debug_assertions)] + fn peek(&self) -> Option> { + self.fresh(&self.state.lock()) + } + /// Debug builds only: forgets the last listing, and any listing under /// way, so the next one reads the adapters as they are now. #[cfg(debug_assertions)] @@ -489,6 +501,14 @@ pub fn home_addresses() -> Vec { HOME.get() } +/// Debug builds only: the home network addresses of a fresh listing, if +/// there is one, without starting a listing as [`home_addresses`] does. +#[cfg(debug_assertions)] +#[doc(hidden)] +pub fn listed_home_addresses() -> Option> { + HOME.peek() +} + static HOME: std::sync::LazyLock = std::sync::LazyLock::new(|| HomeCache::with(listed_home, HOME_TTL)); @@ -1275,8 +1295,11 @@ mod tests { let closed = SWITCHED_GATE.lock(); cache.clear(); assert!(cache.get().is_empty(), "a cleared listing was given out"); + // The first listing may have published before its worker let go: + // asking again starts the second once it has. let deadline = Instant::now() + Duration::from_secs(10); while SWITCHED_CALLS.load(Ordering::SeqCst) < 2 { + assert!(cache.get().is_empty(), "a cleared listing was given out"); assert!(Instant::now() < deadline, "no listing started"); std::thread::sleep(Duration::from_millis(5)); } diff --git a/host/tests/hello_standby.rs b/host/tests/hello_standby.rs index 8a7d59cb..b79632b4 100644 --- a/host/tests/hello_standby.rs +++ b/host/tests/hello_standby.rs @@ -540,7 +540,9 @@ fn a_stopped_pc_follows_the_chosen_network() { #[cfg(debug_assertions)] mod identify { use super::*; - use eternal_host::network::{home_addresses, set_test_adapters, Adapter, Kind}; + use eternal_host::network::{ + home_addresses, listed_home_addresses, set_test_adapters, Adapter, Kind, + }; use eternal_wire::v2::control::{identity_message, Identify, Identity, IDENTIFY_VERSION}; impl FakeIpad { @@ -798,13 +800,21 @@ mod identify { host.wait_until_stopped(); let mut ipad = FakeIpad::new(&host, 1, NEW); ipad.auth_token = host.shared.pairing.lock().token(); - // Answered: the standby holds the port. Its listing runs on a - // thread of its own: give it a moment. + // Answered: the standby holds the port. let ack = ipad .ask_until_answered(1, Duration::from_secs(10)) .expect("a stopped PC must answer"); assert_eq!(ack.status, HelloStatus::Stopped); - std::thread::sleep(Duration::from_millis(300)); + // Its listing runs on a thread of its own. Waiting for it starts + // none, so only the standby's can come. + let deadline = Instant::now() + Duration::from_secs(10); + while listed_home_addresses().is_none() { + assert!( + Instant::now() < deadline, + "the standby never listed the home network addresses" + ); + std::thread::sleep(Duration::from_millis(5)); + } ipad.identify([1; 16]); assert!( ipad.identity([1; 16], Duration::from_secs(2)).is_some(), From ac118061b22ba5e7c6886792485fe84e03472e20 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 19:59:40 -0500 Subject: [PATCH 099/226] Keep a proven second screen proven across reopens, and time tests on paused clocks Review r2 of #205. A second screen reopened by the stage (a display that moved, or one that showed no new image for 15 s after its duplication came back) was reconciled again, and reconciling started a new proof. While it proved itself the watchdog skipped its missing-frame check, and 5 s later the proof failed back to the mirror, so the restart meant for a display gone dark never came. The generation whose second screen captured its first image is now remembered: reconciling again in that generation does not prove it again. A new generation, or a display attached anew, proves itself as before. A reopen during the first proof was noted as a capture error, which fails the proof: a display that moved while the second screen started went back to the mirror. The stage now notes a reopen as an interruption only; the proof keeps its deadline. The no-display wait and the stage's reopen spacing take their clock (and the wait its sleep) as arguments. Their tests run on paused clocks: the display comes back between two looks and is seen at the next one, at 750 ms; a reopen 999 ms after the last waits at the blocked pace, and reopens 1,000 ms or more apart do not. --- host/src/capture/mod.rs | 94 ++++++++++++++++++++++--------- host/src/capture/recovery.rs | 43 +++++++++++--- host/src/capture/second_screen.rs | 94 +++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 34 deletions(-) diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 615c590e..a1008be3 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -948,12 +948,7 @@ pub fn run_capture_stage( |frame_number| { let result = run_platform_loop(&guard.0, shared.clone(), generation, frame_number); if let Err(error) = &result { - if recovery::passing_error(error.as_ref()) { - second_screen::note_interruption(&shared, generation); - } else { - second_screen::note_persistent_error(&shared, generation); - } - second_screen::note_capture_error(&shared, generation); + note_stage_error(&shared, generation, error.as_ref()); } result }, @@ -965,6 +960,26 @@ pub fn run_capture_stage( result } +/// What a capture attempt's error says about a second screen proving +/// itself. A [`recovery::Reopen`] is not a capture error: the stage opens +/// the display again at once, and the proof keeps its deadline. Noting it +/// as one failed the proof, and a display that moved during the first +/// proof went back to the mirror. +fn note_stage_error( + shared: &SharedControl, + generation: u64, + error: &(dyn std::error::Error + Send + Sync + 'static), +) { + if recovery::passing_error(error) { + second_screen::note_interruption(shared, generation); + } else { + second_screen::note_persistent_error(shared, generation); + } + if error.downcast_ref::().is_none() { + second_screen::note_capture_error(shared, generation); + } +} + /// While no display is on, how often the wait looks for one to come back. const DISPLAY_LOOK: Duration = Duration::from_millis(250); @@ -985,14 +1000,34 @@ fn display_attached() -> bool { /// and a display that came back ends the wait: the retries settle 5 s /// apart, and the picture came back up to 5 s after the screen did. fn wait_while_blocked( + shared: &SharedControl, + generation: u64, + delay: Duration, + display_back: impl FnMut() -> bool, +) -> bool { + wait_while_blocked_on( + shared, + generation, + delay, + display_back, + Instant::now, + std::thread::sleep, + ) +} + +/// [`wait_while_blocked`] on the clock `now`, sleeping with `sleep`. +fn wait_while_blocked_on( shared: &SharedControl, generation: u64, delay: Duration, mut display_back: impl FnMut() -> bool, + mut now: impl FnMut() -> Instant, + mut sleep: impl FnMut(Duration), ) -> bool { use std::sync::atomic::Ordering; - let deadline = Instant::now() + delay; - let mut next_look = Instant::now() + DISPLAY_LOOK; + let start = now(); + let deadline = start + delay; + let mut next_look = start + DISPLAY_LOOK; loop { if !shared.running.load(Ordering::SeqCst) || !crate::supervisor::generation_is_current(generation) @@ -1000,19 +1035,19 @@ fn wait_while_blocked( return false; } hold_heartbeats(shared); - let now = Instant::now(); - if now >= deadline { + let at = now(); + if at >= deadline { return true; } - if now >= next_look { - next_look = now + DISPLAY_LOOK; + if at >= next_look { + next_look = at + DISPLAY_LOOK; if *shared.capture_block.lock() == Some(recovery::CaptureBlock::NoDisplay) && display_back() { return true; } } - std::thread::sleep((deadline - now).min(Duration::from_millis(50))); + sleep((deadline - at).min(Duration::from_millis(50))); } } @@ -1744,36 +1779,43 @@ mod tests { shared.running.store(true, SeqCst); let generation = crate::supervisor::CURRENT_GENERATION.load(SeqCst); *shared.capture_block.lock() = Some(recovery::CaptureBlock::NoDisplay); - let started = Instant::now(); + // A paused clock: the wait's sleeps move it. + let start = Instant::now(); + let elapsed = Cell::new(Duration::ZERO); + let now = || start + elapsed.get(); + let sleep = |by: Duration| elapsed.set(elapsed.get() + by); let looks = Cell::new(0); let back = || { looks.set(looks.get() + 1); - started.elapsed() >= Duration::from_millis(600) + elapsed.get() >= Duration::from_millis(600) }; - assert!(wait_while_blocked( + assert!(wait_while_blocked_on( &shared, generation, Duration::from_secs(5), - back + back, + now, + sleep )); - let waited = started.elapsed(); - // Well before the 5 s wait ends, with room for a loaded test machine. - assert!(waited < Duration::from_millis(4_000), "{waited:?}"); - assert!(looks.get() >= 2, "{} looks", looks.get()); + // Looks at 250, 500 and 750 ms: back at the first look after 600. + assert_eq!(elapsed.get(), Duration::from_millis(750)); + assert_eq!(looks.get(), 3); // Other blocked states clear by the user's hand; nothing is polled. *shared.capture_block.lock() = Some(recovery::CaptureBlock::SecureDesktop); - let looks = Cell::new(0); - let started = Instant::now(); - assert!(wait_while_blocked( + elapsed.set(Duration::ZERO); + looks.set(0); + assert!(wait_while_blocked_on( &shared, generation, Duration::from_millis(600), || { looks.set(looks.get() + 1); true - } + }, + now, + sleep )); - assert!(started.elapsed() >= Duration::from_millis(600)); + assert_eq!(elapsed.get(), Duration::from_millis(600)); assert_eq!(looks.get(), 0); } diff --git a/host/src/capture/recovery.rs b/host/src/capture/recovery.rs index ec561528..6e4881dd 100644 --- a/host/src/capture/recovery.rs +++ b/host/src/capture/recovery.rs @@ -376,9 +376,20 @@ impl BlockedRetry { /// stream epoch: an attempt that started again at 1 sent sequence 1 after /// 200, and the iPad dropped every frame until the count passed 200 again. pub fn run_until_unblocked( + shared: &SharedControl, + capture: impl FnMut(&mut u64) -> Result<(), Box>, + wait: impl FnMut(Duration) -> bool, +) -> Result<(), Box> { + run_until_unblocked_on(shared, capture, wait, Instant::now) +} + +/// [`run_until_unblocked`] on the clock `clock`, which times the +/// [`REOPEN_SPACING`]. +pub(crate) fn run_until_unblocked_on( shared: &SharedControl, mut capture: impl FnMut(&mut u64) -> Result<(), Box>, mut wait: impl FnMut(Duration) -> bool, + mut clock: impl FnMut() -> Instant, ) -> Result<(), Box> { let mut retry = BlockedRetry::default(); let mut stalled = 0; @@ -391,7 +402,7 @@ pub fn run_until_unblocked( }; let error: Box = match error.downcast::() { Ok(reopen) => { - let now = Instant::now(); + let now = clock(); if last_reopen.is_none_or(|at| now.duration_since(at) >= REOPEN_SPACING) { last_reopen = Some(now); info!(detail = %reopen.detail, "Opening the display to capture again"); @@ -1302,14 +1313,23 @@ mod tests { /// after waits at that pace instead of reopening in a loop. #[test] fn a_moved_display_is_reopened_at_once_but_not_in_a_loop() { + use std::cell::Cell; + let ms = Duration::from_millis; let shared = SharedControl::new(0, 1); - let mut attempts = 0; + // A paused clock: each attempt and each wait moves it. + let start = Instant::now(); + let elapsed = Cell::new(Duration::ZERO); + // Before each attempt, how far the clock moves; every attempt but + // the last asks to reopen. + let steps = [ms(0), ms(999), ms(0), ms(1_000), ms(0)]; + let mut attempt = 0; let mut waits = Vec::new(); - let result = run_until_unblocked( + let result = run_until_unblocked_on( &shared, |_| { - attempts += 1; - if attempts <= 2 { + elapsed.set(elapsed.get() + steps[attempt]); + attempt += 1; + if attempt < steps.len() { Err(Reopen { detail: "the display layout changed".into(), } @@ -1319,13 +1339,20 @@ mod tests { } }, |delay| { - waits.push(delay.as_millis()); + waits.push(delay); + elapsed.set(elapsed.get() + delay); true }, + || start + elapsed.get(), ); assert!(result.is_ok()); - assert_eq!(attempts, 3); - assert_eq!(waits, [500]); + assert_eq!(attempt, steps.len()); + // At 0 ms: at once. At 999 ms, under a second after it: the blocked + // pace, 500 ms. At 1,499 ms, a second and more after the last one + // opened at once: at once. At 2,499 ms, exactly a second after + // that: at once. + assert_eq!(waits, [ms(500)]); + assert_eq!(elapsed.get(), ms(2_499)); assert_eq!( *shared.capture_block.lock(), Some(CaptureBlock::DisplayChanging) diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 7638c802..0933ca4e 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -172,6 +172,11 @@ pub(crate) struct State { /// A start is claimed: its Attach is queued or running. attaching: bool, proving: Option, + /// The capture generation whose second screen captured its first image. + /// The stage opening the display again in that generation (a recovery + /// reopen) does not prove it again: a new proof hid a display gone dark + /// from the watchdog and then fell back to the mirror. + proven: Option, /// Why the second screen last failed, for the log and the backend. failure: Option<&'static str>, /// The failure saved for later app runs. @@ -746,6 +751,7 @@ pub fn note_attached(shared: &SharedControl, device_name: String) { let mut state = shared.second_screen.lock(); state.switch_pending = true; state.proving = None; + state.proven = None; } /// [`note_attached`], unless the mode changed or quitting began. The mode @@ -765,8 +771,12 @@ fn note_attached_if_selected(shared: &SharedControl, device_name: String) -> boo /// Capture `generation` streams the second screen from `now`. The first /// generation after the switch starts the [`PROVE_TIMEOUT`] deadline; a /// later one inherits it, and any capture failure, until an image arrives. +/// The generation that proved it is not asked again. pub fn start_proving(shared: &SharedControl, generation: u64, now: Instant) { let mut state = shared.second_screen.lock(); + if state.proving.is_none() && state.proven == Some(generation) { + return; + } match state.proving.as_mut() { // A call the earlier generation still waits on is not this one's, // and its guard can no longer clear the flag. @@ -824,6 +834,7 @@ pub fn note_image(shared: &SharedControl, generation: u64) { let mut state = shared.second_screen.lock(); if let Some(proving) = state.proving.filter(|p| p.generation == generation) { state.proving = None; + state.proven = Some(generation); tracing::info!( after_ms = proving.since.elapsed().as_millis() as u64, "Second screen captured its first image — streaming it" @@ -2434,6 +2445,89 @@ mod tests { assert_eq!(remembered_failure(&shared), Some("duplication_failed")); } + /// Review r2 of #205 (P1): a second screen that showed no new image for + /// a while after its duplication came back is opened again by the stage. + /// Reconciling it again started a new proof: the watchdog skipped its + /// missing-frame check while the display proved itself, and 5 s later + /// the proof failed back to the mirror. In the generation that proved + /// it, it stays proven, so the watchdog judges a display gone dark. + #[test] + fn a_recovery_reopen_does_not_prove_a_proven_second_screen_again() { + use std::sync::atomic::Ordering::Relaxed; + let dir = std::env::temp_dir().join(format!( + "eternal-capture-recovery-proven-{}", + std::process::id() + )); + let shared = second_screen_selected(); + let display = fake::FakeDisplay::new(Some(Duration::ZERO), secs(10.0)); + display.install(&shared, &dir, std::time::SystemTime::now()); + assert!(Driver::set_enabled(&display, true)); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let streamed = Some(CaptureTarget::Output("virtual".into())); + let reconcile = + || reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation); + assert_eq!(reconcile(), streamed); + assert!(proving(&shared)); + note_image(&shared, generation); + assert!(!proving(&shared)); + // The loop ended with a reopen; the stage reconciles again. + assert_eq!(reconcile(), streamed); + assert!(!proving(&shared), "proven in this generation"); + // Nothing comes from it: the watchdog restarts the pipeline. + let mut machine = crate::supervisor::Machine::new(1); + let now_ms = 100_000; + shared.hb_capture_loop_ms.store(now_ms, Relaxed); + shared.hb_capture_frame_ms.store(now_ms - 5_100, Relaxed); + shared.hb_encode_frame_ms.store(now_ms - 5_100, Relaxed); + let reason = crate::supervisor::wedge_reason_at(&shared, &mut machine, now_ms); + assert!( + reason + .as_deref() + .is_some_and(|reason| reason.starts_with("no captured frame")), + "{reason:?}" + ); + // The next generation, after that restart, proves it again; so does + // a display attached anew. + start_proving(&shared, generation.wrapping_add(1), Instant::now()); + assert!(proving(&shared)); + note_attached(&shared, "virtual".into()); + assert_eq!(reconcile(), streamed); + assert!(proving(&shared)); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Review r2 of #205 (P2): a display that moved during the first proof + /// ends the capture loop with a reopen, and the stage noted it as a + /// capture error: the proof failed, and the second screen went back to + /// the mirror at once. A reopen keeps the proof, its deadline and the + /// interruption. + #[test] + fn a_reopen_during_the_first_proof_keeps_it_going() { + let start = Instant::now(); + let shared = second_screen_selected(); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + assert_eq!(next_step(&shared, start), Step::Switch); + start_proving(&shared, 7, start); + let reopen = crate::capture::recovery::Reopen { + detail: "the display layout changed".into(), + }; + super::super::note_stage_error(&shared, 7, &reopen); + let proving = shared.second_screen.lock().proving.unwrap(); + assert!(proving.interrupted && !proving.capture_failed); + assert_eq!(proving.since, start); + assert_eq!(next_step(&shared, start + secs(4.9)), Step::Nothing); + // Any other capture error still ends the proof at once. + let failed = std::io::Error::other("DuplicateOutput failed"); + super::super::note_stage_error(&shared, 7, &failed); + assert_eq!( + next_step(&shared, start + secs(4.9)), + Step::Fallback("duplication_failed") + ); + } + #[test] fn a_second_screen_that_shows_an_image_stays() { let shared = second_screen_selected(); From 19f54b11681fcb9b2a50a023f7100a752973695a Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:07:03 -0500 Subject: [PATCH 100/226] List the pointer messages in the architecture notes --- ARCHITECTURE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2b663b49..63cfc75d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -75,6 +75,7 @@ Rust and Swift share golden vectors and stable packet-type registry tests. | KEYFRAME_REQUEST | Recovery after repair expiry or decoder error | | PING / PONG | Min-RTT clock estimate for end-to-end latency | | INPUT_EVENT | Normalized mouse/touch events, HID keys and UTF-16 text | +| POINTER_SHAPE / POINTER_STATE | The Mirror's pointer image (chunked premultiplied BGRA and an invert mask) and its position, at most once a frame budget, for an iPad that draws the pointer itself (feature bit 5, host capability bit 6); the video then leaves the pointer out. The Second screen keeps the pointer in its video. | | Audio | Sequenced Opus packets with discontinuity and capture timing | | BYE | User, background or shutdown teardown; the host ending a silent session | From 7961ceb7b29de1df04368e0e7caa41456e21cd1d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:07:15 -0500 Subject: [PATCH 101/226] Name the panic hook test's folder so Windows accepts it The folder name used the Debug form of an Instant, which has characters Windows doesn't allow in a path, so the test failed there before it ran. --- host/src/startup/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/host/src/startup/mod.rs b/host/src/startup/mod.rs index d582e7ed..acbf96ba 100644 --- a/host/src/startup/mod.rs +++ b/host/src/startup/mod.rs @@ -978,9 +978,12 @@ mod tests { fn a_panic_making_a_window_device_leaves_the_virtual_display_on() { use eframe::egui_wgpu::winit::{Replacement, REPLACEMENT_THREAD}; let directory = std::env::temp_dir().join(format!( - "eternal-panic-hook-{}-{:?}", + "eternal-panic-hook-{}-{}", std::process::id(), - Instant::now() + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() )); fs::create_dir_all(&directory).unwrap(); let notes = directory.join("notes.txt"); From ddb03a8656bdd50a4c6ca87747701d281ed0526a Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:07:15 -0500 Subject: [PATCH 102/226] Let the window close while a device replacement is stuck eframe leaves its event loop only when the closed window is destroyed. The replacement thread's surface held the window, so with that thread stuck in a driver the app never quit: the native hang run had to be killed. The surface is now made from the window's handles and doesn't keep it alive. --- vendor/egui-wgpu/src/winit.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs index 9a560a3d..ea9be482 100644 --- a/vendor/egui-wgpu/src/winit.rs +++ b/vendor/egui-wgpu/src/winit.rs @@ -551,7 +551,19 @@ impl Painter { } Some(_) => {} } - let surface = match self.instance.create_surface(window) { + // The surface goes to the replacement thread, which a driver can hold forever. Made + // from the window's handles, it doesn't keep the window alive: eframe ends its loop + // only once the window is destroyed, so a surface holding it would keep the app from + // quitting. + // SAFETY: the surface is used while the window lives: here and by the replacement + // thread, and on this thread once installed. The window is the root viewport, + // destroyed only as the app quits, after which a stuck thread can only drop the + // surface, never configured. + let made = || -> Result, crate::WgpuError> { + let target = unsafe { wgpu::SurfaceTargetUnsafe::from_window(&*window) }?; + Ok(unsafe { self.instance.create_surface_unsafe(target) }?) + }; + let surface = match made() { Ok(surface) => surface, Err(err) => { self.replacement_failed(now, &err); From 4856cacbf6204db28714da5ae2bdeb01f626909c Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:07:27 -0500 Subject: [PATCH 103/226] Say in vendor/README.md why the replacement surface doesn't hold the window --- vendor/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vendor/README.md b/vendor/README.md index 586f3a21..9c00fcdf 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -35,8 +35,10 @@ The patch: Listing adapters and making the device, where a driver can stall, run on their own thread (`winit::Answer`), which the window polls each frame. One that hasn't answered in 10 s counts as a failed try, and no other starts - while it is out; a late device is still taken. The thread is named - `winit::REPLACEMENT_THREAD`, so the host's panic hook treats a panic there + while it is out; a late device is still taken. The surface it gets is + made from the window's handles, so a stuck thread doesn't keep the window + alive: eframe ends its loop only once the window is destroyed. The thread + is named `winit::REPLACEMENT_THREAD`, so the host's panic hook treats a panic there as one failed try: it doesn't turn the virtual display off as it does for other background threads. Configuring the new surface, which makes the swapchain, stays on the window's thread, because DXGI sends From 09cb15cbd5454fa35212adb8a7b0f1e9e863fbd9 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:09:25 -0500 Subject: [PATCH 104/226] Run no Disable task at an idle quit when Windows reports the display disabled Review r3: since the startup skip records nothing, a quit without a second screen ran a Disable task, up to its 30 s limit when Task Scheduler stalled, which held up exit and the update helper. Quitting (and a panic's cleanup) now decide as startup does, on the Disable's own turn: when this host has nothing recorded (the state is unknown and no Enable ran since the last completed Disable) and Configuration Manager reads the device disabled (problem 22), no task runs and nothing is recorded. A display that is on or unreadable is turned off as before. After an Enable this host ran, completed or timed out, the Disable always runs, whatever the device reads. The failure reporter's cleanup is unchanged: it only runs for a host that turned the display on, where its Disable waits for an Enable that may still be finishing. Tests: an idle quit with the device disabled runs no task; one with the device on or unreadable runs the Disable; after an Enable, completed or timed out, it runs without reading the device. --- host/src/vdd.rs | 150 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 121 insertions(+), 29 deletions(-) diff --git a/host/src/vdd.rs b/host/src/vdd.rs index 4b15eab8..888e47e0 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -214,18 +214,26 @@ impl DriverTasks { self.gate.lock().health } - /// The startup cleanup: a Disable on its own turn, unless `already_off` - /// says, on that turn, that the display is off. That skip records - /// nothing: the state stays unknown, so every later Disable (Stop, a - /// mode change, quitting, the failure reporter) still runs the task, and - /// turns off a display an Enable left from before turns on meanwhile. - fn disable_at_startup( + /// The Disable at startup and at quit: on its own turn, no task runs + /// when this host has nothing recorded (the state is unknown, and no + /// Enable ran since the last completed Disable) and `already_off` then + /// says the display is off. The skip records nothing, so the state stays + /// unknown and the next Disable decides afresh. After an Enable, even + /// one that timed out and may still be running, the Disable always runs. + fn disable_unless_off( &self, on_turn: impl FnOnce(), already_off: impl FnOnce() -> bool, run: impl FnOnce(bool, &mut dyn FnMut()) -> TaskOutcome, ) -> bool { - self.set_enabled_unless(false, Purpose::Cleanup, || {}, on_turn, already_off, run) + let skip = || { + let idle = { + let gate = self.gate.lock(); + gate.known_enabled.is_none() && !gate.enable_ran + }; + idle && already_off() + }; + self.set_enabled_unless(false, Purpose::Cleanup, || {}, on_turn, skip, run) } #[cfg(test)] @@ -528,9 +536,18 @@ fn run_on_turn(enabled: bool, progress: &mut dyn FnMut()) -> TaskOutcome { /// Disable the virtual display device (best effort — failures are non-fatal). /// For startup, quit and panics: a hung or refused earlier task does not stop /// it, and it waits for an Enable still running, within the task's bounds. -/// True when the display is known to be off. +/// A host that never turned the display on runs no task when Windows reports +/// it disabled, so an idle quit doesn't wait on Task Scheduler. True when the +/// display is known or read to be off. pub fn disable() -> bool { - request(false, Purpose::Cleanup, || {}, &mut None) + #[cfg(windows)] + { + TASKS.disable_unless_off(|| {}, device_off, run_on_turn) + } + #[cfg(not(windows))] + { + request(false, Purpose::Cleanup, || {}, &mut None) + } } /// `disable` at startup, without holding up the first frame: the Disable @@ -551,17 +568,7 @@ pub fn disable_in_background() { let _ = held.send(()); }; #[cfg(windows)] - TASKS.disable_at_startup( - on_turn, - || { - let off = reports_disabled(crate::vdd_device::query()); - if off { - info!("Virtual display already off — not running the Disable task"); - } - off - }, - run_on_turn, - ); + TASKS.disable_unless_off(on_turn, device_off, run_on_turn); #[cfg(not(windows))] request_on_turn(false, Purpose::Cleanup, || {}, on_turn, &mut None); }); @@ -575,8 +582,18 @@ pub fn disable_in_background() { } } -/// Whether the device reads disabled (problem 22): the startup cleanup has -/// nothing to turn off. +/// Whether Configuration Manager reads the device disabled, said in the log. +#[cfg(windows)] +fn device_off() -> bool { + let off = reports_disabled(crate::vdd_device::query()); + if off { + info!("Virtual display already off — not running the Disable task"); + } + off +} + +/// Whether the device reads disabled (problem 22): a Disable has nothing to +/// turn off. #[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. fn reports_disabled(device: crate::vdd_device::DeviceState) -> bool { device == crate::vdd_device::DeviceState::Problem(22) @@ -641,7 +658,7 @@ mod tests { use std::cell::Cell; let tasks = DriverTasks::new(); let off = || reports_disabled(DeviceState::Problem(22)); - assert!(tasks.disable_at_startup( + assert!(tasks.disable_unless_off( || {}, off, |_, _| { panic!("ran the Disable task for a display that is off") } @@ -678,7 +695,7 @@ mod tests { ] { let tasks = DriverTasks::new(); let ran = Cell::new(false); - tasks.disable_at_startup( + tasks.disable_unless_off( || {}, || reports_disabled(device), |_, _| { @@ -711,19 +728,19 @@ mod tests { DeviceState::Problem(22) } }; - assert!(tasks.disable_at_startup( + assert!(tasks.disable_unless_off( || {}, || reports_disabled(device()), |_, _| { panic!("ran the startup Disable for a display that reads disabled") } )); - // The surviving Enable ends whenever it ends; then the host quits. + // The surviving Enable ends whenever it ends; then the host quits, + // and the quit reads the device as `disable` does. if enable_running.replace(false) { display_on.set(true); } - let disabled = tasks.set_enabled( - false, - Purpose::Cleanup, + let disabled = tasks.disable_unless_off( || {}, + || reports_disabled(device()), |enabled, _| { assert!(!enabled); display_on.set(false); @@ -734,6 +751,81 @@ mod tests { assert!(!display_on.get(), "a phantom monitor stayed after quit"); } + /// Review r3: with the startup skip recording nothing, an idle quit ran + /// a Disable task, up to the task's 30 s limit when Task Scheduler + /// stalled, holding up exit and the update helper. + #[test] + fn an_idle_quit_runs_the_disable_task_only_when_the_display_is_on() { + use crate::vdd_device::DeviceState; + use std::cell::Cell; + let never = |_: bool, _: &mut dyn FnMut()| -> TaskOutcome { + panic!("ran the Disable task for a display that reads disabled") + }; + // Started and quit without a second screen: the device reads + // disabled both times, and no task runs. + let tasks = DriverTasks::new(); + assert!(tasks.disable_unless_off( + || {}, + || reports_disabled(DeviceState::Problem(22)), + never + )); + assert!(tasks.disable_unless_off( + || {}, + || reports_disabled(DeviceState::Problem(22)), + never + )); + assert_eq!(tasks.gate.lock().known_enabled, None); + + // A display that is on, or unreadable, is turned off as before. + for device in [ + DeviceState::Ready, + DeviceState::NoMonitor, + DeviceState::Missing, + ] { + let tasks = DriverTasks::new(); + let ran = Cell::new(false); + tasks.disable_unless_off( + || {}, + || reports_disabled(device), + |enabled, _| { + assert!(!enabled); + ran.set(true); + TaskOutcome::Done + }, + ); + assert!(ran.get(), "no Disable at quit for {device:?}"); + } + + // After an Enable this host ran, completed or timed out, the device + // may read disabled while the display is still coming: the Disable + // runs whatever it reads. + for enable in [TaskOutcome::Done, TaskOutcome::TimedOut] { + let tasks = DriverTasks::new(); + tasks.set_enabled(true, Purpose::Attempt, || {}, |_, _| enable); + let ran = Cell::new(false); + let read = Cell::new(false); + tasks.disable_unless_off( + || {}, + || { + read.set(true); + reports_disabled(DeviceState::Problem(22)) + }, + |_, _| { + ran.set(true); + TaskOutcome::Done + }, + ); + assert!( + ran.get(), + "no Disable at quit after an Enable that {enable:?}" + ); + assert!( + !read.get(), + "read the device after an Enable that {enable:?}" + ); + } + } + #[test] fn a_hung_task_is_not_waited_out_again_until_the_cooldown_ends() { let mut gate = TaskGate::new(); From 88a16b1987a73009cc5fbc6c184b0145cce5d8b1 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:18:32 -0500 Subject: [PATCH 105/226] Send the second screen's pointer apart, keep old shapes out, clip to the picture The virtual display driver leaves its hardware cursor off unless its settings file turns it on, and the file this host writes had no block, so Windows drew the pointer into the second screen's image. The host now writes the driver's own cursor block (hardware cursor on, 128x128, alpha and XOR support), and the second screen sends the pointer apart whenever the file the driver reads enables it, as Mirror does. A user's own file without it keeps the pointer in the picture. On the iPad, a late chunk of an older pointer image no longer replaces the image held, and both pointer passes are clipped to the picture so a pointer past its edge stays out of Fit's bars. --- host/src/capture/dxgi.rs | 20 ++++--- host/src/capture/mod.rs | 60 +++++++++++++------ host/src/control.rs | 5 ++ host/src/vdd_settings.rs | 58 ++++++++++++++++++ ios/EternalMonitor/Render/MetalView.swift | 6 +- .../Render/PointerOverlay.swift | 28 ++++++++- .../PointerOverlayTests.swift | 51 ++++++++++++++++ 7 files changed, 198 insertions(+), 30 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 3f9cb5d7..35166c03 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -17,9 +17,9 @@ use super::recovery::{ }; use super::{ acquired, client_connected, frame_budget_for, frame_buffer, heartbeat, idle_heartbeat, - keepalive_allowed, keepalive_due, pointer_apart, read_back_image, reconcile_virtual_display, - recovery_poll, resolve_target, Acquired, FrameSlot, FrameUse, IdleGate, OutputInfo, RawFrame, - ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, + keepalive_allowed, keepalive_due, pointer_apart, pointer_in_picture, read_back_image, + reconcile_virtual_display, recovery_poll, resolve_target, Acquired, FrameSlot, FrameUse, + IdleGate, OutputInfo, RawFrame, ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, }; use crate::control::{CaptureTarget, SharedControl}; use crate::gpu_preference::Outcome; @@ -319,7 +319,11 @@ pub fn run_capture_loop( // Whether `last_frame` has the pointer drawn in: it has unless the // client draws the pointer itself (`SharedControl::client_draws_pointer`). let mut last_frame_has_pointer = true; - if second_screen { + let pointer_in_image = pointer_in_picture( + second_screen, + shared.vdd_hardware_cursor.load(Ordering::Relaxed), + ); + if pointer_in_image { // A position from a mirror before this one would stay drawn. shared.pointer.set_position(super::pointer::Position { visible: false, @@ -362,7 +366,7 @@ pub fn run_capture_loop( // other client, or none, one with it. The staging texture still // holds the last desktop image, so the cached frame is redrawn from // it when the connected client's answer differs from the cache's. - let draws_pointer = pointer_apart(shared.client_draws_pointer(), second_screen); + let draws_pointer = pointer_apart(shared.client_draws_pointer(), pointer_in_image); if seen_image && last_frame_has_pointer == draws_pointer { let mut pixel_buf = frame_buffer(frame_bytes); copy_staging( @@ -450,8 +454,8 @@ pub fn run_capture_loop( cursor_x = frame_info.PointerPosition.Position.x; cursor_y = frame_info.PointerPosition.Position.y; shared.pointer.set_position(super::pointer::Position { - // The second screen's picture has the pointer in it. - visible: cursor_visible && !second_screen, + // A picture with the pointer in it has none apart. + visible: cursor_visible && !pointer_in_image, x: cursor_x, y: cursor_y, host_time_us: crate::clock::host_now_us(), @@ -459,7 +463,7 @@ pub fn run_capture_loop( } if frame_info.PointerShapeBufferSize > 0 && cursor.update_shape(dupl, frame_info.PointerShapeBufferSize) - && !second_screen + && !pointer_in_image { if let Some(image) = super::pointer::convert_shape( cursor.shape_type, diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 4fccce95..d97c8b40 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -108,17 +108,22 @@ pub(crate) fn frame_has_image(last_present_time: i64, seen_image: bool) -> bool last_present_time != 0 || seen_image } -/// Whether the pointer goes apart from the video for this capture. On the -/// second screen it never does: the virtual display has no hardware -/// pointer, so Windows draws the pointer into its image, and Desktop -/// Duplication kept reporting a stale position there (measured on the -/// reference PC: every pointer move was a new image, and the reported -/// position never moved). An iPad that draws the pointer would show that -/// stale one too. There the pointer stays in the video, as before, and the -/// iPad is told it is hidden. +/// Whether Windows draws the pointer into the captured picture: on the +/// second screen, unless the virtual display driver was given a hardware +/// cursor (`vdd_settings::hardware_cursor`). Without one, measured on the +/// reference PC, every pointer move there was a new image and Desktop +/// Duplication kept reporting the pointer at one stale position. #[cfg(any(windows, test))] -pub(crate) fn pointer_apart(client_draws_pointer: bool, second_screen: bool) -> bool { - client_draws_pointer && !second_screen +pub(crate) fn pointer_in_picture(second_screen: bool, driver_cursor: bool) -> bool { + second_screen && !driver_cursor +} + +/// Whether the pointer goes apart from the video for this capture: to an +/// iPad that draws it, unless Windows draws it into the picture. Then it +/// stays in the video, as before, and the iPad is told it is hidden. +#[cfg(any(windows, test))] +pub(crate) fn pointer_apart(client_draws_pointer: bool, pointer_in_picture: bool) -> bool { + client_draws_pointer && !pointer_in_picture } /// What the capture loop does with an acquired frame. @@ -498,6 +503,15 @@ fn prepare_vdd_settings(shared: &SharedControl, outputs: &[OutputInfo]) -> VddRe } Err(_) => {} } + // Whatever wrote it, the file the driver reads next decides whether it + // gives Windows a hardware cursor, and so whether the second screen's + // pointer is reported apart from its picture. + let hardware_cursor = + std::fs::read_to_string(path).is_ok_and(|xml| crate::vdd_settings::hardware_cursor(&xml)); + shared + .vdd_hardware_cursor + .store(hardware_cursor, std::sync::atomic::Ordering::Relaxed); + info!(hardware_cursor, "The virtual display's pointer"); VddRequest { pin: crate::vdd_settings::pin_result(wanted, &applied), mode, @@ -1061,15 +1075,25 @@ mod tests { ); } + /// #183 is a second screen report: with the driver's hardware cursor + /// the second screen sends the pointer apart, as Mirror does. A driver + /// file without it keeps the pointer in the picture. #[test] - fn the_second_screen_keeps_the_pointer_in_the_video() { - assert!(pointer_apart(true, false), "Mirror, an iPad that draws it"); - assert!( - !pointer_apart(true, true), - "second screen: Windows draws it" - ); - assert!(!pointer_apart(false, false), "an older iPad"); - assert!(!pointer_apart(false, true)); + fn the_second_screen_sends_the_pointer_apart_with_the_drivers_cursor() { + for (second_screen, driver_cursor, in_picture) in [ + (false, false, false), + (false, true, false), + (true, true, false), + (true, false, true), + ] { + assert_eq!( + pointer_in_picture(second_screen, driver_cursor), + in_picture, + "second_screen={second_screen} driver_cursor={driver_cursor}" + ); + assert_eq!(pointer_apart(true, in_picture), !in_picture); + assert!(!pointer_apart(false, in_picture), "an older iPad"); + } } pub(super) fn connect_viewer(shared: &SharedControl) { diff --git a/host/src/control.rs b/host/src/control.rs index 4f8e3979..b1ace7b9 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -72,6 +72,10 @@ pub struct SharedControl { pub(crate) vdd_worker: Arc, /// Output owned by the enabled driver, retained across pipeline restarts. pub(crate) vdd_output: Arc>>, + /// The virtual display driver was given a hardware cursor, so the + /// second screen's pointer is reported apart from its picture. + #[cfg_attr(not(windows), allow(dead_code))] + pub(crate) vdd_hardware_cursor: Arc, /// Second screen timing across pipeline restarts: the pending switch onto /// it, the grace period after its viewer left, the last driver change. pub(crate) second_screen: Arc>, @@ -199,6 +203,7 @@ impl SharedControl { handoff: Default::default(), vdd_worker: Arc::new(Default::default()), vdd_output: Arc::new(Mutex::new(None)), + vdd_hardware_cursor: Arc::new(AtomicBool::new(false)), second_screen: Arc::new(Mutex::new(Default::default())), hb_capture_loop_ms: Arc::new(AtomicU64::new(0)), hb_capture_frame_ms: Arc::new(AtomicU64::new(0)), diff --git a/host/src/vdd_settings.rs b/host/src/vdd_settings.rs index 9f132195..9c904f27 100644 --- a/host/src/vdd_settings.rs +++ b/host/src/vdd_settings.rs @@ -126,10 +126,35 @@ pub fn render(modes: &[VddMode], gpu: Option<&str>) -> String { )); } xml.push_str(" \n"); + xml.push_str(CURSOR); xml.push_str("\n"); xml } +/// The driver's hardware cursor, as its own sample file sets it. Without +/// `` the driver leaves it off, and Windows draws the pointer into +/// the virtual display's image, so Desktop Duplication has no pointer to +/// report apart from it and an iPad can't draw it. With it on, the second +/// screen reports the pointer like any other screen. The driver reads a +/// missing size as -1, so the whole block is written. +const CURSOR: &str = " \n true\n \ + 128\n 128\n \ + true\n \ + 2\n \n"; + +/// Whether a settings document turns the driver's hardware cursor on, as +/// the driver reads it: the text of ``, `true` or `1`. +pub fn hardware_cursor(xml: &str) -> bool { + let Some(start) = xml.find("") else { + return false; + }; + let rest = &xml[start + "".len()..]; + let Some(end) = rest.find("") else { + return false; + }; + matches!(rest[..end].trim(), "true" | "1") +} + fn escape(text: &str) -> String { text.replace('&', "&") .replace('<', "<") @@ -792,6 +817,39 @@ mod tests { 120" )); assert!(xml.ends_with("\n")); + assert!( + hardware_cursor(&xml), + "the second screen's pointer is reported apart" + ); + assert!(xml.contains( + " \n true\n \ + 128\n 128\n \ + true\n \ + 2\n \n" + )); + } + + /// The driver turns its hardware cursor on only when the file says so; + /// a file from before this version, or the vendor's with it off, keeps + /// the pointer in the second screen's picture. + #[test] + fn the_hardware_cursor_is_read_as_the_driver_reads_it() { + assert!(!hardware_cursor(VENDOR)); + let v032 = render(&modes_for_client(1668, 2420, 60), None).replace(CURSOR, ""); + assert!(!hardware_cursor(&v032)); + for (text, on) in [ + ("true", true), + (" 1 ", true), + ("false", false), + ("0", false), + ("", false), + ] { + let xml = format!("{text}"); + assert_eq!(hardware_cursor(&xml), on, "{text:?}"); + } + assert!(!hardware_cursor( + "true" + )); } #[test] diff --git a/ios/EternalMonitor/Render/MetalView.swift b/ios/EternalMonitor/Render/MetalView.swift index bb11da5a..674f8c53 100644 --- a/ios/EternalMonitor/Render/MetalView.swift +++ b/ios/EternalMonitor/Render/MetalView.swift @@ -314,10 +314,14 @@ final class MetalRenderer { private func drawPointer(_ pointer: PointerOverlay, encoder: MTLRenderCommandEncoder, video: CGSize, drawable: CGSize) { let snapshot = pointer.snapshot() - guard snapshot.visible, let image = snapshot.image, let textures = textures(for: image) else { return } + guard snapshot.visible, let image = snapshot.image, let textures = textures(for: image), + let clip = PointerGeometry.clip(video: video, view: drawable, fit: pictureFit) + else { return } let rect = PointerGeometry.rect(x: snapshot.x, y: snapshot.y, width: image.width, height: image.height, video: video, view: drawable, fit: pictureFit) var uniforms = PointerUniforms(rect: PointerGeometry.ndc(rect, view: drawable)) + // Both passes stay on the picture, out of the letterbox bars. + encoder.setScissorRect(MTLScissorRect(x: clip.x, y: clip.y, width: clip.width, height: clip.height)) encoder.setRenderPipelineState(pointerPipeline) encoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 0) encoder.setFragmentTexture(textures.image, index: 0) diff --git a/ios/EternalMonitor/Render/PointerOverlay.swift b/ios/EternalMonitor/Render/PointerOverlay.swift index 3494e274..2ddc860b 100644 --- a/ios/EternalMonitor/Render/PointerOverlay.swift +++ b/ios/EternalMonitor/Render/PointerOverlay.swift @@ -103,10 +103,11 @@ final class PointerOverlay { func handle(_ chunk: PointerShapeChunk) { let completed = state.withLock { s -> Bool in - if s.snapshot.image?.id == chunk.shapeId { return false } + // A chunk of the held image or an older one changes nothing: a + // late, reordered chunk must not bring back an old image. + if let held = s.snapshot.image?.id, !Self.isNewer(chunk.shapeId, than: held) { return false } if s.assembly?.chunk.shapeId != chunk.shapeId { - // A late chunk of an older image doesn't replace a newer one. - if let pending = s.assembly, Int32(bitPattern: chunk.shapeId &- pending.chunk.shapeId) < 0 { + if let pending = s.assembly, !Self.isNewer(chunk.shapeId, than: pending.chunk.shapeId) { return false } s.assembly = Assembly(chunk: chunk, bytes: [UInt8](repeating: 0, count: Int(chunk.totalLen))) @@ -132,6 +133,12 @@ final class PointerOverlay { if completed { onChange?() } } + /// Image ids count up from 1 and wrap: `id` is newer when it is less + /// than half the counter ahead. + static func isNewer(_ id: UInt32, than other: UInt32) -> Bool { + Int32(bitPattern: id &- other) > 0 + } + static func image(_ chunk: PointerShapeChunk, bytes: [UInt8]) -> Image { let (w, h) = (Int(chunk.width), Int(chunk.height)) let pixels = Data(bytes[0.. (x: Int, y: Int, width: Int, height: Int)? { + let content = ContentRectMapper(viewSize: view, videoSize: video, fit: fit).contentRect + let visible = content.intersection(CGRect(origin: .zero, size: view)) + guard !visible.isNull, visible.width > 0, visible.height > 0 else { return nil } + let x0 = max(0, Int(visible.minX.rounded(.down))) + let y0 = max(0, Int(visible.minY.rounded(.down))) + let x1 = min(Int(view.width), Int(visible.maxX.rounded(.up))) + let y1 = min(Int(view.height), Int(visible.maxY.rounded(.up))) + guard x1 > x0, y1 > y0 else { return nil } + return (x0, y0, x1 - x0, y1 - y0) + } + /// `rect` in Metal's normalized device coordinates: (minX, minY, maxX, /// maxY) with y up, as the renderer's quad takes it. static func ndc(_ rect: CGRect, view: CGSize) -> SIMD4 { diff --git a/ios/EternalMonitorTests/PointerOverlayTests.swift b/ios/EternalMonitorTests/PointerOverlayTests.swift index cf92cece..a8320e6a 100644 --- a/ios/EternalMonitorTests/PointerOverlayTests.swift +++ b/ios/EternalMonitorTests/PointerOverlayTests.swift @@ -57,6 +57,28 @@ final class PointerOverlayTests: XCTestCase { XCTAssertEqual(overlay.snapshot().image?.id, 4) } + /// Review r1: image 1's chunks arriving after image 2 completed brought + /// image 1 back. Older ids lose against the held image as well as the + /// one assembling, across the counter's wrap too. + func testAReorderedOlderImageNeverReplacesTheHeldOne() { + let overlay = PointerOverlay() + for chunk in chunks(id: 2, fill: 9) { overlay.handle(chunk) } + for chunk in chunks(id: 1, fill: 7) { overlay.handle(chunk) } + XCTAssertEqual(overlay.snapshot().image?.id, 2) + XCTAssertEqual(overlay.snapshot().image?.pixels.first, 9) + XCTAssertEqual(overlay.completeShapeId, 2) + XCTAssertEqual(overlay.counters.shapes, 1) + + let wrapped = PointerOverlay() + for chunk in chunks(id: 3) { wrapped.handle(chunk) } + for chunk in chunks(id: UInt32.max) { wrapped.handle(chunk) } + XCTAssertEqual(wrapped.snapshot().image?.id, 3, "max is older than 3 after the wrap") + for chunk in chunks(id: 4) { wrapped.handle(chunk) } + XCTAssertEqual(wrapped.snapshot().image?.id, 4, "a newer image still replaces it") + XCTAssertTrue(PointerOverlay.isNewer(1, than: UInt32.max)) + XCTAssertFalse(PointerOverlay.isNewer(UInt32.max, than: 1)) + } + func testThePositionIsLatestWinsByMessageNumber() { let overlay = PointerOverlay() overlay.handle(state(10), msgSeq: 5) @@ -139,6 +161,35 @@ final class PointerOverlayTests: XCTestCase { XCTAssertEqual(PointerGeometry.rect(x: 0, y: 0, width: 8, height: 8, video: .zero, view: view, fit: .fit), .zero) } + /// Review r1: a pointer past the picture's edge in Fit drew into the + /// letterbox bar. Both passes are clipped to the picture. + func testAPointerCrossingThePicturesEdgeStaysOutOfTheBars() throws { + let view = CGSize(width: 2388, height: 1668) + let video = CGSize(width: 3440, height: 1440) + let content = ContentRectMapper(viewSize: view, videoSize: video, fit: .fit).contentRect + let clip = try XCTUnwrap(PointerGeometry.clip(video: video, view: view, fit: .fit)) + // Letterboxed top and bottom: the clip is the picture, not the view. + XCTAssertEqual(clip.x, 0) + XCTAssertEqual(clip.width, 2388) + XCTAssertEqual(clip.y, Int(content.minY.rounded(.down))) + XCTAssertEqual(clip.y + clip.height, Int(content.maxY.rounded(.up))) + XCTAssertGreaterThan(clip.y, 0) + // A pointer hanging over the bottom edge: part of its quad is in the + // bar, and the clip cuts it at the picture's edge. + let rect = PointerGeometry.rect(x: 1000, y: 1430, width: 32, height: 32, video: video, view: view, fit: .fit) + XCTAssertGreaterThan(rect.maxY, content.maxY) + let shown = rect.intersection(CGRect(x: clip.x, y: clip.y, width: clip.width, height: clip.height)) + XCTAssertLessThanOrEqual(shown.maxY, CGFloat(clip.y + clip.height)) + XCTAssertGreaterThan(shown.height, 0, "the part on the picture still shows") + // Pillarboxed, and Fill (the picture covers the view). + let tall = try XCTUnwrap(PointerGeometry.clip(video: CGSize(width: 1080, height: 1920), view: view, fit: .fit)) + XCTAssertGreaterThan(tall.x, 0) + XCTAssertEqual(tall.height, 1668) + let fill = try XCTUnwrap(PointerGeometry.clip(video: video, view: view, fit: .fill)) + XCTAssertEqual([fill.x, fill.y, fill.width, fill.height], [0, 0, 2388, 1668]) + XCTAssertNil(PointerGeometry.clip(video: .zero, view: view, fit: .fit)) + } + // MARK: Channel /// The channel hands pointer messages on only for its session, and only From c83bc305d1f68bc92d4c753892f208d734967748 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:18:34 -0500 Subject: [PATCH 106/226] Wait up to 3 s for a stopped stream's port, on the address it bound input_stall checked that a Stop frees the port with one bind to 127.0.0.1, right after the loop ended. The transport binds the wildcard. On a macOS CI runner that one bind failed once and never reproduced locally (0 of 45 runs). Nothing in the transport keeps its socket past start_sender: it lives in TransportLinks and drops with it. The check now binds the address the transport bound, via listen_ip_now, retrying for up to 3 s. If the port is still held then, it fails with the address and the error. --- host/tests/input_stall.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/host/tests/input_stall.rs b/host/tests/input_stall.rs index b4b5f3b3..e26b76f9 100644 --- a/host/tests/input_stall.rs +++ b/host/tests/input_stall.rs @@ -234,6 +234,19 @@ fn input_windows_holds_never_stops_the_transport_loop() { std::thread::sleep(Duration::from_millis(10)); } transport.join().unwrap(); - assert!(UdpSocket::bind(("127.0.0.1", port)).is_ok()); + // On the address the transport bound, the wildcard by default: a + // narrower bind can clash with a socket the wildcard one left closing. + let ip = eternal_host::network::listen_ip_now(&shared.listen_on.lock()); + let freeing = Instant::now(); + let freed = loop { + match UdpSocket::bind((ip, port)) { + Ok(_) => break Ok(()), + Err(error) if freeing.elapsed() >= Duration::from_secs(3) => break Err(error), + Err(_) => std::thread::sleep(Duration::from_millis(20)), + } + }; + if let Err(error) = freed { + panic!("the stopped stream's port {ip}:{port} is still held 3 s after it ended: {error}"); + } hold::release(); } From 7523805b04c774ccfb7091cc5052c26a832f99d1 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:24:44 -0500 Subject: [PATCH 107/226] Tie a second screen's proof to its output and attachment Review r3 of #205. A proven second screen stayed proven for its whole capture generation, whatever display it was. A virtual display back under the same name on another adapter made capture reopen it; the reconciliation matched it by name and skipped its proof, so a dark replacement would have cost a watchdog restart. The proof now records the output it ran on: the adapter's LUID and the device name. Reconciling the same output again in the same generation keeps it, as a recovery reopen should. Another adapter or name proves itself again, and so does a display that was seen off the desktop in between, by the reconciliation or by capture's check of the duplicated output, even when it comes back under the same name without a new attachment. --- host/src/capture/dxgi.rs | 35 +++--- host/src/capture/mod.rs | 29 +++-- host/src/capture/second_screen.rs | 153 +++++++++++++++++++++++-- host/src/capture/second_screen/fake.rs | 19 ++- 4 files changed, 202 insertions(+), 34 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 0f3affe9..020bc3f7 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -628,7 +628,7 @@ pub fn run_capture_loop( let allowed = keepalive_allowed(picture.real(), second_screen); let dims = (tex_width, tex_height); let call = super::second_screen::driver_call(&shared, generation); - let moved = target_moved(&output1, &chosen, &target, false, || { + let moved = target_moved(&shared, &output1, &chosen, &target, false, || { if crate::supervisor::generation_is_current(generation) { keep_picture_alive( slot, @@ -756,19 +756,20 @@ pub fn run_capture_loop( let allowed = keepalive_allowed(picture.real(), second_screen); let dims = (tex_width, tex_height); let call = super::second_screen::driver_call(&shared, generation); - let moved = target_moved(&output1, &chosen, &target, true, || { - if crate::supervisor::generation_is_current(generation) { - keep_picture_alive( - slot, - &shared, - frame_number, - &mut last_send, - &last_frame, - dims, - allowed, - ); - } - }); + let moved = + target_moved(&shared, &output1, &chosen, &target, true, || { + if crate::supervisor::generation_is_current(generation) { + keep_picture_alive( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + dims, + allowed, + ); + } + }); drop(call); if moved { return Err(moved_away(&chosen)); @@ -971,8 +972,11 @@ fn access_lost_or( /// primary for a missing display, or when `list` asks after a failed /// re-creation. Both run under the stall limits while `progress` keeps the /// picture alive; one that does not return in time counts as no move, and -/// the re-creation goes on. +/// the re-creation goes on. An output seen off the desktop or renamed is a +/// detach for a second screen's proof (see +/// [`super::second_screen::note_detached`]). fn target_moved( + shared: &SharedControl, output: &IDXGIOutput1, chosen: &OutputInfo, target: &CaptureTarget, @@ -994,6 +998,7 @@ fn target_moved( if same.unwrap_or(true) { return false; } + super::second_screen::note_detached(shared); } let (target, chosen) = (target.clone(), chosen.clone()); let started = Instant::now(); diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index a1008be3..fa0b7ed7 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -776,18 +776,29 @@ pub(crate) fn reconcile_virtual_display( // Settings and codec changes restart capture without removing the // managed monitor, and the stream moves onto it this way once it // attaches. Reuse that output rather than turning it on again. - let attached = owned.clone().filter(|name| { + let attached = owned.as_ref().and_then(|name| { outputs(shared) - .iter() - .any(|output| !output.is_primary && &output.device_name == name) + .into_iter() + .find(|output| !output.is_primary && &output.device_name == name) }); - if let Some(name) = attached { - if status == VddStatus::Active - && (connected || shared.second_screen.lock().keep_without_viewer()) - { - second_screen::start_proving(shared, generation, Instant::now()); - return Some(CaptureTarget::Output(name)); + match attached { + Some(output) => { + if status == VddStatus::Active + && (connected || shared.second_screen.lock().keep_without_viewer()) + { + let proof_output = (output.adapter_id, output.device_name.clone()); + second_screen::start_proving_on( + shared, + generation, + Instant::now(), + Some(proof_output), + ); + return Some(CaptureTarget::Output(output.device_name)); + } } + // Gone from the desktop: if it comes back, it proves itself. + None if owned.is_some() => second_screen::note_detached(shared), + None => {} } // The transport's next tick starts the display beside this mirror. // A viewer that arrived meanwhile may already have claimed it. diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 0933ca4e..041b460f 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -159,6 +159,11 @@ struct Proving { /// a secure screen or with the displays off don't count. const LOSS_LIMIT: u32 = 8; +/// The output a second screen proves itself on: its adapter's LUID and its +/// device name. A display back under the same name on another adapter is +/// another display. +pub(crate) type ProofOutput = (crate::gpu_preference::AdapterId, String); + #[derive(Debug, Default)] pub(crate) struct State { /// The display attached; the stream has not moved onto it yet. @@ -172,11 +177,15 @@ pub(crate) struct State { /// A start is claimed: its Attach is queued or running. attaching: bool, proving: Option, - /// The capture generation whose second screen captured its first image. - /// The stage opening the display again in that generation (a recovery - /// reopen) does not prove it again: a new proof hid a display gone dark - /// from the watchdog and then fell back to the mirror. - proven: Option, + /// The second screen that captured its first image: the capture + /// generation and the output. The stage opening that same output again + /// in that generation (a recovery reopen) does not prove it again: a new + /// proof hid a display gone dark from the watchdog and then fell back + /// to the mirror. Another output, a new attachment or a detach seen in + /// between clears it, and the display proves itself again. + proven: Option<(u64, ProofOutput)>, + /// The output the running proof is for, when the reconciliation named it. + proving_output: Option, /// Why the second screen last failed, for the log and the backend. failure: Option<&'static str>, /// The failure saved for later app runs. @@ -752,6 +761,13 @@ pub fn note_attached(shared: &SharedControl, device_name: String) { state.switch_pending = true; state.proving = None; state.proven = None; + state.proving_output = None; +} + +/// The managed display was seen off the desktop: one that comes back, even +/// under the same name, proves itself again. +pub fn note_detached(shared: &SharedControl) { + shared.second_screen.lock().proven = None; } /// [`note_attached`], unless the mode changed or quitting began. The mode @@ -771,12 +787,32 @@ fn note_attached_if_selected(shared: &SharedControl, device_name: String) -> boo /// Capture `generation` streams the second screen from `now`. The first /// generation after the switch starts the [`PROVE_TIMEOUT`] deadline; a /// later one inherits it, and any capture failure, until an image arrives. -/// The generation that proved it is not asked again. pub fn start_proving(shared: &SharedControl, generation: u64, now: Instant) { + start_proving_on(shared, generation, now, None); +} + +/// [`start_proving`] for the second screen on `output`. The generation that +/// proved that output is not asked again. +pub(crate) fn start_proving_on( + shared: &SharedControl, + generation: u64, + now: Instant, + output: Option, +) { let mut state = shared.second_screen.lock(); - if state.proving.is_none() && state.proven == Some(generation) { - return; + if state.proving.is_none() { + let proven = output.as_ref().is_some_and(|output| { + state + .proven + .as_ref() + .is_some_and(|(by, on)| *by == generation && on == output) + }); + if proven { + return; + } + state.proven = None; } + state.proving_output = output; match state.proving.as_mut() { // A call the earlier generation still waits on is not this one's, // and its guard can no longer clear the flag. @@ -834,7 +870,10 @@ pub fn note_image(shared: &SharedControl, generation: u64) { let mut state = shared.second_screen.lock(); if let Some(proving) = state.proving.filter(|p| p.generation == generation) { state.proving = None; - state.proven = Some(generation); + state.proven = state + .proving_output + .take() + .map(|output| (generation, output)); tracing::info!( after_ms = proving.since.elapsed().as_millis() as u64, "Second screen captured its first image — streaming it" @@ -2498,6 +2537,102 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// A second screen on the fake display, streamed and proven in the + /// current generation. Returns the display, its memory folder and the + /// generation. + fn proven_fake_second_screen( + shared: &SharedControl, + label: &str, + ) -> (fake::FakeDisplay, std::path::PathBuf, u64) { + let dir = std::env::temp_dir().join(format!( + "eternal-capture-recovery-{label}-{}", + std::process::id() + )); + let display = fake::FakeDisplay::new(Some(Duration::ZERO), secs(10.0)); + display.install(shared, &dir, std::time::SystemTime::now()); + assert!(Driver::set_enabled(&display, true)); + connect_viewer(shared); + note_attached(shared, "virtual".into()); + let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(shared)); + note_image(shared, generation); + assert!(!proving(shared)); + (display, dir, generation) + } + + /// Review r3 of #205: the virtual display came back under the same name + /// on another adapter, and capture reopened it. Reconciling matched it by + /// name, and the new display skipped its proof: dark, it would have cost + /// a watchdog restart. Another adapter is another display. + #[test] + fn a_virtual_display_back_on_another_adapter_proves_itself_again() { + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, "moved-luid"); + display.move_virtual_display(true); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(&shared), "same name, another adapter"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Review r3 of #205: the virtual display left the desktop and came back + /// within one generation, without a new attachment from the worker. Seen + /// gone by the reconciliation, or by capture's check of its output, it + /// proves itself again when it is back. + #[test] + fn a_virtual_display_detached_and_back_proves_itself_again() { + // The reconciliation sees it gone. + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, "detached"); + assert!(Driver::set_enabled(&display, false)); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::PrimaryAuto) + ); + assert!(Driver::set_enabled(&display, true)); + *shared.vdd_status.lock() = VddStatus::Active; + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(&shared), "back after a detach"); + let _ = std::fs::remove_dir_all(&dir); + + // Capture's check of the duplicated output sees it off the desktop. + let shared = second_screen_selected(); + let (_display, dir, generation) = proven_fake_second_screen(&shared, "detached-capture"); + note_detached(&shared); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(&shared), "back after a detach capture saw"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Review r3 of #205: the same output reopened in the generation that + /// proved it keeps its proof, so a display gone dark reaches the + /// watchdog (see the test above for the restart itself). + #[test] + fn a_reopen_of_the_same_output_keeps_its_proof() { + let shared = second_screen_selected(); + let (_display, dir, generation) = proven_fake_second_screen(&shared, "same-output"); + for _ in 0..3 { + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(!proving(&shared)); + } + let _ = std::fs::remove_dir_all(&dir); + } + /// Review r2 of #205 (P2): a display that moved during the first proof /// ends the capture loop with a reopen, and the stage noted it as a /// capture error: the proof failed, and the second screen went back to diff --git a/host/src/capture/second_screen/fake.rs b/host/src/capture/second_screen/fake.rs index d065ba2c..f407d4b1 100644 --- a/host/src/capture/second_screen/fake.rs +++ b/host/src/capture/second_screen/fake.rs @@ -55,6 +55,8 @@ struct Inner { desktop_away: Arc, /// The display configuration: see [`FakeDisplay::move_screen`]. facts: Arc>>, + /// See [`FakeDisplay::move_virtual_display`]. + virtual_on_other_card: AtomicBool, } impl FakeDisplay { @@ -72,6 +74,7 @@ impl FakeDisplay { toggles: Mutex::new(Vec::new()), desktop_away: Arc::new(AtomicBool::new(false)), facts: Arc::new(Mutex::new(Some(screen_on(None)))), + virtual_on_other_card: AtomicBool::new(false), })) } @@ -131,6 +134,15 @@ impl FakeDisplay { *self.0.facts.lock() = Some(screen_on(card)); } + /// List the virtual display, under the same name, on the PC's other + /// card (true) or on [`APP_GPU`] (false, at first), as when its driver + /// is restarted on another adapter. + pub fn move_virtual_display(&self, other_card: bool) { + self.0 + .virtual_on_other_card + .store(other_card, Ordering::Relaxed); + } + /// Each time it was turned on (true) or off, on its clock, since it was /// made. pub fn toggles(&self) -> Vec<(Duration, bool)> { @@ -146,6 +158,11 @@ impl Driver for FakeDisplay { .lock() .zip(*self.0.attach_after.lock()) .is_some_and(|(on, after)| self.now() >= on + after); + let virtual_card = if self.0.virtual_on_other_card.load(Ordering::Relaxed) { + OTHER_CARD + } else { + APP_CARD + }; let output = |name: &str, primary: bool| OutputInfo { adapter_index: 0, output_index: u32::from(!primary), @@ -156,7 +173,7 @@ impl Driver for FakeDisplay { top: 0, is_primary: primary, adapter_name: APP_GPU.to_string(), - adapter_id: APP_CARD, + adapter_id: if primary { APP_CARD } else { virtual_card }, }; let mut outputs = vec![output("primary", true)]; if attached { From d21bbd0b533585a6e0216eace6a472d583b87ad6 Mon Sep 17 00:00:00 2001 From: Ali Younes <159104116+whoisaldo@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:26:53 -0500 Subject: [PATCH 108/226] Send the first picture, then wait to hear from the iPad (#200) Before the iPad answers, the PC sends the first picture and then waits for the iPad's reply before it sends more video. Replies to an iPad that hasn't answered yet are kept no larger than its request. USB sessions behave as before. Golden vectors are updated on both sides, and the standby replies follow the same rule. --- host/src/transport/audio.rs | 24 + host/src/transport/mod.rs | 21 +- host/src/transport/session.rs | 777 ++++++++++++++++-- host/src/transport/standby.rs | 11 +- host/tests/e2e_synthetic.rs | 24 +- host/tests/e2e_synthetic/second_screen.rs | 5 +- host/tests/send_budget.rs | 599 ++++++++++++++ .../WireProtocolGoldenTests.swift | 17 +- proto/testdata/v2_vectors.txt | 1 + proto/tests/golden.rs | 23 + 10 files changed, 1422 insertions(+), 80 deletions(-) create mode 100644 host/tests/send_budget.rs diff --git a/host/src/transport/audio.rs b/host/src/transport/audio.rs index b3d39a7e..9f62b74b 100644 --- a/host/src/transport/audio.rs +++ b/host/src/transport/audio.rs @@ -129,7 +129,26 @@ mod tests { use std::sync::atomic::Ordering; use std::time::Instant; + /// A session whose iPad answered, as the app does at once. fn session(caps: u16) -> (SharedControl, u32) { + let (shared, id) = unanswered_session(caps); + let config = super::super::SharedConfigSource { + shared: &shared, + stream_epoch: 7, + pen_available: false, + listen: "127.0.0.1:0".parse().unwrap(), + }; + shared.session.lock().handle_control( + PeerId::udp("127.0.0.1:19891".parse().unwrap()), + id, + ControlMessage::Ping(Ping { t1_us: 1 }), + &config, + Instant::now(), + ); + (shared, id) + } + + fn unanswered_session(caps: u16) -> (SharedControl, u32) { let shared = SharedControl::new(9876, 15_000_000); shared.pairing.lock().required = false; let config = super::super::SharedConfigSource { @@ -190,6 +209,11 @@ mod tests { let (legacy, id) = session(0); assert!(crate::audio::wanted_session(&legacy).is_none()); assert!(sender.prepare(packet(id, 100), &legacy, 7, 100).is_none()); + let (unanswered, id) = unanswered_session(FEATURE_WANTS_AUDIO); + assert!(crate::audio::wanted_session(&unanswered).is_none()); + assert!(sender + .prepare(packet(id, 100), &unanswered, 7, 100) + .is_none()); let (shared, id) = session(FEATURE_WANTS_AUDIO); assert_eq!(crate::audio::wanted_session(&shared), Some(id)); shared.audio_enabled.store(false, Ordering::SeqCst); diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index c6050db6..c30e94e3 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -444,10 +444,7 @@ pub async fn start_sender( if awaiting_keyframe { let started = shared.session.lock().client_info().map(|info| info.connected_at); match first_frame_gate(nal.is_keyframe, nal.timestamp, started) { - FirstFrame::Send => { - awaiting_keyframe = false; - shared.session.lock().note_video_sent(session_id, now()); - } + FirstFrame::Send => {} FirstFrame::Skip => continue, // the forced IDR is coming FirstFrame::SkipAndRequestKeyframe => { shared.force_next_idr.store(true, Ordering::SeqCst); @@ -456,11 +453,23 @@ pub async fn start_sender( } } - frame_seq = frame_seq.wrapping_add(1); - let send_start = Instant::now(); let payload = &nal.data; let payload_size = media_payload_size(shared.max_dgram.load(Ordering::SeqCst) as usize).unwrap_or(MAX_MEDIA_PAYLOAD); let frag_count_usize = payload.len().div_ceil(payload_size).max(1); + // Held back until the client answers; the frames after it + // wait for the keyframe its answer asks for. + let wire_bytes = payload.len() + frag_count_usize * MEDIA_HEADER_SIZE; + if !shared.session.lock().may_send_media(session_id, wire_bytes, now()) { + awaiting_keyframe = true; + continue; + } + if awaiting_keyframe { + awaiting_keyframe = false; + shared.session.lock().note_video_sent(session_id, now()); + } + + frame_seq = frame_seq.wrapping_add(1); + let send_start = Instant::now(); if frag_count_usize > usize::from(MAX_FRAG_COUNT) { warn!( seq = nal.sequence, diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index b667ed13..15eb4280 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -108,6 +108,21 @@ struct RefusalMinute { /// Honor at most one keyframe request per this window (PLI storm guard). pub const KEYFRAME_REQUEST_MIN_INTERVAL: Duration = Duration::from_millis(500); +/// Media bytes, headers included, a Wi-Fi client's address gets before +/// anything of its session comes back from it: room for the first picture, +/// even a large keyframe of a high-resolution stream. Heartbeats go on. An +/// iPad pings the moment its HELLO_ACK lands, so it answers long before +/// this runs out. +pub const UNANSWERED_MEDIA_BUDGET: usize = 512 * 1024; +/// Media to every client not yet heard counts against one host-wide budget +/// of the same size, which comes back over [`UNANSWERED_FOR`]: a client +/// that moves to another address, or many addresses in turn, get no more. +/// +/// Addresses whose session ended before they answered, and for how long a +/// new session there starts with its budget spent. +const UNANSWERED_KEPT: usize = 64; +const UNANSWERED_FOR: Duration = Duration::from_secs(60); + /// What the transport must do after feeding the session an event. #[derive(Debug, Default)] pub struct Actions { @@ -266,6 +281,13 @@ struct ActiveSession { bye: Option, /// The msg_seq of input packets taken in this session. input_seqs: SeqWindow, + /// Something of this session came back from the client after its + /// HELLO2. Always for USB, where the PC dials the iPad on the cable. + answered: bool, + /// Media bytes sent before it answered (see [`UNANSWERED_MEDIA_BUDGET`]). + unanswered_sent: usize, + /// A frame was held back for the budget: the answer asks for a keyframe. + withheld: bool, } /// What a client sent last, for the liveness log lines. @@ -436,10 +458,17 @@ pub struct Session { /// Recently refused HELLO2 nonces: a retransmit is the same attempt. refused: std::collections::VecDeque<(u32, Instant)>, refusal_minute: Option, + /// Wi-Fi addresses whose session ended before they answered (see + /// [`UNANSWERED_FOR`]). + unanswered: std::collections::VecDeque<(std::net::IpAddr, Instant)>, + /// The host-wide media sent to clients before they answered, as of when + /// it was last counted (see [`UNANSWERED_MEDIA_BUDGET`]). + unanswered_media: (usize, Instant), identify_limiter: IdentifyLimiter, - /// Injected so tests control randomness; production passes a seeded value - /// derived from process entropy. + /// Counts the sessions. Session ids come from a per-launch keyed hash + /// of this count ([`Self::session_id_keys`]). next_session_id: u32, + session_id_keys: std::collections::hash_map::RandomState, legacy_client_warned: bool, } @@ -456,8 +485,11 @@ impl Session { ended_for_liveness: None, refused: std::collections::VecDeque::new(), refusal_minute: None, + unanswered: std::collections::VecDeque::new(), + unanswered_media: (0, Instant::now()), identify_limiter: IdentifyLimiter::default(), - next_session_id: session_id_seed.max(1), + next_session_id: session_id_seed, + session_id_keys: std::collections::hash_map::RandomState::new(), legacy_client_warned: false, } } @@ -484,9 +516,54 @@ impl Session { fn end(&mut self, reason: EndReason, now: Instant) -> Option { let session = self.active.take()?; self.last_end = Some((session.session_id, reason)); + // One entry per address, its time refreshed. Once the memory is + // full the oldest goes; the host-wide budget still counts it. + if let Some(addr) = session.peer.addr.filter(|_| !session.answered) { + self.unanswered.retain(|(ip, at)| { + *ip != addr.ip() && now.saturating_duration_since(*at) < UNANSWERED_FOR + }); + if self.unanswered.len() == UNANSWERED_KEPT { + self.unanswered.pop_front(); + } + self.unanswered.push_back((addr.ip(), now)); + } Some(session.end(reason, now)) } + /// Whether a frame of `bytes` may go to `session_id`'s client now. Until + /// it answers, frames go while they fit in its budget; the first that + /// doesn't is held back, and so is every later one until it answers, + /// which asks for a keyframe to start the picture again. + pub fn may_send_media(&mut self, session_id: u32, bytes: usize, now: Instant) -> bool { + let host_spent = self.unanswered_media_at(now); + let Some(session) = self + .active + .as_mut() + .filter(|session| session.session_id == session_id && !session.answered) + else { + return true; + }; + if session.withheld + || session.unanswered_sent + bytes > UNANSWERED_MEDIA_BUDGET + || host_spent + bytes > UNANSWERED_MEDIA_BUDGET + { + session.withheld = true; + return false; + } + session.unanswered_sent += bytes; + self.unanswered_media = (host_spent + bytes, now); + true + } + + /// The host-wide media to clients not yet heard that still counts at + /// `now`. + fn unanswered_media_at(&self, now: Instant) -> usize { + let (spent, at) = self.unanswered_media; + let back = UNANSWERED_MEDIA_BUDGET as u128 * now.saturating_duration_since(at).as_millis() + / UNANSWERED_FOR.as_millis(); + spent.saturating_sub(usize::try_from(back).unwrap_or(usize::MAX)) + } + /// Starts an update handoff. Atomic with admission: call with the session /// lock held; a client either got in before (and this returns `false`) or /// is refused until [`Self::end_update`]. With `stop_stream` (the user @@ -637,6 +714,7 @@ impl Session { .as_ref() .filter(|session| { !session.awaiting_takeover + && session.answered && session.info.feature_caps & eternal_wire::v2::control::FEATURE_WANTS_AUDIO != 0 }) @@ -648,9 +726,21 @@ impl Session { } fn allocate_session_id(&mut self) -> u32 { - let id = self.next_session_id; - self.next_session_id = self.next_session_id.wrapping_add(0x9E37_79B9).max(1); - id + use std::hash::BuildHasher; + let taken = [ + self.session_id(), + self.last_end.map(|(id, _)| id), + self.ended_for_liveness + .as_ref() + .map(|ended| ended.session_id), + ]; + loop { + self.next_session_id = self.next_session_id.wrapping_add(1); + let id = self.session_id_keys.hash_one(self.next_session_id) as u32; + if id != 0 && !taken.contains(&Some(id)) { + return id; + } + } } fn next_msg_seq(&mut self) -> u32 { @@ -666,8 +756,9 @@ impl Session { /// Feed one control datagram off a link, with its header. Input whose /// msg_seq this session took already, or that is too old, is dropped. /// - /// `datagram_len` is the datagram's size: an answer saying the PC isn't - /// streaming is never larger than the HELLO2 it answers. + /// `datagram_len` is the datagram's size: over Wi-Fi, a refused HELLO2, + /// or one the PC answers while it isn't streaming, gets no answer larger + /// than itself. pub fn handle_datagram( &mut self, source: PeerId, @@ -733,18 +824,13 @@ impl Session { _ => return self.answer_ended(source, header_session_id, now), } } + // Answers over USB stay whole. + let hello_len = datagram + .map(|(_, len)| len) + .filter(|_| source.link == LinkId::Udp); match message { ControlMessage::Hello2(hello) => { - let len = datagram.map(|(_, len)| len); - let mut actions = self.handle_hello(source, hello, len, config, now); - // While no stream runs, no answer to a HELLO2 is larger than - // it: refusals included, after the checks that chose them. - if self.not_streaming.is_some() { - if let Some(len) = len { - actions.replies.retain(|(_, reply)| reply.len() <= len); - } - } - actions + self.handle_hello(source, hello, hello_len, config, now) } ControlMessage::Identify(request) => self.answer_identify(source, request, config, now), ControlMessage::Bye(reason) => self.handle_bye(source, reason, now), @@ -827,7 +913,7 @@ impl Session { &mut self, source: PeerId, hello: eternal_wire::v2::control::Hello2, - datagram_len: Option, + hello_len: Option, config: &impl ConfigSource, now: Instant, ) -> Actions { @@ -836,14 +922,21 @@ impl Session { // Version gate first. if hello.proto_min > 2 || hello.proto_max < 2 { crate::updates::note_ipad_protocol(hello.proto_min, hello.proto_max); - return self.refuse(source, HelloStatus::VersionUnsupported, &hello, config, now); + return self.refuse( + source, + HelloStatus::VersionUnsupported, + &hello, + hello_len, + config, + now, + ); } // Don't start a stream nothing will serve. let closed = self.closed_to_clients(); if let Some(Closed::Busy(why)) = closed { info!(peer = %source, "{why}"); - return self.refuse(source, HelloStatus::Busy, &hello, config, now); + return self.refuse(source, HelloStatus::Busy, &hello, hello_len, config, now); } // Keep credentials locked through authorization, rotation and ACK creation: @@ -859,14 +952,8 @@ impl Session { // first code-based ACK must not require the now-rotated code. if let Some(Closed::NotStreaming(state)) = closed { drop(pairing); - return self.answer_not_streaming( - source, - state, - &hello, - datagram_len, - config, - now, - ); + return self + .answer_not_streaming(source, state, &hello, hello_len, config, now); } actions.replies.push((source, session.ack.clone())); return actions; @@ -878,7 +965,7 @@ impl Session { { Some(Ok(grant)) => grant, Some(Err(status)) => { - return self.refuse(source, status, &hello, config, now); + return self.refuse(source, status, &hello, hello_len, config, now); } None => crate::pairing::Grant { token: [0; 16], @@ -890,7 +977,7 @@ impl Session { // A right pairing code isn't spent: it still works after Start. if let Some(Closed::NotStreaming(state)) = closed { drop(pairing); - return self.answer_not_streaming(source, state, &hello, datagram_len, config, now); + return self.answer_not_streaming(source, state, &hello, hello_len, config, now); } // The media target: the client's advertised listen port at its source IP @@ -912,7 +999,7 @@ impl Session { if let Some(session) = self.active.as_ref() { if session.same_device(source, hello.device_id) { if matches!(session.peer.link, LinkId::Usb { .. }) && source.link == LinkId::Udp { - return self.refuse(source, HelloStatus::Busy, &hello, config, now); + return self.refuse(source, HelloStatus::Busy, &hello, hello_len, config, now); } // Same device, new connect attempt: supersede in place. info!(peer = %source, "Client reconnected — superseding session in place"); @@ -922,7 +1009,7 @@ impl Session { } else { // A different device while one is streaming: reject. info!(peer = %source, "Second client rejected while a session is active"); - return self.refuse(source, HelloStatus::Busy, &hello, config, now); + return self.refuse(source, HelloStatus::Busy, &hello, hello_len, config, now); } } @@ -933,10 +1020,16 @@ impl Session { .rotate_code() { warn!(%error, "Pairing code rotation failed"); - return self.refuse(source, HelloStatus::Error, &hello, config, now); + return self.refuse(source, HelloStatus::Error, &hello, hello_len, config, now); } } + // A session taken over before it was heard passes on what it spent. + let inherited = self + .active + .as_ref() + .filter(|session| !session.answered) + .map_or(0, |session| session.unanswered_sent); let silent_taken_over = self .active .as_ref() @@ -949,6 +1042,17 @@ impl Session { actions.ended = self.end(EndReason::Superseded, now); } let session_id = self.allocate_session_id(); + // An address whose last session ended before it answered (this + // HELLO2 may have just superseded it) gets no media until it does. + self.unanswered + .retain(|(_, at)| now.saturating_duration_since(*at) < UNANSWERED_FOR); + let answered = source.link != LinkId::Udp; + let unanswered_sent = match source.addr { + Some(addr) if self.unanswered.iter().any(|(ip, _)| *ip == addr.ip()) => { + UNANSWERED_MEDIA_BUDGET + } + _ => inherited, + }; crate::telemetry::note_client_device(&hello.device_name); info!( peer = %source, @@ -991,6 +1095,9 @@ impl Session { frames_received: false, bye: None, input_seqs: SeqWindow::default(), + answered, + unanswered_sent, + withheld: false, }); // Only a new pairing carries the token. iPads keep the token they @@ -1346,12 +1453,17 @@ impl Session { /// as before. A stopped or failed PC waits for someone at the PC, so it /// turns them down with Error: "Your PC turned down the connection", /// which also ends their reconnect cycle. + /// + /// Over Wi-Fi the answer is never larger than the HELLO2: iPads that + /// understand these answers pad theirs so every answer fits whole. A + /// stopping stream answers here too, until its transport ends, so the + /// bound is here as well as in the standby responder. fn answer_not_streaming( &mut self, source: PeerId, state: NotStreaming, hello: &eternal_wire::v2::control::Hello2, - datagram_len: Option, + hello_len: Option, config: &impl ConfigSource, now: Instant, ) -> Actions { @@ -1367,41 +1479,65 @@ impl Session { NotStreaming::Failed(reason) => (HelloStatus::Failed, reason.token()), } }; - let mut actions = self.refuse_with_reason(source, status, reason, hello, config, now); - // Never larger than the HELLO2: iPads that understand these answers - // pad theirs so every answer fits. A stopping stream answers here - // too, until its transport ends, so the bound is here as well as in - // the standby responder. - actions - .replies - .retain(|(_, reply)| datagram_len.is_none_or(|len| reply.len() <= len)); + let actions = + self.refuse_with_reason(source, status, reason, hello, hello_len, config, now); if actions.ended.is_some() && !actions.replies.is_empty() { info!(peer = %source, ?status, reason, "Told an iPad the PC isn't streaming"); } actions } + /// Answers `hello` with a refusal. The first refusal of a nonce ends + /// that attempt; a retransmit of it gets the answer again. + /// + /// With `hello_len`, the answer is no larger than the HELLO2: without + /// the PC name and host caps if need be, which iPads read only from an + /// accepted HELLO2, or not sent at all. A not-streaming reason stays. + /// Every iPad's HELLO2 fits the bare answer. fn refuse( &mut self, source: PeerId, status: HelloStatus, hello: &eternal_wire::v2::control::Hello2, + hello_len: Option, config: &impl ConfigSource, now: Instant, ) -> Actions { - self.refuse_with_reason(source, status, "", hello, config, now) + self.refuse_with_reason(source, status, "", hello, hello_len, config, now) } + #[allow(clippy::too_many_arguments)] fn refuse_with_reason( &mut self, source: PeerId, status: HelloStatus, reason: &str, hello: &eternal_wire::v2::control::Hello2, + hello_len: Option, config: &impl ConfigSource, now: Instant, ) -> Actions { - let ack = self.make_reply(status, hello.client_nonce, 0, config, [0; 16], reason); + let msg_seq = self.next_msg_seq(); + let mut answer = self.hello_ack(status, hello.client_nonce, 0, config, [0; 16], reason); + let encode = |answer: &HelloAck| { + eternal_wire::v2::control::encode_control( + 0, + msg_seq, + &ControlMessage::HelloAck(answer.clone()), + ) + }; + let mut ack = encode(&answer); + let fits = |ack: &[u8]| hello_len.is_none_or(|len| ack.len() <= len); + if !fits(&ack) { + answer.host_name.clear(); + answer.host_caps = 0; + ack = encode(&answer); + } + let replies = if fits(&ack) { + vec![(source, ack)] + } else { + Vec::new() + }; self.refused .retain(|(_, at)| now.saturating_duration_since(*at) < REFUSED_FOR); let first = !self @@ -1416,7 +1552,7 @@ impl Session { } let summarized = first && self.summarize_refusal(now); Actions { - replies: vec![(source, ack)], + replies, ended: summarized.then_some(SessionEnd { session_id: None, client_nonce: hello.client_nonce, @@ -1482,7 +1618,24 @@ impl Session { reason: &str, ) -> Vec { let msg_seq = self.next_msg_seq(); - let ack = ControlMessage::HelloAck(HelloAck { + let ack = self.hello_ack(status, client_nonce, session_id, config, auth_token, reason); + eternal_wire::v2::control::encode_control( + session_id, + msg_seq, + &ControlMessage::HelloAck(ack), + ) + } + + fn hello_ack( + &self, + status: HelloStatus, + client_nonce: u32, + session_id: u32, + config: &impl ConfigSource, + auth_token: [u8; 16], + reason: &str, + ) -> HelloAck { + HelloAck { status, accepted_version: 2, client_nonce, @@ -1495,8 +1648,7 @@ impl Session { auth_token, host_caps: HOSTCAP_NACK | config.host_caps(), reason: reason.to_string(), - }); - eternal_wire::v2::control::encode_control(session_id, msg_seq, &ack) + } } fn negotiated_config(&self, source: &impl ConfigSource) -> StreamConfig { @@ -1575,11 +1727,28 @@ impl ActiveSession { /// The client was heard: it gets a full liveness window again, and a /// session on hold carries on. Returns whether to force an IDR: reports - /// and keyframe requests sent during the silence may not have arrived. + /// and keyframe requests sent during the silence may not have arrived, + /// and a client that answers after frames were held back from it needs + /// a keyframe to start its picture. fn heard(&mut self, kind: Heard, now: Instant) -> bool { let silent = now.saturating_duration_since(self.heard.0); self.heard = (now, kind); self.liveness_deadline = now + LIVENESS_TIMEOUT; + if !self.answered { + self.answered = true; + info!( + session_id = self.session_id, + answered_ms = now + .saturating_duration_since(self.info.connected_at) + .as_millis() as u64, + heard = kind.as_str(), + withheld = self.withheld, + "Client answered" + ); + if self.withheld { + return self.grant_keyframe(now); + } + } let Some(since) = self.held_since.take() else { return false; }; @@ -2474,7 +2643,9 @@ mod tests { msg_seq, }; let message = ControlMessage::InputEvent(event); - let actions = session.handle_datagram(peer, &header, message, 64, &TestConfig, now); + let len = + eternal_wire::v2::control::encode_control(session_id, msg_seq, &message).len(); + let actions = session.handle_datagram(peer, &header, message, len, &TestConfig, now); actions .input .map(|(_, event)| event) @@ -4069,6 +4240,444 @@ mod tests { } } + /// Feeds `message` as the datagram a Wi-Fi or USB link delivers. + fn datagram( + session: &mut Session, + peer: PeerId, + session_id: u32, + message: ControlMessage, + now: Instant, + ) -> Actions { + let bytes = eternal_wire::v2::control::encode_control(session_id, 1, &message); + let (header, message) = eternal_wire::v2::control::parse_control(&bytes).unwrap(); + session.handle_datagram(peer, &header, message, bytes.len(), &TestConfig, now) + } + + fn ping() -> ControlMessage { + ControlMessage::Ping(eternal_wire::v2::control::Ping { t1_us: 1 }) + } + + /// Until something of its session comes back, a Wi-Fi client gets its + /// first picture and heartbeats, but no more media. Its answer asks for + /// a keyframe, as frames were held back, and the stream goes on. + #[test] + fn a_wifi_client_gets_its_first_picture_then_waits_to_be_heard() { + let mut session = Session::new(1234); + let now = Instant::now(); + let ipad = addr([10, 0, 0, 5], 50000); + session.handle_control(ipad, 0, identified_hello(1, 77), &TestConfig, now); + let id = session.session_id().unwrap(); + + assert!( + session.may_send_media(id, 300 * 1024, now), + "the first keyframe" + ); + assert!(session.may_send_media(id, UNANSWERED_MEDIA_BUDGET - 300 * 1024, now)); + assert!(!session.may_send_media(id, 1, now), "the budget is spent"); + let heartbeat = session.tick(&TestConfig, true, now + Duration::from_secs(1)); + assert_eq!(heartbeat.replies.len(), 1, "heartbeats go on"); + + let answer = datagram(&mut session, ipad, id, ping(), now + Duration::from_secs(1)); + assert!(answer.force_idr, "a keyframe starts the picture again"); + assert!(session.may_send_media(id, 4 * UNANSWERED_MEDIA_BUDGET, now)); + } + + /// A frame that doesn't fit is held back, and so is every one after it + /// until the client answers, even one that would fit: a frame after a + /// gap can't be decoded. + #[test] + fn nothing_follows_a_held_back_frame_until_the_answer() { + let mut session = Session::new(1234); + let now = Instant::now(); + let ipad = addr([10, 0, 0, 5], 50000); + session.handle_control(ipad, 0, identified_hello(1, 77), &TestConfig, now); + let id = session.session_id().unwrap(); + assert!(!session.may_send_media(id, UNANSWERED_MEDIA_BUDGET + 1, now)); + assert!(!session.may_send_media(id, 100, now)); + datagram(&mut session, ipad, id, ping(), now); + assert!(session.may_send_media(id, 100, now)); + } + + /// Whatever the iPad sends first counts, and an answer before the + /// budget ran out needs no keyframe. A packet with another session's + /// id, from another address, or a repeat of the HELLO2 is no answer. + #[test] + fn any_packet_of_the_session_from_its_address_is_its_answer() { + let answers = [ + ping(), + ControlMessage::ReceiverReport(ReceiverReport::default()), + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 7, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + }), + ControlMessage::Nack(Nack { + stream_epoch: 7, + frame_seq: 1, + frag_count: 2, + missing: vec![1], + }), + ]; + for answer in answers { + let mut session = Session::new(1234); + let now = Instant::now(); + let ipad = addr([10, 0, 0, 5], 50000); + session.handle_control(ipad, 0, identified_hello(1, 77), &TestConfig, now); + let id = session.session_id().unwrap(); + datagram(&mut session, ipad, id, identified_hello(1, 77), now); + datagram(&mut session, ipad, id.wrapping_add(1), ping(), now); + datagram(&mut session, addr([10, 0, 0, 6], 50000), id, ping(), now); + assert!(!session.may_send_media(id, UNANSWERED_MEDIA_BUDGET + 1, now)); + let heard = datagram(&mut session, ipad, id, answer.clone(), now); + assert!( + session.may_send_media(id, UNANSWERED_MEDIA_BUDGET + 1, now), + "{answer:?}" + ); + + let mut early = Session::new(1234); + early.handle_control(ipad, 0, identified_hello(1, 77), &TestConfig, now); + let id = early.session_id().unwrap(); + assert!(early.may_send_media(id, 1000, now)); + let heard_early = datagram(&mut early, ipad, id, answer.clone(), now); + assert_eq!( + heard_early.force_idr, + heard.force_idr && matches!(answer, ControlMessage::KeyframeRequest(_)), + "{answer:?}" + ); + assert!(heard.force_idr, "{answer:?}"); + } + } + + /// The PC opens USB tunnels itself, to an iPad on the cable: they + /// stream from the start. + #[test] + fn usb_streams_before_the_ipad_is_heard() { + let mut session = Session::new(1234); + let usb = PeerId::usb(17); + let actions = datagram( + &mut session, + usb, + 0, + identified_hello(1, 77), + Instant::now(), + ); + let id = session.session_id().unwrap(); + assert_eq!(actions.new_target, Some(usb)); + assert!(session.may_send_media(id, 10 * UNANSWERED_MEDIA_BUDGET, Instant::now())); + } + + /// A new session from an address whose last one ended before it was + /// heard, by a new HELLO2 or the liveness timeout, gets no media until + /// it is, for a minute. An address that was heard starts afresh. + #[test] + fn an_address_that_was_never_heard_gets_no_new_budget_for_a_minute() { + let mut session = Session::new(1234); + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let ipad = addr([10, 0, 0, 5], 50000); + datagram(&mut session, ipad, 0, identified_hello(1, 77), at(0)); + let first = session.session_id().unwrap(); + datagram(&mut session, ipad, 0, identified_hello(2, 77), at(100)); + let second = session.session_id().unwrap(); + assert_ne!(first, second); + assert!( + !session.may_send_media(second, 1, at(100)), + "superseded unheard" + ); + + let expired = session.tick(&TestConfig, false, at(3200)); + assert_eq!(expired.ended.unwrap().reason, EndReason::Liveness); + datagram(&mut session, ipad, 0, identified_hello(3, 77), at(5000)); + let third = session.session_id().unwrap(); + assert!( + !session.may_send_media(third, 1, at(5000)), + "expired unheard" + ); + session.tick(&TestConfig, false, at(8200)); + assert_eq!(session.last_end(), Some((third, EndReason::Liveness))); + + datagram(&mut session, ipad, 0, identified_hello(4, 77), at(68_000)); + let early = session.session_id().unwrap(); + assert!( + !session.may_send_media(early, 1, at(68_000)), + "not yet a minute" + ); + session.tick(&TestConfig, false, at(71_200)); + + datagram(&mut session, ipad, 0, identified_hello(5, 77), at(132_000)); + let fresh = session.session_id().unwrap(); + assert!( + session.may_send_media(fresh, 1000, at(132_000)), + "a minute later" + ); + + datagram(&mut session, ipad, fresh, ping(), at(132_010)); + datagram(&mut session, ipad, 0, identified_hello(6, 77), at(133_000)); + let reconnect = session.session_id().unwrap(); + assert!( + session.may_send_media(reconnect, 1000, at(133_000)), + "it was heard" + ); + } + + /// A HELLO2 with the same device id from another address takes over a + /// session that was never heard, with what it already spent. + #[test] + fn an_address_change_with_the_same_device_id_brings_no_new_budget() { + let mut session = Session::new(1234); + let now = Instant::now(); + datagram( + &mut session, + addr([10, 0, 0, 5], 50000), + 0, + identified_hello(1, 77), + now, + ); + let first = session.session_id().unwrap(); + assert!(session.may_send_media(first, UNANSWERED_MEDIA_BUDGET, now)); + + let later = now + Duration::from_millis(50); + let moved = addr([10, 0, 0, 9], 50000); + let actions = datagram(&mut session, moved, 0, identified_hello(2, 77), later); + assert_eq!(actions.new_target, Some(moved)); + let second = session.session_id().unwrap(); + assert_ne!(first, second); + assert!(!session.may_send_media(second, 1, later)); + } + + /// Media to clients not yet heard counts against one host-wide budget, + /// whatever their address, which comes back over a minute. It still + /// holds once more addresses than the memory keeps have come and gone. + #[test] + fn the_host_budget_covers_every_address_even_with_the_memory_full() { + let mut session = Session::new(1234); + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let first = addr([10, 0, 0, 5], 50000); + datagram(&mut session, first, 0, identified_hello(1, 77), at(0)); + let id = session.session_id().unwrap(); + assert!(session.may_send_media(id, UNANSWERED_MEDIA_BUDGET, at(0))); + session.tick(&TestConfig, false, at(3200)); + assert_eq!(session.last_end(), Some((id, EndReason::Liveness))); + + // Every slot of the memory taken by other addresses. + for host in 0..UNANSWERED_KEPT as u8 { + let other = addr([10, 0, 1, host], 50000); + datagram( + &mut session, + other, + 0, + identified_hello(10 + u32::from(host), 88), + at(4000 + u64::from(host)), + ); + } + session.tick(&TestConfig, false, at(7200)); + assert!(session.session_id().is_none()); + + datagram(&mut session, first, 0, identified_hello(2, 77), at(9000)); + let back = session.session_id().unwrap(); + assert!(!session.may_send_media(back, 200 * 1024, at(9000))); + session.tick(&TestConfig, false, at(12_200)); + + let fresh = addr([10, 0, 2, 1], 50000); + datagram(&mut session, fresh, 0, identified_hello(3, 99), at(13_000)); + let other = session.session_id().unwrap(); + assert!( + !session.may_send_media(other, 200 * 1024, at(13_000)), + "another address" + ); + session.tick(&TestConfig, false, at(16_200)); + + datagram(&mut session, fresh, 0, identified_hello(4, 99), at(80_000)); + let refilled = session.session_id().unwrap(); + assert!( + session.may_send_media(refilled, UNANSWERED_MEDIA_BUDGET, at(80_000)), + "a minute after the last spend" + ); + } + + /// An address that keeps ending sessions before it is heard holds one + /// entry in the memory, refreshed in place, and can't push another + /// address out of it. + #[test] + fn a_churning_address_keeps_one_entry_in_the_memory() { + let mut session = Session::new(1234); + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let first = addr([10, 0, 0, 5], 50000); + datagram(&mut session, first, 0, identified_hello(1, 77), at(0)); + session.tick(&TestConfig, false, at(3200)); + + let churn = addr([10, 0, 0, 6], 50000); + for nonce in 0..2 * UNANSWERED_KEPT as u32 { + datagram( + &mut session, + churn, + 0, + identified_hello(10 + nonce, 88), + at(4000 + u64::from(nonce)), + ); + } + session.tick(&TestConfig, false, at(7200)); + assert_eq!(session.unanswered.len(), 2); + + datagram(&mut session, first, 0, identified_hello(2, 77), at(8000)); + let back = session.session_id().unwrap(); + assert!(!session.may_send_media(back, 1000, at(8000))); + } + + /// An iPad's HELLO2 as shipped since 0.3.0: the generic "iPad" name and + /// its device id. + fn shipped_hello(nonce: u32, device_id: u64) -> ControlMessage { + let ControlMessage::Hello2(mut hello) = identified_hello(nonce, device_id) else { + unreachable!() + }; + hello.device_name = "iPad".to_string(); + ControlMessage::Hello2(hello) + } + + /// A refusal over Wi-Fi is never larger than the HELLO2 it answers. It + /// leaves out the PC name and host caps when they don't fit (iPads read + /// only the status and version of a refusal), and a HELLO2 too small + /// for even that gets no answer. USB answers stay whole. + #[test] + fn a_refusal_is_never_larger_than_its_hello() { + let mut session = Session::new(1234); + let now = Instant::now(); + datagram( + &mut session, + addr([10, 0, 0, 5], 50000), + 0, + shipped_hello(1, 77), + now, + ); + + let other = addr([10, 0, 0, 6], 50000); + let shipped = shipped_hello(2, 88); + let shipped_len = eternal_wire::v2::control::encode_control(0, 1, &shipped).len(); + assert_eq!(shipped_len, 71); + let busy = datagram(&mut session, other, 0, shipped, now); + let (to, reply) = &busy.replies[0]; + assert_eq!(*to, other); + assert!( + reply.len() <= shipped_len, + "{} > {shipped_len}", + reply.len() + ); + let ack = parse_ack(reply); + assert_eq!( + (ack.status, ack.client_nonce, ack.accepted_version), + (HelloStatus::Busy, 2, 2) + ); + assert_eq!((ack.host_name.as_str(), ack.host_caps), ("", 0)); + + let ControlMessage::Hello2(mut named) = shipped_hello(3, 88) else { + unreachable!() + }; + named.device_name = "A much longer iPad name here".to_string(); + let named = ControlMessage::Hello2(named); + let named_len = eternal_wire::v2::control::encode_control(0, 1, &named).len(); + let busy = datagram(&mut session, other, 0, named, now); + let reply = &busy.replies[0].1; + assert!(reply.len() <= named_len); + assert_eq!(parse_ack(reply).host_name, "TEST-HOST", "it fits whole"); + + let ControlMessage::Hello2(mut bare) = hello(4, 50000) else { + unreachable!() + }; + bare.device_name.clear(); + let bare = ControlMessage::Hello2(bare); + assert_eq!( + eternal_wire::v2::control::encode_control(0, 1, &bare).len(), + 38 + ); + let unanswered = datagram(&mut session, other, 0, bare.clone(), now); + assert!(unanswered.replies.is_empty()); + assert_eq!( + unanswered.ended.unwrap().reason, + EndReason::Refused(HelloStatus::Busy), + "still one refused attempt" + ); + + let usb = datagram(&mut session, PeerId::usb(17), 0, bare, now); + assert_eq!(parse_ack(&usb.replies[0].1).host_name, "TEST-HOST"); + } + + /// The golden HELLO2 of a 0.3.0 iPad, byte for byte, as the second + /// device: a refusal that fits it, which that iPad's parser reads (the + /// `hello_ack_busy_bare` vector). As the only one: a session that + /// streams once its first ping arrives. + #[test] + fn a_shipped_ipads_golden_hello_is_answered_within_its_size() { + let golden = include_str!("../../../proto/testdata/v2_vectors.txt") + .lines() + .find_map(|line| line.strip_prefix("hello2_v030 ")) + .unwrap(); + let bytes: Vec = (0..golden.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&golden[at..at + 2], 16).unwrap()) + .collect(); + let feed = |session: &mut Session, peer: PeerId, now: Instant| { + let (header, message) = eternal_wire::v2::control::parse_control(&bytes).unwrap(); + session.handle_datagram(peer, &header, message, bytes.len(), &TestConfig, now) + }; + let now = Instant::now(); + let ipad = addr([10, 0, 0, 6], 50000); + + let mut session = Session::new(1234); + datagram( + &mut session, + addr([10, 0, 0, 5], 50000), + 0, + shipped_hello(1, 77), + now, + ); + let busy = feed(&mut session, ipad, now); + let reply = &busy.replies[0].1; + assert!( + reply.len() <= bytes.len(), + "{} > {}", + reply.len(), + bytes.len() + ); + assert_eq!(parse_ack(reply).status, HelloStatus::Busy); + + let mut session = Session::new(1234); + let accepted = feed(&mut session, ipad, now); + let ack = parse_ack(&accepted.replies[0].1); + assert_eq!( + (ack.status, ack.host_name.as_str()), + (HelloStatus::Ok, "TEST-HOST") + ); + datagram(&mut session, ipad, ack.session_id, ping(), now); + assert!(session.may_send_media(ack.session_id, 10 * UNANSWERED_MEDIA_BUDGET, now)); + } + + /// Session ids come from a keyed hash, so two instances with the same + /// seed hand out different ids. + #[test] + fn session_ids_differ_between_instances() { + let ipad = addr([10, 0, 0, 5], 50000); + let ids = |session: &mut Session| -> Vec { + (1..=4) + .map(|nonce| { + session.handle_control( + ipad, + 0, + identified_hello(nonce, 77), + &TestConfig, + Instant::now(), + ); + session.session_id().unwrap() + }) + .collect() + }; + let first = ids(&mut Session::new(1234)); + let steps: Vec = first.windows(2).map(|p| p[1].wrapping_sub(p[0])).collect(); + assert!(steps.windows(2).any(|p| p[0] != p[1]), "{first:?}"); + assert_ne!(first, ids(&mut Session::new(1234)), "same seed, other ids"); + } + fn paired_hello(nonce: u32, feature_caps: u16) -> Hello2 { let mut hello = pair_hello(nonce); hello.auth_token = [7; 16]; @@ -4346,9 +4955,11 @@ mod tests { ); } - /// While no stream runs, every answer to a HELLO2 is bounded by its - /// size, the version and pairing refusals too. A padded HELLO2 still - /// hears them; while streaming, nothing changes. + /// Every answer to a Wi-Fi HELLO2 is bounded by its size, the version + /// and pairing refusals too, while no stream runs and while one does. A + /// guess at the code carries the HELLO2's extension, so it hears the + /// short refusal; a padded HELLO2 hears whole answers. Over USB the + /// answers stay whole. #[test] fn every_hello_answer_is_bounded_while_not_streaming() { use eternal_wire::v2::control::{encode_control, parse_control}; @@ -4406,9 +5017,14 @@ mod tests { for attempt in 0..6 { let mut guess = small(10 + attempt, 0); guess.pairing_code = 1_000_000 + attempt; + let expected = if attempt < 3 { + HelloStatus::Unauthorized + } else { + HelloStatus::RateLimited + }; assert_eq!( send(&mut session, &config, &guess), - None, + Some(expected), "attempt {attempt}" ); } @@ -4419,19 +5035,48 @@ mod tests { Some(HelloStatus::RateLimited) ); - // Streaming: the refusals are as they always were. + // Streaming: the same bound, and an attempt with no answer that + // fits is still one refused attempt. let config = pair_config(true); let mut session = Session::new(10); - let datagram = encode_control(0, 1, &ControlMessage::Hello2(small(30, 0))); - let (header, message) = parse_control(&datagram).unwrap(); - let actions = session.handle_datagram( - peer, - &header, - message, - datagram.len(), - &config, - Instant::now(), + assert_eq!(send(&mut session, &config, &small(30, 0)), None); + let mut guess = small(31, 0); + guess.pairing_code = 1_000_031; + assert_eq!( + send(&mut session, &config, &guess), + Some(HelloStatus::Unauthorized) + ); + + // Over USB, stopped or streaming, the answers stay whole. + let usb = PeerId::usb(17); + let over_usb = |session: &mut Session, hello: &Hello2| { + let datagram = encode_control(0, 1, &ControlMessage::Hello2(hello.clone())); + let (header, message) = parse_control(&datagram).unwrap(); + let actions = session.handle_datagram( + usb, + &header, + message, + datagram.len(), + &config, + Instant::now(), + ); + assert!(actions.replies[0].1.len() > datagram.len()); + pair_ack(&actions) + }; + let mut session = Session::new(10); + let mut old_version = small(40, 0); + old_version.proto_min = 1; + old_version.proto_max = 1; + let refused = over_usb(&mut session, &old_version); + assert_eq!(refused.status, HelloStatus::VersionUnsupported); + assert_eq!(refused.host_name, "TEST-HOST"); + let mut paired = small(41, 0); + paired.auth_token = [7; 16]; + session.set_not_streaming(Some(NotStreaming::Stopped)); + let stopped = over_usb(&mut session, &paired); + assert_eq!( + (stopped.status, stopped.host_name.as_str()), + (HelloStatus::Error, "TEST-HOST") ); - assert_eq!(pair_ack(&actions).status, HelloStatus::Unauthorized); } } diff --git a/host/src/transport/standby.rs b/host/src/transport/standby.rs index e7f36077..4ba89185 100644 --- a/host/src/transport/standby.rs +++ b/host/src/transport/standby.rs @@ -22,7 +22,7 @@ use tokio::net::UdpSocket; use tokio::sync::oneshot; use tracing::{debug, info, warn}; -use super::link::PeerId; +use super::link::{LinkId, PeerId}; use super::usb::{LinkEvent, TransportLinks}; use crate::control::SharedControl; @@ -224,10 +224,11 @@ async fn answer( crate::diagnostics::log_session_end(end, shared); } for (destination, reply) in &actions.replies { - // An answer is never larger than the HELLO2 it answers. iPads that - // understand these answers pad their HELLO2 so every answer fits - // (`STANDBY_HELLO2_LEN`); an older one may get today's silence. - if reply.len() > datagram.len() { + // Over Wi-Fi an answer is never larger than the HELLO2 it answers. + // iPads that understand these answers pad their HELLO2 so every + // answer fits (`STANDBY_HELLO2_LEN`); an older one gets the short + // refusal, or today's silence. Answers over USB stay whole. + if destination.link == LinkId::Udp && reply.len() > datagram.len() { debug!( peer = %destination, reply = reply.len(), diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index 4187355c..d9bc5f80 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -26,7 +26,7 @@ use eternal_wire::h264::contains_nal_type; use eternal_wire::reassembly::{AddOutcome, Reassembler, MAX_STALL_ALLOWANCE}; use eternal_wire::v2::control::{ encode_control, parse_control, ByeReason, ControlMessage, Hello2, HelloAck, HelloStatus, - KeyframeReason, KeyframeRequest, ReceiverReport, CAP_DECODE_H264, + KeyframeReason, KeyframeRequest, Ping, ReceiverReport, CAP_DECODE_H264, }; use eternal_wire::v2::media::MediaHeader; use eternal_wire::v2::{classify, Classified}; @@ -1210,12 +1210,34 @@ impl FakeReceiver { ) } + /// The handshake, then a ping, as every iPad since 0.3.0 sends the + /// moment its HELLO_ACK lands. fn connect_as( host_port: u16, decoder_caps: u16, feature_caps: u16, preferred_fps: u8, client_nonce: u32, + ) -> Self { + let mut receiver = Self::handshake( + host_port, + decoder_caps, + feature_caps, + preferred_fps, + client_nonce, + ); + receiver.send(&ControlMessage::Ping(Ping { t1_us: 1 })); + receiver + } + + /// The handshake alone: the first report, 400 ms later, is the first + /// thing the PC hears from this receiver. + fn handshake( + host_port: u16, + decoder_caps: u16, + feature_caps: u16, + preferred_fps: u8, + client_nonce: u32, ) -> Self { let socket = UdpSocket::bind("127.0.0.1:0").expect("receiver socket"); socket diff --git a/host/tests/e2e_synthetic/second_screen.rs b/host/tests/e2e_synthetic/second_screen.rs index eac94743..e2437c49 100644 --- a/host/tests/e2e_synthetic/second_screen.rs +++ b/host/tests/e2e_synthetic/second_screen.rs @@ -71,8 +71,11 @@ impl Host { } } + /// Unanswered until its first report: an answer runs the second screen's + /// next step at once, and with an instant attach these tests would see + /// no mirror before the switch. fn connect(&self, nonce: u32) -> FakeReceiver { - FakeReceiver::connect_as(self.port, CAP_DECODE_H264, 0, 0, nonce) + FakeReceiver::handshake(self.port, CAP_DECODE_H264, 0, 0, nonce) } fn status(&self) -> VddStatus { diff --git a/host/tests/send_budget.rs b/host/tests/send_budget.rs new file mode 100644 index 00000000..c097aed1 --- /dev/null +++ b/host/tests/send_budget.rs @@ -0,0 +1,599 @@ +//! The PC sends the first picture, then waits to hear back from the iPad +//! before it streams more. +//! +//! Paused-clock simulations of the real transport loop, with a fake encoder +//! and fake clients on loopback. The tokio clock only moves when every task +//! waits, and nothing waits while a datagram is on the loopback, so a busy +//! machine cannot change the result. Frames stay at 16 fragments or fewer, +//! which go out unpaced: pacing waits on a real-time thread. + +use std::net::SocketAddr; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; + +use eternal_host::control::SharedControl; +use eternal_host::encoder::NALUnit; +use eternal_host::transport; +use eternal_host::transport::probe; +use eternal_host::transport::session::{EndReason, UNANSWERED_MEDIA_BUDGET}; +use eternal_wire::v2::control::{ + encode_control, parse_control, ControlMessage, Hello2, HelloStatus, Ping, ReceiverReport, + CAP_DECODE_H264, FEATURE_SUPPORTS_NACK, +}; +use eternal_wire::v2::media::MediaHeader; +use eternal_wire::v2::{classify, Classified}; +use parking_lot::Mutex; +use tokio::sync::mpsc; +use tokio::time::Instant; + +/// The tests share ETERNAL_USB_DIRECT and the transport probe; run them one +/// at a time. +static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +const FRAME_INTERVAL: Duration = Duration::from_micros(16_667); +const STEP: Duration = Duration::from_millis(10); +/// 16 fragments: the largest frame that goes out unpaced. +const KEYFRAME_BYTES: usize = 21_000; +const DELTA_BYTES: usize = 8_000; +/// Real time a datagram may take across the loopback before a test fails. +const LOOPBACK_LIMIT: Duration = Duration::from_secs(10); + +/// Waits until `done`, without moving the paused clock: a yielding task +/// keeps the runtime busy, so tokio polls the sockets but advances no time. +async fn settle(what: &str, mut done: impl FnMut() -> bool) { + let limit = std::time::Instant::now() + LOOPBACK_LIMIT; + while !done() { + assert!( + std::time::Instant::now() < limit, + "{what} still in flight after {LOOPBACK_LIMIT:?}" + ); + tokio::task::yield_now().await; + } +} + +fn secs(seconds: f64) -> Duration { + Duration::from_secs_f64(seconds) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Heard { + /// Session, frame number, keyframe. + Media(u32, u32, bool), + Ack(HelloStatus, u32), + Heartbeat, + Bye, + Other, +} + +/// How a fake client answers its session. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Answers { + /// As the app does since 0.3.0: a ping the moment the HELLO_ACK lands, + /// four more 200 ms apart, then one every 2 s, and a report every + /// 500 ms. + AtOnce, + /// Its first report, this long after the HELLO_ACK. + After(Duration), + Never, +} + +struct Client { + socket: std::net::UdpSocket, + host: SocketAddr, + answers: Answers, + nonce: u32, + msg_seq: u32, + session: Option<(u32, Duration)>, + /// What arrived, when, and its size. + received: Vec<(Duration, Heard, usize)>, + /// When each packet reached the PC's socket. + delivered: Vec, + sent_before: u64, + taken_before: u64, + next_ping: Option<(Duration, u32)>, + next_report: Option, + start: Instant, +} + +impl Client { + fn new(host: SocketAddr, answers: Answers, start: Instant) -> Self { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.set_nonblocking(true).unwrap(); + let address = socket.local_addr().unwrap(); + // A port an earlier test used may come round again. + let (sent_before, taken_before) = (probe::sent_to(address), probe::taken_from(address)); + Self { + socket, + host, + answers, + nonce: 0, + msg_seq: 0, + session: None, + received: Vec::new(), + delivered: Vec::new(), + sent_before, + taken_before, + next_ping: None, + next_report: None, + start, + } + } + + fn address(&self) -> SocketAddr { + self.socket.local_addr().unwrap() + } + + /// Sends, and waits until the PC took it. + async fn send(&mut self, bytes: Vec) { + self.socket.send_to(&bytes, self.host).unwrap(); + self.delivered.push(self.start.elapsed()); + let (address, sent) = ( + self.address(), + self.taken_before + self.delivered.len() as u64, + ); + settle("a packet to the PC", || probe::taken_from(address) >= sent).await; + } + + async fn send_message(&mut self, message: &ControlMessage) { + self.msg_seq += 1; + let session_id = self.session.map_or(0, |(id, _)| id); + let bytes = encode_control(session_id, self.msg_seq, message); + self.send(bytes).await; + } + + /// The shipped iPad's HELLO2: the generic name, a device id, and + /// `listen_port` its own. + async fn hello(&mut self) { + self.nonce += 1; + self.session = None; + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: self.nonce, + listen_port: self.address().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_SUPPORTS_NACK, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "iPad".into(), + device_id: 77, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + self.send_message(&hello).await; + } + + /// The smallest HELLO2 there is: no name and no device id. 38 bytes. + async fn bare_hello(&mut self) { + self.nonce += 1; + let bytes = encode_control( + 0, + 1, + &ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: self.nonce, + listen_port: self.address().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: 0, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: String::new(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }), + ); + assert_eq!(bytes.len(), 38); + self.send(bytes).await; + } + + /// Takes everything the PC has sent so far, as received `now`. + async fn drain(&mut self, now: Duration) { + let address = self.address(); + let sent_before = self.sent_before; + let mut bytes = [0; 2048]; + loop { + while let Ok((len, _)) = self.socket.recv_from(&mut bytes) { + let datagram = &bytes[..len]; + self.received.push((now, heard(datagram), len)); + } + if self.received.len() as u64 >= probe::sent_to(address) - sent_before { + break; + } + tokio::task::yield_now().await; + } + } + + /// What the client does every step: take the session from an ACK, then + /// answer as it does. + async fn step(&mut self, now: Duration) { + self.drain(now).await; + if self.session.is_none() { + let accepted = self + .received + .iter() + .rev() + .find_map(|(_, heard, _)| match heard { + Heard::Ack(HelloStatus::Ok, id) => Some(*id), + _ => None, + }); + if let Some(id) = accepted { + self.session = Some((id, now)); + match self.answers { + Answers::AtOnce => { + self.next_ping = Some((now, 5)); + self.next_report = Some(now + Duration::from_millis(500)); + } + Answers::After(delay) => self.next_report = Some(now + delay), + Answers::Never => {} + } + } + } + if let Some((at, burst)) = self.next_ping.filter(|(at, _)| now >= *at) { + self.next_ping = Some(if burst > 1 { + (at + Duration::from_millis(200), burst - 1) + } else { + (at + Duration::from_secs(2), 1) + }); + self.send_message(&ControlMessage::Ping(Ping { t1_us: 1 })) + .await; + } + if let Some(at) = self.next_report.filter(|at| now >= *at) { + self.next_report = Some(at + Duration::from_millis(500)); + let frames = self.frames().len() as u32; + self.send_message(&ControlMessage::ReceiverReport(ReceiverReport { + stream_epoch: 1, + highest_seq: frames, + frames_complete: frames, + ..ReceiverReport::default() + })) + .await; + } + } + + /// Each video frame's first fragment: when, its number, keyframe. + fn frames(&self) -> Vec<(Duration, u32, bool)> { + let mut seen = std::collections::BTreeSet::new(); + self.received + .iter() + .filter_map(|(at, heard, _)| match heard { + Heard::Media(_, seq, key) if seen.insert(*seq) => Some((*at, *seq, *key)), + _ => None, + }) + .collect() + } + + fn media_bytes(&self) -> usize { + self.received + .iter() + .filter(|(_, heard, _)| matches!(heard, Heard::Media(..))) + .map(|(_, _, len)| len) + .sum() + } + + fn count(&self, kind: fn(&Heard) -> bool) -> usize { + self.received + .iter() + .filter(|(_, heard, _)| kind(heard)) + .count() + } +} + +fn heard(bytes: &[u8]) -> Heard { + match classify(bytes) { + Classified::Media { .. } => { + let header = MediaHeader::decode(bytes).unwrap().0; + Heard::Media(header.session_id, header.frame_seq, header.is_keyframe) + } + Classified::Control(_) => match parse_control(bytes) { + Ok((_, ControlMessage::HelloAck(ack))) => Heard::Ack(ack.status, ack.session_id), + Ok((_, ControlMessage::Heartbeat(_))) => Heard::Heartbeat, + Ok((_, ControlMessage::Bye(_))) => Heard::Bye, + _ => Heard::Other, + }, + _ => Heard::Other, + } +} + +/// The real transport loop and a fake encoder at 60 fps that makes a +/// keyframe whenever one is asked for. +struct Sim { + start: Instant, + shared: SharedControl, + host: SocketAddr, + transport: tokio::task::JoinHandle>>, + encoder: tokio::task::JoinHandle<()>, + /// When the encoder found an IDR asked for. + forced: Arc>>, + clients: Vec, +} + +impl Sim { + async fn start() -> Self { + let start = Instant::now(); + // Another process can take the free port before the transport binds it. + let (port, shared, nal_tx, transport) = loop { + let port = std::net::UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap(); + let shared = SharedControl::new(port, 10_000_000); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + let (nal_tx, nal_rx) = mpsc::channel(4); + let (audio_tx, audio_rx) = mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + let control = shared.clone(); + let transport = tokio::spawn(async move { + let _audio = audio_tx; + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx).await + }); + // The transport binds before its first wait. + for _ in 0..10 { + tokio::task::yield_now().await; + } + if !transport.is_finished() { + break (port, shared, nal_tx, transport); + } + let error = transport.await.unwrap().unwrap_err(); + assert!( + error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::AddrInUse), + "{error}" + ); + }; + let forced = Arc::new(Mutex::new(Vec::new())); + let encoder = { + let (force, forced) = (shared.force_next_idr.clone(), forced.clone()); + tokio::spawn(async move { + let mut next = Instant::now(); + for sequence in 0.. { + tokio::time::sleep_until(next).await; + next += FRAME_INTERVAL; + let is_keyframe = force.swap(false, Ordering::SeqCst); + if is_keyframe { + forced.lock().push(start.elapsed()); + } + let mut data = vec![ + 0xAA; + if is_keyframe { + KEYFRAME_BYTES + } else { + DELTA_BYTES + } + ]; + data[..5].copy_from_slice(&[0, 0, 0, 1, if is_keyframe { 0x65 } else { 0x41 }]); + let nal = NALUnit { + data, + sequence, + timestamp: Instant::now().into_std(), + encode_duration_us: 0, + is_keyframe, + }; + if nal_tx.send(nal).await.is_err() { + break; + } + } + }) + }; + Self { + start, + shared, + host: SocketAddr::from(([127, 0, 0, 1], port)), + transport, + encoder, + forced, + clients: Vec::new(), + } + } + + fn add_client(&mut self, answers: Answers) -> usize { + self.clients + .push(Client::new(self.host, answers, self.start)); + self.clients.len() - 1 + } + + fn now(&self) -> Duration { + self.start.elapsed() + } + + async fn run_until(&mut self, at: Duration) { + while self.start.elapsed() < at { + let now = self.start.elapsed(); + for client in &mut self.clients { + client.step(now).await; + } + tokio::time::sleep(STEP).await; + let now = self.start.elapsed(); + for client in &mut self.clients { + client.drain(now).await; + } + } + } + + fn last_end(&self) -> Option<(u32, EndReason)> { + self.shared.session.lock().last_end() + } + + async fn stop(self) { + self.shared.stop(); + tokio::time::timeout(Duration::from_secs(5), self.transport) + .await + .expect("the transport stops") + .unwrap() + .unwrap(); + self.encoder.abort(); + } +} + +fn run(test: impl std::future::Future) { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + eternal_host::clock::init(); + // Never reach a real iPad: this unreachable endpoint stands in for USB. + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .unwrap() + .block_on(test); +} + +/// A HELLO2 from an address that never answers gets the first picture, at +/// most [`UNANSWERED_MEDIA_BUDGET`] of media, and nothing more however +/// often it asks again: a new HELLO2 there brings no new budget for a +/// minute. Its session ends at the liveness timeout, as before. +#[test] +fn an_address_that_never_answers_gets_only_the_first_picture() { + run(async { + let mut sim = Sim::start().await; + let silent = sim.add_client(Answers::Never); + sim.clients[silent].hello().await; + sim.run_until(secs(2.0)).await; + // It asks again and again, from the same address. + for _ in 0..10 { + sim.clients[silent].hello().await; + let next = sim.now() + secs(0.1); + sim.run_until(next).await; + } + sim.run_until(secs(8.0)).await; + let (ended, reason) = sim.last_end().expect("the session ended"); + assert_eq!(reason, EndReason::Liveness, "session {ended}"); + sim.clients[silent].bare_hello().await; + sim.run_until(secs(12.0)).await; + + let client = &sim.clients[silent]; + let frames = client.frames(); + eprintln!( + "UNANSWERED_SIM media_bytes={} frames={} first={:?} last={:?} acks={} heartbeats={}", + client.media_bytes(), + frames.len(), + frames.first(), + frames.last(), + client.count(|h| matches!(h, Heard::Ack(..))), + client.count(|h| *h == Heard::Heartbeat), + ); + assert!( + client.media_bytes() <= UNANSWERED_MEDIA_BUDGET, + "{} bytes of media", + client.media_bytes() + ); + let (_, first_seq, first_key) = frames[0]; + assert!( + first_key && first_seq == 1, + "the first picture: {:?}", + frames[0] + ); + let (last_at, _, _) = *frames.last().unwrap(); + assert!(last_at < secs(1.2), "media at {last_at:?}"); + sim.stop().await; + }); +} + +/// An iPad that answers as the app does streams exactly as before: one +/// keyframe for its session, and every frame from the first on, with no +/// pause while the PC waits to hear from it. +#[test] +fn an_ipad_that_answers_at_once_streams_as_before() { + run(async { + let mut sim = Sim::start().await; + let ipad = sim.add_client(Answers::AtOnce); + sim.clients[ipad].hello().await; + sim.run_until(secs(4.0)).await; + + let client = &sim.clients[ipad]; + let (id, accepted) = client.session.expect("a session"); + let frames = client.frames(); + eprintln!( + "ANSWERED_SIM accepted={accepted:?} media_bytes={} frames={} first={:?} forced={:?}", + client.media_bytes(), + frames.len(), + frames.first(), + sim.forced.lock() + ); + assert_eq!(sim.shared.session.lock().session_id(), Some(id)); + assert_eq!(sim.forced.lock().len(), 1, "one keyframe"); + assert!(frames[0].2 && frames[0].1 == 1); + assert!( + frames.windows(2).all(|pair| pair[1].1 == pair[0].1 + 1), + "frames missing" + ); + let gap = frames + .windows(2) + .map(|pair| pair[1].0 - pair[0].0) + .max() + .unwrap(); + assert!(gap <= FRAME_INTERVAL + STEP, "video stopped for {gap:?}"); + assert!(frames.last().unwrap().0 >= secs(3.9)); + assert!(client.media_bytes() > 3 * UNANSWERED_MEDIA_BUDGET); + sim.stop().await; + }); +} + +/// An iPad first heard after its budget ran out gets nothing more until +/// then, and then a keyframe and the stream, numbered on from the last +/// frame it got. +#[test] +fn an_ipad_heard_late_gets_a_keyframe_then_the_stream() { + run(async { + let mut sim = Sim::start().await; + let ipad = sim.add_client(Answers::After(secs(2.0))); + sim.clients[ipad].hello().await; + sim.run_until(secs(4.0)).await; + + let client = &sim.clients[ipad]; + let (id, accepted) = client.session.expect("a session"); + let answered = accepted + secs(2.0); + let frames = client.frames(); + let before: Vec<_> = frames.iter().filter(|(at, _, _)| *at < answered).collect(); + let after: Vec<_> = frames.iter().filter(|(at, _, _)| *at >= answered).collect(); + let held_bytes: usize = client + .received + .iter() + .filter(|(at, heard, _)| *at < answered && matches!(heard, Heard::Media(..))) + .map(|(_, _, len)| len) + .sum(); + eprintln!( + "LATE_SIM accepted={accepted:?} before={} ({held_bytes} bytes, last {:?}) \ + after_first={:?} forced={:?}", + before.len(), + before.last(), + after.first(), + sim.forced.lock() + ); + assert_eq!(sim.shared.session.lock().session_id(), Some(id)); + assert!(held_bytes <= UNANSWERED_MEDIA_BUDGET, "{held_bytes}"); + assert!(before.last().unwrap().0 < secs(1.2), "{:?}", before.last()); + let (resumed, seq, key) = *after[0]; + assert!(key, "the stream starts again with a keyframe"); + assert_eq!(seq, before.last().unwrap().1 + 1, "numbered on"); + assert!( + resumed - answered <= FRAME_INTERVAL + 2 * STEP, + "{resumed:?}" + ); + assert_eq!( + sim.forced + .lock() + .iter() + .filter(|at| **at >= answered) + .count(), + 1 + ); + assert!( + frames.windows(2).all(|pair| pair[1].1 == pair[0].1 + 1), + "frames missing" + ); + sim.stop().await; + }); +} diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index 888bd53d..b7d2273e 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,7 +35,7 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 36, + Self.vectors.count, 37, "golden vector count drifted — update both test suites together" ) } @@ -365,6 +365,21 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(ack.streamConfig, StreamConfig()) } + /// A refusal without the PC name and host caps, as a HELLO2 too small + /// for the whole answer gets. Every iPad reads only its status. + func testHelloAckBusyBareVector() throws { + let data = try vector("hello_ack_busy_bare") + XCTAssertEqual(data.count, 49) + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + guard case .helloAck(let ack) = message else { return XCTFail("wrong type") } + XCTAssertEqual(ack.status, .busy) + XCTAssertEqual(ack.acceptedVersion, 2) + XCTAssertEqual(ack.clientNonce, 0xDEAD_BEEF) + XCTAssertEqual(ack.hostName, "") + XCTAssertEqual(ack.hostCaps, 0) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + } + func testHeartbeatVector() throws { let data = try vector("heartbeat") let (header, message) = try XCTUnwrap(Wire.parseControl(data)) diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 4c62e3c4..82419b11 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -39,3 +39,4 @@ identity 454d021b00004400000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf94260 hello_ack_stopped 454d02110000390000000000050000000602efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d5043000000000000000000000000000000000100 hello_ack_restarting 454d02110000390000000000050000000702efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d5043000000000000000000000000000000000100 hello_ack_failed 454d021100004a0000000000050000000802efbeadde00000000e803f401b80b0000000000000000000000000000000006414c492d504300000000000000000000000000000000010010646973706c61795f6368616e67696e67 +hello_ack_busy_bare 454d02110000210000000000020000000102efbeadde00000000e803f401b80b0000000000000000000000000000000000 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index cbe81264..968c6cc1 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -487,6 +487,29 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { ), )); } + // A refusal without the PC name and host caps, as a HELLO2 too small + // for the whole answer gets: every iPad reads only its status. + vectors.push(( + "hello_ack_busy_bare", + encode_control( + 0, + 2, + &ControlMessage::HelloAck(HelloAck { + status: HelloStatus::Busy, + accepted_version: 2, + client_nonce: 0xDEAD_BEEF, + session_id: 0, + heartbeat_interval_ms: 1000, + report_interval_ms: 500, + liveness_timeout_ms: 3000, + stream_config: StreamConfig::default(), + host_name: String::new(), + auth_token: [0; 16], + host_caps: 0, + reason: String::new(), + }), + ), + )); vectors } From c77b22068b3da594b753fea29c6490cdaaa731a3 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:39:04 -0500 Subject: [PATCH 109/226] Bring PC audio back when another app lets go of the output When another app has the PC's output device to itself, loopback capture can't open it. Audio used to stay off for the rest of the session. It is now tried again in the background, soon at first and then every 2 s, and starts as soon as the app lets go. Video is not touched. The same applies when the default output changes to a device another app holds. The PC window says why while it lasts, and the log notes once when audio becomes unavailable and once when it is back. Other audio errors behave as before. --- host/src/audio/mod.rs | 25 ++++ host/src/audio/stage.rs | 242 +++++++++++++++++++++++++++++++++--- host/src/audio/wasapi.rs | 66 ++++++++-- host/src/gui/page_stream.rs | 7 ++ host/src/gui/tests.rs | 19 +++ host/src/supervisor.rs | 5 + host/src/transport/audio.rs | 8 +- host/src/transport/mod.rs | 2 + 8 files changed, 345 insertions(+), 29 deletions(-) diff --git a/host/src/audio/mod.rs b/host/src/audio/mod.rs index a0d30698..5af1b7ae 100644 --- a/host/src/audio/mod.rs +++ b/host/src/audio/mod.rs @@ -50,6 +50,31 @@ pub fn wanted_session(shared: &SharedControl) -> Option { } } +/// Tests that replace the global audio stats run one at a time. +#[cfg(test)] +pub(crate) static STATS_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); + +/// The audio status while another app has the PC's output to itself. +pub const DEVICE_IN_USE_STATUS: &str = "Output in use by another app"; + +/// Another app has the PC's output device to itself (exclusive mode), so +/// loopback capture can't open it. Not the session's failure: the audio +/// stage tries again until the app lets go. +#[derive(Debug)] +pub struct DeviceInUse; + +impl std::fmt::Display for DeviceInUse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("AUDCLNT_E_DEVICE_IN_USE: another app has exclusive use of the output device") + } +} + +impl std::error::Error for DeviceInUse {} + +pub fn is_device_in_use(error: &(dyn std::error::Error + Send + Sync + 'static)) -> bool { + error.downcast_ref::().is_some() +} + pub fn codec_available() -> bool { ffmpeg_next::encoder::find_by_name("libopus").is_some() } diff --git a/host/src/audio/stage.rs b/host/src/audio/stage.rs index 8659b5ba..6775b7de 100644 --- a/host/src/audio/stage.rs +++ b/host/src/audio/stage.rs @@ -6,7 +6,9 @@ use tokio::sync::mpsc::{error::TrySendError, Sender}; use super::encoder::OpusEncoder; use super::resample::{PcmBlock, PcmFormat, Resampler}; use super::synthetic::SyntheticSource; -use super::{wanted_session, AudioResult, AudioStats, SessionPacket, SAMPLE_RATE}; +use super::{ + wanted_session, AudioResult, AudioStats, SessionPacket, DEVICE_IN_USE_STATUS, SAMPLE_RATE, +}; use crate::capture::timing::FrameTimer; use crate::control::SharedControl; use crate::stats::PIPELINE_STATS; @@ -121,10 +123,10 @@ struct CaptureSession { } impl CaptureSession { - fn new(id: u32) -> AudioResult { + fn new(id: u32, source: Source) -> AudioResult { Ok(Self { id, - source: Source::open()?, + source, resampler: None, encoder: OpusEncoder::new(MAX_OPUS_PAYLOAD)?, mark_discontinuity: true, @@ -302,25 +304,38 @@ pub(crate) fn restart_stats( } } -/// When to try again to open audio while FFmpeg's encoder lane is stuck: -/// soon at first, then at most every 8 s, so a long stall neither spins nor +/// When to try again to open audio that waits on something outside it: +/// soon at first, then at most every `max`, so a long wait neither spins nor /// floods the log. -#[derive(Default)] -struct StallRetry { +struct Retry { next: Option, wait: Duration, + first: Duration, + max: Duration, } -impl StallRetry { - const FIRST: Duration = Duration::from_millis(500); - const MAX: Duration = Duration::from_secs(8); +impl Retry { + /// FFmpeg's encoder lane is stuck in a driver call. + const STALL: Self = Self::new(Duration::from_millis(500), Duration::from_secs(8)); + /// Another app has the output to itself. A try is a few WASAPI calls and + /// opens no encoder, so audio is back within 2 s of the app letting go. + const DEVICE_IN_USE: Self = Self::new(Duration::from_millis(250), Duration::from_secs(2)); + + const fn new(first: Duration, max: Duration) -> Self { + Self { + next: None, + wait: Duration::ZERO, + first, + max, + } + } fn due(&self, now: Instant) -> bool { self.next.is_none_or(|next| now >= next) } fn failed(&mut self, now: Instant) -> Duration { - self.wait = (self.wait * 2).clamp(Self::FIRST, Self::MAX); + self.wait = (self.wait * 2).clamp(self.first, self.max); self.next = Some(now + self.wait); self.wait } @@ -328,16 +343,30 @@ impl StallRetry { /// An endpoint is opened only after both peers opt in. A failed capture /// attempt stays disabled for that session; reconnecting or toggling the -/// host audio setting off and on retries without restarting video. +/// host audio setting off and on retries without restarting video. An output +/// another app has to itself is tried again until it lets go. pub fn run_audio_stage( tx: Sender, shared: SharedControl, generation: u64, reporter: HealthReporter, +) { + run_stage(tx, shared, generation, reporter, Source::open); +} + +fn run_stage( + tx: Sender, + shared: SharedControl, + generation: u64, + reporter: HealthReporter, + mut open: impl FnMut() -> AudioResult, ) { let mut active: Option = None; let mut failed_session = None; - let mut stalled = StallRetry::default(); + let mut stalled = Retry::STALL; + let mut in_use = Retry::DEVICE_IN_USE; + // Logged once when an app takes the output, once when audio is back. + let mut waiting_for_output = false; let mut next_log = Instant::now() + Duration::from_secs(1); while shared.running.load(Ordering::SeqCst) && generation_is_current(generation) @@ -363,7 +392,8 @@ pub fn run_audio_stage( continue; } let session_id = wanted.unwrap(); - if failed_session == Some(session_id) || !stalled.due(Instant::now()) { + let now = Instant::now(); + if failed_session == Some(session_id) || !stalled.due(now) || !in_use.due(now) { std::thread::sleep(Duration::from_millis(20)); continue; } @@ -373,7 +403,7 @@ pub fn run_audio_stage( .is_none_or(|session| session.id != session_id) { active = None; - let mut session = CaptureSession::new(session_id)?; + let mut session = CaptureSession::new(session_id, open()?)?; if !shared.running.load(Ordering::SeqCst) { return Ok(false); } @@ -394,6 +424,12 @@ pub fn run_audio_stage( bitrate = super::BITRATE, "PC audio session started (Opus, 48 kHz stereo, 20 ms)" ); + if std::mem::take(&mut waiting_for_output) { + tracing::info!( + session_id, + "PC audio is back: the other app let go of the output" + ); + } active = Some(session); } active.as_mut().unwrap().tick(&tx, &shared, generation) @@ -402,7 +438,10 @@ pub fn run_audio_stage( break; } match result { - Ok(true) => stalled = StallRetry::default(), + Ok(true) => { + stalled = Retry::STALL; + in_use = Retry::DEVICE_IN_USE; + } Ok(false) => break, // Video's encoder holds FFmpeg's lock in a stuck driver call: // not this session's failure. Video goes on; the open is tried @@ -420,6 +459,24 @@ pub fn run_audio_stage( audio.error = Some(error.to_string()); }); } + // Another app has the output to itself: not this session's + // failure either. Video goes on; the open is tried again until + // the app lets go, and the window says why meanwhile. + Err(error) if super::is_device_in_use(error.as_ref()) => { + log_final(active.take()); + in_use.failed(Instant::now()); + if !std::mem::replace(&mut waiting_for_output, true) { + tracing::warn!( + session_id, + reason = %error, + "PC audio unavailable while another app has the output to itself; video continues, and audio starts when it lets go" + ); + } + update_current(generation, |audio| { + audio.stopped(DEVICE_IN_USE_STATUS); + audio.error = None; + }); + } Err(error) => { log_final(active.take()); failed_session = Some(session_id); @@ -453,7 +510,7 @@ mod tests { #[test] fn audio_retries_a_stalled_open_with_a_bounded_backoff() { - let mut retry = StallRetry::default(); + let mut retry = Retry::STALL; let now = Instant::now(); assert!(retry.due(now)); let waits: Vec = (0..7) @@ -533,6 +590,7 @@ mod tests { #[test] fn loopback_backlog_from_before_the_session_is_not_sent_stale() { + let _stats = crate::audio::STATS_TEST_LOCK.lock(); ffmpeg_next::init().unwrap(); // WASAPI buffers from Start() while the session opens its encoder. A // 0.3.1 host on an i7-5500U took 105 ms, and its first six packets @@ -751,6 +809,156 @@ mod tests { assert_eq!(logged(|| restart_stats(&target, |_| {})), ""); } + /// Runs the stage on this thread, opening its source with `open`, until + /// `control` returns on another. Gives the stage's log lines, its health + /// reports and what `control` returned. + fn run_with( + open: impl FnMut() -> AudioResult, + control: impl FnOnce(&mut tokio::sync::mpsc::Receiver) -> T + Send + 'static, + ) -> (String, Vec, T) { + let (shared, _) = + crate::transport::test_session(eternal_wire::v2::control::FEATURE_WANTS_AUDIO); + shared.running.store(true, Ordering::SeqCst); + let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let (tx, mut rx) = tokio::sync::mpsc::channel(super::super::QUEUE_CAPACITY); + let (health_tx, health_rx) = std::sync::mpsc::channel(); + let controller = { + let shared = shared.clone(); + std::thread::spawn(move || { + let out = control(&mut rx); + shared.running.store(false, Ordering::SeqCst); + out + }) + }; + let reporter = HealthReporter::new(health_tx, generation); + let lines = logged(|| run_stage(tx, shared.clone(), generation, reporter, open)); + let out = controller.join().unwrap(); + // Video goes on: nothing the stage reported stops or restarts it. + let health: Vec<_> = health_rx.try_iter().collect(); + let mut machine = crate::supervisor::Machine::new(generation); + for message in &health { + let crate::supervisor::SupMsg::Health { + generation, + stage, + outcome, + } = message + else { + unreachable!() + }; + assert_eq!( + machine.on_stage_exit(*generation, *stage, outcome, Instant::now()), + crate::supervisor::Effect::None + ); + } + assert_eq!( + machine.state, + crate::supervisor::SupervisorState::Running { generation } + ); + assert!(shared.audio_enabled.load(Ordering::SeqCst)); + (lines, health, out) + } + + fn only_completed(health: &[crate::supervisor::SupMsg]) -> bool { + health.iter().all(|message| { + matches!( + message, + crate::supervisor::SupMsg::Health { + stage: Stage::Audio, + outcome: StageOutcome::Completed, + .. + } + ) + }) + } + + const UNAVAILABLE: &str = "PC audio unavailable while another app has the output to itself"; + const BACK: &str = "PC audio is back"; + + #[test] + fn audio_starts_once_another_app_lets_go_of_the_output() { + let _stats = crate::audio::STATS_TEST_LOCK.lock(); + ffmpeg_next::init().unwrap(); + const BUSY: usize = 3; + // The audio status each time the stage opens its source. + let seen = std::cell::RefCell::new(Vec::new()); + let open = || { + let mut seen = seen.borrow_mut(); + seen.push(PIPELINE_STATS.lock().audio.status.clone()); + if seen.len() <= BUSY { + return Err(crate::audio::DeviceInUse.into()); + } + let start = crate::clock::host_now_us() + 50_000; + Ok(Source::Scripted( + (0..10u64) + .map(|index| PcmBlock { + format: PcmFormat::STEREO_48K, + samples: vec![0.25; 2 * super::super::FRAME_SAMPLES], + capture_ts_us: start + index * FRAME_US, + discontinuity: index == 0, + }) + .collect(), + )) + }; + let (lines, health, (packet, status)) = run_with(open, |rx| { + let until = Instant::now() + Duration::from_secs(10); + let mut packet = None; + while packet.is_none() && Instant::now() < until { + packet = rx.try_recv().ok(); + std::thread::sleep(Duration::from_millis(5)); + } + let audio = PIPELINE_STATS.lock().audio.clone(); + (packet, (audio.status, audio.error)) + }); + let seen = seen.into_inner(); + assert_eq!(seen.len(), BUSY + 1, "opens: {seen:?}"); + assert!( + seen[1..] + .iter() + .all(|status| status == DEVICE_IN_USE_STATUS), + "the window said why while it waited: {seen:?}" + ); + assert!( + packet.is_some(), + "audio never started +{lines}" + ); + assert_eq!(status, ("Streaming".to_string(), None)); + assert_eq!(lines.matches(UNAVAILABLE).count(), 1, "{lines}"); + assert_eq!(lines.matches(BACK).count(), 1, "{lines}"); + assert!(only_completed(&health), "{health:?}"); + } + + #[test] + fn an_output_that_stays_in_use_is_tried_on_a_bounded_backoff() { + let _stats = crate::audio::STATS_TEST_LOCK.lock(); + let now = Instant::now(); + let mut retry = Retry::DEVICE_IN_USE; + let waits: Vec = (0..6) + .map(|_| retry.failed(now).as_millis() as u64) + .collect(); + assert_eq!(waits, [250, 500, 1000, 2000, 2000, 2000]); + + let opens = std::cell::Cell::new(0); + let open = || { + opens.set(opens.get() + 1); + Err(crate::audio::DeviceInUse.into()) + }; + // Tries at 0, 0.25, 0.75 and 1.75 s. + let (lines, health, status) = run_with(open, |_| { + std::thread::sleep(Duration::from_millis(2200)); + PIPELINE_STATS.lock().audio.status.clone() + }); + assert!( + (3..=5).contains(&opens.get()), + "{} opens in 2.2 s", + opens.get() + ); + assert_eq!(status, DEVICE_IN_USE_STATUS); + assert_eq!(lines.matches(UNAVAILABLE).count(), 1, "{lines}"); + assert_eq!(lines.matches(BACK).count(), 0, "{lines}"); + assert!(only_completed(&health), "{health:?}"); + } + /// Native check of the same startup path with the PC's real loopback. /// The delay stands in for the encoder open that follows WASAPI Start(). /// Inaudible: it only reads the mix. Something must be rendering (even diff --git a/host/src/audio/wasapi.rs b/host/src/audio/wasapi.rs index 320646ab..0f70f057 100644 --- a/host/src/audio/wasapi.rs +++ b/host/src/audio/wasapi.rs @@ -17,7 +17,7 @@ use windows::Win32::System::Threading::{CreateEventW, WaitForSingleObject}; use super::pcm::SampleEncoding; use super::resample::{PcmBlock, PcmFormat}; -use super::AudioResult; +use super::{AudioResult, DeviceInUse}; const PCM_GUID: GUID = GUID::from_u128(0x00000001_0000_0010_8000_00aa00389b71); const FLOAT_GUID: GUID = GUID::from_u128(0x00000003_0000_0010_8000_00aa00389b71); @@ -122,6 +122,21 @@ fn parse_format(mix: &MixFormat) -> AudioResult<(PcmFormat, SampleEncoding)> { Ok((format, encoding)) } +/// Another app took the output in exclusive mode: shared-mode loopback +/// can't open until it lets go, which the audio stage waits out. +fn in_use( + error: Box, +) -> Box { + if error + .downcast_ref::() + .is_some_and(|error| error.code() == AUDCLNT_E_DEVICE_IN_USE) + { + Box::new(DeviceInUse) + } else { + error + } +} + struct Loopback { capture: IAudioCaptureClient, client: IAudioClient, @@ -297,7 +312,7 @@ impl WasapiSource { pub fn new() -> AudioResult { let apartment = ComApartment::new()?; let enumerator = unsafe { CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)? }; - let stream = Loopback::open(&enumerator)?; + let stream = Loopback::open(&enumerator).map_err(in_use)?; Ok(Self { stream: Some(stream), enumerator, @@ -325,11 +340,15 @@ impl WasapiSource { self.stream = Some(stream); } Err(error) => { - if self - .unavailable_since - .get_or_insert_with(Instant::now) - .elapsed() - >= Duration::from_secs(5) + // An app holding the output to itself is waited out by + // the audio stage, which shows why. + let error = in_use(error); + if super::is_device_in_use(error.as_ref()) + || self + .unavailable_since + .get_or_insert_with(Instant::now) + .elapsed() + >= Duration::from_secs(5) { return Err(error); } @@ -373,7 +392,7 @@ impl WasapiSource { self.reopen(); Ok(None) } else { - Err(error) + Err(in_use(error)) } } } @@ -386,3 +405,34 @@ impl WasapiSource { tracing::info!("Default PC audio endpoint changed; reopening loopback"); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_an_output_in_use_waits_for_the_other_app() { + let busy = in_use(windows::core::Error::from(AUDCLNT_E_DEVICE_IN_USE).into()); + assert!(super::super::is_device_in_use(busy.as_ref())); + // Reopened in place, or not something waiting fixes: unchanged. + for code in [ + AUDCLNT_E_DEVICE_INVALIDATED, + AUDCLNT_E_RESOURCES_INVALIDATED, + AUDCLNT_E_SERVICE_NOT_RUNNING, + AUDCLNT_E_ENDPOINT_CREATE_FAILED, + AUDCLNT_E_UNSUPPORTED_FORMAT, + AUDCLNT_E_EXCLUSIVE_MODE_NOT_ALLOWED, + ] { + let other = in_use(windows::core::Error::from(code).into()); + assert!(!super::super::is_device_in_use(other.as_ref()), "{code:?}"); + assert_eq!( + other + .downcast_ref::() + .map(|error| error.code()), + Some(code) + ); + } + let text = in_use("invalid WASAPI mix geometry".into()); + assert_eq!(text.to_string(), "invalid WASAPI mix geometry"); + } +} diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index c4134488..1d4a36be 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -1496,6 +1496,13 @@ pub(super) fn audio_summary( ACCENT, ); } + if audio.status == crate::audio::DEVICE_IN_USE_STATUS { + return ( + "Another app is using your speakers on its own. PC audio comes back when it lets go." + .into(), + WARNING, + ); + } // The audio stage's idle status names the protocol, not the device. if audio.status == "Waiting for client" { return ("Starts when an iPad connects.".into(), TEXT_MUTED); diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index 9699c452..14237733 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -140,6 +140,25 @@ fn audio_row_reports_rate_and_keeps_failure_separate_from_video() { assert_eq!(color, TEXT_MUTED); } +#[test] +fn audio_row_says_why_while_another_app_has_the_output() { + let mut audio = crate::audio::AudioStats::started("Speakers"); + audio.stopped(crate::audio::DEVICE_IN_USE_STATUS); + let (text, color) = audio_summary(&audio, true); + assert_eq!( + text, + "Another app is using your speakers on its own. PC audio comes back when it lets go." + ); + assert_eq!(color, WARNING); + + // Gone once audio is back. + let mut audio = crate::audio::AudioStats::started("Speakers"); + audio.kbps = 128.0; + let (text, color) = audio_summary(&audio, true); + assert!(text.starts_with("Streaming Speakers"), "{text}"); + assert_eq!(color, ACCENT); +} + #[test] fn usb_row_shows_install_hint_only_when_the_service_is_unavailable() { let missing = usb_summary(false, 2, "Connected", 0, false); diff --git a/host/src/supervisor.rs b/host/src/supervisor.rs index bf2bd173..895d9b86 100644 --- a/host/src/supervisor.rs +++ b/host/src/supervisor.rs @@ -299,6 +299,11 @@ pub struct HealthReporter { } impl HealthReporter { + #[cfg(test)] + pub(crate) fn new(tx: mpsc::Sender, generation: u64) -> Self { + Self { tx, generation } + } + pub fn stage_exited(&self, stage: Stage, outcome: StageOutcome) { let _ = self.tx.send(SupMsg::Health { generation: self.generation, diff --git a/host/src/transport/audio.rs b/host/src/transport/audio.rs index b3d39a7e..fb450773 100644 --- a/host/src/transport/audio.rs +++ b/host/src/transport/audio.rs @@ -123,13 +123,14 @@ impl AudioSender { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use eternal_wire::v2::control::*; use std::sync::atomic::Ordering; use std::time::Instant; - fn session(caps: u16) -> (SharedControl, u32) { + /// A session whose iPad advertised `caps`, and its id. + pub(crate) fn session(caps: u16) -> (SharedControl, u32) { let shared = SharedControl::new(9876, 15_000_000); shared.pairing.lock().required = false; let config = super::super::SharedConfigSource { @@ -167,8 +168,7 @@ mod tests { (shared, id) } - /// Tests that replace the global audio stats run one at a time. - static AUDIO_STATS: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); + use crate::audio::STATS_TEST_LOCK as AUDIO_STATS; fn packet(id: u32, time: u64) -> SessionPacket { SessionPacket { diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index c6050db6..8d43d446 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -1,6 +1,8 @@ pub mod abr; pub mod apple_usb; mod audio; +#[cfg(test)] +pub(crate) use audio::tests::session as test_session; pub mod beat; pub mod fault; pub mod link; From 9cb3a2cb76dbe5656f7234ea63e72b65d37e382e Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:39:50 -0500 Subject: [PATCH 110/226] Wait for the window in two steps, and watch its first frame Review of the first version found five gaps; this closes them. - Crash avoidance read an empty GPU list while detection was stuck: its DXGI list was refused in the startup lane, so a known-bad backend was picked again and a crash recorded under an empty key. The window now waits for the startup lane to free first, then configures the renderer, then runs the wgpu probe, in a lane of its own (stall=gpu_probe). - The window's thread still creates its device and swap chain unwatched. A watcher now runs from run_native until the first frame is presented; past the 4 s limit it shows the same notice, closed by the first frame. Cancel there runs the app's quit path, now one closure main.rs and the watcher share, and ends the process. - After OK the notice never came back, so quitting was impossible while the driver stayed stuck. OK now hides it for 15 s. A Cancel that lands as the driver answers still quits. - After the first frame, the renderer's DXGI list (as a renderer rebuilt after a lost device makes it) uses the gpu_facts lane, not startup's. While it lists, the crash marker names its thread, so a driver crash there is still the window's. - The tests no longer depend on timing: the notice, probe and watcher tests order events on channels and a hand-moved clock, and the in-time detection test has a 1 s warning threshold. --- host/src/gpu.rs | 39 +- host/src/gui/driver_wait.rs | 694 +++++++++++++++++++++++++++--------- host/src/gui/mod.rs | 55 ++- host/src/gui/renderer.rs | 94 ++++- host/src/main.rs | 137 ++++--- host/src/stall.rs | 21 +- 6 files changed, 779 insertions(+), 261 deletions(-) diff --git a/host/src/gpu.rs b/host/src/gpu.rs index 7c0dda08..7558e3d3 100644 --- a/host/src/gpu.rs +++ b/host/src/gpu.rs @@ -371,10 +371,24 @@ mod tests { }) } + /// A 300 ms limit, for a detection that hangs. + const HELD: crate::stall::Limits = crate::stall::Limits { + warn: std::time::Duration::from_millis(100), + limit: std::time::Duration::from_millis(300), + }; + + /// Limits a detection that returns at once never comes near, however + /// slowly the machine runs. + const ROOMY: crate::stall::Limits = crate::stall::Limits { + warn: std::time::Duration::from_secs(1), + limit: std::time::Duration::from_secs(30), + }; + /// `detect_within` over `detect` in lanes and a slot of its own, under - /// a 300 ms limit: what it found, how long it took, what it logged, the - /// lanes and the slot. + /// `limits`: what it found, how long it took, what it logged, the lanes + /// and the slot. fn detect_held( + limits: crate::stall::Limits, detect: impl FnOnce() -> Result + Send + 'static, ) -> ( GpuInfo, @@ -387,10 +401,6 @@ mod tests { let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); let found: &'static parking_lot::Mutex> = Box::leak(Box::new(parking_lot::Mutex::new(None))); - let limits = crate::stall::Limits { - warn: std::time::Duration::from_millis(100), - limit: std::time::Duration::from_millis(300), - }; let out = Arc::new(parking_lot::Mutex::new(Vec::::new())); struct Sink(Arc>>); impl std::io::Write for Sink { @@ -432,7 +442,9 @@ mod tests { // The detector waits on a driver that does not return until the // test lets it. let (release, held) = std::sync::mpsc::channel::<()>(); - let (info, took, text, lanes, slot) = detect_held(move || { + let (started, detecting) = std::sync::mpsc::channel::<()>(); + let (info, took, text, lanes, slot) = detect_held(HELD, move || { + let _ = started.send(()); let _ = held.recv(); Ok(found("late")) }); @@ -442,11 +454,9 @@ mod tests { assert_eq!(info.encoder_name, "libx264"); assert_eq!(info.vendor, GpuVendor::Unknown); assert!(info.detection_stalled); - assert!( - took >= std::time::Duration::from_millis(300) - && took < std::time::Duration::from_secs(3), - "{took:?}" - ); + // Never before the limit; that it returned at all while the detector + // still blocks shows it did not wait for it. + assert!(took >= HELD.limit, "{took:?}"); // One WARN, saying detection timed out and after how long. let warns: Vec<_> = text.lines().filter(|l| l.contains("WARN")).collect(); assert_eq!(warns.len(), 1, "{text}"); @@ -455,6 +465,7 @@ mod tests { // Until it returns, the lane holds it and nothing was found. assert_eq!(lanes.abandoned(Call::GpuDetect), 1); assert!(slot.lock().is_none()); + detecting.recv().unwrap(); release.send(()).unwrap(); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); while lanes.abandoned(Call::GpuDetect) != 0 { @@ -475,7 +486,7 @@ mod tests { #[test] fn a_detection_that_returns_in_time_keeps_its_facts() { - let (info, _, text, _, _) = detect_held(|| Ok(found("RTX"))); + let (info, _, text, _, _) = detect_held(ROOMY, || Ok(found("RTX"))); assert_eq!( (info.name.as_str(), info.encoder_name.as_str()), ("RTX", "h264_nvenc") @@ -484,7 +495,7 @@ mod tests { assert!(!text.contains("WARN"), "{text}"); // A failed one falls back with its own line, as before, and is not // a stall: no notice blames a driver that answered. - let (info, _, text, _, slot) = detect_held(|| Err("no adapters".to_string())); + let (info, _, text, _, slot) = detect_held(ROOMY, || Err("no adapters".to_string())); assert_eq!(info.encoder_name, "libx264"); assert!(!info.detection_stalled); assert!(slot.lock().is_none()); diff --git a/host/src/gui/driver_wait.rs b/host/src/gui/driver_wait.rs index c3bab2a4..5b5185d4 100644 --- a/host/src/gui/driver_wait.rs +++ b/host/src/gui/driver_wait.rs @@ -1,18 +1,29 @@ -//! The window waits for a graphics driver that answers before it opens. +//! The window waits for a graphics driver that answers, and says so. //! //! The window's thread lists adapters, creates its device and draws through -//! the graphics driver, and Windows cannot stop a thread stuck inside it. So -//! before that thread makes its first graphics call, the same calls are made -//! once on a thread of their own, in the startup graphics lane -//! ([`Call::GpuDetect`]), under its limit. While they do not return, or an -//! earlier startup call in that lane is still stuck (GPU detection, the -//! screen list), a notice drawn by Windows without the graphics card says -//! why the window is not there yet, and the window opens once they return. +//! the graphics driver, and Windows cannot stop a thread stuck inside it. +//! So before that thread makes its first graphics call ([`before_window`]): +//! - the startup graphics lane ([`Call::GpuDetect`]) must be free: a GPU +//! detection or screen list given up on may still be stuck there, and the +//! renderer's crash avoidance reads the GPU list through the same lane; +//! - then the same wgpu calls the window's thread makes first are made once +//! on a thread of their own ([`Call::GpuProbe`]), under its limit. +//! +//! Past that, the window's thread can still stall in calls that cannot be +//! moved off it (its own device and swap chain), so [`watch_first_paint`] +//! watches from another thread until its first frame is presented. +//! +//! Whenever the window waits on the driver, a notice drawn by Windows +//! without the graphics card says why ([`Prompt`]). It stays until the +//! first frame. OK hides it for a while; Cancel quits. // Only Windows waits; the logic is tested everywhere. #![cfg_attr(not(windows), allow(dead_code))] +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use std::time::{Duration, Instant}; +use parking_lot::Mutex; use tracing::{info, warn}; use crate::stall::{Call, Lanes, Limits}; @@ -20,36 +31,131 @@ use crate::stall::{Call, Lanes, Limits}; /// How often a waiting launch looks whether the stuck call has returned. pub(super) const POLL: Duration = Duration::from_millis(250); +/// How long OK hides the notice while the wait goes on. +pub(super) const REMIND_AFTER: Duration = Duration::from_secs(15); + +/// How the user closed the notice. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Answer { + /// OK: keep waiting. + Wait, + /// Cancel: quit EternalMonitor. + Quit, +} + /// What the user sees while the window waits. -pub(super) trait Notice { +pub(super) trait Notice: Send { fn show(&mut self); - /// The user chose to quit instead of waiting. - fn quit(&mut self) -> bool; + /// How the user closed the shown notice, once. + fn answer(&mut self) -> Option; + /// Takes the notice down, if it is up. fn close(&mut self); } -/// Runs a fresh `probe` in `lanes` until one returns within `limits`, with -/// `notice` up from the first one that does not. False when the user quit -/// from the notice. -pub(super) fn wait_for_driver( - lanes: &'static Lanes, - limits: Limits, - mut probe: impl FnMut() -> F, - notice: &mut impl Notice, +/// The notice while startup waits on the driver, shared by the steps on +/// the window's thread and the first-paint watcher: up from the first wait, +/// back [`REMIND_AFTER`] after OK, down for good at the first frame. +#[derive(Clone)] +pub(super) struct Prompt(Arc>); + +struct PromptState { + notice: Box, + up: bool, + raised: bool, + hidden_at: Option, + quit: bool, + done: bool, +} + +impl Prompt { + pub(super) fn new(notice: impl Notice + 'static) -> Self { + Self(Arc::new(Mutex::new(PromptState { + notice: Box::new(notice), + up: false, + raised: false, + hidden_at: None, + quit: false, + done: false, + }))) + } + + /// Show the notice at `now`, unless it is up, OK hid it less than + /// [`REMIND_AFTER`] ago, or the wait is over. + pub(super) fn raise(&self, now: Instant) { + let mut state = self.0.lock(); + state.read_answer(now); + if state.up || state.quit || state.done { + return; + } + if state + .hidden_at + .is_some_and(|at| now.saturating_duration_since(at) < REMIND_AFTER) + { + return; + } + state.notice.show(); + state.up = true; + state.raised = true; + state.hidden_at = None; + } + + /// The user chose Cancel, by `now`. + pub(super) fn quit(&self, now: Instant) -> bool { + let mut state = self.0.lock(); + state.read_answer(now); + state.quit + } + + /// The notice was ever shown. + pub(super) fn raised(&self) -> bool { + self.0.lock().raised + } + + /// The wait is over: take the notice down and keep it down. + pub(super) fn close(&self) { + let mut state = self.0.lock(); + state.done = true; + state.up = false; + state.notice.close(); + } +} + +impl PromptState { + fn read_answer(&mut self, now: Instant) { + if !self.up { + return; + } + match self.notice.answer() { + Some(Answer::Wait) => { + self.up = false; + self.hidden_at = Some(now); + } + Some(Answer::Quit) => { + self.up = false; + self.quit = true; + } + None => {} + } + } +} + +/// Waits until none of `free` is stuck in `lanes` and `attempt` succeeds, +/// with the notice up from the first time either fails. False when the +/// user quit, also by a Cancel that came as the driver answered. +fn wait_until( + lanes: &Lanes, + free: &[Call], + mut attempt: impl FnMut() -> bool, + prompt: &Prompt, poll: Duration, -) -> bool -where - F: FnOnce() + Send + 'static, -{ +) -> bool { let started = Instant::now(); - let mut shown = false; loop { - // While an earlier call is stuck, a new one would only be refused. - if lanes.abandoned(Call::GpuDetect) == 0 - && crate::stall::run_on(lanes, Call::GpuDetect, limits, probe(), || {}).is_ok() - { - if shown { - notice.close(); + if free.iter().all(|&call| lanes.abandoned(call) == 0) && attempt() { + if prompt.quit(Instant::now()) { + break; + } + if prompt.raised() { info!( waited_ms = started.elapsed().as_millis() as u64, "The graphics driver answered; opening the window" @@ -57,20 +163,114 @@ where } return true; } - if !shown { - shown = true; - notice.show(); + if !prompt.raised() { warn!("The graphics driver is not answering; the window opens once it does"); } - if notice.quit() { - info!( - waited_ms = started.elapsed().as_millis() as u64, - "Quit while the window waited for the graphics driver" - ); - return false; + prompt.raise(Instant::now()); + if prompt.quit(Instant::now()) { + break; } std::thread::sleep(poll); } + info!( + waited_ms = started.elapsed().as_millis() as u64, + "Quit while the window waited for the graphics driver" + ); + false +} + +/// The steps before the window's thread makes its first graphics call. +/// First the startup lane frees, so `configure` reads the GPU list crash +/// avoidance keys its history by. Then fresh probes from `probe`, over what +/// `configure` returned, run in [`Call::GpuProbe`] under `limits` until one +/// returns in time. False when the user quit from the notice. +pub(super) fn before_window( + lanes: &'static Lanes, + limits: Limits, + prompt: &Prompt, + poll: Duration, + configure: impl FnOnce() -> S, + probe: impl Fn(&S) -> F, +) -> bool +where + F: FnOnce() + Send + 'static, +{ + if !wait_until(lanes, &[Call::GpuDetect], || true, prompt, poll) { + return false; + } + let spec = configure(); + wait_until( + lanes, + &[Call::GpuDetect, Call::GpuProbe], + || crate::stall::run_on(lanes, Call::GpuProbe, limits, probe(&spec), || {}).is_ok(), + prompt, + poll, + ) +} + +/// Watches the window's first frame from another thread, from now: the +/// notice goes up if `painted` is still false after `limit` (or already +/// was), and down for good once it is true. Cancel runs `quit`, which ends +/// the process: the window's thread may never return. +pub(super) fn watch_first_paint( + prompt: Prompt, + limit: Duration, + poll: Duration, + now: impl Fn() -> Instant + Send + 'static, + painted: impl Fn() -> bool + Send + 'static, + quit: impl FnOnce() + Send + 'static, +) -> FirstPaintWatch { + let stop = Arc::new(AtomicBool::new(false)); + let stopped = Arc::clone(&stop); + let started = now(); + let watch = move || { + let mut warned = false; + loop { + if painted() || stopped.load(Ordering::SeqCst) { + prompt.close(); + return; + } + let at = now(); + if prompt.raised() || at.saturating_duration_since(started) >= limit { + if !warned && !prompt.raised() { + warn!( + limit_ms = limit.as_millis() as u64, + "The window has not drawn its first frame; the graphics driver may not be answering" + ); + } + warned = true; + prompt.raise(at); + } + if prompt.quit(at) { + info!("Quit while the window waited for its first frame"); + quit(); + return; + } + std::thread::sleep(poll); + } + }; + let thread = std::thread::Builder::new() + .name("first-paint-watch".into()) + .spawn(watch) + .map_err(|error| warn!(%error, "Could not watch the window's first frame")) + .ok(); + FirstPaintWatch { stop, thread } +} + +/// A running [`watch_first_paint`]. +pub(super) struct FirstPaintWatch { + stop: Arc, + thread: Option>, +} + +impl FirstPaintWatch { + /// The window ended without a first frame: take the notice down. + pub(super) fn stop(self) { + self.stop.store(true, Ordering::SeqCst); + if let Some(thread) = self.thread { + let _ = thread.join(); + } + } } /// A message box: Windows draws it without the graphics card's driver. @@ -83,7 +283,7 @@ pub(super) struct MessageNotice { #[cfg(windows)] struct Shown { /// The box's thread, to find its window by. - thread: std::sync::Arc, + thread: Arc, answer: std::thread::JoinHandle, } @@ -93,8 +293,7 @@ const NOTICE: &str = "The graphics driver is not responding, so the EternalMonit #[cfg(windows)] impl Notice for MessageNotice { fn show(&mut self) { - use std::sync::atomic::{AtomicU32, Ordering}; - use std::sync::Arc; + use std::sync::atomic::AtomicU32; use windows::core::HSTRING; use windows::Win32::UI::WindowsAndMessaging::{ MessageBoxW, MB_ICONWARNING, MB_OKCANCEL, MB_SETFOREGROUND, @@ -121,18 +320,20 @@ impl Notice for MessageNotice { } } - fn quit(&mut self) -> bool { + fn answer(&mut self) -> Option { use windows::Win32::UI::WindowsAndMessaging::IDCANCEL; - // OK, or a box that could not be shown, keeps waiting. - if !self.shown.as_ref().is_some_and(|s| s.answer.is_finished()) { - return false; + if !self.shown.as_ref()?.answer.is_finished() { + return None; } - let shown = self.shown.take().expect("checked above"); - shown.answer.join().is_ok_and(|answer| answer == IDCANCEL) + let shown = self.shown.take()?; + // A box that could not be shown keeps waiting, as OK does. + Some(match shown.answer.join() { + Ok(answer) if answer == IDCANCEL => Answer::Quit, + _ => Answer::Wait, + }) } fn close(&mut self) { - use std::sync::atomic::Ordering; use windows::Win32::Foundation::{BOOL, HWND, LPARAM, WPARAM}; use windows::Win32::UI::WindowsAndMessaging::{EnumThreadWindows, PostMessageW, WM_CLOSE}; unsafe extern "system" fn close_window(hwnd: HWND, _: LPARAM) -> BOOL { @@ -160,44 +361,72 @@ impl Notice for MessageNotice { #[cfg(test)] mod tests { use super::*; - use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; - use std::sync::{mpsc, Arc}; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering::SeqCst}; + use std::sync::mpsc; const LIMITS: Limits = Limits { warn: Duration::from_millis(100), limit: Duration::from_millis(300), }; - const SLICE: Duration = Duration::from_millis(10); + const SLICE: Duration = Duration::from_millis(1); fn lanes() -> &'static Lanes { Box::leak(Box::new(Lanes::new())) } - /// When the notice was shown and closed, from the start of the test. - #[derive(Default)] - struct Recorded { - started: Option, - shown: Option, - closed: Option, - quit_after_shown: bool, - } + /// What happened, in order. + #[derive(Clone, Default)] + struct Events(Arc>>); - impl Notice for Recorded { - fn show(&mut self) { - self.shown = Some(self.started.unwrap().elapsed()); + impl Events { + fn push(&self, event: &'static str) { + self.0.lock().push(event); } - fn quit(&mut self) -> bool { - self.quit_after_shown + fn all(&self) -> Vec<&'static str> { + self.0.lock().clone() } - fn close(&mut self) { - self.closed = Some(self.started.unwrap().elapsed()); + fn has(&self, event: &str) -> bool { + self.0.lock().contains(&event) + } + /// Backstop only; each wait ends as soon as the event happens. + fn wait_for(&self, event: &str) { + let deadline = Instant::now() + Duration::from_secs(60); + while !self.has(event) { + assert!(Instant::now() < deadline, "no {event}: {:?}", self.all()); + std::thread::sleep(Duration::from_millis(1)); + } } } - fn notice() -> Recorded { - Recorded { - started: Some(Instant::now()), - ..Default::default() + /// A notice the test answers by setting `answer` while it is up. + #[derive(Clone, Default)] + struct Fake { + events: Events, + answer: Arc>>, + up: Arc, + } + + impl Notice for Fake { + fn show(&mut self) { + self.up.store(true, SeqCst); + self.events.push("shown"); + } + fn answer(&mut self) -> Option { + if !self.up.load(SeqCst) { + return None; + } + let answer = self.answer.lock().take()?; + self.up.store(false, SeqCst); + self.events.push(match answer { + Answer::Wait => "ok", + Answer::Quit => "cancel", + }); + Some(answer) + } + fn close(&mut self) { + if self.up.swap(false, SeqCst) { + self.events.push("closed"); + } } } @@ -218,116 +447,259 @@ mod tests { release } + /// [`before_window`] on its own thread, with `configure` and each probe + /// recorded. + fn start_before_window( + lanes: &'static Lanes, + fake: &Fake, + probe: impl Fn() -> Box + Send + 'static, + ) -> std::thread::JoinHandle { + let prompt = Prompt::new(fake.clone()); + let events = fake.events.clone(); + std::thread::spawn(move || { + let configured = events.clone(); + before_window( + lanes, + LIMITS, + &prompt, + SLICE, + move || { + // Crash avoidance reads the GPU list only once the + // startup lane is free. + assert_eq!(lanes.abandoned(Call::GpuDetect), 0); + configured.push("configure"); + }, + |_| probe(), + ) + }) + } + #[test] fn a_driver_that_answers_opens_the_window_without_a_notice() { - let mut notice = notice(); - let probes = Arc::new(AtomicUsize::new(0)); - let count = Arc::clone(&probes); - let opened = wait_for_driver( - lanes(), - LIMITS, - || { - let count = Arc::clone(&count); - move || { - count.fetch_add(1, SeqCst); - } - }, - &mut notice, - SLICE, - ); - assert!(opened); - assert_eq!(probes.load(SeqCst), 1); - assert!(notice.shown.is_none() && notice.closed.is_none()); + let fake = Fake::default(); + let events = fake.events.clone(); + let opened = start_before_window(lanes(), &fake, move || { + let events = events.clone(); + Box::new(move || events.push("probe")) + }); + assert!(opened.join().unwrap()); + assert_eq!(fake.events.all(), ["configure", "probe"]); } #[test] - fn while_detection_is_stuck_the_notice_shows_at_once_and_the_window_follows_it() { + fn while_detection_is_stuck_nothing_reads_the_gpus_and_the_notice_is_up() { let lanes = lanes(); let release = stuck_detection(lanes); - let probes = Arc::new(AtomicUsize::new(0)); - let count = Arc::clone(&probes); - let waiter = std::thread::spawn(move || { - let mut notice = notice(); - let opened = wait_for_driver( - lanes, - LIMITS, - || { - let count = Arc::clone(&count); - move || { - count.fetch_add(1, SeqCst); - } - }, - &mut notice, - SLICE, - ); - (opened, notice) + let fake = Fake::default(); + let events = fake.events.clone(); + let opened = start_before_window(lanes, &fake, move || { + let events = events.clone(); + Box::new(move || events.push("probe")) }); - // No probe joins the stuck driver. - std::thread::sleep(LIMITS.limit * 2); - assert_eq!(probes.load(SeqCst), 0); + fake.events.wait_for("shown"); + // Nothing went near the stuck driver. + assert_eq!(fake.events.all(), ["shown"]); release.send(()).unwrap(); - let (opened, notice) = waiter.join().unwrap(); - assert!(opened); - // Shown without waiting out another limit. - let shown = notice.shown.expect("no notice"); - assert!(shown < LIMITS.limit, "{shown:?}"); - // Closed once the driver answered, and the window's probe ran then. - assert!(notice.closed.expect("not closed") >= LIMITS.limit); - assert_eq!(probes.load(SeqCst), 1); + assert!(opened.join().unwrap()); + // The notice stays up for the first-paint watcher to take down. + assert_eq!(fake.events.all(), ["shown", "configure", "probe"]); } #[test] - fn a_probe_that_hangs_shows_the_notice_at_its_limit() { + fn a_probe_that_hangs_raises_the_notice_and_the_next_one_opens_the_window() { let lanes = lanes(); + let fake = Fake::default(); + let (started, probe_started) = mpsc::channel::<()>(); let (release, held) = mpsc::channel::<()>(); - let held = Arc::new(parking_lot::Mutex::new(Some(held))); - let waiter = std::thread::spawn(move || { - let mut notice = notice(); - let opened = wait_for_driver( - lanes, - LIMITS, - || { - // The first probe blocks in the driver; the next answers. - let held = held.lock().take(); - move || { - if let Some(held) = held { - let _ = held.recv(); - } - } - }, - &mut notice, - SLICE, - ); - (opened, notice) + let held = Arc::new(Mutex::new(Some(held))); + let events = fake.events.clone(); + let opened = start_before_window(lanes, &fake, move || { + let events = events.clone(); + let started = started.clone(); + // The first probe blocks in the driver; the next answers. + let held = held.lock().take(); + Box::new(move || { + events.push("probe"); + let _ = started.send(()); + if let Some(held) = held { + let _ = held.recv(); + events.push("probe returned"); + } + }) }); - std::thread::sleep(LIMITS.limit * 3); + probe_started.recv().unwrap(); + fake.events.wait_for("shown"); + assert!(!fake.events.has("probe returned")); release.send(()).unwrap(); - let (opened, notice) = waiter.join().unwrap(); - assert!(opened); - let shown = notice.shown.expect("no notice"); - assert!( - shown >= LIMITS.limit && shown < LIMITS.limit + Duration::from_secs(1), - "{shown:?}" - ); - assert!(notice.closed.expect("not closed") >= LIMITS.limit * 2); + assert!(opened.join().unwrap()); + let events = fake.events.all(); + assert_eq!(events[..3], ["configure", "probe", "shown"], "{events:?}"); + // The second probe ran once the first had returned and freed its lane. + assert_eq!(events.iter().filter(|e| **e == "probe").count(), 2); + let returned = events.iter().position(|e| *e == "probe returned").unwrap(); + assert_eq!(events[returned + 1], "probe", "{events:?}"); } #[test] - fn quitting_from_the_notice_opens_no_window() { + fn cancel_opens_no_window() { let lanes = lanes(); let _release = stuck_detection(lanes); - let mut notice = Recorded { - quit_after_shown: true, - ..notice() - }; - assert!(!wait_for_driver( - lanes, - LIMITS, - || || {}, - &mut notice, - SLICE - )); - assert!(notice.shown.is_some()); - assert!(notice.closed.is_none()); + let fake = Fake::default(); + *fake.answer.lock() = Some(Answer::Quit); + let opened = start_before_window(lanes, &fake, || Box::new(|| {})); + assert!(!opened.join().unwrap()); + assert_eq!(fake.events.all(), ["shown", "cancel"]); + } + + #[test] + fn a_cancel_as_the_driver_answers_still_quits() { + let lanes = lanes(); + let release = stuck_detection(lanes); + let fake = Fake::default(); + let answer = Arc::clone(&fake.answer); + // The user clicks Cancel while the probe runs, the moment before it + // returns in time. + let opened = start_before_window(lanes, &fake, move || { + let answer = Arc::clone(&answer); + Box::new(move || *answer.lock() = Some(Answer::Quit)) + }); + fake.events.wait_for("shown"); + release.send(()).unwrap(); + assert!(!opened.join().unwrap()); + assert_eq!(fake.events.all(), ["shown", "configure", "cancel"]); + } + + #[test] + fn ok_hides_the_notice_until_the_reminder_and_closing_ends_it() { + let fake = Fake::default(); + let prompt = Prompt::new(fake.clone()); + let start = Instant::now(); + prompt.raise(start); + prompt.raise(start + Duration::from_secs(1)); + assert_eq!(fake.events.all(), ["shown"]); + *fake.answer.lock() = Some(Answer::Wait); + let ok_at = start + Duration::from_secs(2); + assert!(!prompt.quit(ok_at)); + prompt.raise(ok_at + REMIND_AFTER - Duration::from_millis(1)); + assert_eq!(fake.events.all(), ["shown", "ok"]); + prompt.raise(ok_at + REMIND_AFTER); + assert_eq!(fake.events.all(), ["shown", "ok", "shown"]); + // Cancel on the second showing quits. + *fake.answer.lock() = Some(Answer::Quit); + assert!(prompt.quit(ok_at + REMIND_AFTER)); + // Once the wait is over, nothing shows it again. + let fake = Fake::default(); + let prompt = Prompt::new(fake.clone()); + prompt.raise(start); + prompt.close(); + prompt.raise(start + REMIND_AFTER * 2); + assert_eq!(fake.events.all(), ["shown", "closed"]); + } + + /// A clock the test moves on by hand. + fn manual_clock() -> (Arc, impl Fn() -> Instant + Send + 'static) { + let start = Instant::now(); + let ms = Arc::new(AtomicU64::new(0)); + let read = Arc::clone(&ms); + (ms, move || start + Duration::from_millis(read.load(SeqCst))) + } + + #[test] + fn a_window_that_paints_in_time_shows_no_notice() { + let fake = Fake::default(); + let (_clock, now) = manual_clock(); + let painted = Arc::new(AtomicBool::new(true)); + let seen = Arc::clone(&painted); + let watch = watch_first_paint( + Prompt::new(fake.clone()), + LIMITS.limit, + SLICE, + now, + move || seen.load(SeqCst), + || panic!("quit"), + ); + watch.thread.unwrap().join().unwrap(); + assert!(fake.events.all().is_empty()); + } + + #[test] + fn a_window_stuck_before_its_first_frame_gets_the_notice_until_it_paints() { + let fake = Fake::default(); + let (clock, now) = manual_clock(); + let painted = Arc::new(AtomicBool::new(false)); + let seen = Arc::clone(&painted); + let watch = watch_first_paint( + Prompt::new(fake.clone()), + LIMITS.limit, + SLICE, + now, + move || seen.load(SeqCst), + || panic!("quit"), + ); + // Just under the limit: nothing yet. + clock.store(LIMITS.limit.as_millis() as u64 - 1, SeqCst); + std::thread::sleep(Duration::from_millis(20)); + assert!(fake.events.all().is_empty()); + clock.store(LIMITS.limit.as_millis() as u64, SeqCst); + fake.events.wait_for("shown"); + // OK, then the reminder. + *fake.answer.lock() = Some(Answer::Wait); + fake.events.wait_for("ok"); + clock.fetch_add(REMIND_AFTER.as_millis() as u64, SeqCst); + let deadline = Instant::now() + Duration::from_secs(60); + while fake.events.all().len() < 3 { + assert!(Instant::now() < deadline, "{:?}", fake.events.all()); + std::thread::sleep(Duration::from_millis(1)); + } + // The first frame takes it down. + painted.store(true, SeqCst); + watch.thread.unwrap().join().unwrap(); + assert_eq!(fake.events.all(), ["shown", "ok", "shown", "closed"]); + } + + #[test] + fn cancel_while_the_first_frame_is_stuck_quits_from_the_watcher() { + let fake = Fake::default(); + let (clock, now) = manual_clock(); + clock.store(LIMITS.limit.as_millis() as u64, SeqCst); + let quits = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&quits); + *fake.answer.lock() = Some(Answer::Quit); + let watch = watch_first_paint( + Prompt::new(fake.clone()), + Duration::ZERO, + SLICE, + now, + || false, + move || { + count.fetch_add(1, SeqCst); + }, + ); + watch.thread.unwrap().join().unwrap(); + assert_eq!(quits.load(SeqCst), 1); + assert_eq!(fake.events.all(), ["shown", "cancel"]); + } + + #[test] + fn a_notice_raised_before_the_window_stays_until_its_first_frame() { + let fake = Fake::default(); + let prompt = Prompt::new(fake.clone()); + prompt.raise(Instant::now()); + let (_clock, now) = manual_clock(); + let painted = Arc::new(AtomicBool::new(false)); + let seen = Arc::clone(&painted); + let watch = watch_first_paint( + prompt, + REMIND_AFTER * 4, + SLICE, + now, + move || seen.load(SeqCst), + || panic!("quit"), + ); + std::thread::sleep(Duration::from_millis(20)); + assert_eq!(fake.events.all(), ["shown"]); + painted.store(true, SeqCst); + watch.thread.unwrap().join().unwrap(); + assert_eq!(fake.events.all(), ["shown", "closed"]); } } diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index f9b70d30..cd2f6fca 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -1775,7 +1775,13 @@ pub fn shown_address(ip: std::net::Ipv4Addr, listen_port: u16) -> String { } /// Launch the GUI window. Blocks the calling thread. -pub fn run_gui(control: GuiControl) -> eframe::Result<()> { +/// Runs the window until it closes. `quit_while_stuck` is the app's quit +/// path, for a Cancel in the graphics driver notice while the window's +/// thread is stuck before its first frame; it ends the process. +pub fn run_gui( + control: GuiControl, + quit_while_stuck: Box, +) -> eframe::Result<()> { let icon = eframe::icon_data::from_png_bytes(include_bytes!("../../assets/icon_256.png")) .expect("embedded icon PNG is valid"); @@ -1787,19 +1793,42 @@ pub fn run_gui(control: GuiControl) -> eframe::Result<()> { .with_icon(std::sync::Arc::new(icon)), ..Default::default() }; - renderer::configure(&mut options.wgpu_options); // The window's thread makes its first graphics call only once the - // driver has answered the same calls on another thread. + // driver has answered the same calls on another thread, and a stall + // after that is watched until the first frame. #[cfg(windows)] - if !driver_wait::wait_for_driver( - crate::stall::lanes(), - crate::stall::Call::GpuDetect.limits(), - || renderer::adapter_probe(&options.wgpu_options), - &mut driver_wait::MessageNotice::default(), - driver_wait::POLL, - ) { - renderer::window_closed(); - return Ok(()); + let watch = { + let lanes = crate::stall::lanes(); + let limits = crate::stall::Call::GpuProbe.limits(); + let prompt = driver_wait::Prompt::new(driver_wait::MessageNotice::default()); + let opened = driver_wait::before_window( + lanes, + limits, + &prompt, + driver_wait::POLL, + || { + renderer::configure(&mut options.wgpu_options); + renderer::probe_spec(&options.wgpu_options) + }, + |spec| renderer::adapter_probe(spec.clone()), + ); + if !opened { + renderer::window_closed(); + return Ok(()); + } + driver_wait::watch_first_paint( + prompt, + limits.limit, + driver_wait::POLL, + std::time::Instant::now, + renderer::first_frame_painted, + quit_while_stuck, + ) + }; + #[cfg(not(windows))] + { + renderer::configure(&mut options.wgpu_options); + drop(quit_while_stuck); } let result = eframe::run_native( @@ -1807,6 +1836,8 @@ pub fn run_gui(control: GuiControl) -> eframe::Result<()> { options, Box::new(|cc| Ok(Box::new(AnalyzerApp::new(cc, control)))), ); + #[cfg(windows)] + watch.stop(); if result.is_ok() { renderer::window_closed(); } diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 8326e3e7..260a56b0 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -46,7 +46,7 @@ use std::fs; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; @@ -119,6 +119,9 @@ static PRESENT: Mutex = Mutex::new(PresentTracker::new()); /// Frames whose surface texture could not be acquired, so nothing was presented. static FAILED_ACQUIRES: AtomicU64 = AtomicU64::new(0); static STATUS: Mutex> = Mutex::new(None); +/// The window's first frame was presented: its thread is past every +/// graphics call of startup. +static FIRST_PAINT: AtomicBool = AtomicBool::new(false); /// How this launch draws its window, for Settings > Troubleshooting. #[derive(Clone, Debug, Default, PartialEq)] @@ -305,16 +308,22 @@ pub(super) fn configure(config: &mut WgpuConfiguration) { }); } -/// The wgpu calls the window's thread makes first, for another thread to -/// make before it: an instance with this launch's backends lists its -/// adapters (DX12 creates a device on each to describe it). Built on the -/// window's thread, run on another by -/// [`super::driver_wait::wait_for_driver`]. While it lists, the marker -/// names that thread, so a driver crash there is still the window's. +/// The instance the window's renderer will create, for [`adapter_probe`]. +/// None when it is given an existing device. +#[derive(Clone)] +pub(super) struct ProbeSpec( + Option<( + Backends, + wgpu::InstanceFlags, + wgpu::BackendOptions, + wgpu::MemoryBudgetThresholds, + )>, +); + +/// What `config`, as [`configure`] left it, will create. #[cfg_attr(not(windows), allow(dead_code))] -pub(super) fn adapter_probe(config: &WgpuConfiguration) -> impl FnOnce() + Send + 'static { - let window = current_thread(); - let descriptor = match &config.wgpu_setup { +pub(super) fn probe_spec(config: &WgpuConfiguration) -> ProbeSpec { + ProbeSpec(match &config.wgpu_setup { WgpuSetup::CreateNew(setup) => Some(( setup.instance_descriptor.backends, setup.instance_descriptor.flags, @@ -322,9 +331,20 @@ pub(super) fn adapter_probe(config: &WgpuConfiguration) -> impl FnOnce() + Send setup.instance_descriptor.memory_budget_thresholds, )), WgpuSetup::Existing(_) => None, - }; + }) +} + +/// The wgpu calls the window's thread makes first, for another thread to +/// make before it: an instance with this launch's backends lists its +/// adapters (DX12 creates a device on each to describe it). Built on the +/// window's thread, run on another by [`super::driver_wait::before_window`]. +/// While it lists, the marker names that thread, so a driver crash there is +/// still the window's. +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn adapter_probe(spec: ProbeSpec) -> impl FnOnce() + Send + 'static { + let window = current_thread(); move || { - let Some((backends, flags, backend_options, memory_budget_thresholds)) = descriptor else { + let Some((backends, flags, backend_options, memory_budget_thresholds)) = spec.0 else { return; }; mark_thread(current_thread()); @@ -493,6 +513,10 @@ fn save_marker(directory: &Path, owner: ProcessId, marker: &Marker) -> bool { /// Called at the start of each UI pass; clears this launch's marker once the /// window has kept drawing without a surface error. pub(super) fn frame_started(ctx: &egui::Context) { + // A pass after the first: the first frame was presented. + if ctx.cumulative_frame_nr() >= 1 { + FIRST_PAINT.store(true, Ordering::SeqCst); + } crate::startup::simulate_surface_error(&FAILED_ACQUIRES); let visible = ctx.input(|input| input.viewport().minimized != Some(true)); let Ok(mut tracker) = PRESENT.lock() else { @@ -515,6 +539,12 @@ pub(super) fn frame_started(ctx: &egui::Context) { } } +/// The window's first frame was presented. +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn first_frame_painted() -> bool { + FIRST_PAINT.load(Ordering::SeqCst) +} + /// The window closed normally. A launch that quits before its window is /// confirmed did not fail, so its marker goes without counting either way. pub(super) fn window_closed() { @@ -876,11 +906,35 @@ fn system_gpus() -> String { } /// Vendor, device and driver version of each DXGI adapter. Creates no device. -/// Runs in the startup graphics lane: empty if it does not return within -/// the limit, or while an earlier call there is still stuck. +/// Runs under a limit on a thread of its own: empty if it does not return +/// in time, or while an earlier call of its lane is still stuck. While it +/// lists, the marker names that thread, so a driver crash there is still +/// the window's. #[cfg(windows)] fn dxgi_adapters() -> Vec<(u32, u32, String)> { - crate::stall::run(crate::stall::Call::GpuDetect, list_dxgi_adapters, || {}).unwrap_or_default() + let window = current_thread(); + let listed = crate::stall::run( + adapters_lane(first_frame_painted()), + || { + mark_thread(current_thread()); + list_dxgi_adapters() + }, + || {}, + ); + mark_thread(window); + listed.unwrap_or_default() +} + +/// The lane of the window's DXGI adapter list: startup's until the first +/// frame, then the one the second screen's graphics reads use, so a later +/// renderer (after a lost device) neither waits behind nor holds startup's. +#[cfg_attr(not(windows), allow(dead_code))] +fn adapters_lane(painted: bool) -> crate::stall::Call { + if painted { + crate::stall::Call::GpuFacts + } else { + crate::stall::Call::GpuDetect + } } #[cfg(windows)] @@ -1686,6 +1740,16 @@ mod tests { fs::remove_dir_all(directory).unwrap(); } + #[test] + fn after_the_first_frame_the_adapter_list_leaves_the_startup_lane() { + use crate::stall::Call; + // Before it, a stuck startup call refuses it at once. + assert_eq!(adapters_lane(false), Call::GpuDetect); + // A renderer started later (after a lost device) neither waits behind + // nor holds startup's lane, nor logs its stalls as startup's. + assert_eq!(adapters_lane(true), Call::GpuFacts); + } + #[test] fn the_windows_instance_leaves_out_opengl_unless_wgpu_backend_asks() { let full = instance_backends(None, Plan::Full); diff --git a/host/src/main.rs b/host/src/main.rs index 1b412536..7090d77e 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -274,6 +274,70 @@ fn main() -> Result<(), Box> { let mdns = discovery::advertise_service(listen_port); + // The quit path, once: after the window closes (or the headless loop + // ends), or from the first-paint watcher when the user quits while the + // window's thread is stuck in the graphics driver and may never return. + let quit_control = gui_control.clone(); + let quit_once: QuitOnce = std::sync::Arc::new(parking_lot::Mutex::new(Some(Box::new( + move |window_error: Option| { + let stop_streaming = || { + eternal_host::diagnostics::log_session_end_at_quit(&quit_control.shared); + quit_control.shared.stop(); + quit_control.request_shutdown(); + + // Never leave the virtual display attached after we exit — it should only exist while + // EternalMonitor is actively using it. No Enable starts once the second screen worker + // is shut down, so this Disable is the last change. If it may still be on, the failure + // reporter turns it off after this process exits. + eternal_host::capture::second_screen::shut_down(&quit_control.shared); + eternal_host::startup::display_off_at_quit(vdd::disable); + }; + // A window error is saved first and shown after the cleanup, as a panic is. + match window_error { + Some(detail) => eternal_host::startup::report_after(&detail, stop_streaming), + None => stop_streaming(), + } + + // Give the supervisor a bounded amount of time to shut down cleanly, + // exiting the moment it finishes instead of always burning the full + // window. If it doesn't finish (blocked on DXGI acquire, etc.), + // force-exit so the process never lingers as an invisible zombie. + let (supervisor_done_tx, supervisor_done_rx) = mpsc::channel(); + std::thread::spawn(move || { + if let Err(error) = supervisor_thread.join() { + error!(error = ?error, "Supervisor thread panicked"); + } + let _ = supervisor_done_tx.send(()); + }); + + // Say goodbye on mDNS while the supervisor winds down, so iPads drop the + // host from their scan list promptly (process::exit skips destructors — + // without this no goodbye is ever sent). + if let Some(mdns) = mdns { + discovery::say_goodbye(mdns); + } + + if supervisor_done_rx + .recv_timeout(std::time::Duration::from_secs(3)) + .is_err() + { + tracing::warn!("Supervisor did not shut down within 3s — forcing exit"); + } + + // The window is gone and the display is off, so a slow disk keeps no one + // waiting on screen: give the second screen record up to 30 s to land. + eternal_host::capture::second_screen::finish_saving(&quit_control.shared); + + eternal_host::startup::simulate_slow_exit(); + info!("EternalMonitor shutting down"); + // process::exit skips destructors. Flush the log writers and the upload + // outbox first, so a normal quit is told apart from a crash. + logging::flush_before_exit(_session_guard, _uploads, _stdout_guard); + eternal_host::startup::finish(); + std::process::exit(0); + }, + )))); + // Headless mode for automation (the iOS E2E harness runs the host without // a window): stream until SIGTERM/SIGINT, then fall through to the same // clean teardown as a GUI quit — mDNS goodbye, VDD disable, bounded @@ -319,64 +383,29 @@ fn main() -> Result<(), Box> { } } info!("Termination signal received — shutting down cleanly"); - } else if let Err(e) = gui::run_gui(gui_control.clone()) { + } else if let Err(e) = gui::run_gui(gui_control.clone(), { + let quit_once = std::sync::Arc::clone(&quit_once); + Box::new(move || { + let quit = quit_once.lock().take(); + if let Some(quit) = quit { + quit(None); + } + }) + }) { error!(error = %e, "GUI exited with error"); window_error = Some(format!("Window error: {e}")); } - let stop_streaming = || { - eternal_host::diagnostics::log_session_end_at_quit(&gui_control.shared); - gui_control.shared.stop(); - gui_control.request_shutdown(); - - // Never leave the virtual display attached after we exit — it should only exist while - // EternalMonitor is actively using it. No Enable starts once the second screen worker - // is shut down, so this Disable is the last change. If it may still be on, the failure - // reporter turns it off after this process exits. - eternal_host::capture::second_screen::shut_down(&gui_control.shared); - eternal_host::startup::display_off_at_quit(vdd::disable); - }; - // A window error is saved first and shown after the cleanup, as a panic is. - match window_error { - Some(detail) => eternal_host::startup::report_after(&detail, stop_streaming), - None => stop_streaming(), - } - - // Give the supervisor a bounded amount of time to shut down cleanly, - // exiting the moment it finishes instead of always burning the full - // window. If it doesn't finish (blocked on DXGI acquire, etc.), - // force-exit so the process never lingers as an invisible zombie. - let (supervisor_done_tx, supervisor_done_rx) = mpsc::channel(); - std::thread::spawn(move || { - if let Err(error) = supervisor_thread.join() { - error!(error = ?error, "Supervisor thread panicked"); - } - let _ = supervisor_done_tx.send(()); - }); - - // Say goodbye on mDNS while the supervisor winds down, so iPads drop the - // host from their scan list promptly (process::exit skips destructors — - // without this no goodbye is ever sent). - if let Some(mdns) = mdns { - discovery::say_goodbye(mdns); + // The quit path ends the process. Without it, the first-paint watcher + // took it and is ending the process. + let quit = quit_once.lock().take(); + if let Some(quit) = quit { + quit(window_error); } - - if supervisor_done_rx - .recv_timeout(std::time::Duration::from_secs(3)) - .is_err() - { - tracing::warn!("Supervisor did not shut down within 3s — forcing exit"); + loop { + std::thread::park(); } - - // The window is gone and the display is off, so a slow disk keeps no one - // waiting on screen: give the second screen record up to 30 s to land. - eternal_host::capture::second_screen::finish_saving(&gui_control.shared); - - eternal_host::startup::simulate_slow_exit(); - info!("EternalMonitor shutting down"); - // process::exit skips destructors. Flush the log writers and the upload - // outbox first, so a normal quit is told apart from a crash. - logging::flush_before_exit(_session_guard, _uploads, _stdout_guard); - eternal_host::startup::finish(); - std::process::exit(0); } + +/// The quit path; taken by whichever ends the process first. +type QuitOnce = std::sync::Arc) + Send>>>>; diff --git a/host/src/stall.rs b/host/src/stall.rs index 72f8ac61..76623915 100644 --- a/host/src/stall.rs +++ b/host/src/stall.rs @@ -41,14 +41,18 @@ pub enum Call { /// screen's decisions (`capture::second_screen::facts`). GpuFacts, /// The graphics calls before the window's first paint: finding the - /// graphics cards (`gpu::GpuInfo::detect`), listing the screens, the - /// window renderer's adapter list and its wgpu probe. They share one - /// lane: they wait on the same driver, so after one hangs the others - /// are refused at once instead of each waiting out its own limit. + /// graphics cards (`gpu::GpuInfo::detect`), listing the screens and the + /// window renderer's adapter list. They share one lane: they wait on the + /// same driver, so after one hangs the others are refused at once + /// instead of each waiting out its own limit. GpuDetect, + /// The window's adapters listed through wgpu on another thread before + /// the window's thread makes its first graphics call + /// (`gui::renderer::adapter_probe`). + GpuProbe, } -const CALLS: usize = 7; +const CALLS: usize = 8; /// When a call is slow enough to log, and when it is given up on. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -77,6 +81,7 @@ impl Call { Self::VddDisable => "vdd_disable", Self::GpuFacts => "gpu_facts", Self::GpuDetect => "gpu_detect", + Self::GpuProbe => "gpu_probe", } } @@ -112,6 +117,10 @@ impl Call { // GpuFacts and keeps its limits. One AMD laptop's dxgkrnl // deadlock held it 62 s, and the window with it. Self::GpuDetect => Limits::ms(1_000, 4_000), + // Not timed in the field yet: on a test PC (RTX 5080) 340-440 ms + // in debug builds, which run wgpu's validation. It makes the + // window's own first calls, and keeps detection's limits. + Self::GpuProbe => Limits::ms(1_000, 4_000), } } } @@ -795,6 +804,7 @@ mod tests { Call::VddDisable, Call::GpuFacts, Call::GpuDetect, + Call::GpuProbe, ] .map(Call::name) .into(); @@ -813,6 +823,7 @@ mod tests { Call::VddEnable, Call::GpuFacts, Call::GpuDetect, + Call::GpuProbe, ] { let limits = call.limits(); assert!(limits.limit >= Duration::from_secs(3), "{call:?}"); From 4d8805fc156e94646ccd1e2731286f75c2c914bd Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:40:20 -0500 Subject: [PATCH 111/226] Back off and step the bitrate down when the system has no buffer space to send Windows refuses a UDP send with WSAENOBUFS (10055), and macOS and Linux with ENOBUFS, when a queue below the socket is full. Each refused send logged its own line, and the bitrate fell only once the iPad reported the losses. A refused video datagram now waits a few milliseconds, at most once per frame, while repairs and audio are still served, and is sent once more. A refusal after that gives up the rest of the frame, which repair recovers. The bitrate steps down one rung at once, once per episode. A refused control reply is retried once. The log says one line when an episode starts and one with its count and duration when it ends. The port wait now logs at info when the holder is this copy's previous stream, whose socket is still closing. --- host/src/startup/port.rs | 28 +++ host/src/transport/abr.rs | 49 ++++ host/src/transport/congestion.rs | 200 +++++++++++++++++ host/src/transport/link.rs | 4 + host/src/transport/mod.rs | 148 ++++++++++--- host/src/transport/pacer.rs | 11 +- host/src/transport/usb.rs | 14 +- host/tests/send_congestion.rs | 368 +++++++++++++++++++++++++++++++ 8 files changed, 789 insertions(+), 33 deletions(-) create mode 100644 host/src/transport/congestion.rs create mode 100644 host/tests/send_congestion.rs diff --git a/host/src/startup/port.rs b/host/src/startup/port.rs index d58562e3..2f29acc9 100644 --- a/host/src/startup/port.rs +++ b/host/src/startup/port.rs @@ -195,6 +195,11 @@ fn note(waiting: &mut Waiting, held: Option) { port, code, "Windows is blocking the stream port; checking again every 2 s" ), + // Its socket is still closing after a restart: no fault. + HeldBy::ThisCopy => info!( + port, + "This copy's previous stream still has the stream port; the stream starts once it lets go" + ), by => warn!( port, holder = category(by), @@ -826,6 +831,29 @@ mod tests { assert!(log.contains("Windows is blocking"), "{log}"); } + /// This copy's own previous stream, whose socket is still closing, is + /// no fault: said at info. Any other holder is still a warning. + #[test] + fn this_copys_previous_stream_is_said_at_info() { + let _serial = HELD_TESTS.lock().unwrap_or_else(|e| e.into_inner()); + let log = logged(|| { + check_with(9876, Bind::InUse, || Some(HeldBy::ThisCopy), || None); + check_with(9876, Bind::Free, || None, || None); + check_with(9876, Bind::InUse, || Some(HeldBy::OtherCopy), || None); + forget(); + }); + let lines: Vec<&str> = log.lines().collect(); + assert_eq!(lines.len(), 3, "{log}"); + assert!( + lines[0].contains(" INFO ") && lines[0].contains("previous stream still has"), + "{log}" + ); + assert!( + lines[2].contains(" WARN ") && lines[2].contains("Another program"), + "{log}" + ); + } + fn image(path: &'static str) -> impl FnOnce(u32) -> Option<(String, Option)> { started(path, 1) } diff --git a/host/src/transport/abr.rs b/host/src/transport/abr.rs index 5735cb48..85188523 100644 --- a/host/src/transport/abr.rs +++ b/host/src/transport/abr.rs @@ -246,6 +246,36 @@ impl AbrController { unchanged } + + /// The system had no buffer space for a datagram: one rung down at + /// once, without waiting for the iPad to report the loss. It counts as a + /// downgrade, so the loss reports that follow wait out the cooldown. + pub fn on_send_refused(&mut self, now: Instant) -> AbrDecision { + let unchanged = AbrDecision { + target_bps: self.current_bps, + changed: false, + }; + if !self.enabled { + return unchanged; + } + self.clean_since = None; + let target = step(self.current_bps, -1, self.ceiling_bps); + if target == self.current_bps { + return unchanged; + } + info!( + from = self.current_bps, + to = target, + reason = "no_buffer_space", + "ABR stepping bitrate down" + ); + self.current_bps = target; + self.last_change = Some((now, true)); + AbrDecision { + target_bps: target, + changed: true, + } + } } /// Nearest ladder rung at or below `bps`, capped by `ceiling`. @@ -580,4 +610,23 @@ mod tests { assert!(stepped_up); assert_eq!(abr.current_bps(), 15_000_000); } + + /// A refused send steps one rung down at once, and the loss reports of + /// the datagrams it lost wait out the cooldown instead of stepping again. + #[test] + fn a_refused_send_steps_down_at_once_and_reports_wait_the_cooldown() { + let mut abr = AbrController::new(20_000_000, true); + let t0 = Instant::now(); + let d = abr.on_send_refused(t0); + assert_eq!((d.target_bps, d.changed), (12_000_000, true)); + let d = abr.on_report(&report(950, 50, 0), None, t0 + Duration::from_secs(1)); + assert!(!d.changed, "the same loss, inside the cooldown"); + assert_eq!(abr.current_bps(), 12_000_000); + + let mut floor = AbrController::new(4_000_000, true); + assert!(!floor.on_send_refused(t0).changed); + let mut disabled = AbrController::new(20_000_000, false); + assert!(!disabled.on_send_refused(t0).changed); + assert_eq!(disabled.current_bps(), 20_000_000); + } } diff --git a/host/src/transport/congestion.rs b/host/src/transport/congestion.rs new file mode 100644 index 00000000..72bcbdc3 --- /dev/null +++ b/host/src/transport/congestion.rs @@ -0,0 +1,200 @@ +//! Windows refuses a UDP send with WSAENOBUFS (10055) when it has no buffer +//! space for the datagram: a queue in the network adapter, its driver or a +//! filter driver is full. macOS and Linux say ENOBUFS. The socket's own send +//! buffer (4 MiB) is not the one that ran out. In the field each refused send +//! logged its own line, and the stream lowered its bitrate only after the +//! iPad reported the losses. A refused send now counts as congestion: video +//! backs off for a moment, the bitrate steps down at once, and the log says +//! one line when an episode starts and one when it ends. + +use std::io; +use std::time::{Duration, Instant}; + +use tracing::{info, warn}; + +/// How long a frame's sending pauses after the system refused a datagram, +/// before that datagram is sent once more. A frame pauses once at most. +pub const BACKOFF: Duration = Duration::from_millis(5); + +/// How long a refused control reply waits before its one retry. +pub const REPLY_RETRY: Duration = Duration::from_millis(2); + +/// An episode ends once sends go this long without a refusal. +pub const EPISODE_QUIET: Duration = Duration::from_secs(1); + +#[cfg(windows)] +const NO_BUFFER_SPACE: i32 = 10055; // WSAENOBUFS +#[cfg(not(windows))] +const NO_BUFFER_SPACE: i32 = libc::ENOBUFS; + +/// Whether the system refused a send for lack of buffer space. +pub fn is_no_buffer_space(error: &io::Error) -> bool { + error.raw_os_error() == Some(NO_BUFFER_SPACE) +} + +/// The error a send the system had no buffer space for returns. +pub fn no_buffer_space() -> io::Error { + io::Error::from_raw_os_error(NO_BUFFER_SPACE) +} + +/// Refused sends from the first until sends go [`EPISODE_QUIET`] without one. +struct Episode { + started: Instant, + last: Instant, + refused: u64, +} + +/// Every refused UDP send of one transport, by episode. +#[derive(Default)] +pub struct SendCongestion { + episode: Option, + step_down: bool, +} + +impl SendCongestion { + /// Notes a send refused for lack of buffer space. The first of an + /// episode is logged, and asks for one step down. + pub fn refused(&mut self, error: &io::Error, now: Instant) { + self.settle(now); + match &mut self.episode { + Some(episode) => { + episode.refused += 1; + episode.last = now; + } + None => { + warn!(link = %"wifi", error = %error, "UDP send failed: no buffer space, backing off"); + self.episode = Some(Episode { + started: now, + last: now, + refused: 1, + }); + self.step_down = true; + } + } + } + + /// Whether an episode started since the last call, once per episode. + pub fn take_step_down(&mut self) -> bool { + std::mem::take(&mut self.step_down) + } + + /// Ends an episode with no refusal for [`EPISODE_QUIET`], logging how + /// many sends it refused and how long it lasted. + pub fn settle(&mut self, now: Instant) { + if self + .episode + .as_ref() + .is_some_and(|episode| now.saturating_duration_since(episode.last) >= EPISODE_QUIET) + { + self.end(); + } + } + + fn end(&mut self) { + if let Some(episode) = self.episode.take() { + info!( + link = %"wifi", + refused = episode.refused, + took_ms = episode.last.duration_since(episode.started).as_millis() as u64, + "UDP sends have buffer space again" + ); + } + } +} + +/// A transport that stops during an episode still says how it went. +impl Drop for SendCongestion { + fn drop(&mut self) { + self.end(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn logged(run: impl FnOnce()) -> Vec { + let out = std::sync::Arc::new(parking_lot::Mutex::new(Vec::::new())); + struct Sink(std::sync::Arc>>); + impl io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.lock().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let writer = std::sync::Arc::clone(&out); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_target(false) + .with_writer(move || Sink(std::sync::Arc::clone(&writer))) + .finish(); + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, run); + let text = String::from_utf8(out.lock().clone()).unwrap(); + text.lines().map(|line| line.trim().to_owned()).collect() + } + + #[test] + fn only_no_buffer_space_counts_as_congestion() { + assert!(is_no_buffer_space(&no_buffer_space())); + #[cfg(windows)] + assert!(is_no_buffer_space(&io::Error::from_raw_os_error(10055))); + #[cfg(target_os = "macos")] + assert!(is_no_buffer_space(&io::Error::from_raw_os_error(55))); + #[cfg(target_os = "linux")] + assert!(is_no_buffer_space(&io::Error::from_raw_os_error(105))); + for other in [ + io::Error::from(io::ErrorKind::ConnectionReset), + io::Error::from(io::ErrorKind::WouldBlock), + io::Error::from_raw_os_error(10054), + ] { + assert!(!is_no_buffer_space(&other), "{other}"); + } + } + + /// Hundreds of refused sends close together are one episode: one line + /// when it starts, one step down, and one summary once it has been quiet. + #[test] + fn an_episode_logs_one_line_and_its_end() { + let start = Instant::now(); + let mut congestion = SendCongestion::default(); + let lines = logged(|| { + for n in 0..300u32 { + congestion.refused(&no_buffer_space(), start + Duration::from_millis(n.into())); + congestion.settle(start + Duration::from_millis(n.into())); + } + assert!(congestion.take_step_down()); + assert!(!congestion.take_step_down(), "one step per episode"); + congestion.settle(start + Duration::from_millis(299) + EPISODE_QUIET / 2); + congestion.settle(start + Duration::from_millis(299) + EPISODE_QUIET); + }); + assert_eq!(lines.len(), 2, "{lines:#?}"); + assert!( + lines[0] + .starts_with("WARN UDP send failed: no buffer space, backing off link=wifi error="), + "{lines:#?}" + ); + assert_eq!( + lines[1], + "INFO UDP sends have buffer space again link=wifi refused=300 took_ms=299" + ); + + // A refusal after the quiet starts a new episode, with its own step. + let later = start + Duration::from_secs(5); + let lines = logged(|| { + congestion.refused(&no_buffer_space(), later); + assert!(congestion.take_step_down()); + drop(congestion); + }); + assert_eq!(lines.len(), 2, "{lines:#?}"); + assert!(lines[0].starts_with("WARN UDP send failed: no buffer space")); + assert_eq!( + lines[1], "INFO UDP sends have buffer space again link=wifi refused=1 took_ms=0", + "a transport that stops ends its episode" + ); + } +} diff --git a/host/src/transport/link.rs b/host/src/transport/link.rs index 2d3e73d8..ae0f53e8 100644 --- a/host/src/transport/link.rs +++ b/host/src/transport/link.rs @@ -103,6 +103,10 @@ impl Link for UdpLink { return Err(invalid("UDP destination")); }; check_length(datagram.len())?; + #[cfg(feature = "test-hooks")] + if super::probe::refuse(addr, datagram) { + return Err(super::congestion::no_buffer_space()); + } self.socket.send_to(datagram, addr).await?; #[cfg(feature = "test-hooks")] super::probe::sent(addr); diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index c6050db6..ae734b7d 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -2,6 +2,7 @@ pub mod abr; pub mod apple_usb; mod audio; pub mod beat; +pub mod congestion; pub mod fault; pub mod link; pub mod pacer; @@ -63,6 +64,7 @@ pub mod probe { static SENT: Mutex> = Mutex::new(BTreeMap::new()); static TAKEN: Mutex> = Mutex::new(BTreeMap::new()); + static REFUSE: Mutex> = Mutex::new(BTreeMap::new()); pub(crate) fn sent(addr: SocketAddr) { *SENT.lock().entry(addr).or_default() += 1; @@ -79,6 +81,47 @@ pub mod probe { pub fn taken_from(addr: SocketAddr) -> u64 { TAKEN.lock().get(&addr).copied().unwrap_or(0) } + + /// The datagrams a refusal applies to. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum Kind { + Media, + Control, + } + + /// Of the UDP sends of `kind` to `addr`, the next `after` go out and + /// the `count` after them fail as if the system had no buffer space for + /// them (WSAENOBUFS, ENOBUFS). + pub fn refuse_sends(addr: SocketAddr, kind: Kind, after: u64, count: u64) { + REFUSE.lock().insert(addr, (kind, after, count)); + } + + /// Sends of the kind given to [`refuse_sends`] still to be refused. + pub fn refusals_left(addr: SocketAddr) -> u64 { + REFUSE.lock().get(&addr).map_or(0, |&(_, _, count)| count) + } + + /// Whether this send of `datagram` to `addr` is refused, counting it. + pub(crate) fn refuse(addr: SocketAddr, datagram: &[u8]) -> bool { + use eternal_wire::v2::{classify, Classified}; + let kind = match classify(datagram) { + Classified::Media { .. } => Kind::Media, + Classified::Control(_) => Kind::Control, + _ => return false, + }; + let mut refuse = REFUSE.lock(); + match refuse.get_mut(&addr) { + Some((refused, after, _)) if *refused == kind && *after > 0 => { + *after -= 1; + false + } + Some((refused, _, count)) if *refused == kind && *count > 0 => { + *count -= 1; + true + } + _ => false, + } + } } /// What an identity proof signs for a socket on `listen`: that address, or @@ -369,6 +412,8 @@ pub async fn start_sender( _ = liveness_tick.tick() => { beat.mark(Phase::Tick); links.update_stats(); + links.congestion.lock().settle(now()); + ease_congestion(&links, &mut state, &shared, &config).await; if !shared.running.load(Ordering::SeqCst) { info!("Transport loop stopping on running=false"); break; @@ -501,31 +546,44 @@ pub async fn start_sender( } let mut frame_pacer = FramePacer::new(frag_count_usize, Instant::now()); + let mut backed_off = false; for datagram in &stored { // A busy machine can exhaust the pacing budget mid-frame. // Keep serving audio even when there are no more pauses; // otherwise a large IDR can fill its 60 ms capture queue. audio_sender.send_queued(&mut audio_rx, &links, &shared, stream_epoch).await; - if target_addr.link == LinkId::Udp && state.fault.enabled() { - for due in state.fault.push(datagram, Instant::now()) { - if let Err(e) = links.send_to(&due, target_addr).await { - if !send_failed { - log_send_failed(target_addr.link, &e); - send_failed = true; - } - } - } - } else if let Err(e) = links.send_to(datagram, target_addr).await { - if !send_failed { + let mut refused = false; + let mut note = |sent: std::io::Result<()>| match sent { + Err(e) if congestion::is_no_buffer_space(&e) => refused = true, + Err(e) if !send_failed => { log_send_failed(target_addr.link, &e); send_failed = true; } + _ => {} + }; + if target_addr.link == LinkId::Udp && state.fault.enabled() { + for due in state.fault.push(datagram, Instant::now()) { + note(links.send_to(&due, target_addr).await); + } + } else { + note(links.send_to(datagram, target_addr).await); + } + // The system had no buffer space for it. A frame backs + // off once and sends it again; a refusal after that gives + // up the rest of the frame, which repair recovers. + if refused && backed_off { + break; + } + if refused { + backed_off = true; + ease_congestion(&links, &mut state, &shared, &config).await; } - let pause = frame_pacer.after_send(Instant::now()); + let paced = frame_pacer.after_send(Instant::now()); + let pause = if refused { congestion::BACKOFF } else { paced }; if target_addr.link == LinkId::Udp && pause > Duration::ZERO { let repairs = repair_while_pacing(&links, &mut state, &shared, &config, &supervisor_tx, - pacing_clock.wait(pause), &mut deferred_control); + pacing_clock.until(Instant::now() + pause), &mut deferred_control); tokio::pin!(repairs); loop { tokio::select! { @@ -537,6 +595,9 @@ pub async fn start_sender( } } } + if refused && links.send_to(datagram, target_addr).await.is_err() { + break; + } } let total_bytes = payload.len(); @@ -874,22 +935,45 @@ async fn act( target_bps: reported.target_bps, changed: clamped.changed || reported.changed, }; - if decision.changed { - shared - .abr_current_bps - .store(decision.target_bps, Ordering::SeqCst); - PIPELINE_STATS.lock().set_bitrate(decision.target_bps); - // One immediate notify; the 1s heartbeat's embedded config - // self-heals any loss. - let notify = shared.session.lock().stream_config_notify(config); - for (destination, datagram) in notify { - let _ = links.send_to(&datagram, destination).await; - } - } + apply_bitrate(decision, links, shared, config).await; } execute_actions(actions, links, shared, supervisor_tx).await; } +/// Steps the bitrate down once for each episode of sends the system had no +/// buffer space for, without waiting for the iPad to report the loss. +async fn ease_congestion( + links: &TransportLinks, + state: &mut LoopState, + shared: &SharedControl, + config: &impl ConfigSource, +) { + if links.congestion.lock().take_step_down() { + let decision = state.abr.on_send_refused(Instant::now()); + apply_bitrate(decision, links, shared, config).await; + } +} + +async fn apply_bitrate( + decision: abr::AbrDecision, + links: &TransportLinks, + shared: &SharedControl, + config: &impl ConfigSource, +) { + if decision.changed { + shared + .abr_current_bps + .store(decision.target_bps, Ordering::SeqCst); + PIPELINE_STATS.lock().set_bitrate(decision.target_bps); + // One immediate notify; the 1s heartbeat's embedded config + // self-heals any loss. + let notify = shared.session.lock().stream_config_notify(config); + for (destination, datagram) in notify { + let _ = links.send_to(&datagram, destination).await; + } + } +} + async fn execute_actions( mut actions: Actions, links: &TransportLinks, @@ -910,8 +994,18 @@ async fn execute_actions( } } for (destination, datagram) in &actions.replies { - if let Err(e) = links.send_to(datagram, *destination).await { - warn!(peer = %destination, error = %e, "Failed to send control reply"); + let mut sent = links.send_to(datagram, *destination).await; + // The system had no buffer space for it: one short retry. The + // episode's own lines report it if that fails too. + if sent.as_ref().is_err_and(congestion::is_no_buffer_space) { + tokio::time::sleep(congestion::REPLY_RETRY).await; + sent = links.send_to(datagram, *destination).await; + } + match sent { + Err(e) if !congestion::is_no_buffer_space(&e) => { + warn!(peer = %destination, error = %e, "Failed to send control reply"); + } + _ => {} } } diff --git a/host/src/transport/pacer.rs b/host/src/transport/pacer.rs index a44643c9..0ab4b159 100644 --- a/host/src/transport/pacer.rs +++ b/host/src/transport/pacer.rs @@ -34,12 +34,13 @@ impl PacingClock { } pub async fn wait(&self, duration: Duration) { + self.until(Instant::now() + duration.min(MAX_SPREAD)).await; + } + + /// Waits until `deadline`, or not at all if a wait is already queued. + pub async fn until(&self, deadline: Instant) { let (done, ready) = tokio::sync::oneshot::channel(); - if self - .tx - .try_send((Instant::now() + duration.min(MAX_SPREAD), done)) - .is_ok() - { + if self.tx.try_send((deadline, done)).is_ok() { let _ = ready.await; } } diff --git a/host/src/transport/usb.rs b/host/src/transport/usb.rs index b3a3e94c..7f96c4f6 100644 --- a/host/src/transport/usb.rs +++ b/host/src/transport/usb.rs @@ -14,6 +14,7 @@ use tokio::task::{AbortHandle, JoinSet}; use tracing::{debug, info, warn}; use super::apple_usb::{self, AppleUsbState}; +use super::congestion::{is_no_buffer_space, SendCongestion}; use super::link::{FramedLink, Link, LinkId, PeerId, UdpLink}; use super::usbmuxd::{self, BoxedTunnel, Client}; use crate::stats::PIPELINE_STATS; @@ -233,6 +234,8 @@ impl Drop for ActiveLink { pub struct TransportLinks { pub udp: UdpLink, + /// UDP sends the system had no buffer space for. + pub congestion: Mutex, usb: HashMap, events: mpsc::Receiver, _subscription: HubToken, @@ -252,6 +255,7 @@ impl TransportLinks { let subscription = UsbHub::get().subscribe(&force_idr); let mut links = Self { udp: UdpLink::new(socket), + congestion: Mutex::default(), usb: HashMap::new(), events: subscription.events, _subscription: HubToken(subscription.token), @@ -279,7 +283,15 @@ impl TransportLinks { pub async fn send_to(&self, bytes: &[u8], peer: PeerId) -> io::Result<()> { match peer.link { - LinkId::Udp => self.udp.send(peer, bytes).await, + LinkId::Udp => { + let sent = self.udp.send(peer, bytes).await; + if let Err(error) = &sent { + if is_no_buffer_space(error) { + self.congestion.lock().refused(error, super::now()); + } + } + sent + } id => match self.usb.get(&id) { Some(active) => active.link.send(peer, bytes).await, None => Err(io::Error::new( diff --git a/host/tests/send_congestion.rs b/host/tests/send_congestion.rs new file mode 100644 index 00000000..77714344 --- /dev/null +++ b/host/tests/send_congestion.rs @@ -0,0 +1,368 @@ +//! When the system has no buffer space for a datagram (WSAENOBUFS 10055 on +//! Windows, ENOBUFS elsewhere), the real transport loop backs off, steps the +//! bitrate down once per episode, gives up a frame it can't send rather than +//! wait on it, retries a control reply once, and logs one line per episode +//! plus its end. The sends are refused through the test-hooks seam. + +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::Ordering; +use std::sync::mpsc as std_mpsc; +use std::time::{Duration, Instant}; + +use eternal_host::control::SharedControl; +use eternal_host::encoder::NALUnit; +use eternal_host::transport::congestion::{BACKOFF, EPISODE_QUIET}; +use eternal_host::transport::probe::{self, Kind}; +use eternal_wire::v2::control::{ + encode_control, parse_control, ControlMessage, Hello2, Nack, Ping, CAP_DECODE_H264, + FEATURE_SUPPORTS_NACK, +}; +use eternal_wire::v2::media::{media_payload_size, MediaHeader}; +use parking_lot::Mutex; + +static LOG: Mutex> = Mutex::new(Vec::new()); + +struct Sink; + +impl std::io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + LOG.lock().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn log_lines(containing: &str) -> Vec { + String::from_utf8_lossy(&LOG.lock()) + .lines() + .filter(|line| line.contains(containing)) + .map(|line| line.trim().to_owned()) + .collect() +} + +fn free_udp_port() -> u16 { + UdpSocket::bind("127.0.0.1:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap() +} + +struct Ipad { + socket: UdpSocket, + host: String, + session_id: u32, + msg_seq: u32, + pinged: Instant, +} + +impl Ipad { + fn addr(&self) -> SocketAddr { + self.socket.local_addr().unwrap() + } + + fn send(&mut self, message: &ControlMessage) { + self.msg_seq += 1; + self.socket + .send_to( + &encode_control(self.session_id, self.msg_seq, message), + &self.host, + ) + .unwrap(); + } + + /// Keeps the session alive while the test waits. + fn keep_alive(&mut self) { + if self.pinged.elapsed() >= Duration::from_millis(250) { + self.pinged = Instant::now(); + self.send(&ControlMessage::Ping(Ping { t1_us: 1 })); + } + } + + /// Every datagram for `wait`, with when it came. + fn receive(&mut self, wait: Duration) -> Vec<(Instant, Vec)> { + let mut buffer = [0; 2048]; + let mut received = Vec::new(); + let until = Instant::now() + wait; + while Instant::now() < until { + self.keep_alive(); + if let Ok(len) = self.socket.recv(&mut buffer) { + received.push((Instant::now(), buffer[..len].to_vec())); + } + } + received + } + + /// Frame `frame_seq`'s fragments among `received`, with when each came. + fn fragments(received: &[(Instant, Vec)], frame_seq: u32) -> Vec<(Instant, u16)> { + received + .iter() + .filter_map(|(at, bytes)| { + let (header, _) = MediaHeader::decode(bytes).ok()?; + (header.frame_seq == frame_seq && !header.is_retransmit) + .then_some((*at, header.frag_index)) + }) + .collect() + } + + /// Waits for a heartbeat, so the next second has none. + fn after_heartbeat(&mut self) { + let mut buffer = [0; 2048]; + let until = Instant::now() + Duration::from_secs(3); + while Instant::now() < until { + self.keep_alive(); + if let Ok(len) = self.socket.recv(&mut buffer) { + if let Ok((_, ControlMessage::Heartbeat(_))) = parse_control(&buffer[..len]) { + return; + } + } + } + panic!("no heartbeat"); + } +} + +/// A keyframe of exactly `fragments` datagrams. +fn keyframe(shared: &SharedControl, fragments: usize) -> NALUnit { + let payload = media_payload_size(shared.max_dgram.load(Ordering::SeqCst) as usize).unwrap(); + let mut data = vec![0x41; fragments * payload]; + data[..5].copy_from_slice(&[0, 0, 0, 1, 0x65]); + NALUnit { + data, + sequence: 0, + timestamp: Instant::now(), + encode_duration_us: 0, + is_keyframe: true, + } +} + +#[test] +fn refused_sends_back_off_step_down_once_and_log_one_episode() { + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_target(false) + .with_max_level(tracing::Level::INFO) + .with_writer(|| Sink) + .finish(); + tracing::subscriber::set_global_default(subscriber).unwrap(); + + let port = free_udp_port(); + let shared = SharedControl::new(port, 15_000_000); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + let (nal_tx, nal_rx) = tokio::sync::mpsc::channel(16); + let (_audio_tx, audio_rx) = tokio::sync::mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + let sender_shared = shared.clone(); + let sender = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(eternal_host::transport::start_sender( + nal_rx, + audio_rx, + port, + sender_shared, + supervisor_tx, + )) + .unwrap(); + }); + + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(5))) + .unwrap(); + let mut ipad = Ipad { + host: format!("127.0.0.1:{port}"), + session_id: 0, + msg_seq: 0, + pinged: Instant::now(), + socket, + }; + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 9, + listen_port: ipad.addr().port(), + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_SUPPORTS_NACK, + screen_px_w: 640, + screen_px_h: 360, + screen_pt_w: 640, + screen_pt_h: 360, + refresh_hz: 60, + device_name: "Congested iPad".into(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + let mut buffer = [0; 2048]; + let deadline = Instant::now() + Duration::from_secs(10); + ipad.session_id = loop { + assert!(Instant::now() < deadline, "no HELLO_ACK"); + ipad.send(&hello); + if let Ok(len) = ipad.socket.recv(&mut buffer) { + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&buffer[..len]) { + break ack.session_id; + } + } + }; + let addr = ipad.addr(); + assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 15_000_000); + + // Frame 1: fragment 10 is refused once. The frame backs off and sends it + // again, and the bitrate steps down without waiting for a loss report. + probe::refuse_sends(addr, Kind::Media, 10, 1); + nal_tx.blocking_send(keyframe(&shared, 40)).unwrap(); + let received = ipad.receive(Duration::from_millis(300)); + let frame = Ipad::fragments(&received, 1); + let indexes: Vec = frame.iter().map(|&(_, index)| index).collect(); + assert_eq!( + indexes, + (0..40).collect::>(), + "the refused fragment is sent again, not lost" + ); + let paused = frame[10].0.duration_since(frame[9].0); + assert!( + paused >= BACKOFF - Duration::from_millis(1), + "it waited {paused:?} before sending again" + ); + assert_eq!( + shared.abr_current_bps.load(Ordering::SeqCst), + 12_000_000, + "one rung down at once" + ); + + // Frame 2, in the same episode: fragment 5 is refused, and so is its + // retry. The frame gives up its rest, which repair recovers, and the + // bitrate stays: one step per episode. + probe::refuse_sends(addr, Kind::Media, 5, 2); + nal_tx.blocking_send(keyframe(&shared, 40)).unwrap(); + let received = ipad.receive(Duration::from_millis(300)); + let indexes: Vec = Ipad::fragments(&received, 2) + .iter() + .map(|&(_, index)| index) + .collect(); + assert_eq!( + indexes, + (0..5).collect::>(), + "the rest is given up" + ); + assert_eq!(probe::refusals_left(addr), 0); + assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 12_000_000); + let epoch = received + .iter() + .find_map(|(_, bytes)| MediaHeader::decode(bytes).ok()) + .unwrap() + .0 + .stream_epoch; + ipad.send(&ControlMessage::Nack(Nack { + stream_epoch: epoch, + frame_seq: 2, + frag_count: 40, + missing: (5..40).collect(), + })); + let repaired = ipad + .receive(Duration::from_millis(300)) + .iter() + .filter(|(_, bytes)| { + MediaHeader::decode(bytes) + .is_ok_and(|(header, _)| header.frame_seq == 2 && header.is_retransmit) + }) + .count(); + assert_eq!(repaired, 35, "repair recovers what was given up"); + + // Five frames of 100 fragments while every send is refused: each frame + // backs off once and gives up, so none holds the loop for more than one + // back-off. + probe::refuse_sends(addr, Kind::Media, 0, 1_000_000); + let pushed = Instant::now(); + for _ in 0..5 { + nal_tx.blocking_send(keyframe(&shared, 100)).unwrap(); + } + while 1_000_000 - probe::refusals_left(addr) < 10 { + assert!( + pushed.elapsed() < Duration::from_secs(2), + "refused {} sends", + 1_000_000 - probe::refusals_left(addr) + ); + std::thread::sleep(Duration::from_millis(1)); + } + let took = pushed.elapsed(); + ipad.receive(Duration::from_millis(100)); + assert_eq!( + 1_000_000 - probe::refusals_left(addr), + 10, + "two refusals per frame, the first and its retry" + ); + assert!( + took < 5 * (BACKOFF + Duration::from_millis(30)), + "five frames took {took:?}" + ); + probe::refuse_sends(addr, Kind::Media, 0, 0); + assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 12_000_000); + + // The episode ends after a quiet second, with one summary line. + let until = Instant::now() + EPISODE_QUIET + Duration::from_secs(2); + while log_lines("buffer space again").is_empty() && Instant::now() < until { + ipad.receive(Duration::from_millis(50)); + } + let started = log_lines("no buffer space"); + assert_eq!(started.len(), 1, "{started:#?}"); + assert!( + started[0].starts_with("WARN UDP send failed: no buffer space, backing off link=wifi"), + "{started:#?}" + ); + let ended = log_lines("buffer space again"); + assert_eq!(ended.len(), 1, "{ended:#?}"); + assert!( + ended[0] + .starts_with("INFO UDP sends have buffer space again link=wifi refused=13 took_ms="), + "{ended:#?}" + ); + let steps = log_lines("ABR stepping bitrate down"); + assert_eq!(steps.len(), 1, "{steps:#?}"); + assert!( + steps[0].contains("reason=\"no_buffer_space\""), + "{steps:#?}" + ); + + // A refused control reply is sent again once. This is a new episode, + // with its own step down. + ipad.after_heartbeat(); + // No keep-alive Pong still on its way to take the refusal. + ipad.pinged = Instant::now(); + ipad.receive(Duration::from_millis(30)); + probe::refuse_sends(addr, Kind::Control, 0, 1); + ipad.send(&ControlMessage::Ping(Ping { t1_us: 0xC0FFEE })); + let ponged = ipad + .receive(Duration::from_millis(200)) + .iter() + .any(|(_, bytes)| { + matches!( + parse_control(bytes), + Ok((_, ControlMessage::Pong(pong))) if pong.t1_us == 0xC0FFEE + ) + }); + assert!(ponged, "the reply went on its retry"); + assert_eq!(probe::refusals_left(addr), 0); + // Stepped on the next liveness tick. + let until = Instant::now() + Duration::from_secs(1); + while shared.abr_current_bps.load(Ordering::SeqCst) != 10_000_000 && Instant::now() < until { + ipad.receive(Duration::from_millis(20)); + } + assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 10_000_000); + assert_eq!(log_lines("no buffer space").len(), 2); + assert!(log_lines("Failed to send control reply").is_empty()); + assert!( + log_lines("UDP send failed link=").is_empty(), + "no line per send" + ); + + drop(nal_tx); + shared.stop(); + sender.join().unwrap(); +} From 6d268cc7ff3ffa98ad62258443f880176af8c480 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 20:48:04 -0500 Subject: [PATCH 112/226] Tell iPad users when their PC app is out of date Most slow first pictures on the iPad come from PCs that still run 0.3.2 or 0.3.3, and 0.3.2 never offers its users an update. The iPad is the one place they can learn about it. The iPad now tells a PC older than 0.3.4 apart using what PCs already send: - A PC found on the network announces its version in its Bonjour TXT record. Only a plain release version counts. - Every 0.3.4 and later PC sets the contiguous frame numbering bit in every HELLO_ACK, whatever its hardware or settings, and no earlier PC ever set it. This covers PCs reached over USB or by address. A PC counts as old only when nothing it sent says it is current. Betas and PCs that send nothing are never called old. For such a PC, the Connect screen shows a quiet card, "Your PC app is out of date", that names the PC and points to eternalmonitor.dev. It shows at most once per PC every three days, never over a stream, and doesn't block connecting. Close hides it. Each showing writes one info line, old_pc_prompt shown=true version=. The two strings are in English and wait for translation. The tests use HELLO_ACKs recorded from hosts built at v0.3.2, v0.3.3 and v0.3.4. An injected clock covers the three-day rule, and a UI test covers the card. --- ios/EternalMonitor/App/ConnectView.swift | 21 ++ .../App/ConnectionManager.swift | 5 + ios/EternalMonitor/App/UIPreview.swift | 4 +- ios/EternalMonitor/App/UpdateBanner.swift | 48 ++++ ios/EternalMonitor/Core/OldPC.swift | 111 +++++++++ ios/EternalMonitor/Network/WireProtocol.swift | 1 + .../Resources/Localizable.xcstrings | 24 ++ .../LocalizationTests.swift | 2 + ios/EternalMonitorTests/OldPCTests.swift | 212 ++++++++++++++++++ .../ConnectScreenTests.swift | 24 ++ proto/src/v2/control.rs | 2 + 11 files changed, 453 insertions(+), 1 deletion(-) create mode 100644 ios/EternalMonitor/Core/OldPC.swift create mode 100644 ios/EternalMonitorTests/OldPCTests.swift diff --git a/ios/EternalMonitor/App/ConnectView.swift b/ios/EternalMonitor/App/ConnectView.swift index 92a33ccb..b3b45843 100644 --- a/ios/EternalMonitor/App/ConnectView.swift +++ b/ios/EternalMonitor/App/ConnectView.swift @@ -9,6 +9,7 @@ struct ConnectView: View { @ObservedObject private var pairings = PairingStore.shared @ObservedObject private var language = LanguageStore.shared @ObservedObject private var updates = UpdateChecker.shared + @ObservedObject private var oldPC = OldPCPrompt.shared @State private var hostIP: String = "" @State private var port: String = "9876" @@ -96,6 +97,10 @@ struct ConnectView: View { ) } + if let card = oldPC.card { + OldPCCard(card: card, close: { withAnimation(Motion.fade) { oldPC.close() } }) + } + if isConnecting { connectingCard .transition(.opacity.combined(with: .scale(scale: 0.97))) @@ -229,6 +234,12 @@ struct ConnectView: View { .onChange(of: scenePhase) { _, phase in if phase == .active && scanner.searchProblem == .localNetworkOff { scanner.startScan() } } + // The old-PC card shows only here, never over a stream. + .onAppear(perform: offerOldPC) + .onChange(of: connectionManager.state) { _, _ in offerOldPC() } + .onChange(of: scanner.hosts) { _, _ in offerOldPC() } + .onChange(of: showSetup) { _, _ in offerOldPC() } + .onDisappear { oldPC.close() } .animation(.easeInOut(duration: 0.2), value: isConnecting) .animation(.easeInOut(duration: 0.2), value: connectionManager.connectionError) .animation(.easeInOut(duration: 0.2), value: connectionManager.problem) @@ -735,6 +746,16 @@ struct ConnectView: View { : "Couldn't get its address. Type the Address shown on the PC." } + /// Notes the PCs the search found and shows a waiting old-PC card + /// while nothing is streaming and setup is closed. + private func offerOldPC() { + for host in scanner.hosts { + oldPC.saw(pc: OldPC.pcName(bonjour: host.name), + announced: PCVersions.version(address: host.address, name: host.name), hostCaps: nil) + } + if connectionManager.state == .disconnected && !showSetup { oldPC.show() } + } + private var unlistedRecents: [RecentConnection] { RecentConnection.unlisted(recentStore.connections, discovered: scanner.hosts) } diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 7a920dab..9780dc4d 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -163,6 +163,8 @@ final class ConnectionManager: ObservableObject { private var appleDevicesExpiry: Task? /// The wall clock for the Apple Devices sighting. Tests replace it. var now: () -> Date = { Date() } + /// Hears whether each PC that accepts a session is older than 0.3.4. + var oldPCPrompt = OldPCPrompt.shared private var mediaLink: (any MediaLink)? private var usbListener: USBListener? private var isForeground = true @@ -681,6 +683,9 @@ final class ConnectionManager: ObservableObject { at: isUSB ? nil : "\(normalizedHost):\(port)") self.livenessTimeoutUs = UInt64(info.livenessTimeoutMs) * 1000 self.hostInfo = info + self.oldPCPrompt.saw(pc: info.hostName, + announced: isUSB ? nil : PCVersions.version(address: normalizedHost, name: nil), + hostCaps: info.hostCaps) if self.waitingForStream { self.waitingForStream = false if case .pcNotStreaming = self.problem { self.problem = nil } diff --git a/ios/EternalMonitor/App/UIPreview.swift b/ios/EternalMonitor/App/UIPreview.swift index 2d35d2df..4c446737 100644 --- a/ios/EternalMonitor/App/UIPreview.swift +++ b/ios/EternalMonitor/App/UIPreview.swift @@ -4,7 +4,7 @@ import SwiftUI /// Debug builds only: put the UI in a state that normally needs a live PC, /// for screenshots and design review. Never opens a connection. /// -/// `EM_UI_PREVIEW=display|quality|restoring|connecting|error|error-no-picture|error-pairing-code|error-router|error-updating|error-local-network|error-wifi-off|mismatch|pairing|settings|settings-about|feedback|recent|searchFailed|searchBlocked|setup|setup-blocked`. +/// `EM_UI_PREVIEW=display|quality|restoring|connecting|error|error-no-picture|error-pairing-code|error-router|error-updating|error-local-network|error-wifi-off|mismatch|pairing|settings|settings-about|feedback|recent|old-pc|searchFailed|searchBlocked|setup|setup-blocked`. /// `setup` opens first-run setup whatever the saved state (launch with /// `-didSeeOnboarding NO`); `setup-blocked` also fails its search as if /// Local Network were off. @@ -71,6 +71,8 @@ enum UIPreview { manager.connectionError = VersionMismatch.iPadOlder.message case "pairing": manager.pairingPrompt = PairingSheetModel() + case "old-pc": + OldPCPrompt.shared.preview(OldPCPrompt.Card(pc: "Studio PC", version: "0.3.2")) case "recent": RecentConnectionStore.shared.add(host: "192.168.1.20", port: 9876, isUSB: false) RecentConnectionStore.shared.add(host: "Studio PC", port: 9877, isUSB: true) diff --git a/ios/EternalMonitor/App/UpdateBanner.swift b/ios/EternalMonitor/App/UpdateBanner.swift index 5b2149e5..a22a516c 100644 --- a/ios/EternalMonitor/App/UpdateBanner.swift +++ b/ios/EternalMonitor/App/UpdateBanner.swift @@ -65,6 +65,54 @@ struct UpdateBanner: View { } } +/// Connect screen: the PC this iPad found or streamed from runs an +/// EternalMonitor older than 0.3.4. Quiet, and only in the screen's flow. +struct OldPCCard: View { + let card: OldPCPrompt.Card + let close: () -> Void + + var body: some View { + let title = L("Your PC app is out of date") + let detail = L("Update EternalMonitor on \(card.pc) for faster, steadier streaming. Download it on the PC from eternalmonitor.dev.") + HStack(alignment: .top, spacing: 12) { + Image(systemName: "desktopcomputer") + .font(.system(size: 15, weight: .semibold)) + .foregroundStyle(Theme.accent) + .frame(width: 34, height: 34) + .background( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .fill(Theme.accent.opacity(0.1)) + ) + .accessibilityHidden(true) + VStack(alignment: .leading, spacing: 2) { + Text(title) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + Text(detail) + .font(.app(13, relativeTo: .footnote)) + .foregroundStyle(Theme.textMuted) + .fixedSize(horizontal: false, vertical: true) + } + .accessibilityElement(children: .combine) + .accessibilityIdentifier("connect.oldPC") + Spacer(minLength: 0) + Button(action: close) { + Image(systemName: "xmark") + .font(.system(size: 12, weight: .semibold)) + .foregroundStyle(Theme.textFaint) + .frame(width: 44, height: 44) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .padding(.top, -6) + .accessibilityLabel(Text(verbatim: L("Close"))) + .accessibilityIdentifier("connect.oldPCClose") + } + .card(padding: 12) + .transition(.opacity) + } +} + /// Connect screen: a small card over the screen when a newer build is ready. /// `allowed` is false while the user is busy (connecting, typing, or in a /// sheet). The Connect screen is never on screen during a stream. diff --git a/ios/EternalMonitor/Core/OldPC.swift b/ios/EternalMonitor/Core/OldPC.swift new file mode 100644 index 00000000..1aaba72a --- /dev/null +++ b/ios/EternalMonitor/Core/OldPC.swift @@ -0,0 +1,111 @@ +import Foundation + +/// Tells a PC whose EternalMonitor is older than 0.3.4 from a current one, +/// using only what PCs already send. +/// +/// - Bonjour: every PC puts its version in the TXT record's `version`. +/// Only a plain release version counts; a beta's or a missing one doesn't. +/// - HELLO_ACK: every 0.3.4 and later PC sets `hostCapContiguousFrameSeq` in +/// every answer, whatever its hardware or settings, and no earlier PC ever +/// set that bit. This covers PCs reached over USB or by address. +/// +/// A PC is old only when nothing it sent says it is current. +enum OldPC { + static let current = AppVersion("0.3.4")! + + /// True for an old PC, false for a current one, nil when it can't tell. + static func isOld(announced: String?, hostCaps: UInt16?) -> Bool? { + if let hostCaps, hostCaps & HelloAck.hostCapContiguousFrameSeq != 0 { return false } + if let version = announced.flatMap(AppVersion.init) { return version < current } + return hostCaps == nil ? nil : true + } + + /// The name a PC gives itself, from its Bonjour service name. + static func pcName(bonjour name: String) -> String { + let prefix = "EternalMonitor (", suffix = ")" + guard name.hasPrefix(prefix), name.hasSuffix(suffix), name.count > prefix.count + suffix.count else { return name } + return String(name.dropFirst(prefix.count).dropLast(suffix.count)) + } +} + +/// The Connect screen's card asking to update an old PC. It shows at most +/// once per PC every three days, only on the Connect screen, and goes away +/// when closed or when a stream starts. +@MainActor +final class OldPCPrompt: ObservableObject { + static let shared = OldPCPrompt() + static let interval: TimeInterval = 3 * 86_400 + static let defaultsKey = "oldPCPromptShownAt" + /// PCs whose last showing is kept. + static let kept = 16 + + struct Card: Equatable { + /// The name the PC gives itself. It also keys the three days. + let pc: String + /// The release version it announced, if any. + let version: String? + } + + /// The card on screen. + @Published private(set) var card: Card? + /// An old PC waiting for the Connect screen. + private var waiting: Card? + private let defaults: UserDefaults + private let now: () -> Date + private let log: (String) -> Void + + init(defaults: UserDefaults = .standard, now: @escaping () -> Date = { Date() }, + log: @escaping (String) -> Void = { SupportDiagnostics.shared.record("info", "update", $0) }) { + self.defaults = defaults + self.now = now + self.log = log + } + + /// A PC this iPad found or connected to. An old one waits for `show`; a + /// current one takes its card away. + func saw(pc: String, announced: String?, hostCaps: UInt16?) { + switch OldPC.isOld(announced: announced, hostCaps: hostCaps) { + case true?: + guard card == nil, due(pc) else { return } + waiting = Card(pc: pc, version: announced.flatMap(AppVersion.init) == nil ? nil : announced) + case false?: + if waiting?.pc == pc { waiting = nil } + if card?.pc == pc { card = nil } + case nil: + break + } + } + + /// The Connect screen is in view with nothing streaming: shows the + /// waiting card, if its PC is due. + func show() { + guard card == nil, let next = waiting else { return } + waiting = nil + guard due(next.pc) else { return } + var shown = defaults.dictionary(forKey: Self.defaultsKey) as? [String: Date] ?? [:] + shown[next.pc] = now() + for (pc, _) in shown.sorted(by: { $0.value > $1.value }).dropFirst(Self.kept) { shown[pc] = nil } + defaults.set(shown, forKey: Self.defaultsKey) + card = next + log("old_pc_prompt shown=true version=\(next.version ?? "unknown")") + } + + /// Closed, or the Connect screen gave way to a stream. + func close() { + card = nil + } + + /// Whether `pc`'s card may show again. A showing dated after now, from a + /// clock that was ahead, doesn't hold it back. + func due(_ pc: String) -> Bool { + guard let last = (defaults.dictionary(forKey: Self.defaultsKey) as? [String: Date])?[pc] else { return true } + let now = now() + return last > now || now.timeIntervalSince(last) >= Self.interval + } + + #if DEBUG + func preview(_ card: Card) { + self.card = card + } + #endif +} diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index 88cce1da..6b7b3d0e 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -310,6 +310,7 @@ struct HelloAck: Equatable { static let hostCapPen: UInt16 = 1 << 4 /// Media frame_seq counts the frames sent in a stream epoch without gaps, /// so a forward gap is a frame lost in transit. Older iPads ignore it. + /// Every 0.3.4 and later PC sets it; see `OldPC`. static let hostCapContiguousFrameSeq: UInt16 = 1 << 5 var status: HelloStatus var acceptedVersion: UInt8 diff --git a/ios/EternalMonitor/Resources/Localizable.xcstrings b/ios/EternalMonitor/Resources/Localizable.xcstrings index 9d27ffc2..75d48faa 100644 --- a/ios/EternalMonitor/Resources/Localizable.xcstrings +++ b/ios/EternalMonitor/Resources/Localizable.xcstrings @@ -8544,6 +8544,18 @@ } } }, + "Update EternalMonitor on %@ for faster, steadier streaming. Download it on the PC from eternalmonitor.dev." : { + "comment" : "Connect screen card under \"Your PC app is out of date\". The value is the PC's name. Keep eternalmonitor.dev as is", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Update EternalMonitor on %@ for faster, steadier streaming. Download it on the PC from eternalmonitor.dev." + } + } + } + }, "Updates" : { "extractionState" : "manual", "localizations" : { @@ -9043,6 +9055,18 @@ } } }, + "Your PC app is out of date" : { + "comment" : "Title of the Connect screen card shown when the PC runs an EternalMonitor older than 0.3.4", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Your PC app is out of date" + } + } + } + }, "Your PC is locked. Unlock it to continue." : { "comment" : "Over the held picture while the PC shows its lock screen or a permission prompt", "extractionState" : "manual", diff --git a/ios/EternalMonitorTests/LocalizationTests.swift b/ios/EternalMonitorTests/LocalizationTests.swift index 5a33593f..d17d441c 100644 --- a/ios/EternalMonitorTests/LocalizationTests.swift +++ b/ios/EternalMonitorTests/LocalizationTests.swift @@ -24,6 +24,8 @@ final class LocalizationTests: XCTestCase { /// then every language shows them in English. private let awaitingTranslation: Set = [ "Your PC is locked. Unlock it to continue.", + "Your PC app is out of date", + "Update EternalMonitor on %@ for faster, steadier streaming. Download it on the PC from eternalmonitor.dev.", ] /// Every language has exactly English's strings, none empty, each with diff --git a/ios/EternalMonitorTests/OldPCTests.swift b/ios/EternalMonitorTests/OldPCTests.swift new file mode 100644 index 00000000..2a612eff --- /dev/null +++ b/ios/EternalMonitorTests/OldPCTests.swift @@ -0,0 +1,212 @@ +import XCTest +@testable import EternalMonitor + +/// Telling a PC older than 0.3.4 from a current one, and the Connect +/// screen's card that asks to update it. +@MainActor +final class OldPCTests: XCTestCase { + /// HELLO_ACKs recorded from hosts built at the tags v0.3.2, v0.3.3 and + /// v0.3.4 (macOS, synthetic capture, pairing off, PC name OLD-PC), + /// answering this app's HELLO2. 0.3.2 and 0.3.3 answer with host caps + /// 0x000F, 0.3.4 with 0x002F. + static let recordedAcks = [ + "0.3.2": "454d0211000039002fe161dd010000000002443322112fe161dde803f401b80b010000000005d0023c000000c0e1e400064f4c442d5043000000000000000000000000000000000f00", + "0.3.3": "454d0211000039001d64ed60010000000002443322111d64ed60e803f401b80b010000000005d0023c000000c0e1e400064f4c442d5043000000000000000000000000000000000f00", + "0.3.4": "454d0211000039000c8c4e43010000000002443322110c8c4e43e803f401b80b010000000005d0023c000000c0e1e400064f4c442d5043000000000000000000000000000000002f00", + ] + + /// The same hosts' Bonjour TXT records, as `dns-sd -L` showed them. + static func recordedTXT(_ version: String) -> Data { + NetService.data(fromTXTRecord: ["version": Data(version.utf8), "proto": Data("2".utf8), + "platform": Data("other".utf8)]) + } + + private struct NotAnAck: Error {} + + private func recordedAck(_ version: String) throws -> HelloAck { + let hex = Array(try XCTUnwrap(Self.recordedAcks[version])) + let data = Data(stride(from: 0, to: hex.count, by: 2).map { UInt8(String(hex[$0...$0 + 1]), radix: 16)! }) + guard case .helloAck(let ack) = Wire.parseControl(data)?.message else { throw NotAnAck() } + return ack + } + + func testRecordedRepliesFromTaggedPCs() throws { + for (version, old) in [("0.3.2", true), ("0.3.3", true), ("0.3.4", false)] { + let ack = try recordedAck(version) + XCTAssertEqual(ack.status, .ok, version) + XCTAssertEqual(ack.hostName, "OLD-PC", version) + // Over USB or a typed address: the HELLO_ACK alone. + XCTAssertEqual(OldPC.isOld(announced: nil, hostCaps: ack.hostCaps), old, version) + // Found on the network: the TXT record alone, and both. + let announced = PCVersions.version(fromTXT: Self.recordedTXT(version)) + XCTAssertEqual(announced, version) + XCTAssertEqual(OldPC.isOld(announced: announced, hostCaps: nil), old, version) + XCTAssertEqual(OldPC.isOld(announced: announced, hostCaps: ack.hostCaps), old, version) + } + } + + /// Before 0.3.4 a PC's optional capabilities (audio, USB, keyboard, pen) + /// vary with its hardware and settings. None makes it current; the + /// numbering bit always does. + func testOnlyTheNumberingBitMarksACurrentPC() { + for caps in UInt16(0)..<32 { + XCTAssertEqual(OldPC.isOld(announced: nil, hostCaps: caps), true, "caps \(caps)") + XCTAssertEqual(OldPC.isOld(announced: nil, hostCaps: caps | HelloAck.hostCapContiguousFrameSeq), false, "caps \(caps)") + } + XCTAssertEqual(OldPC.isOld(announced: nil, hostCaps: 0xFFFF), false, "bits a later PC adds") + } + + /// Never call a current PC old: whatever says current wins, and a beta's + /// or a missing version decides nothing by itself. + func testACurrentPCIsNeverOld() { + let current = HelloAck.hostCapNack | HelloAck.hostCapContiguousFrameSeq + XCTAssertEqual(OldPC.isOld(announced: "0.3.2", hostCaps: current), false, "another PC's version at a reused address") + XCTAssertEqual(OldPC.isOld(announced: "0.3.4", hostCaps: HelloAck.hostCapNack), false) + XCTAssertEqual(OldPC.isOld(announced: "0.3.5", hostCaps: nil), false) + XCTAssertEqual(OldPC.isOld(announced: "1.0.0", hostCaps: nil), false) + XCTAssertNil(OldPC.isOld(announced: "0.3.4-beta.20261029.1", hostCaps: nil)) + XCTAssertNil(OldPC.isOld(announced: "0.3.3-beta.20261020.2", hostCaps: nil), "a beta from before the bump") + XCTAssertNil(OldPC.isOld(announced: "", hostCaps: nil)) + XCTAssertNil(OldPC.isOld(announced: nil, hostCaps: nil)) + XCTAssertEqual(OldPC.isOld(announced: "0.3.3-beta.20261020.2", hostCaps: HelloAck.hostCapNack), true) + } + + func testThePCsNameFromItsBonjourName() { + XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor (OLD-PC)"), "OLD-PC") + XCTAssertEqual(OldPC.pcName(bonjour: "Studio"), "Studio") + XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor ()"), "EternalMonitor ()") + } + + private func defaults() -> UserDefaults { + let name = "OldPCTests-\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: name)! + addTeardownBlock { UserDefaults().removePersistentDomain(forName: name) } + return defaults + } + + /// At most once per PC every three days, on an injected clock, across + /// launches; a clock that was ahead doesn't hold it back. + func testTheCardShowsOncePerPCEveryThreeDays() { + let store = defaults() + let start = Date(timeIntervalSince1970: 1_000_000) + var now = start + var lines: [String] = [] + let make = { OldPCPrompt(defaults: store, now: { now }, log: { lines.append($0) }) } + var prompt = make() + let old = HelloAck.hostCapNack + + prompt.show() + XCTAssertNil(prompt.card, "nothing seen yet") + prompt.saw(pc: "Studio", announced: "0.3.2", hostCaps: nil) + XCTAssertNil(prompt.card, "waits for the Connect screen") + prompt.show() + XCTAssertEqual(prompt.card, OldPCPrompt.Card(pc: "Studio", version: "0.3.2")) + XCTAssertEqual(lines, ["old_pc_prompt shown=true version=0.3.2"]) + prompt.show() + XCTAssertEqual(lines.count, 1, "one line per showing") + + // Another old PC waits while a card is up, and isn't kept. + prompt.saw(pc: "Office", announced: nil, hostCaps: old) + prompt.close() + XCTAssertNil(prompt.card) + prompt.show() + XCTAssertNil(prompt.card) + + // The same PC again, later that day and in a new launch: no card. + now = start + 3_600 + prompt = make() + prompt.saw(pc: "Studio", announced: nil, hostCaps: old) + prompt.show() + XCTAssertNil(prompt.card) + + // Another PC has its own three days. Known only by its HELLO_ACK. + prompt.saw(pc: "Office", announced: nil, hostCaps: old) + prompt.show() + XCTAssertEqual(prompt.card, OldPCPrompt.Card(pc: "Office", version: nil)) + XCTAssertEqual(lines.last, "old_pc_prompt shown=true version=unknown") + prompt.close() + + now = start + OldPCPrompt.interval - 1 + prompt.saw(pc: "Studio", announced: "0.3.2", hostCaps: nil) + prompt.show() + XCTAssertNil(prompt.card, "a second short of three days") + now = start + OldPCPrompt.interval + prompt.saw(pc: "Studio", announced: "0.3.2", hostCaps: nil) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Studio") + prompt.close() + + // Shown "tomorrow" by a clock that was ahead, then the clock is fixed. + now = start + 30 * 86_400 + prompt.saw(pc: "Lab", announced: "0.3.3", hostCaps: nil) + prompt.show() + prompt.close() + now = start + 29 * 86_400 + prompt.saw(pc: "Lab", announced: "0.3.3", hostCaps: nil) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Lab") + XCTAssertEqual(lines.count, 5) + } + + /// A current PC never gets a card, and its answer takes a waiting or + /// showing card for that PC away. + func testACurrentPCTakesItsCardAway() { + var lines: [String] = [] + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { lines.append($0) }) + let current = HelloAck.hostCapNack | HelloAck.hostCapContiguousFrameSeq + prompt.saw(pc: "Studio", announced: "0.3.4", hostCaps: nil) + prompt.saw(pc: "Office", announced: nil, hostCaps: current) + prompt.show() + XCTAssertNil(prompt.card) + + prompt.saw(pc: "Studio", announced: "0.3.3", hostCaps: nil) + prompt.saw(pc: "Studio", announced: nil, hostCaps: current) + prompt.show() + XCTAssertNil(prompt.card, "updated before the card showed") + + prompt.saw(pc: "Office", announced: "0.3.2", hostCaps: nil) + prompt.show() + XCTAssertNotNil(prompt.card) + prompt.saw(pc: "Office", announced: nil, hostCaps: current) + XCTAssertNil(prompt.card) + XCTAssertEqual(lines, ["old_pc_prompt shown=true version=0.3.2"]) + } + + /// Only the last showings of a few PCs are kept. + func testTheShowingsKeptAreBounded() { + let store = defaults() + var now = Date(timeIntervalSince1970: 1_000_000) + let prompt = OldPCPrompt(defaults: store, now: { now }, log: { _ in }) + for index in 0..<(OldPCPrompt.kept + 4) { + now += 60 + prompt.saw(pc: "PC \(index)", announced: "0.3.2", hostCaps: nil) + prompt.show() + prompt.close() + } + let kept = store.dictionary(forKey: OldPCPrompt.defaultsKey) ?? [:] + XCTAssertEqual(kept.count, OldPCPrompt.kept) + XCTAssertNil(kept["PC 0"]) + XCTAssertNotNil(kept["PC \(OldPCPrompt.kept + 3)"]) + } + + /// A session's HELLO_ACK reaches the prompt: an old PC's card waits for + /// the Connect screen, a current PC's doesn't. + func testASessionsAnswerReachesThePrompt() async throws { + for (caps, old) in [(UInt16(0x000F), true), (UInt16(0x002F), false)] { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + let manager = ConnectionManager() + manager.oldPCPrompt = prompt + addTeardownBlock { await MainActor.run { manager.cancel() } } + let host = FakeUDPHost() + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession(hostCaps: caps) }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + let deadline = Date().addingTimeInterval(10) + while manager.hostInfo == nil, Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertEqual(manager.hostInfo?.hostCaps, caps) + manager.cancel() + prompt.show() + XCTAssertEqual(prompt.card, old ? OldPCPrompt.Card(pc: "Test PC", version: nil) : nil, "caps \(caps)") + } + } +} diff --git a/ios/EternalMonitorUITests/ConnectScreenTests.swift b/ios/EternalMonitorUITests/ConnectScreenTests.swift index 8385b6d8..7c9e75d5 100644 --- a/ios/EternalMonitorUITests/ConnectScreenTests.swift +++ b/ios/EternalMonitorUITests/ConnectScreenTests.swift @@ -61,6 +61,30 @@ final class ConnectScreenTests: UITestCase { XCTAssertTrue(gone(star, within: 5), "Close did not hide the card") } + /// A PC older than 0.3.4: a quiet card on the Connect screen, above the + /// connect controls, that names the PC and closes. When it shows is a + /// unit test (OldPCTests). + func testAnOldPCsCardNamesThePCAndCloses() { + let app = XCUIApplication() + app.launchArguments = ["-didSeeOnboarding", "YES", "-allowUSB", "NO", "-checkForUpdates", ""] + app.launchEnvironment["EM_UI_PREVIEW"] = "old-pc" + app.launchEnvironment["EM_USB_PORT"] = "0" + launchCleanly(app) + let card = app.descendants(matching: .any)["connect.oldPC"] + XCTAssertTrue(card.waitForExistence(timeout: 15), app.debugDescription) + XCTAssertTrue(card.label.contains("Your PC app is out of date"), card.label) + XCTAssertTrue(card.label.contains("Update EternalMonitor on Studio PC for faster, steadier streaming."), card.label) + XCTAssertTrue(card.label.contains("eternalmonitor.dev"), card.label) + XCTAssertLessThan(card.frame.maxY, app.buttons["connect.button"].frame.minY, "above the connect controls") + XCTAssertTrue(app.buttons["connect.button"].isEnabled, "connecting isn't blocked") + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = "old-pc-card"; shot.lifetime = .keepAlways; add(shot) + let close = app.buttons["connect.oldPCClose"] + close.tap() + if !gone(card, within: 5), close.exists { close.tap() } + XCTAssertTrue(gone(card, within: 5), "Close did not hide the card") + } + private func gone(_ element: XCUIElement, within timeout: TimeInterval) -> Bool { let gone = XCTNSPredicateExpectation(predicate: NSPredicate(format: "exists == false"), object: element) return XCTWaiter().wait(for: [gone], timeout: timeout) == .completed diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index f9528fe2..17f2dba1 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -53,6 +53,8 @@ pub const HOSTCAP_PEN: u16 = 1 << 4; /// Media `frame_seq` counts the access units sent in a stream epoch without /// gaps, so a forward gap is a frame lost in transit. Without it, numbers /// may skip frames that were never sent. Older iPads ignore the bit. +/// Every 0.3.4 and later host sets it, and iPads read an answer without it +/// as a PC to update, so it stays set. pub const HOSTCAP_CONTIGUOUS_FRAME_SEQ: u16 = 1 << 5; /// [`StreamConfig::flags`] bits. From 8f617065658d45d01e64fdb1abc1eec325f285fb Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 21:09:20 -0500 Subject: [PATCH 113/226] Know each PC by its pairing and remember what its session proved A PC that a session proved current could get the card back from a later Bonjour record with an old or stale version. A version cached by address could also land on another PC that later used the same address. - Each found service now carries the version from its own TXT record, through the IPv4 lookup too. The card never takes a version from another service or by address. - A PC is known by a hash of its pairing token when it has one, otherwise by its Bonjour service name, and last by the name it answers with. The service the user tapped, or the one listed at the session's address, is linked to the PC that answered there. A link can only keep a card away. - The last showing and what the accepted HELLO_ACK said are kept per PC across launches, for the last 16 PCs. A PC that proved current gets no card from any later Bonjour record. Refusals and not-streaming replies, which may carry no caps at all, count for nothing. - One tap on Close hides the card. --- ios/EternalMonitor/App/ConnectView.swift | 11 +- .../App/ConnectionManager.swift | 7 +- ios/EternalMonitor/App/UIPreview.swift | 2 +- ios/EternalMonitor/Core/OldPC.swift | 171 ++++++++++-- .../Network/NetworkScanner.swift | 17 +- ios/EternalMonitor/Network/WireProtocol.swift | 3 +- ios/EternalMonitorTests/OldPCTests.swift | 254 +++++++++++++++--- .../ConnectScreenTests.swift | 6 +- proto/src/v2/control.rs | 5 +- 9 files changed, 383 insertions(+), 93 deletions(-) diff --git a/ios/EternalMonitor/App/ConnectView.swift b/ios/EternalMonitor/App/ConnectView.swift index b3b45843..c2fe400f 100644 --- a/ios/EternalMonitor/App/ConnectView.swift +++ b/ios/EternalMonitor/App/ConnectView.swift @@ -712,6 +712,7 @@ struct ConnectView: View { guard let address = host.address else { return } hostIP = address port = "\(host.port)" + oldPC.connecting(service: host.name, address: address) connectionManager.connect(host: address, port: host.port, found: true) } label: { hostRow( @@ -746,13 +747,11 @@ struct ConnectView: View { : "Couldn't get its address. Type the Address shown on the PC." } - /// Notes the PCs the search found and shows a waiting old-PC card - /// while nothing is streaming and setup is closed. + /// Notes the PCs the search lists, each with its own TXT version, and + /// shows a waiting old-PC card while nothing is streaming and setup is + /// closed. private func offerOldPC() { - for host in scanner.hosts { - oldPC.saw(pc: OldPC.pcName(bonjour: host.name), - announced: PCVersions.version(address: host.address, name: host.name), hostCaps: nil) - } + oldPC.discovered(scanner.hosts) if connectionManager.state == .disconnected && !showSetup { oldPC.show() } } diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 9780dc4d..a8381af5 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -163,7 +163,7 @@ final class ConnectionManager: ObservableObject { private var appleDevicesExpiry: Task? /// The wall clock for the Apple Devices sighting. Tests replace it. var now: () -> Date = { Date() } - /// Hears whether each PC that accepts a session is older than 0.3.4. + /// Hears, from each accepted HELLO_ACK, whether the PC is older than 0.3.4. var oldPCPrompt = OldPCPrompt.shared private var mediaLink: (any MediaLink)? private var usbListener: USBListener? @@ -683,9 +683,8 @@ final class ConnectionManager: ObservableObject { at: isUSB ? nil : "\(normalizedHost):\(port)") self.livenessTimeoutUs = UInt64(info.livenessTimeoutMs) * 1000 self.hostInfo = info - self.oldPCPrompt.saw(pc: info.hostName, - announced: isUSB ? nil : PCVersions.version(address: normalizedHost, name: nil), - hostCaps: info.hostCaps) + self.oldPCPrompt.answered(hostName: info.hostName, pairingToken: info.authToken, + hostCaps: info.hostCaps, address: isUSB ? nil : normalizedHost) if self.waitingForStream { self.waitingForStream = false if case .pcNotStreaming = self.problem { self.problem = nil } diff --git a/ios/EternalMonitor/App/UIPreview.swift b/ios/EternalMonitor/App/UIPreview.swift index 4c446737..5e5522f6 100644 --- a/ios/EternalMonitor/App/UIPreview.swift +++ b/ios/EternalMonitor/App/UIPreview.swift @@ -72,7 +72,7 @@ enum UIPreview { case "pairing": manager.pairingPrompt = PairingSheetModel() case "old-pc": - OldPCPrompt.shared.preview(OldPCPrompt.Card(pc: "Studio PC", version: "0.3.2")) + OldPCPrompt.shared.preview(OldPCPrompt.Card(key: "preview", pc: "Studio PC", version: "0.3.2")) case "recent": RecentConnectionStore.shared.add(host: "192.168.1.20", port: 9876, isUSB: false) RecentConnectionStore.shared.add(host: "Studio PC", port: 9877, isUSB: true) diff --git a/ios/EternalMonitor/Core/OldPC.swift b/ios/EternalMonitor/Core/OldPC.swift index 1aaba72a..01f343ed 100644 --- a/ios/EternalMonitor/Core/OldPC.swift +++ b/ios/EternalMonitor/Core/OldPC.swift @@ -1,3 +1,4 @@ +import CryptoKit import Foundation /// Tells a PC whose EternalMonitor is older than 0.3.4 from a current one, @@ -6,14 +7,17 @@ import Foundation /// - Bonjour: every PC puts its version in the TXT record's `version`. /// Only a plain release version counts; a beta's or a missing one doesn't. /// - HELLO_ACK: every 0.3.4 and later PC sets `hostCapContiguousFrameSeq` in -/// every answer, whatever its hardware or settings, and no earlier PC ever -/// set that bit. This covers PCs reached over USB or by address. +/// every reply that accepts a session, whatever its hardware or settings, +/// and no earlier PC ever set that bit. This covers PCs reached over USB or +/// by address. Refusals and not-streaming replies may leave every cap out, +/// so they count for nothing either way. /// /// A PC is old only when nothing it sent says it is current. enum OldPC { static let current = AppVersion("0.3.4")! /// True for an old PC, false for a current one, nil when it can't tell. + /// `hostCaps` must come from a reply that accepted a session. static func isOld(announced: String?, hostCaps: UInt16?) -> Bool? { if let hostCaps, hostCaps & HelloAck.hostCapContiguousFrameSeq != 0 { return false } if let version = announced.flatMap(AppVersion.init) { return version < current } @@ -31,25 +35,51 @@ enum OldPC { /// The Connect screen's card asking to update an old PC. It shows at most /// once per PC every three days, only on the Connect screen, and goes away /// when closed or when a stream starts. +/// +/// A PC is known by its pairing token (hashed) when it has one, otherwise by +/// its Bonjour service name, which is unique on a network, and last by the +/// name it answers with. A service the user tapped, or the one the search +/// lists at the session's address right now, is linked to the PC that +/// answered there. What a session showed about a PC outlasts the session: +/// a PC that proved current gets no card from a later Bonjour record. Links +/// only ever keep a card away. @MainActor final class OldPCPrompt: ObservableObject { static let shared = OldPCPrompt() static let interval: TimeInterval = 3 * 86_400 - static let defaultsKey = "oldPCPromptShownAt" - /// PCs whose last showing is kept. + static let defaultsKey = "oldPCPromptMemory" + /// PCs remembered. static let kept = 16 struct Card: Equatable { - /// The name the PC gives itself. It also keys the three days. + /// Who the PC is to this memory. + let key: String + /// The name shown. let pc: String /// The release version it announced, if any. let version: String? } + struct Memory: Codable, Equatable { + struct PC: Codable, Equatable { + var shownAt: Date? + /// What a session's accepted HELLO_ACK said: current or not. + var current: Bool? + var seenAt: Date + } + var pcs: [String: PC] = [:] + /// Bonjour service name to the PC a session through it answered as. + var services: [String: String] = [:] + } + /// The card on screen. @Published private(set) var card: Card? /// An old PC waiting for the Connect screen. private var waiting: Card? + /// The services the search lists now. + private var listed: [DiscoveredHost] = [] + /// The service the user tapped, and its address. + private var tapped: (service: String, address: String)? private let defaults: UserDefaults private let now: () -> Date private let log: (String) -> Void @@ -61,31 +91,118 @@ final class OldPCPrompt: ObservableObject { self.log = log } - /// A PC this iPad found or connected to. An old one waits for `show`; a - /// current one takes its card away. - func saw(pc: String, announced: String?, hostCaps: UInt16?) { - switch OldPC.isOld(announced: announced, hostCaps: hostCaps) { - case true?: - guard card == nil, due(pc) else { return } - waiting = Card(pc: pc, version: announced.flatMap(AppVersion.init) == nil ? nil : announced) - case false?: - if waiting?.pc == pc { waiting = nil } - if card?.pc == pc { card = nil } - case nil: - break + var memory: Memory { + defaults.data(forKey: Self.defaultsKey).flatMap { try? JSONDecoder().decode(Memory.self, from: $0) } ?? Memory() + } + + private func save(_ memory: Memory) { + var memory = memory + let dropped = memory.pcs.sorted { $0.value.seenAt > $1.value.seenAt }.dropFirst(Self.kept).map(\.key) + for key in dropped { memory.pcs[key] = nil } + memory.services = memory.services.filter { memory.pcs[$0.value] != nil } + defaults.set(try? JSONEncoder().encode(memory), forKey: Self.defaultsKey) + } + + static func serviceKey(_ name: String) -> String { "service:\(name)" } + + /// The PC behind a Bonjour service. + private func key(service: String, in memory: Memory) -> String { + memory.services[service] ?? Self.serviceKey(service) + } + + /// The services the search lists now, each with the version its own + /// TXT record announced. + func discovered(_ hosts: [DiscoveredHost]) { + listed = hosts + let memory = self.memory + for host in hosts { + let key = key(service: host.name, in: memory) + switch OldPC.isOld(announced: host.version, hostCaps: nil) { + case true? where memory.pcs[key]?.current != true: + queue(Card(key: key, pc: OldPC.pcName(bonjour: host.name), version: host.version), memory: memory) + case false?: + drop(key) + default: + break + } + } + } + + /// The user tapped a found PC at `address`. + func connecting(service: String, address: String) { + tapped = (service, address) + } + + /// A PC accepted a session. `hostCaps` is from that accepted HELLO_ACK; + /// `address` is nil over USB. + func answered(hostName: String, pairingToken: Data, hostCaps: UInt16, address: String?) { + var memory = self.memory + let service = address.flatMap { address in + tapped.flatMap { $0.address == address ? $0.service : nil } + ?? listed.first { $0.address == address }?.name + } + tapped = nil + let key = Self.key(hostName: hostName, pairingToken: pairingToken, service: service) + var pc = memory.pcs[key] ?? Memory.PC(seenAt: now()) + if let service { + let serviceKey = Self.serviceKey(service) + let before = [memory.services[service], serviceKey].compactMap { $0 }.filter { $0 != key } + for old in before { + if let shown = memory.pcs[old]?.shownAt { pc.shownAt = max(pc.shownAt ?? shown, shown) } + if waiting?.key == old { waiting = nil } + if let shown = card, shown.key == old { card = Card(key: key, pc: shown.pc, version: shown.version) } + } + if key == serviceKey { + memory.services[service] = nil + } else { + memory.pcs[serviceKey] = nil + memory.services[service] = key + } + } + let old = OldPC.isOld(announced: nil, hostCaps: hostCaps) == true + pc.current = !old + pc.seenAt = now() + memory.pcs[key] = pc + save(memory) + if old { + queue(Card(key: key, pc: hostName, version: nil), memory: memory) + } else { + drop(key) } } + /// The pairing token's hash when the PC has one, otherwise the service + /// it was found as, otherwise its name. + static func key(hostName: String, pairingToken: Data, service: String?) -> String { + if PairingStore.valid(pairingToken) { + return "pair:" + SHA256.hash(data: pairingToken).prefix(12).map { String(format: "%02x", $0) }.joined() + } + if let service { return serviceKey(service) } + return "name:\(hostName)" + } + + private func queue(_ next: Card, memory: Memory) { + guard card == nil, due(next.key, in: memory) else { return } + waiting = next + } + + private func drop(_ key: String) { + if waiting?.key == key { waiting = nil } + if card?.key == key { card = nil } + } + /// The Connect screen is in view with nothing streaming: shows the /// waiting card, if its PC is due. func show() { guard card == nil, let next = waiting else { return } waiting = nil - guard due(next.pc) else { return } - var shown = defaults.dictionary(forKey: Self.defaultsKey) as? [String: Date] ?? [:] - shown[next.pc] = now() - for (pc, _) in shown.sorted(by: { $0.value > $1.value }).dropFirst(Self.kept) { shown[pc] = nil } - defaults.set(shown, forKey: Self.defaultsKey) + var memory = self.memory + guard due(next.key, in: memory) else { return } + var pc = memory.pcs[next.key] ?? Memory.PC(seenAt: now()) + pc.shownAt = now() + pc.seenAt = now() + memory.pcs[next.key] = pc + save(memory) card = next log("old_pc_prompt shown=true version=\(next.version ?? "unknown")") } @@ -95,10 +212,12 @@ final class OldPCPrompt: ObservableObject { card = nil } - /// Whether `pc`'s card may show again. A showing dated after now, from a - /// clock that was ahead, doesn't hold it back. - func due(_ pc: String) -> Bool { - guard let last = (defaults.dictionary(forKey: Self.defaultsKey) as? [String: Date])?[pc] else { return true } + /// Whether the PC `key` may get its card again. A showing dated after + /// now, from a clock that was ahead, doesn't hold it back. + func due(_ key: String) -> Bool { due(key, in: memory) } + + private func due(_ key: String, in memory: Memory) -> Bool { + guard let last = memory.pcs[key]?.shownAt else { return true } let now = now() return last > now || now.timeIntervalSince(last) >= Self.interval } diff --git a/ios/EternalMonitor/Network/NetworkScanner.swift b/ios/EternalMonitor/Network/NetworkScanner.swift index 8c5e0226..bd65a5a5 100644 --- a/ios/EternalMonitor/Network/NetworkScanner.swift +++ b/ios/EternalMonitor/Network/NetworkScanner.swift @@ -11,20 +11,23 @@ struct DiscoveredHost: Identifiable, Equatable { let port: UInt16 /// Still resolving. False once resolution gave up without IPv4. var finding = false + /// The `version` this service's own TXT record announced. + var version: String? - init(name: String, address: String?, port: UInt16, finding: Bool = false) { + init(name: String, address: String?, port: UInt16, finding: Bool = false, version: String? = nil) { self.name = name self.address = address self.port = port self.finding = finding + self.version = version } /// The service as far as it has resolved: its first IPv4 address, or a /// PC still being found when only IPv6 (or nothing usable) came back. - static func resolved(name: String, port: Int, addresses: [Data]) -> DiscoveredHost { + static func resolved(name: String, port: Int, addresses: [Data], version: String? = nil) -> DiscoveredHost { let port = port > 0 ? UInt16(clamping: port) : 9876 let ipv4 = addresses.lazy.compactMap { NetworkScanner.ipAddress(from: $0, family: AF_INET) }.first - return DiscoveredHost(name: name, address: ipv4, port: port, finding: ipv4 == nil) + return DiscoveredHost(name: name, address: ipv4, port: port, finding: ipv4 == nil, version: version) } } @@ -232,11 +235,13 @@ extension NetworkScanner: NetServiceBrowserDelegate, NetServiceDelegate { Task { @MainActor in let key = Self.serviceKey(sender) guard isCurrent(sender), sender.addresses?.isEmpty == false else { return } - let host = DiscoveredHost.resolved(name: sender.name, port: sender.port, addresses: sender.addresses ?? []) + let version = PCVersions.version(fromTXT: sender.txtRecordData()) + let host = DiscoveredHost.resolved(name: sender.name, port: sender.port, addresses: sender.addresses ?? [], + version: version) // discovery diagnostic — surface every resolved host so we // can tell whether mDNS is finding the right machine. print("[mDNS] Found: \(host.name) at \(host.address ?? "no IPv4 yet"):\(host.port)") - if let version = PCVersions.version(fromTXT: sender.txtRecordData()) { + if let version { PCVersions.record(version, name: host.name, address: host.address) } merge(host) @@ -277,7 +282,7 @@ extension NetworkScanner: NetServiceBrowserDelegate, NetServiceDelegate { guard self.ipv4Lookups[service.name] == lookup else { return } self.ipv4Lookups.removeValue(forKey: service.name) if let address { - self.merge(DiscoveredHost(name: service.name, address: address, port: service.port)) + self.merge(DiscoveredHost(name: service.name, address: address, port: service.port, version: service.version)) } else if !self.isScanning, let index = self.hosts.firstIndex(where: { $0.name == service.name && $0.address == nil }) { self.hosts[index].finding = false } diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index 6b7b3d0e..95342fcb 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -310,7 +310,8 @@ struct HelloAck: Equatable { static let hostCapPen: UInt16 = 1 << 4 /// Media frame_seq counts the frames sent in a stream epoch without gaps, /// so a forward gap is a frame lost in transit. Older iPads ignore it. - /// Every 0.3.4 and later PC sets it; see `OldPC`. + /// Every 0.3.4 and later PC sets it in replies that accept a session; + /// see `OldPC`. static let hostCapContiguousFrameSeq: UInt16 = 1 << 5 var status: HelloStatus var acceptedVersion: UInt8 diff --git a/ios/EternalMonitorTests/OldPCTests.swift b/ios/EternalMonitorTests/OldPCTests.swift index 2a612eff..d58e1db9 100644 --- a/ios/EternalMonitorTests/OldPCTests.swift +++ b/ios/EternalMonitorTests/OldPCTests.swift @@ -1,3 +1,4 @@ +import Darwin import XCTest @testable import EternalMonitor @@ -84,6 +85,13 @@ final class OldPCTests: XCTestCase { return defaults } + private let tokenA = Data(1...16), tokenB = Data(17...32), tokenC = Data(33...48) + private let oldCaps: UInt16 = 0x000F, currentCaps: UInt16 = 0x002F + + private func service(_ pc: String, at address: String?, version: String?) -> DiscoveredHost { + DiscoveredHost(name: "EternalMonitor (\(pc))", address: address, port: 9876, finding: address == nil, version: version) + } + /// At most once per PC every three days, on an injected clock, across /// launches; a clock that was ahead doesn't hold it back. func testTheCardShowsOncePerPCEveryThreeDays() { @@ -93,20 +101,20 @@ final class OldPCTests: XCTestCase { var lines: [String] = [] let make = { OldPCPrompt(defaults: store, now: { now }, log: { lines.append($0) }) } var prompt = make() - let old = HelloAck.hostCapNack + let studio = OldPCPrompt.key(hostName: "Studio", pairingToken: tokenA, service: nil) prompt.show() XCTAssertNil(prompt.card, "nothing seen yet") - prompt.saw(pc: "Studio", announced: "0.3.2", hostCaps: nil) + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: oldCaps, address: nil) XCTAssertNil(prompt.card, "waits for the Connect screen") prompt.show() - XCTAssertEqual(prompt.card, OldPCPrompt.Card(pc: "Studio", version: "0.3.2")) - XCTAssertEqual(lines, ["old_pc_prompt shown=true version=0.3.2"]) + XCTAssertEqual(prompt.card, OldPCPrompt.Card(key: studio, pc: "Studio", version: nil)) + XCTAssertEqual(lines, ["old_pc_prompt shown=true version=unknown"]) prompt.show() XCTAssertEqual(lines.count, 1, "one line per showing") // Another old PC waits while a card is up, and isn't kept. - prompt.saw(pc: "Office", announced: nil, hostCaps: old) + prompt.discovered([service("Office", at: "192.0.2.20", version: "0.3.2")]) prompt.close() XCTAssertNil(prompt.card) prompt.show() @@ -115,98 +123,258 @@ final class OldPCTests: XCTestCase { // The same PC again, later that day and in a new launch: no card. now = start + 3_600 prompt = make() - prompt.saw(pc: "Studio", announced: nil, hostCaps: old) + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: oldCaps, address: nil) prompt.show() XCTAssertNil(prompt.card) - // Another PC has its own three days. Known only by its HELLO_ACK. - prompt.saw(pc: "Office", announced: nil, hostCaps: old) + // Another PC has its own three days. Known only by its TXT record. + prompt.discovered([service("Office", at: "192.0.2.20", version: "0.3.2")]) prompt.show() - XCTAssertEqual(prompt.card, OldPCPrompt.Card(pc: "Office", version: nil)) - XCTAssertEqual(lines.last, "old_pc_prompt shown=true version=unknown") + XCTAssertEqual(prompt.card, OldPCPrompt.Card(key: OldPCPrompt.serviceKey("EternalMonitor (Office)"), + pc: "Office", version: "0.3.2")) + XCTAssertEqual(lines.last, "old_pc_prompt shown=true version=0.3.2") prompt.close() now = start + OldPCPrompt.interval - 1 - prompt.saw(pc: "Studio", announced: "0.3.2", hostCaps: nil) + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: oldCaps, address: nil) prompt.show() XCTAssertNil(prompt.card, "a second short of three days") now = start + OldPCPrompt.interval - prompt.saw(pc: "Studio", announced: "0.3.2", hostCaps: nil) + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: oldCaps, address: nil) prompt.show() - XCTAssertEqual(prompt.card?.pc, "Studio") + XCTAssertEqual(prompt.card?.key, studio) prompt.close() // Shown "tomorrow" by a clock that was ahead, then the clock is fixed. now = start + 30 * 86_400 - prompt.saw(pc: "Lab", announced: "0.3.3", hostCaps: nil) + prompt.discovered([service("Lab", at: "192.0.2.30", version: "0.3.3")]) prompt.show() prompt.close() now = start + 29 * 86_400 - prompt.saw(pc: "Lab", announced: "0.3.3", hostCaps: nil) + prompt.discovered([service("Lab", at: "192.0.2.30", version: "0.3.3")]) prompt.show() XCTAssertEqual(prompt.card?.pc, "Lab") XCTAssertEqual(lines.count, 5) } + /// r1 P1: a PC whose accepted HELLO_ACK proved it current gets no card + /// from a Bonjour record that says otherwise afterwards, in this launch + /// or a later one. + func testProofThatAPCIsCurrentOutlastsItsSession() { + let store = defaults() + var lines: [String] = [] + var prompt = OldPCPrompt(defaults: store, now: { Date() }, log: { lines.append($0) }) + // Tapped in the list: the HELLO_ACK says current, then the TXT 0.3.3. + prompt.connecting(service: "EternalMonitor (Studio)", address: "192.0.2.10") + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: currentCaps, address: "192.0.2.10") + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.3")]) + prompt.show() + XCTAssertNil(prompt.card) + + prompt = OldPCPrompt(defaults: store, now: { Date() }, log: { lines.append($0) }) + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.3")]) + prompt.show() + XCTAssertNil(prompt.card, "a new launch") + + // Typed address, while the search lists that PC there; no pairing. + prompt.discovered([service("Lab", at: "192.0.2.11", version: nil)]) + prompt.answered(hostName: "Lab", pairingToken: Data(count: 16), hostCaps: currentCaps, address: "192.0.2.11") + prompt.discovered([service("Lab", at: "192.0.2.11", version: "0.3.2")]) + prompt.show() + XCTAssertNil(prompt.card) + XCTAssertEqual(lines, []) + + // Only those PCs: another old one still gets its card. + prompt.discovered([service("Lab", at: "192.0.2.11", version: "0.3.2"), + service("Office", at: "192.0.2.12", version: "0.3.3")]) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Office") + XCTAssertEqual(lines, ["old_pc_prompt shown=true version=0.3.3"]) + } + + /// r1 P2: a version stays with the service whose TXT record announced it. + /// A service first seen over IPv6 keeps its own version when its IPv4 + /// address resolves, and a service at an address another PC used before + /// never takes that PC's version. + func testAVersionStaysWithItsOwnService() async throws { + var addr = sockaddr_in6() + addr.sin6_len = UInt8(MemoryLayout.size) + addr.sin6_family = sa_family_t(AF_INET6) + inet_pton(AF_INET6, "fe80::1", &addr.sin6_addr) + let ipv6 = withUnsafeBytes(of: &addr) { Data($0) } + + // An old PC was found at 192.0.2.40 earlier and left. + PCVersions.record("0.3.2", name: "EternalMonitor (Gone)", address: "192.0.2.40") + let scanner = NetworkScanner() + scanner.resolveIPv4 = { _ in "192.0.2.40" } + let beta = DiscoveredHost.resolved(name: "EternalMonitor (New)", port: 9876, addresses: [ipv6], + version: "0.3.4-beta.20261107.1") + XCTAssertNil(beta.address) + scanner.merge(beta) + scanner.lookUpIPv4(hostName: "new.local.", service: beta) + let deadline = Date().addingTimeInterval(10) + while scanner.hosts.first?.address == nil, Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertEqual(scanner.hosts.map(\.address), ["192.0.2.40"]) + XCTAssertEqual(scanner.hosts.first?.version, "0.3.4-beta.20261107.1") + + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + prompt.discovered(scanner.hosts) + prompt.show() + XCTAssertNil(prompt.card, "a beta at a reused address") + prompt.discovered([service("Quiet", at: "192.0.2.40", version: nil)]) + prompt.show() + XCTAssertNil(prompt.card, "no TXT version at a reused address") + + // The same path keeps an old version too. + let old = DiscoveredHost.resolved(name: "EternalMonitor (Old)", port: 9876, addresses: [ipv6], version: "0.3.2") + scanner.resolveIPv4 = { _ in "192.0.2.41" } + scanner.merge(old) + scanner.lookUpIPv4(hostName: "old.local.", service: old) + while scanner.hosts.first(where: { $0.name == old.name })?.address == nil, Date() < deadline { + try await Task.sleep(nanoseconds: 10_000_000) + } + prompt.discovered(scanner.hosts) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Old") + XCTAssertEqual(prompt.card?.version, "0.3.2") + } + + /// r1 P3: a PC is its pairing, not its name. Two PCs with one name each + /// get a card; a renamed PC doesn't get another; Bonjour's collision + /// suffix names a second PC, and a session links it to its pairing. + func testAPCIsItsPairingNotItsName() { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Studio") + prompt.close() + prompt.answered(hostName: "Studio", pairingToken: tokenB, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertNotNil(prompt.card, "another PC with the same name") + prompt.close() + prompt.answered(hostName: "Studio Renamed", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertNil(prompt.card, "the first PC, renamed") + + // Two PCs called Lab on one network: Bonjour lists the second as "Lab (2)". + let lab = service("Lab", at: "192.0.2.50", version: "0.3.2") + let lab2 = DiscoveredHost(name: "EternalMonitor (Lab) (2)", address: "192.0.2.51", port: 9876, version: "0.3.2") + prompt.discovered([lab]) + prompt.show() + XCTAssertEqual(prompt.card?.key, OldPCPrompt.serviceKey(lab.name)) + prompt.close() + prompt.discovered([lab, lab2]) + prompt.show() + XCTAssertEqual(prompt.card?.key, OldPCPrompt.serviceKey(lab2.name), "the second Lab is another PC") + prompt.close() + // Connecting to the second links its service to its pairing: no new card. + prompt.connecting(service: lab2.name, address: "192.0.2.51") + prompt.answered(hostName: "Lab", pairingToken: tokenC, hostCaps: oldCaps, address: "192.0.2.51") + prompt.show() + XCTAssertNil(prompt.card, "shown already, as its service") + let memory = prompt.memory + let keyC = OldPCPrompt.key(hostName: "Lab", pairingToken: tokenC, service: nil) + XCTAssertEqual(memory.services[lab2.name], keyC) + XCTAssertNotNil(memory.pcs[keyC]?.shownAt) + XCTAssertNil(memory.pcs[OldPCPrompt.serviceKey(lab2.name)]) + XCTAssertFalse(memory.pcs.keys.contains { $0.contains(tokenC.map { String(format: "%02x", $0) }.joined()) }, + "the token itself isn't kept") + } + /// A current PC never gets a card, and its answer takes a waiting or /// showing card for that PC away. func testACurrentPCTakesItsCardAway() { var lines: [String] = [] let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { lines.append($0) }) - let current = HelloAck.hostCapNack | HelloAck.hostCapContiguousFrameSeq - prompt.saw(pc: "Studio", announced: "0.3.4", hostCaps: nil) - prompt.saw(pc: "Office", announced: nil, hostCaps: current) + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.4")]) + prompt.answered(hostName: "Office", pairingToken: tokenB, hostCaps: currentCaps, address: nil) prompt.show() XCTAssertNil(prompt.card) - prompt.saw(pc: "Studio", announced: "0.3.3", hostCaps: nil) - prompt.saw(pc: "Studio", announced: nil, hostCaps: current) + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.3")]) + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.4")]) prompt.show() XCTAssertNil(prompt.card, "updated before the card showed") - prompt.saw(pc: "Office", announced: "0.3.2", hostCaps: nil) + prompt.discovered([service("Office", at: "192.0.2.20", version: "0.3.2")]) prompt.show() XCTAssertNotNil(prompt.card) - prompt.saw(pc: "Office", announced: nil, hostCaps: current) + prompt.connecting(service: "EternalMonitor (Office)", address: "192.0.2.20") + prompt.answered(hostName: "Office", pairingToken: tokenB, hostCaps: currentCaps, address: "192.0.2.20") XCTAssertNil(prompt.card) XCTAssertEqual(lines, ["old_pc_prompt shown=true version=0.3.2"]) } - /// Only the last showings of a few PCs are kept. - func testTheShowingsKeptAreBounded() { - let store = defaults() + /// Only the last few PCs are kept, with their links. + func testTheMemoryIsBounded() { var now = Date(timeIntervalSince1970: 1_000_000) - let prompt = OldPCPrompt(defaults: store, now: { now }, log: { _ in }) + let prompt = OldPCPrompt(defaults: defaults(), now: { now }, log: { _ in }) for index in 0..<(OldPCPrompt.kept + 4) { now += 60 - prompt.saw(pc: "PC \(index)", announced: "0.3.2", hostCaps: nil) + prompt.connecting(service: "EternalMonitor (PC \(index))", address: "192.0.2.\(index)") + prompt.answered(hostName: "PC \(index)", pairingToken: Data(repeating: UInt8(index + 1), count: 16), + hostCaps: oldCaps, address: "192.0.2.\(index)") prompt.show() prompt.close() } - let kept = store.dictionary(forKey: OldPCPrompt.defaultsKey) ?? [:] - XCTAssertEqual(kept.count, OldPCPrompt.kept) - XCTAssertNil(kept["PC 0"]) - XCTAssertNotNil(kept["PC \(OldPCPrompt.kept + 3)"]) + let memory = prompt.memory + XCTAssertEqual(memory.pcs.count, OldPCPrompt.kept) + XCTAssertEqual(memory.services.count, OldPCPrompt.kept) + XCTAssertNil(memory.services["EternalMonitor (PC 0)"]) + XCTAssertNotNil(memory.services["EternalMonitor (PC \(OldPCPrompt.kept + 3))"]) } - /// A session's HELLO_ACK reaches the prompt: an old PC's card waits for - /// the Connect screen, a current PC's doesn't. + /// One session with `host`, answered with `status` and `caps`. + private func session(_ manager: ConnectionManager, _ host: FakeUDPHost, status: HelloStatus = .ok, + caps: UInt16) async throws { + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession(status: status, hostCaps: caps) }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + let deadline = Date().addingTimeInterval(10) + if status == .ok { + while manager.hostInfo == nil, Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertEqual(manager.hostInfo?.hostCaps, caps) + } else { + try await Task.sleep(nanoseconds: 300_000_000) + XCTAssertNil(manager.hostInfo) + } + manager.cancel() + } + + /// A session's accepted HELLO_ACK reaches the prompt: an old PC's card + /// waits for the Connect screen, a current PC's doesn't. func testASessionsAnswerReachesThePrompt() async throws { - for (caps, old) in [(UInt16(0x000F), true), (UInt16(0x002F), false)] { + for (caps, old) in [(oldCaps, true), (currentCaps, false)] { let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) let manager = ConnectionManager() manager.oldPCPrompt = prompt addTeardownBlock { await MainActor.run { manager.cancel() } } - let host = FakeUDPHost() - manager.connect(host: "127.0.0.1", port: host.port) - let accepted = await Task.detached { host.acceptSession(hostCaps: caps) }.value - XCTAssertTrue(accepted, "the iPad never sent HELLO2") - let deadline = Date().addingTimeInterval(10) - while manager.hostInfo == nil, Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } - XCTAssertEqual(manager.hostInfo?.hostCaps, caps) - manager.cancel() + try await session(manager, FakeUDPHost(), caps: caps) prompt.show() - XCTAssertEqual(prompt.card, old ? OldPCPrompt.Card(pc: "Test PC", version: nil) : nil, "caps \(caps)") + XCTAssertEqual(prompt.card?.pc, old ? "Test PC" : nil, "caps \(caps)") + XCTAssertEqual(prompt.card?.key.hasPrefix("pair:"), old ? true : nil) + XCTAssertEqual(prompt.memory.pcs.values.map(\.current), [!old]) } } + + /// Refusals and not-streaming replies may carry no caps at all (the PC + /// keeps them within the HELLO2's size), so they say nothing about the + /// PC's age either way. + func testRefusalsAndNotStreamingRepliesCountForNothing() async throws { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + let manager = ConnectionManager() + manager.oldPCPrompt = prompt + addTeardownBlock { await MainActor.run { manager.cancel() } } + for status in [HelloStatus.stopped, .busy, .unauthorized] { + try await session(manager, FakeUDPHost(), status: status, caps: 0) + } + prompt.show() + XCTAssertNil(prompt.card) + XCTAssertEqual(prompt.memory, OldPCPrompt.Memory()) + + // The same PC accepting a session is what counts. + try await session(manager, FakeUDPHost(), caps: oldCaps) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Test PC") + } } diff --git a/ios/EternalMonitorUITests/ConnectScreenTests.swift b/ios/EternalMonitorUITests/ConnectScreenTests.swift index 7c9e75d5..6d84fc24 100644 --- a/ios/EternalMonitorUITests/ConnectScreenTests.swift +++ b/ios/EternalMonitorUITests/ConnectScreenTests.swift @@ -79,10 +79,8 @@ final class ConnectScreenTests: UITestCase { XCTAssertTrue(app.buttons["connect.button"].isEnabled, "connecting isn't blocked") let shot = XCTAttachment(screenshot: app.screenshot()) shot.name = "old-pc-card"; shot.lifetime = .keepAlways; add(shot) - let close = app.buttons["connect.oldPCClose"] - close.tap() - if !gone(card, within: 5), close.exists { close.tap() } - XCTAssertTrue(gone(card, within: 5), "Close did not hide the card") + app.buttons["connect.oldPCClose"].tap() + XCTAssertTrue(gone(card, within: 5), "one tap on Close hides the card") } private func gone(_ element: XCUIElement, within timeout: TimeInterval) -> Bool { diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index 17f2dba1..e3df226f 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -53,8 +53,9 @@ pub const HOSTCAP_PEN: u16 = 1 << 4; /// Media `frame_seq` counts the access units sent in a stream epoch without /// gaps, so a forward gap is a frame lost in transit. Without it, numbers /// may skip frames that were never sent. Older iPads ignore the bit. -/// Every 0.3.4 and later host sets it, and iPads read an answer without it -/// as a PC to update, so it stays set. +/// Every 0.3.4 and later host sets it in every reply that accepts a +/// session, and iPads read an accepted reply without it as a PC to update, +/// so it stays set there. pub const HOSTCAP_CONTIGUOUS_FRAME_SEQ: u16 = 1 << 5; /// [`StreamConfig::flags`] bits. From c56d89376e5b7bdcd39044b0620a27a717a9385c Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 21:14:38 -0500 Subject: [PATCH 114/226] Expect a stopped stream to hand its port on, as #196 made it input_stall checked that a Stop frees the stream port. Since #196 a transport that ends leaves its socket, still bound, in SharedControl, for the next generation or the responder that answers while nothing streams. So the port is never free after a Stop, and the check failed on macOS and Windows CI. What the test guards is unchanged: a loop held by Windows' input calls must not keep the port. The loop must still end within 3 s of the Stop, and its socket must now be the one parked for the next owner. --- host/tests/input_stall.rs | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/host/tests/input_stall.rs b/host/tests/input_stall.rs index e26b76f9..bfba0f50 100644 --- a/host/tests/input_stall.rs +++ b/host/tests/input_stall.rs @@ -222,7 +222,9 @@ fn input_windows_holds_never_stops_the_transport_loop() { assert!(answered, "the other iPad's HELLO went unanswered"); assert_eq!(verdicts, []); - // A stream stopped meanwhile ends at once, and frees its port. + // A stream stopped meanwhile ends at once, and hands its port on: the + // socket stays bound for the next generation or the responder that + // answers while nothing streams, not with a loop waiting for Windows. shared.stop(); drop(nal_tx); let stopping = Instant::now(); @@ -234,19 +236,12 @@ fn input_windows_holds_never_stops_the_transport_loop() { std::thread::sleep(Duration::from_millis(10)); } transport.join().unwrap(); - // On the address the transport bound, the wildcard by default: a - // narrower bind can clash with a socket the wildcard one left closing. - let ip = eternal_host::network::listen_ip_now(&shared.listen_on.lock()); - let freeing = Instant::now(); - let freed = loop { - match UdpSocket::bind((ip, port)) { - Ok(_) => break Ok(()), - Err(error) if freeing.elapsed() >= Duration::from_secs(3) => break Err(error), - Err(_) => std::thread::sleep(Duration::from_millis(20)), - } - }; - if let Err(error) = freed { - panic!("the stopped stream's port {ip}:{port} is still held 3 s after it ended: {error}"); - } + let parked = shared + .stream_socket + .lock() + .as_ref() + .and_then(|socket| socket.local_addr().ok()) + .map(|addr| addr.port()); + assert_eq!(parked, Some(port), "the stopped loop handed its port on"); hold::release(); } From 668ec4a184395d26271dfc4cf513f01ca36dee02 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 21:27:00 -0500 Subject: [PATCH 115/226] Keep the window's surface out of the device replacement thread The replacement thread got a surface made from the window's handles, which it could still query after a quit destroyed the window, against SurfaceTargetUnsafe's contract. It now gets only the configuration and the instance: RenderState::new_device lists adapters, chooses one as at start but with no surface to check, and makes the device. Back on the window's thread, RenderState::with_new_device makes the surface the safe way, checks the new adapter can draw on it (a failed try if not) and makes the renderer. The unsafe surface is gone, and a stuck thread still holds no window, so the app can quit. --- host/tests/window_device_loss.rs | 21 ++++++- vendor/README.md | 29 +++++----- vendor/egui-wgpu/src/lib.rs | 90 ++++++++++++++++++++++++++++++ vendor/egui-wgpu/src/winit.rs | 95 +++++++++++++++----------------- 4 files changed, 170 insertions(+), 65 deletions(-) diff --git a/host/tests/window_device_loss.rs b/host/tests/window_device_loss.rs index c2b0305a..cacf3fd5 100644 --- a/host/tests/window_device_loss.rs +++ b/host/tests/window_device_loss.rs @@ -8,7 +8,9 @@ use std::time::{Duration, Instant}; use eframe::egui; use eframe::egui_wgpu::winit::{Answer, Asked, Replacement, REPLACEMENT_TIMEOUT}; -use eframe::egui_wgpu::{Renderer, RendererOptions, ScreenDescriptor}; +use eframe::egui_wgpu::{ + RenderState, Renderer, RendererOptions, ScreenDescriptor, WgpuConfiguration, +}; use eframe::wgpu; fn block_on(future: F) -> F::Output { @@ -228,3 +230,20 @@ fn a_replacement_that_never_answers_ends_in_giving_up() { } assert_eq!(timeouts, Replacement::MAX_FAILURES); } + +#[test] +fn a_replacement_device_is_made_without_any_window_or_surface() { + // What the replacement thread runs gets the configuration and the + // instance only, so a thread stuck in a driver can't hold the window or + // touch it after it is gone. + let config = WgpuConfiguration::default(); + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); + let new = block_on(RenderState::new_device(&config, &instance)).expect("a device"); + let buffer = new.device().create_buffer(&wgpu::BufferDescriptor { + label: None, + size: 256, + usage: wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + assert_eq!(buffer.size(), 256); +} diff --git a/vendor/README.md b/vendor/README.md index 9c00fcdf..6f6bbd86 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -30,19 +30,22 @@ The patch: its own devices and is not affected. The new device's first frame uploads the font atlas again. Other textures are the app's to load again (`host/src/gui/renderer.rs` counts devices for that). -- A replacement never blocks the window's thread. The window makes the new - surface (it belongs to the window, and Metal wants it on the main thread). - Listing adapters and making the device, where a driver can stall, run on - their own thread (`winit::Answer`), which the window polls each frame. One - that hasn't answered in 10 s counts as a failed try, and no other starts - while it is out; a late device is still taken. The surface it gets is - made from the window's handles, so a stuck thread doesn't keep the window - alive: eframe ends its loop only once the window is destroyed. The thread - is named `winit::REPLACEMENT_THREAD`, so the host's panic hook treats a panic there - as one failed try: it doesn't turn the virtual display off as it does for - other background threads. Configuring the new surface, - which makes the swapchain, stays on the window's thread, because DXGI sends - the window messages while it does that. +- A replacement never blocks the window's thread. Listing adapters and + making the device, where a driver can stall, run on their own thread + (`winit::Answer`, `RenderState::new_device`), which the window polls each + frame. That thread gets no window or surface: the adapter is chosen as at + start but with no surface to check. So a thread stuck in a driver neither + keeps the window alive (eframe ends its loop only once the window is + destroyed) nor touches it after it is gone; a late answer after quitting is + dropped with the closed channel. One that hasn't answered in 10 s counts as + a failed try, and no other starts while it is out; a late device is still + taken. The window's thread then makes the surface, checks the new adapter + can draw on it (a failed try if not), makes the renderer for its format + (`RenderState::with_new_device`) and configures it. The swapchain stays on + that thread because DXGI sends the window messages while it makes one. + The thread is named `winit::REPLACEMENT_THREAD`, so the host's panic hook + treats a panic there as one failed try: it doesn't turn the virtual display + off as it does for other background threads. - `configure_surface` replaces the device, instead of panicking, when the adapter supports no surface any more. - `Renderer::upload_failed`, `winit::SKIPPED_FRAMES` and `winit::DEVICES_LOST` diff --git a/vendor/egui-wgpu/src/lib.rs b/vendor/egui-wgpu/src/lib.rs index 1eb975ae..da2731e8 100644 --- a/vendor/egui-wgpu/src/lib.rs +++ b/vendor/egui-wgpu/src/lib.rs @@ -296,6 +296,96 @@ impl RenderState { } } +/// EternalMonitor patch: an adapter and device to replace a lost one, made without the +/// window's surface: the thread that makes them, where a driver can stall, never touches +/// the window. The window's thread finishes it with [`RenderState::with_new_device`]. +#[cfg(not(target_arch = "wasm32"))] +pub struct NewDevice { + adapter: wgpu::Adapter, + available_adapters: Vec, + device: wgpu::Device, + queue: wgpu::Queue, +} + +#[cfg(not(target_arch = "wasm32"))] +impl NewDevice { + pub fn device(&self) -> &wgpu::Device { + &self.device + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl RenderState { + /// EternalMonitor patch: list adapters, choose one as [`Self::create`] does but with no + /// surface to check against, and make its device. Takes no window or surface, so it + /// can run on a thread that may outlive the window. + pub async fn new_device( + config: &WgpuConfiguration, + instance: &wgpu::Instance, + ) -> Result { + let WgpuSetup::CreateNew(create_new) = &config.wgpu_setup else { + return Err(WgpuError::CustomNativeAdapterSelectionError( + "a device the app made can't be replaced".to_owned(), + )); + }; + let available_adapters = instance + .enumerate_adapters(create_new.instance_descriptor.backends) + .await; + let adapter = if let Some(selector) = &create_new.native_adapter_selector { + selector(&available_adapters, None) + .map_err(WgpuError::CustomNativeAdapterSelectionError)? + } else { + request_adapter( + instance, + create_new.power_preference, + None, + &available_adapters, + ) + .await? + }; + let (device, queue) = adapter + .request_device(&(*create_new.device_descriptor)(&adapter)) + .await?; + Ok(NewDevice { + adapter, + available_adapters, + device, + queue, + }) + } + + /// EternalMonitor patch: finish `new` for the window's `surface`, on the window's + /// thread: the adapter must be able to draw on it, and the renderer is made for its + /// format. + pub fn with_new_device( + new: NewDevice, + config: &WgpuConfiguration, + instance: &wgpu::Instance, + surface: &wgpu::Surface<'_>, + options: RendererOptions, + ) -> Result { + if !new.adapter.is_surface_supported(surface) { + return Err(WgpuError::CustomNativeAdapterSelectionError( + "the new adapter can't draw on the window".to_owned(), + )); + } + log_adapter_info(&new.adapter.get_info()); + let target_format = + preferred_framebuffer_format(&surface.get_capabilities(&new.adapter).formats)?; + let renderer = Renderer::new(&new.device, target_format, options); + Ok(Self { + instance: instance.clone(), + adapter: new.adapter, + available_adapters: new.available_adapters, + device: new.device, + queue: new.queue, + target_format, + renderer: Arc::new(RwLock::new(renderer)), + surface_config: config.surface, + }) + } +} + fn describe_adapters(adapters: &[wgpu::Adapter]) -> String { if adapters.is_empty() { "(none)".to_owned() diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs index ea9be482..e928ddbf 100644 --- a/vendor/egui-wgpu/src/winit.rs +++ b/vendor/egui-wgpu/src/winit.rs @@ -109,12 +109,8 @@ fn block_on(future: F) -> F::Output { } } -/// EternalMonitor patch: a replacement device being made on another thread, with the -/// surface it was made for. -type Replacing = Answer<( - wgpu::Surface<'static>, - Result, -)>; +/// EternalMonitor patch: a replacement device being made on another thread. +type Replacing = Answer>; /// EternalMonitor patch: when the painter tries to replace a lost device, and when it /// stops trying. @@ -505,13 +501,14 @@ impl Painter { /// EternalMonitor patch: work towards a new device for a lost one, without blocking. /// - /// The surface is made here, on the window's thread: it is the window's (Metal wants - /// it on the main thread) and calls no driver for a device. Listing adapters and - /// making the device, where a driver can stall, run on another thread, which this - /// polls. One that hasn't answered in [`REPLACEMENT_TIMEOUT`] counts as a failed try, - /// and no other starts while it is out; its late device is still taken. Configuring - /// the new surface, which makes the swapchain, stays here too: DXGI sends the window - /// messages while it does. + /// Listing adapters and making the device, where a driver can stall, run on another + /// thread, which this polls. That thread gets no window or surface, so one stuck in a + /// driver neither keeps the window alive (eframe ends its loop only once the window is + /// destroyed) nor touches it after it is gone. One that hasn't answered in + /// [`REPLACEMENT_TIMEOUT`] counts as a failed try, and no other starts while it is out; + /// its late device is still taken. On this thread the window's surface is then made, + /// checked against the new adapter and configured (DXGI sends the window messages while + /// it makes the swapchain), and the renderer is made; a mismatch is a failed try. fn replace_device(&mut self, viewport_id: ViewportId, window: Arc) { let now = Instant::now(); if let Some(replacing) = &mut self.replacing { @@ -522,18 +519,14 @@ impl Painter { self.replacing = None; self.replacement_failed(now, &"the thread making it ended"); } - Asked::Answered((surface, Ok(render_state))) => { + Asked::Answered(Ok(new)) => { self.replacing = None; - self.watch_device(&render_state.device); - self.render_state = Some(render_state); - let size = window.inner_size(); - self.install_surface(surface, viewport_id, size.width, size.height, false); - self.rebuilding = false; - self.restore_fonts = true; - log::info!("Replaced the lost wgpu device"); - return; + match self.adopt(new, viewport_id, window) { + Ok(()) => return, + Err(err) => self.replacement_failed(now, &err), + } } - Asked::Answered((_, Err(err))) => { + Asked::Answered(Err(err)) => { self.replacing = None; self.replacement_failed(now, &err); } @@ -551,35 +544,9 @@ impl Painter { } Some(_) => {} } - // The surface goes to the replacement thread, which a driver can hold forever. Made - // from the window's handles, it doesn't keep the window alive: eframe ends its loop - // only once the window is destroyed, so a surface holding it would keep the app from - // quitting. - // SAFETY: the surface is used while the window lives: here and by the replacement - // thread, and on this thread once installed. The window is the root viewport, - // destroyed only as the app quits, after which a stuck thread can only drop the - // surface, never configured. - let made = || -> Result, crate::WgpuError> { - let target = unsafe { wgpu::SurfaceTargetUnsafe::from_window(&*window) }?; - Ok(unsafe { self.instance.create_surface_unsafe(target) }?) - }; - let surface = match made() { - Ok(surface) => surface, - Err(err) => { - self.replacement_failed(now, &err); - return; - } - }; - let (config, instance, options) = - (self.config.clone(), self.instance.clone(), self.options); + let (config, instance) = (self.config.clone(), self.instance.clone()); let asked = Answer::ask(REPLACEMENT_THREAD, now, REPLACEMENT_TIMEOUT, move || { - let made = block_on(RenderState::create( - &config, - &instance, - Some(&surface), - options, - )); - (surface, made) + block_on(RenderState::new_device(&config, &instance)) }); match asked { Ok(replacing) => { @@ -590,6 +557,32 @@ impl Painter { } } + /// EternalMonitor patch: on the window's thread, make the window's surface for a device + /// from the replacement thread, check them together and draw with them. + fn adopt( + &mut self, + new: crate::NewDevice, + viewport_id: ViewportId, + window: Arc, + ) -> Result<(), crate::WgpuError> { + let size = window.inner_size(); + let surface = self.instance.create_surface(window)?; + let render_state = RenderState::with_new_device( + new, + &self.config, + &self.instance, + &surface, + self.options, + )?; + self.watch_device(&render_state.device); + self.render_state = Some(render_state); + self.install_surface(surface, viewport_id, size.width, size.height, false); + self.rebuilding = false; + self.restore_fonts = true; + log::info!("Replaced the lost wgpu device"); + Ok(()) + } + /// EternalMonitor patch: a try at a new device failed at `now`. fn replacement_failed(&mut self, now: Instant, why: &dyn core::fmt::Display) { if self.replacement.failed(now) { From 57d3203898f4e5d99a67a8f3e9457fa5a8d491d3 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 21:28:32 -0500 Subject: [PATCH 116/226] Link a USB session to the PC's own service, and keep other PCs' proofs Over USB a session has no address, so it linked no service. The same PC seen by the search got a second card within three days, and a stale TXT record could bring a card back after the PC proved current over USB. - A session with no tapped or listed service at its address now links the service every PC registers, "EternalMonitor ()". - A service already linked to a PC with a proof of its own stays with it, and a record with a proof is never removed by a link. - At most twice as many links as PCs are kept, the ones whose PCs were seen last. --- ios/EternalMonitor/Core/Feedback.swift | 7 +++ ios/EternalMonitor/Core/OldPC.swift | 48 ++++++++++++------ ios/EternalMonitorTests/OldPCTests.swift | 63 +++++++++++++++++++++++- 3 files changed, 100 insertions(+), 18 deletions(-) diff --git a/ios/EternalMonitor/Core/Feedback.swift b/ios/EternalMonitor/Core/Feedback.swift index 481597f1..1a3c7e23 100644 --- a/ios/EternalMonitor/Core/Feedback.swift +++ b/ios/EternalMonitor/Core/Feedback.swift @@ -191,6 +191,13 @@ enum PCVersions { address.flatMap { byKey[$0] } ?? name.flatMap { byKey[$0] } } + #if DEBUG + static func forget(name: String, address: String?) { + byKey[name] = nil + if let address { byKey[address] = nil } + } + #endif + /// The `version` entry of a Bonjour TXT record. nonisolated static func version(fromTXT data: Data?) -> String? { guard let data else { return nil } diff --git a/ios/EternalMonitor/Core/OldPC.swift b/ios/EternalMonitor/Core/OldPC.swift index 01f343ed..e3dff1ac 100644 --- a/ios/EternalMonitor/Core/OldPC.swift +++ b/ios/EternalMonitor/Core/OldPC.swift @@ -38,11 +38,12 @@ enum OldPC { /// /// A PC is known by its pairing token (hashed) when it has one, otherwise by /// its Bonjour service name, which is unique on a network, and last by the -/// name it answers with. A service the user tapped, or the one the search -/// lists at the session's address right now, is linked to the PC that -/// answered there. What a session showed about a PC outlasts the session: -/// a PC that proved current gets no card from a later Bonjour record. Links -/// only ever keep a card away. +/// name it answers with. A session links one service to the PC that +/// answered: the one the user tapped, the one the search lists at the +/// session's address right now, or else the PC's own. What a session showed +/// about a PC outlasts the session: a PC that proved current gets no card +/// from a later Bonjour record. Links only ever keep a card away, and never +/// take a service, or a proof, from another PC. @MainActor final class OldPCPrompt: ObservableObject { static let shared = OldPCPrompt() @@ -99,7 +100,10 @@ final class OldPCPrompt: ObservableObject { var memory = memory let dropped = memory.pcs.sorted { $0.value.seenAt > $1.value.seenAt }.dropFirst(Self.kept).map(\.key) for key in dropped { memory.pcs[key] = nil } - memory.services = memory.services.filter { memory.pcs[$0.value] != nil } + // Links to the PCs seen last, a few per PC at most. + let linked = memory.services.filter { memory.pcs[$0.value] != nil } + .sorted { (memory.pcs[$0.value]!.seenAt, $1.key) > (memory.pcs[$1.value]!.seenAt, $0.key) } + memory.services = Dictionary(uniqueKeysWithValues: linked.prefix(Self.kept * 2).map { ($0.key, $0.value) }) defaults.set(try? JSONEncoder().encode(memory), forKey: Self.defaultsKey) } @@ -135,29 +139,41 @@ final class OldPCPrompt: ObservableObject { /// A PC accepted a session. `hostCaps` is from that accepted HELLO_ACK; /// `address` is nil over USB. + /// + /// The session's service is the one the user tapped at that address, + /// else the one the search lists there, else the PC's own: every PC + /// registers "EternalMonitor ()". A service already linked to + /// a PC with a proof of its own stays with it. func answered(hostName: String, pairingToken: Data, hostCaps: UInt16, address: String?) { var memory = self.memory - let service = address.flatMap { address in + let found = address.flatMap { address in tapped.flatMap { $0.address == address ? $0.service : nil } ?? listed.first { $0.address == address }?.name } tapped = nil + var service = found ?? (hostName.isEmpty ? nil : "EternalMonitor (\(hostName))") + var linkedTo: String? + if let candidate = service { + let key = Self.key(hostName: hostName, pairingToken: pairingToken, service: candidate) + let target = memory.services[candidate] ?? Self.serviceKey(candidate) + if target == key || memory.pcs[target]?.current == nil { + linkedTo = target + } else { + service = nil + } + } let key = Self.key(hostName: hostName, pairingToken: pairingToken, service: service) var pc = memory.pcs[key] ?? Memory.PC(seenAt: now()) - if let service { + if let service, let old = linkedTo { let serviceKey = Self.serviceKey(service) - let before = [memory.services[service], serviceKey].compactMap { $0 }.filter { $0 != key } - for old in before { + if old != key { if let shown = memory.pcs[old]?.shownAt { pc.shownAt = max(pc.shownAt ?? shown, shown) } if waiting?.key == old { waiting = nil } if let shown = card, shown.key == old { card = Card(key: key, pc: shown.pc, version: shown.version) } + // Only a record without a proof of its own goes. + if old == serviceKey { memory.pcs[old] = nil } } - if key == serviceKey { - memory.services[service] = nil - } else { - memory.pcs[serviceKey] = nil - memory.services[service] = key - } + memory.services[service] = key == serviceKey ? nil : key } let old = OldPC.isOld(announced: nil, hostCaps: hostCaps) == true pc.current = !old diff --git a/ios/EternalMonitorTests/OldPCTests.swift b/ios/EternalMonitorTests/OldPCTests.swift index d58e1db9..8f862920 100644 --- a/ios/EternalMonitorTests/OldPCTests.swift +++ b/ios/EternalMonitorTests/OldPCTests.swift @@ -205,6 +205,7 @@ final class OldPCTests: XCTestCase { // An old PC was found at 192.0.2.40 earlier and left. PCVersions.record("0.3.2", name: "EternalMonitor (Gone)", address: "192.0.2.40") + addTeardownBlock { await MainActor.run { PCVersions.forget(name: "EternalMonitor (Gone)", address: "192.0.2.40") } } let scanner = NetworkScanner() scanner.resolveIPv4 = { _ in "192.0.2.40" } let beta = DiscoveredHost.resolved(name: "EternalMonitor (New)", port: 9876, addresses: [ipv6], @@ -214,6 +215,7 @@ final class OldPCTests: XCTestCase { scanner.lookUpIPv4(hostName: "new.local.", service: beta) let deadline = Date().addingTimeInterval(10) while scanner.hosts.first?.address == nil, Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertNotNil(scanner.hosts.first?.address, "the IPv4 lookup didn't answer") XCTAssertEqual(scanner.hosts.map(\.address), ["192.0.2.40"]) XCTAssertEqual(scanner.hosts.first?.version, "0.3.4-beta.20261107.1") @@ -230,7 +232,8 @@ final class OldPCTests: XCTestCase { scanner.resolveIPv4 = { _ in "192.0.2.41" } scanner.merge(old) scanner.lookUpIPv4(hostName: "old.local.", service: old) - while scanner.hosts.first(where: { $0.name == old.name })?.address == nil, Date() < deadline { + let oldDeadline = Date().addingTimeInterval(10) + while scanner.hosts.first(where: { $0.name == old.name })?.address == nil, Date() < oldDeadline { try await Task.sleep(nanoseconds: 10_000_000) } prompt.discovered(scanner.hosts) @@ -287,7 +290,7 @@ final class OldPCTests: XCTestCase { var lines: [String] = [] let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { lines.append($0) }) prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.4")]) - prompt.answered(hostName: "Office", pairingToken: tokenB, hostCaps: currentCaps, address: nil) + prompt.answered(hostName: "Den", pairingToken: tokenC, hostCaps: currentCaps, address: nil) prompt.show() XCTAssertNil(prompt.card) @@ -305,6 +308,55 @@ final class OldPCTests: XCTestCase { XCTAssertEqual(lines, ["old_pc_prompt shown=true version=0.3.2"]) } + /// r2: a PC streaming over USB has no address to link, so its session + /// links the service every PC registers, "EternalMonitor ()". + /// The same PC over USB and in the search gets one card, in either + /// order, and a proof over USB holds against its stale TXT record. + func testAUSBSessionAndItsOwnServiceAreOnePC() { + for usbFirst in [true, false] { + var lines: [String] = [] + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { lines.append($0) }) + let steps: [() -> Void] = [ + { prompt.answered(hostName: "Studio", pairingToken: self.tokenA, hostCaps: self.oldCaps, address: nil) }, + { prompt.discovered([self.service("Studio", at: "192.0.2.10", version: "0.3.3")]) }, + ] + for step in usbFirst ? steps : steps.reversed() { + step() + prompt.show() + prompt.close() + } + XCTAssertEqual(lines.count, 1, "usbFirst \(usbFirst): \(lines)") + } + + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: currentCaps, address: nil) + prompt.discovered([service("Studio", at: "192.0.2.10", version: "0.3.3")]) + prompt.show() + XCTAssertNil(prompt.card, "proved current over USB") + } + + /// r2: a link never takes a service, or a proof, from another PC. Here + /// the search still lists A at an address B answers at now. + func testALinkNeverTakesAnotherPCsProof() { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + let a = service("A", at: "192.0.2.60", version: nil) + prompt.discovered([a]) + prompt.answered(hostName: "A", pairingToken: Data(count: 16), hostCaps: currentCaps, address: "192.0.2.60") + let proof = OldPCPrompt.serviceKey(a.name) + XCTAssertEqual(prompt.memory.pcs[proof]?.current, true) + + prompt.answered(hostName: "B", pairingToken: tokenB, hostCaps: oldCaps, address: "192.0.2.60") + let memory = prompt.memory + XCTAssertEqual(memory.pcs[proof]?.current, true, "A's proof stays") + XCTAssertNil(memory.services[a.name], "A's service isn't B's") + prompt.show() + XCTAssertEqual(prompt.card?.pc, "B", "B is old") + prompt.close() + prompt.discovered([service("A", at: "192.0.2.61", version: "0.3.3")]) + prompt.show() + XCTAssertNil(prompt.card, "A proved current") + } + /// Only the last few PCs are kept, with their links. func testTheMemoryIsBounded() { var now = Date(timeIntervalSince1970: 1_000_000) @@ -322,6 +374,13 @@ final class OldPCTests: XCTestCase { XCTAssertEqual(memory.services.count, OldPCPrompt.kept) XCTAssertNil(memory.services["EternalMonitor (PC 0)"]) XCTAssertNotNil(memory.services["EternalMonitor (PC \(OldPCPrompt.kept + 3))"]) + + // One PC found under many names keeps a few links. + for index in 0..<(OldPCPrompt.kept * 3) { + prompt.connecting(service: "EternalMonitor (Alias \(index))", address: "198.51.100.\(index)") + prompt.answered(hostName: "Alias", pairingToken: tokenA, hostCaps: oldCaps, address: "198.51.100.\(index)") + } + XCTAssertEqual(prompt.memory.services.count, OldPCPrompt.kept * 2) } /// One session with `host`, answered with `status` and `caps`. From c1dea61893c1b8e3d3f689469dd8a4af7b81d041 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 21:29:27 -0500 Subject: [PATCH 117/226] Say in the architecture notes when the second screen sends the pointer apart --- ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 63cfc75d..9159a5be 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -75,7 +75,7 @@ Rust and Swift share golden vectors and stable packet-type registry tests. | KEYFRAME_REQUEST | Recovery after repair expiry or decoder error | | PING / PONG | Min-RTT clock estimate for end-to-end latency | | INPUT_EVENT | Normalized mouse/touch events, HID keys and UTF-16 text | -| POINTER_SHAPE / POINTER_STATE | The Mirror's pointer image (chunked premultiplied BGRA and an invert mask) and its position, at most once a frame budget, for an iPad that draws the pointer itself (feature bit 5, host capability bit 6); the video then leaves the pointer out. The Second screen keeps the pointer in its video. | +| POINTER_SHAPE / POINTER_STATE | The Mirror's pointer image (chunked premultiplied BGRA and an invert mask) and its position, at most once a frame budget, for an iPad that draws the pointer itself (feature bit 5, host capability bit 6); the video then leaves the pointer out. The Second screen does the same when the virtual display driver's settings give Windows a hardware cursor, as the host writes them; otherwise its pointer stays in the video. | | Audio | Sequenced Opus packets with discontinuity and capture timing | | BYE | User, background or shutdown teardown; the host ending a silent session | From 7d8fa2ab9f536b0b849920cbe2a1aa4095cd5b2a Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 21:36:25 -0500 Subject: [PATCH 118/226] Link a PC's own service by name only when it is unambiguous No PC sends anything in Bonjour that ties a service to its pairing, so a session without an address match has only the PC's name. Two PCs that both answer as "Lab" are listed as "EternalMonitor (Lab)" and "EternalMonitor (Lab) (2)". A link made by name alone could hand one PC's proof to the other, miss a PC's suffixed service, or outlive a rename. - A name now links a service only while the search lists exactly one "EternalMonitor ()", with or without a collision suffix, and nothing while it lists two or more. The check runs again whenever the list or a session changes. - Links made by name last for this launch only and are never saved. A PC that answers under a new name gives up its old one. - Known limit: two PCs with one name on a network, one of them used over USB, can get one extra or one missed card per PC every three days. - The card shows a suffixed service as "Lab (2)". --- ios/EternalMonitor/Core/OldPC.swift | 93 ++++++++++++++++++++---- ios/EternalMonitorTests/OldPCTests.swift | 73 +++++++++++++++++++ 2 files changed, 150 insertions(+), 16 deletions(-) diff --git a/ios/EternalMonitor/Core/OldPC.swift b/ios/EternalMonitor/Core/OldPC.swift index e3dff1ac..ccf20328 100644 --- a/ios/EternalMonitor/Core/OldPC.swift +++ b/ios/EternalMonitor/Core/OldPC.swift @@ -24,11 +24,21 @@ enum OldPC { return hostCaps == nil ? nil : true } - /// The name a PC gives itself, from its Bonjour service name. + /// The name a PC gives itself, from its Bonjour service name, with + /// Bonjour's collision number if it has one: "Lab (2)". static func pcName(bonjour name: String) -> String { - let prefix = "EternalMonitor (", suffix = ")" - guard name.hasPrefix(prefix), name.hasSuffix(suffix), name.count > prefix.count + suffix.count else { return name } - return String(name.dropFirst(prefix.count).dropLast(suffix.count)) + let prefix = "EternalMonitor (" + guard name.hasPrefix(prefix), name.hasSuffix(")") else { return name } + var inner = name.dropFirst(prefix.count).dropLast() + var number = "" + if let open = inner.range(of: ") (", options: .backwards) { + let digits = inner[open.upperBound...] + if !digits.isEmpty, digits.allSatisfy({ $0.isASCII && $0.isNumber }) { + number = " (\(digits))" + inner = inner[..)", with or without a collision suffix such as +/// " (2)", for this launch only, and nothing while the search lists two or +/// more. Known limit: two PCs with one name on a network, one of them used +/// over USB, can get one extra or one missed card per PC every three days. @MainActor final class OldPCPrompt: ObservableObject { static let shared = OldPCPrompt() @@ -81,6 +99,10 @@ final class OldPCPrompt: ObservableObject { private var listed: [DiscoveredHost] = [] /// The service the user tapped, and its address. private var tapped: (service: String, address: String)? + /// This launch's sessions that no address linked, by PC name. + private var unlinked: [String: String] = [:] + /// Services linked by those sessions' names, for this launch only. + private var named: [String: String] = [:] private let defaults: UserDefaults private let now: () -> Date private let log: (String) -> Void @@ -111,13 +133,48 @@ final class OldPCPrompt: ObservableObject { /// The PC behind a Bonjour service. private func key(service: String, in memory: Memory) -> String { - memory.services[service] ?? Self.serviceKey(service) + memory.services[service] ?? named[service] ?? Self.serviceKey(service) + } + + /// Whether `service` is the name a PC called `hostName` registers, or + /// that name with a Bonjour collision suffix. + static func isOwnService(_ service: String, of hostName: String) -> Bool { + let own = "EternalMonitor (\(hostName))" + if service == own { return true } + guard service.hasPrefix(own + " ("), service.hasSuffix(")") else { return false } + let number = service.dropFirst(own.count + 2).dropLast() + return !number.isEmpty && number.allSatisfy { $0.isASCII && $0.isNumber } + } + + /// Links each unlinked session's PC to its own service while the search + /// lists exactly one, and to nothing otherwise. + private func linkNames() { + var memory = self.memory + var changed = false + named.removeAll() + for (hostName, key) in unlinked { + let matches = listed.map(\.name).filter { Self.isOwnService($0, of: hostName) } + guard matches.count == 1, let service = matches.first, memory.services[service] == nil else { continue } + let target = Self.serviceKey(service) + guard target == key || memory.pcs[target]?.current == nil else { continue } + named[service] = key + guard target != key else { continue } + if let shown = memory.pcs[target]?.shownAt, var pc = memory.pcs[key] { + pc.shownAt = max(pc.shownAt ?? shown, shown) + memory.pcs[key] = pc + changed = true + } + if waiting?.key == target { waiting = nil } + if let shown = card, shown.key == target { card = Card(key: key, pc: shown.pc, version: shown.version) } + } + if changed { save(memory) } } /// The services the search lists now, each with the version its own /// TXT record announced. func discovered(_ hosts: [DiscoveredHost]) { listed = hosts + linkNames() let memory = self.memory for host in hosts { let key = key(service: host.name, in: memory) @@ -141,9 +198,9 @@ final class OldPCPrompt: ObservableObject { /// `address` is nil over USB. /// /// The session's service is the one the user tapped at that address, - /// else the one the search lists there, else the PC's own: every PC - /// registers "EternalMonitor ()". A service already linked to - /// a PC with a proof of its own stays with it. + /// else the one the search lists there. A service already linked to a PC + /// with a proof of its own stays with it. Without either, the PC's name + /// may link its own service for this launch (see the type's note). func answered(hostName: String, pairingToken: Data, hostCaps: UInt16, address: String?) { var memory = self.memory let found = address.flatMap { address in @@ -151,7 +208,7 @@ final class OldPCPrompt: ObservableObject { ?? listed.first { $0.address == address }?.name } tapped = nil - var service = found ?? (hostName.isEmpty ? nil : "EternalMonitor (\(hostName))") + var service = found var linkedTo: String? if let candidate = service { let key = Self.key(hostName: hostName, pairingToken: pairingToken, service: candidate) @@ -180,8 +237,12 @@ final class OldPCPrompt: ObservableObject { pc.seenAt = now() memory.pcs[key] = pc save(memory) + // A PC answers under one name at a time. + unlinked = unlinked.filter { $0.value != key } + if found == nil, !hostName.isEmpty { unlinked[hostName] = key } + linkNames() if old { - queue(Card(key: key, pc: hostName, version: nil), memory: memory) + queue(Card(key: key, pc: hostName, version: nil), memory: self.memory) } else { drop(key) } diff --git a/ios/EternalMonitorTests/OldPCTests.swift b/ios/EternalMonitorTests/OldPCTests.swift index 8f862920..e25ffd5b 100644 --- a/ios/EternalMonitorTests/OldPCTests.swift +++ b/ios/EternalMonitorTests/OldPCTests.swift @@ -76,6 +76,9 @@ final class OldPCTests: XCTestCase { XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor (OLD-PC)"), "OLD-PC") XCTAssertEqual(OldPC.pcName(bonjour: "Studio"), "Studio") XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor ()"), "EternalMonitor ()") + XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor (Lab) (2)"), "Lab (2)") + XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor (Lab (Home)) (12)"), "Lab (Home) (12)") + XCTAssertEqual(OldPC.pcName(bonjour: "EternalMonitor (Lab (Home))"), "Lab (Home)") } private func defaults() -> UserDefaults { @@ -335,6 +338,76 @@ final class OldPCTests: XCTestCase { XCTAssertNil(prompt.card, "proved current over USB") } + /// r3: a name links a service only when the search lists exactly one + /// for it. B and A both answer as "Lab"; Bonjour lists B as "Lab (2)". + /// B proving current over USB never hides A's card, in either order. + func testANameLinksOnlyAnUnambiguousService() { + let a = DiscoveredHost(name: "EternalMonitor (Lab)", address: "192.0.2.70", port: 9876, version: "0.3.2") + let b = DiscoveredHost(name: "EternalMonitor (Lab) (2)", address: "192.0.2.71", port: 9876, version: "0.3.4") + for usbFirst in [true, false] { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + let steps: [() -> Void] = [ + { prompt.answered(hostName: "Lab", pairingToken: self.tokenB, hostCaps: self.currentCaps, address: nil) }, + { prompt.discovered([a, b]) }, + ] + for step in usbFirst ? steps : steps.reversed() { step() } + prompt.show() + XCTAssertEqual(prompt.card?.key, OldPCPrompt.serviceKey(a.name), "usbFirst \(usbFirst): A's card") + } + + // With A gone, B's own suffixed service is the only one: it is B's, + // and B's proof holds against its stale TXT record. + let staleB = DiscoveredHost(name: b.name, address: b.address, port: 9876, version: "0.3.3") + for usbFirst in [true, false] { + let prompt = OldPCPrompt(defaults: defaults(), now: { Date() }, log: { _ in }) + let steps: [() -> Void] = [ + { prompt.answered(hostName: "Lab", pairingToken: self.tokenB, hostCaps: self.currentCaps, address: nil) }, + { prompt.discovered([staleB]) }, + ] + for step in usbFirst ? steps : steps.reversed() { step() } + prompt.show() + XCTAssertNil(prompt.card, "usbFirst \(usbFirst): B proved current") + XCTAssertEqual(prompt.memory.services, [:], "kept for this launch only") + } + + XCTAssertTrue(OldPCPrompt.isOwnService("EternalMonitor (Lab)", of: "Lab")) + XCTAssertTrue(OldPCPrompt.isOwnService("EternalMonitor (Lab) (2)", of: "Lab")) + XCTAssertTrue(OldPCPrompt.isOwnService("EternalMonitor (Lab) (12)", of: "Lab")) + for other in ["EternalMonitor (Lab2)", "EternalMonitor (Lab) ()", "EternalMonitor (Lab) (x)", + "EternalMonitor (Lab) (2) (3)", "EternalMonitor (Lab)(2)", "Lab"] { + XCTAssertFalse(OldPCPrompt.isOwnService(other, of: "Lab"), other) + } + } + + /// r3: a name a PC gave up is free for another PC: after a rename in the + /// same launch, and in any later launch. + func testANameIsNotKeptForAnotherPC() { + let store = defaults() + var prompt = OldPCPrompt(defaults: store, now: { Date() }, log: { _ in }) + prompt.discovered([service("Lab", at: "192.0.2.80", version: nil)]) + prompt.answered(hostName: "Lab", pairingToken: tokenA, hostCaps: currentCaps, address: nil) + prompt.discovered([service("Lab", at: "192.0.2.80", version: "0.3.3")]) + prompt.show() + XCTAssertNil(prompt.card, "linked while A is the only Lab") + XCTAssertEqual(prompt.memory.services, [:]) + + // A is renamed; another PC takes the name "Lab". + prompt.answered(hostName: "Studio", pairingToken: tokenA, hostCaps: currentCaps, address: nil) + prompt.discovered([service("Lab", at: "192.0.2.81", version: "0.3.2")]) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Lab", "same launch, after the rename") + prompt.close() + + let other = defaults() + prompt = OldPCPrompt(defaults: other, now: { Date() }, log: { _ in }) + prompt.discovered([service("Lab", at: "192.0.2.80", version: nil)]) + prompt.answered(hostName: "Lab", pairingToken: tokenA, hostCaps: currentCaps, address: nil) + prompt = OldPCPrompt(defaults: other, now: { Date() }, log: { _ in }) + prompt.discovered([service("Lab", at: "192.0.2.81", version: "0.3.2")]) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Lab", "a later launch") + } + /// r2: a link never takes a service, or a proof, from another PC. Here /// the search still lists A at an address B answers at now. func testALinkNeverTakesAnotherPCsProof() { From e9238ee998c06a21763eec13bcbb48933da2438d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 21:44:55 -0500 Subject: [PATCH 119/226] Hold the first picture's deadline however long the open takes to start The first open's 3 s wait was given as the time left when the open was asked for, and counted from when its thread started. On a loaded Windows PC starting that thread took 2.2 s, so the first picture came at 5.3 s. A BGRA open retried at YUV420 also started a fresh 3 s. The open now carries the instant the picture is due, and is waited for until then. --- host/src/encoder/mod.rs | 106 +++++++++++++++++++++++++++------------- host/src/stall.rs | 14 ++++++ 2 files changed, 85 insertions(+), 35 deletions(-) diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index a53e14dd..be23a719 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -319,7 +319,6 @@ fn run_encode_loop( if encoder_state.is_none() { let picture_by = *first_picture_by.get_or_insert_with(|| Instant::now() + FIRST_PICTURE_WAIT); - let wait = || picture_by.saturating_duration_since(Instant::now()); match open_session( &LAUNCH_FAILURES, &desired_encoder, @@ -328,7 +327,7 @@ fn run_encode_loop( desired_bitrate, target_fps, &mut input_mode, - first_open(&desired_encoder, wait()), + first_open(&desired_encoder, picture_by), &mut opening, ) { Ok(state) => { @@ -405,7 +404,7 @@ fn run_encode_loop( desired_bitrate, target_fps, &mut input_mode, - first_open(name, wait()), + first_open(name, picture_by), &mut opening, ); if let Err(error) = &opened { @@ -1307,9 +1306,11 @@ enum Opening { /// first open but libx264's. FFmpeg's errors while it opens are logged /// as warnings (see [`crate::diagnostics::while_probing`]). One in the /// lane is waited for only until the pipeline's first picture is due, - /// the time given (see [`FIRST_PICTURE_WAIT`]): then it goes on opening - /// in the background while libx264 streams. - Probe(Duration), + /// at the instant given (see [`FIRST_PICTURE_WAIT`]): then it goes on + /// opening in the background while libx264 streams. The instant holds + /// whatever came before the open: a fallback's earlier tries, a BGRA + /// open retried at YUV420, a call's thread that was slow to start. + Probe(Instant), /// The live session, which goes on if the reopen is given up on. The /// picture freezes until then, so the limit is shorter. Reopen, @@ -1395,15 +1396,13 @@ fn wait_or_park Instant>( opening: Opening, progress: &mut impl FnMut(), ) -> Result, crate::stall::Pending> { - let wait = match opening { - Opening::Probe(wait) => wait, - Opening::Behind => Duration::ZERO, - Opening::First | Opening::Reopen => Duration::MAX, + let waited = match opening { + Opening::Probe(by) => pending.wait_until(by, progress), + Opening::Behind => None, + Opening::First | Opening::Reopen => pending.wait_up_to(Duration::MAX, progress), }; - if !wait.is_zero() { - if let Some(result) = pending.wait_up_to(wait, progress) { - return Ok(result); - } + if let Some(result) = waited { + return Ok(result); } match pending.park() { Some(result) => Ok(result), @@ -1411,13 +1410,13 @@ fn wait_or_park Instant>( } } -/// A pipeline's first open of `encoder_name`, `wait` before its first -/// picture is due: anything but libx264 has another encoder after it. -fn first_open(encoder_name: &str, wait: Duration) -> Opening { +/// A pipeline's first open of `encoder_name`, whose first picture is due +/// `by`: anything but libx264 has another encoder after it. +fn first_open(encoder_name: &str, by: Instant) -> Opening { if encoder_name == "libx264" { Opening::First } else { - Opening::Probe(wait) + Opening::Probe(by) } } @@ -3747,7 +3746,7 @@ mod tests { #[test] fn ffmpeg_errors_are_warnings_only_while_an_encoder_is_one_of_several() { - let probed = failing_open(Opening::Probe(FIRST_PICTURE_WAIT)); + let probed = failing_open(Opening::Probe(Instant::now() + FIRST_PICTURE_WAIT)); assert!(!probed.is_empty()); assert!( probed @@ -3770,15 +3769,15 @@ mod tests { ); } // The pipeline's first open probes, except the last resort. - let wait = FIRST_PICTURE_WAIT; - assert_eq!(first_open("h264_nvenc", wait), Opening::Probe(wait)); - assert_eq!(first_open("hevc_amf", wait), Opening::Probe(wait)); - assert_eq!(first_open("libx265", wait), Opening::Probe(wait)); - assert_eq!(first_open("h264_videotoolbox", wait), Opening::Probe(wait)); - assert_eq!(first_open("libx264", wait), Opening::First); + let by = Instant::now() + FIRST_PICTURE_WAIT; + assert_eq!(first_open("h264_nvenc", by), Opening::Probe(by)); + assert_eq!(first_open("hevc_amf", by), Opening::Probe(by)); + assert_eq!(first_open("libx265", by), Opening::Probe(by)); + assert_eq!(first_open("h264_videotoolbox", by), Opening::Probe(by)); + assert_eq!(first_open("libx264", by), Opening::First); // A probe has the first open's limits. assert_eq!( - open_limits("h264_qsv", Opening::Probe(FIRST_PICTURE_WAIT)), + open_limits("h264_qsv", Opening::Probe(Instant::now())), open_limits("h264_qsv", Opening::First) ); } @@ -5132,26 +5131,58 @@ mod tests { (release, pending) } - /// Waits for the pipeline's first open of `name` as the encode loop - /// does, each 1 ms heartbeat moving the paused clock 100 ms on. + /// Waits for the pipeline's first open as the encode loop does, as + /// `opening` says, each 1 ms heartbeat moving the paused clock 100 ms on. fn first_open_of Instant>( pending: crate::stall::Pending<&'static str, N>, + opening: Opening, ms: &std::sync::atomic::AtomicU64, ) -> Result, crate::stall::Pending<&'static str, N>> { - wait_or_park(pending, Opening::Probe(FIRST_PICTURE_WAIT), &mut || { + wait_or_park(pending, opening, &mut || { ms.fetch_add(100, Ordering::SeqCst); }) } + #[test] + fn time_before_the_open_starts_counts_against_the_first_pictures_wait() { + let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); + let (ms, clock) = paused_clock(); + let opening = Opening::Probe(clock() + FIRST_PICTURE_WAIT); + // 2.2 s go by before the open's thread is running, as Windows once + // took to start one on a loaded PC. + ms.store(2_200, Ordering::SeqCst); + let (release, pending) = hung_open(lanes, "h264_nvenc", opening, clock.clone()); + let Err(parked) = first_open_of(pending, opening, &ms) else { + panic!("the hung open returned") + }; + // Parked when the picture was due, 3 s after it was asked for, not + // 3 s after the thread began. + assert_eq!(ms.load(Ordering::SeqCst), 3_000); + assert_eq!(parked.running_for(), Duration::from_millis(800)); + release.send(()).unwrap(); + drop(parked); + wait_for_clear_lane(lanes); + // Due before the open even starts: parked at once. + ms.store(4_000, Ordering::SeqCst); + let (release, pending) = hung_open(lanes, "h264_amf", opening, clock); + let Err(parked) = wait_or_park(pending, opening, &mut || panic!("it waited")) else { + panic!("returned") + }; + assert_eq!(parked.running_for(), Duration::ZERO); + release.send(()).unwrap(); + drop(parked); + wait_for_clear_lane(lanes); + } + #[test] fn a_hung_first_open_leaves_the_picture_to_libx264_when_it_is_due() { let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); let (ms, clock) = paused_clock(); let ((parked, opened_late), lines) = logged(|| { - let opening = Opening::Probe(FIRST_PICTURE_WAIT); + let opening = Opening::Probe(clock() + FIRST_PICTURE_WAIT); let (release, pending) = hung_open(lanes, "h264_qsv", opening, clock.clone()); - let Err(mut parked) = first_open_of(pending, &ms) else { + let Err(mut parked) = first_open_of(pending, opening, &ms) else { panic!("the hung open returned") }; // Given up waiting for when the first picture was due, 7 s @@ -5204,7 +5235,7 @@ mod tests { fn an_open_that_returns_before_the_first_picture_is_due_is_not_parked() { let lanes: &'static crate::stall::Lanes = Box::leak(Box::new(crate::stall::Lanes::new())); let (ms, clock) = paused_clock(); - let opening = Opening::Probe(FIRST_PICTURE_WAIT); + let opening = Opening::Probe(clock() + FIRST_PICTURE_WAIT); let (release, pending) = hung_open(lanes, "h264_nvenc", opening, clock); // NVENC's slowest open in the field: 2.6 s. The clock stops there, // however long the hand-over takes on a loaded machine. @@ -5223,8 +5254,13 @@ mod tests { assert!(beats >= 26, "{beats}"); // A deadline already past (an earlier encoder of the fallback used // the wait up) parks at once; so does a background reopen. - for opening in [Opening::Probe(Duration::ZERO), Opening::Behind] { + for behind in [false, true] { let (_ms, clock) = paused_clock(); + let opening = if behind { + Opening::Behind + } else { + Opening::Probe(clock()) + }; let (release, pending) = hung_open(lanes, "h264_amf", opening, clock); let Err(parked) = wait_or_park(pending, opening, &mut || panic!("it waited")) else { panic!("returned") @@ -5255,9 +5291,9 @@ mod tests { let t0 = Instant::now(); let at = |ms: u64| t0 + Duration::from_millis(ms); let mut recovery = StallRecovery::default(); - let opening = Opening::Probe(FIRST_PICTURE_WAIT); + let opening = Opening::Probe(clock() + FIRST_PICTURE_WAIT); let (release, pending) = hung_open(lanes, "h264_nvenc", opening, clock.clone()); - let Err(mut parked) = first_open_of(pending, &ms) else { + let Err(mut parked) = first_open_of(pending, opening, &ms) else { panic!("the hung open returned") }; recovery.outwaited("h264_nvenc", "libx264", at(3_000)); diff --git a/host/src/stall.rs b/host/src/stall.rs index 5d587bc8..5a263e8d 100644 --- a/host/src/stall.rs +++ b/host/src/stall.rs @@ -517,6 +517,20 @@ impl Instant> Pending { } } + /// [`Self::wait_up_to`] until `by` on the caller's clock, however long + /// the call took to start. None at once if `by` has passed. + pub(crate) fn wait_until( + &mut self, + by: Instant, + progress: &mut impl FnMut(), + ) -> Option> { + let up_to = by.saturating_duration_since(self.started); + if (self.now)() >= by { + return None; + } + self.wait_up_to(up_to, progress) + } + /// How long the call has run, by its caller's clock. pub(crate) fn running_for(&self) -> Duration { (self.now)().saturating_duration_since(self.started) From 4511f28d53e297af78a0db34b85cc72b7692ec49 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 21:54:46 -0500 Subject: [PATCH 120/226] Keep one showing for a PC linked by name across launches A name link lasts one launch, so a single old PC used over USB and seen by the search had two records: its pairing's and its service's. The card stamped only one of them, and the next launch showed the card again under the other. - A showing now also stamps the services this launch links to the PC by name. A name link made after the showing gives both records the later showing time. Only the time is saved, never the link. - Bonjour updates that change nothing no longer write to UserDefaults. --- ios/EternalMonitor/Core/OldPC.swift | 20 ++++++-- ios/EternalMonitorTests/OldPCTests.swift | 60 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/ios/EternalMonitor/Core/OldPC.swift b/ios/EternalMonitor/Core/OldPC.swift index ccf20328..77ef894e 100644 --- a/ios/EternalMonitor/Core/OldPC.swift +++ b/ios/EternalMonitor/Core/OldPC.swift @@ -159,10 +159,18 @@ final class OldPCPrompt: ObservableObject { guard target == key || memory.pcs[target]?.current == nil else { continue } named[service] = key guard target != key else { continue } - if let shown = memory.pcs[target]?.shownAt, var pc = memory.pcs[key] { - pc.shownAt = max(pc.shownAt ?? shown, shown) - memory.pcs[key] = pc - changed = true + // Both records keep the later showing, so a later launch, which + // has no link, holds the card back under either. Only the time + // is saved, never the link. + if let shown = [memory.pcs[target]?.shownAt, memory.pcs[key]?.shownAt].compactMap({ $0 }).max() { + if memory.pcs[key] != nil, memory.pcs[key]?.shownAt != shown { + memory.pcs[key]?.shownAt = shown + changed = true + } + if memory.pcs[target]?.shownAt != shown { + memory.pcs[target, default: Memory.PC(seenAt: now())].shownAt = shown + changed = true + } } if waiting?.key == target { waiting = nil } if let shown = card, shown.key == target { card = Card(key: key, pc: shown.pc, version: shown.version) } @@ -279,6 +287,10 @@ final class OldPCPrompt: ObservableObject { pc.shownAt = now() pc.seenAt = now() memory.pcs[next.key] = pc + // The services this launch links to it by name get the showing too. + for (service, key) in named where key == next.key { + memory.pcs[Self.serviceKey(service), default: Memory.PC(seenAt: now())].shownAt = now() + } save(memory) card = next log("old_pc_prompt shown=true version=\(next.version ?? "unknown")") diff --git a/ios/EternalMonitorTests/OldPCTests.swift b/ios/EternalMonitorTests/OldPCTests.swift index e25ffd5b..8d737852 100644 --- a/ios/EternalMonitorTests/OldPCTests.swift +++ b/ios/EternalMonitorTests/OldPCTests.swift @@ -379,6 +379,66 @@ final class OldPCTests: XCTestCase { } } + /// r4: one old PC used over USB and seen by the search gets one card + /// every three days, also in later launches, which have no name link. + /// The search's record may come before the card or after it. + func testAPCLinkedByNameGetsOneCardAcrossLaunches() { + let lab = service("Lab", at: "192.0.2.90", version: "0.3.3") + for listedFirst in [true, false] { + let store = defaults() + let start = Date(timeIntervalSince1970: 1_000_000) + var now = start + var lines: [String] = [] + let make = { OldPCPrompt(defaults: store, now: { now }, log: { lines.append($0) }) } + var prompt = make() + if listedFirst { prompt.discovered([lab]) } + prompt.answered(hostName: "Lab", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + prompt.show() + XCTAssertEqual(prompt.card?.key, OldPCPrompt.key(hostName: "Lab", pairingToken: tokenA, service: nil)) + prompt.close() + if !listedFirst { prompt.discovered([lab]) } + prompt.show() + XCTAssertNil(prompt.card) + + now = start + 86_400 + prompt = make() + prompt.discovered([lab]) + prompt.show() + XCTAssertNil(prompt.card, "listedFirst \(listedFirst): day 1, the search only") + + now = start + OldPCPrompt.interval + prompt = make() + prompt.discovered([lab]) + prompt.show() + XCTAssertEqual(prompt.card?.pc, "Lab", "listedFirst \(listedFirst): day 3") + XCTAssertEqual(lines.count, 2, "listedFirst \(listedFirst): \(lines)") + XCTAssertEqual(prompt.memory.services, [:]) + } + } + + /// Bonjour updates that change nothing write nothing. + func testUnchangedUpdatesWriteNothing() { + final class Counting: UserDefaults { + var writes = 0 + override func set(_ value: Any?, forKey defaultName: String) { + writes += 1 + super.set(value, forKey: defaultName) + } + } + let name = "OldPCTests-\(UUID().uuidString)" + let store = Counting(suiteName: name)! + addTeardownBlock { UserDefaults().removePersistentDomain(forName: name) } + let prompt = OldPCPrompt(defaults: store, now: { Date() }, log: { _ in }) + let lab = service("Lab", at: "192.0.2.90", version: "0.3.3") + prompt.discovered([lab]) + prompt.show() + prompt.close() + prompt.answered(hostName: "Lab", pairingToken: tokenA, hostCaps: oldCaps, address: nil) + let writes = store.writes + for _ in 0..<5 { prompt.discovered([lab]) } + XCTAssertEqual(store.writes, writes) + } + /// r3: a name a PC gave up is free for another PC: after a rename in the /// same launch, and in any later launch. func testANameIsNotKeptForAnotherPC() { From d24040445c6c4495f9142e9f33f6fbc4ee5df4da Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 21:57:05 -0500 Subject: [PATCH 121/226] Start pointer messages once the iPad has answered, and resume refused image chunks Pointer messages now wait until the iPad has answered, as audio does: until then nothing confirms it received its HELLO_ACK. Capture keeps the pointer in the video for it meanwhile, so it always has one. Over USB the tunnel refuses a datagram when its control queue (64) is full. An image of more chunks than that lost the rest and still counted as sent. The sender now keeps the image's next chunk, resumes there 5 ms after a refusal, and counts the image sent once every chunk went. --- host/src/control.rs | 9 ++ host/src/transport/mod.rs | 159 +++++++++++++++++++++--- host/src/transport/pointer.rs | 226 ++++++++++++++++++++++++++++++---- host/src/transport/session.rs | 37 ++++-- host/src/transport/usb.rs | 6 + host/tests/e2e_synthetic.rs | 96 +++++++++++++++ 6 files changed, 485 insertions(+), 48 deletions(-) diff --git a/host/src/control.rs b/host/src/control.rs index b1ace7b9..a65d12bc 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -382,6 +382,15 @@ mod tests { std::time::Instant::now(), ); assert!(shared.client_connected()); + assert!(!shared.client_draws_pointer(), "not before it answers"); + let session_id = shared.session.lock().session_id().unwrap(); + shared.session.lock().handle_control( + PeerId::udp("10.0.0.5:50000".parse().unwrap()), + session_id, + ControlMessage::ReceiverReport(Default::default()), + &Config, + std::time::Instant::now(), + ); assert_eq!(shared.client_draws_pointer(), draws, "caps {caps:#x}"); } } diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 4d789d8d..38d7439b 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -958,32 +958,48 @@ async fn send_pointer( } let budget = crate::capture::frame_budget_for(shared.effective_fps()); for send in sender.poll(now(), budget, shared.pointer.latest()) { - let messages = match send { - pointer::Send::Shape(shape) => { + match send { + pointer::Send::Shape(shape, from) => { let image = &shape.image; - PIPELINE_STATS.lock().pointer_shapes_sent += 1; - eternal_wire::v2::control::pointer_shape_chunks( + let chunks = eternal_wire::v2::control::pointer_shape_chunks( shape.id, image.source_type, image.flags, (image.width, image.height), image.hot, &image.bytes, - ) - .into_iter() - .map(ControlMessage::PointerShape) - .collect() + ); + let total = chunks.len(); + let mut next = from; + let mut refused = false; + for chunk in chunks.into_iter().skip(from) { + let message = ControlMessage::PointerShape(chunk); + let Some((peer, datagram)) = shared.session.lock().encode_for_client(&message) + else { + return; + }; + // A full USB control queue refuses it: resume here later. + if links.send_to(&datagram, peer).await.is_err() { + refused = true; + break; + } + next += 1; + } + if next >= total { + PIPELINE_STATS.lock().pointer_shapes_sent += 1; + } + sender.shape_progress(shape.id, next, total, refused, now()); } pointer::Send::State(state) => { PIPELINE_STATS.lock().pointer_states_sent += 1; - vec![ControlMessage::PointerState(state)] + let message = ControlMessage::PointerState(state); + let Some((peer, datagram)) = shared.session.lock().encode_for_client(&message) + else { + return; + }; + // A lost position is replaced by the next, or the repeat. + let _ = links.send_to(&datagram, peer).await; } - }; - for message in messages { - let Some((peer, datagram)) = shared.session.lock().encode_for_client(&message) else { - return; - }; - let _ = links.send_to(&datagram, peer).await; } } } @@ -1381,6 +1397,119 @@ mod tests { } } + /// Review r2: the USB tunnel's control queue takes 64 datagrams and + /// refuses more while its writer is blocked. A 256x256 image is 194 + /// chunks: they went out until the queue filled, the rest were dropped, + /// and the image counted as sent, so the iPad never had it whole. The + /// chunks now resume where the link refused them. + #[tokio::test] + async fn an_image_larger_than_the_usb_queue_arrives_whole_after_backpressure() { + use crate::capture::pointer::{PointerImage, Position}; + use eternal_wire::v2::control::{parse_control, FEATURE_DRAWS_POINTER}; + let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let shared = SharedControl::new(socket.local_addr().unwrap().port(), 15_000_000); + shared.pairing.lock().required = false; + let config = SharedConfigSource { + shared: &shared, + stream_epoch: 1, + pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), + }; + let mut links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); + let cable = PeerId::usb(5); + let (host_end, mut ipad_end) = tokio::io::duplex(4096); + let link = link::FramedLink::connect( + host_end, + cable, + std::sync::Arc::clone(&shared.force_next_idr), + ) + .await + .unwrap(); + links.adopt_for_test(std::sync::Arc::new(link)); + link::read_preamble(&mut ipad_end).await.unwrap(); + let hello = ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: 1, + listen_port: 0, + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_DRAWS_POINTER, + screen_px_w: 1920, + screen_px_h: 1080, + screen_pt_w: 960, + screen_pt_h: 540, + refresh_hz: 60, + device_name: "cable test".into(), + device_id: 0, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }); + shared + .session + .lock() + .handle_control(cable, 0, hello, &config, now()); + assert!( + shared.session.lock().pointer_session_id().is_some(), + "a cable counts as answered" + ); + let (side, total_bytes) = (256u16, 256usize * 256 * 4); + shared.pointer.set_shape(PointerImage { + source_type: 2, + flags: 0, + width: side, + height: side, + hot: (0, 0), + bytes: (0..total_bytes).map(|i| i as u8).collect(), + }); + shared.pointer.set_position(Position { + visible: true, + x: 10, + y: 10, + host_time_us: 0, + }); + let chunk_count = total_bytes.div_ceil(eternal_wire::v2::control::POINTER_CHUNK_MAX); + assert!(chunk_count > 64, "{chunk_count} chunks"); + + // Nobody reads the tunnel: the queue fills and refuses the rest. + let mut sender = pointer::PointerSender::default(); + send_pointer(&mut sender, &links, &shared).await; + let mut got = vec![false; chunk_count]; + let mut first = 0; + let deadline = Instant::now() + Duration::from_secs(10); + let mut rounds = 0; + while got.iter().any(|g| !g) { + assert!( + Instant::now() < deadline, + "only {} of {chunk_count} chunks arrived", + got.iter().filter(|g| **g).count() + ); + // Drain what the tunnel holds, then let the sender go on. + while let Ok(Ok(datagram)) = tokio::time::timeout( + Duration::from_millis(20), + link::read_datagram(&mut ipad_end), + ) + .await + { + if let Ok((_, ControlMessage::PointerShape(chunk))) = parse_control(&datagram) { + got[chunk.offset as usize / eternal_wire::v2::control::POINTER_CHUNK_MAX] = + true; + } + } + if rounds == 0 { + first = got.iter().filter(|g| **g).count(); + } + rounds += 1; + tokio::time::sleep(pointer::SHAPE_RETRY).await; + send_pointer(&mut sender, &links, &shared).await; + } + assert!( + first < chunk_count, + "the queue refused some at first ({first} of {chunk_count})" + ); + assert!(rounds > 1); + } + /// A cable pulled while a key and a mouse button are down ends the USB /// session, and both go up, as at every other end of a session. #[cfg(not(windows))] diff --git a/host/src/transport/pointer.rs b/host/src/transport/pointer.rs index f2623fd8..3bb8a61f 100644 --- a/host/src/transport/pointer.rs +++ b/host/src/transport/pointer.rs @@ -12,6 +12,9 @@ //! - An image goes when it changes, when a session starts, with every IDR //! the session asks for, and again when a report the iPad sent //! [`SHAPE_RESEND_AFTER`] or more after it went still names another one. +//! - An image's chunks resume where the link last refused one (the USB +//! tunnel's control queue holds 64), [`SHAPE_RETRY`] later. It counts as +//! sent once every chunk was accepted. use std::sync::Arc; use std::time::{Duration, Instant}; @@ -24,10 +27,14 @@ pub const REPEAT_AFTER: Duration = Duration::from_millis(100); /// Long enough for the image to reach the iPad before a report it sends, /// so a report already on its way doesn't send the image twice. pub const SHAPE_RESEND_AFTER: Duration = Duration::from_millis(250); +/// How long a refused image chunk waits before it is offered again. +pub const SHAPE_RETRY: Duration = Duration::from_millis(5); #[derive(Debug, PartialEq, Eq)] pub enum Send { - Shape(Arc), + /// The image, from this chunk on: report what went with + /// [`PointerSender::shape_progress`]. + Shape(Arc, usize), State(PointerState), } @@ -39,8 +46,12 @@ pub struct PointerSender { /// Where the grid of frame budgets starts. epoch: Option, repeat_at: Option, - /// The image sent last, and when. + /// The image sent last, whole, and when. shape_sent: Option<(u32, Instant)>, + /// The image being sent, and its next chunk. + pending: Option<(Arc, usize)>, + /// A chunk was refused: offer the image again from then. + retry_at: Option, shape_due: bool, /// The image the iPad's last report said it holds, and when it came. client_shape: Option<(u32, Instant)>, @@ -99,8 +110,20 @@ impl PointerSender { }; if changed || lagging || self.shape_due { self.shape_due = false; - self.shape_sent = Some((shape.id, now)); - out.push(Send::Shape(Arc::clone(shape))); + // Sending it already: carry on from where it got to. + if self + .pending + .as_ref() + .is_none_or(|(sending, _)| sending.id != shape.id) + { + self.pending = Some((Arc::clone(shape), 0)); + self.retry_at = None; + } + } + } + if let Some((sending, next)) = self.pending.as_ref() { + if self.retry_at.is_none_or(|at| now >= at) { + out.push(Send::Shape(Arc::clone(sending), *next)); } } let Some(position) = position else { @@ -134,6 +157,33 @@ impl PointerSender { out } + /// The transport sent the pending image's chunks up to `next` of + /// `total`; `refused` when the link refused chunk `next`. The image is + /// sent once every chunk was. + pub fn shape_progress( + &mut self, + id: u32, + next: usize, + total: usize, + refused: bool, + now: Instant, + ) { + let Some((sending, at)) = self.pending.as_mut() else { + return; + }; + if sending.id != id { + return; + } + if next >= total { + self.pending = None; + self.retry_at = None; + self.shape_sent = Some((id, now)); + return; + } + *at = next; + self.retry_at = refused.then_some(now + SHAPE_RETRY); + } + /// When [`Self::poll`] next has something to send without new input: /// the start of the next budget, for a newer position that waits for /// it, or a repeat. @@ -178,9 +228,14 @@ impl PointerSender { waiting.then(|| self.next_slot(budget)).flatten() } - /// When the last position goes again, no earlier than the next budget. + /// When the last position goes again, no earlier than the next budget, + /// or a refused image chunk is offered again. pub fn repeat_due(&self, budget: Duration) -> Option { - Some(self.repeat_at?.max(self.next_slot(budget)?)) + let repeat = self + .repeat_at + .and_then(|due| Some(due.max(self.next_slot(budget)?))); + let retry = self.pending.as_ref().and(self.retry_at); + [repeat, retry].into_iter().flatten().min() } fn next_slot(&self, budget: Duration) -> Option { @@ -223,11 +278,27 @@ mod tests { } } + /// Polls with a link that takes every chunk. + fn poll_all( + sender: &mut PointerSender, + now: Instant, + budget: Duration, + latest: (Option>, Option), + ) -> Vec { + let sent = sender.poll(now, budget, latest); + for send in &sent { + if let Send::Shape(shape, _) = send { + sender.shape_progress(shape.id, 1, 1, false, now); + } + } + sent + } + fn states(sent: &[Send]) -> Vec { sent.iter() .filter_map(|send| match send { Send::State(state) => Some(state.x), - Send::Shape(_) => None, + Send::Shape(..) => None, }) .collect() } @@ -261,7 +332,12 @@ mod tests { latest = Some(at(next.next().unwrap().1)); } let now = start + Duration::from_millis(ms); - for x in states(&sender.poll(now, BUDGET, (Some(shape(1)), latest))) { + for x in states(&poll_all( + &mut sender, + now, + BUDGET, + (Some(shape(1)), latest), + )) { sent.push((ms, x)); } // What the loop's high-resolution timer waits for: within one @@ -323,7 +399,13 @@ mod tests { let mut t = arrived; loop { let now = start + Duration::from_micros(t); - if states(&sender.poll(now, BUDGET, (Some(shape(1)), latest))).contains(&(k as i32)) + if states(&poll_all( + &mut sender, + now, + BUDGET, + (Some(shape(1)), latest), + )) + .contains(&(k as i32)) { delays.push(t - arrived); break; @@ -344,15 +426,23 @@ mod tests { let mut sender = PointerSender::default(); sender.serve(Some(1)); let latest = (Some(shape(1)), Some(at(5))); - assert_eq!(sender.poll(start, BUDGET, latest.clone()).len(), 2); assert_eq!( - states(&sender.poll(start + REPEAT_AFTER, BUDGET, latest.clone())), + poll_all(&mut sender, start, BUDGET, latest.clone()).len(), + 2 + ); + assert_eq!( + states(&poll_all( + &mut sender, + start + REPEAT_AFTER, + BUDGET, + latest.clone() + )), [5] ); for ms in 101..2000 { let now = start + Duration::from_millis(ms); assert!( - sender.poll(now, BUDGET, latest.clone()).is_empty(), + poll_all(&mut sender, now, BUDGET, latest.clone()).is_empty(), "{ms} ms" ); assert_eq!(sender.next_poll(BUDGET, latest.1, Some(1)), None); @@ -360,7 +450,7 @@ mod tests { let now = start + Duration::from_secs(2); sender.refresh(now); assert_eq!(sender.next_poll(BUDGET, latest.1, Some(1)), Some(now)); - assert_eq!(states(&sender.poll(now, BUDGET, latest)), [5]); + assert_eq!(states(&poll_all(&mut sender, now, BUDGET, latest)), [5]); } #[test] @@ -368,7 +458,7 @@ mod tests { let start = Instant::now(); let mut sender = PointerSender::default(); sender.serve(Some(1)); - sender.poll(start, BUDGET, (None, Some(at(5)))); + poll_all(&mut sender, start, BUDGET, (None, Some(at(5)))); let hidden = Position { visible: false, ..at(5) @@ -377,7 +467,7 @@ mod tests { sender.next_poll(BUDGET, Some(hidden), None), Some(start + BUDGET) ); - let sent = sender.poll(start + BUDGET, BUDGET, (None, Some(hidden))); + let sent = poll_all(&mut sender, start + BUDGET, BUDGET, (None, Some(hidden))); assert!(matches!(sent[..], [Send::State(state)] if state.flags & POINTER_VISIBLE == 0)); } @@ -388,7 +478,7 @@ mod tests { let shapes = |sent: &[Send]| { sent.iter() .filter_map(|send| match send { - Send::Shape(shape) => Some(shape.id), + Send::Shape(shape, _) => Some(shape.id), Send::State(_) => None, }) .collect::>() @@ -401,25 +491,40 @@ mod tests { ); sender.serve(Some(7)); assert_eq!( - shapes(&sender.poll(start, BUDGET, (Some(shape(1)), None))), + shapes(&poll_all( + &mut sender, + start, + BUDGET, + (Some(shape(1)), None) + )), [1] ); assert!(sender .poll(start, BUDGET, (Some(shape(1)), None)) .is_empty()); assert_eq!( - shapes(&sender.poll(start, BUDGET, (Some(shape(2)), None))), + shapes(&poll_all( + &mut sender, + start, + BUDGET, + (Some(shape(2)), None) + )), [2] ); // A report sent before image 2 could arrive names image 1: no resend. let soon = start + SHAPE_RESEND_AFTER - Duration::from_millis(1); sender.report(1, soon); - assert!(sender.poll(soon, BUDGET, (Some(shape(2)), None)).is_empty()); + assert!(poll_all(&mut sender, soon, BUDGET, (Some(shape(2)), None)).is_empty()); // A later one still names it: image 2 was lost on the way. let later = start + SHAPE_RESEND_AFTER; sender.report(1, later); assert_eq!( - shapes(&sender.poll(later, BUDGET, (Some(shape(2)), None))), + shapes(&poll_all( + &mut sender, + later, + BUDGET, + (Some(shape(2)), None) + )), [2] ); assert!( @@ -430,19 +535,29 @@ mod tests { ); let next = later + Duration::from_millis(500); sender.report(2, next); - assert!(sender.poll(next, BUDGET, (Some(shape(2)), None)).is_empty()); + assert!(poll_all(&mut sender, next, BUDGET, (Some(shape(2)), None)).is_empty()); let much_later = later + Duration::from_secs(5); assert!(sender .poll(much_later, BUDGET, (Some(shape(2)), None)) .is_empty()); sender.keyframe(much_later); assert_eq!( - shapes(&sender.poll(much_later, BUDGET, (Some(shape(2)), None))), + shapes(&poll_all( + &mut sender, + much_later, + BUDGET, + (Some(shape(2)), None) + )), [2] ); // A new session gets the image and position again. sender.serve(Some(8)); - let sent = sender.poll(much_later, BUDGET, (Some(shape(2)), Some(at(1)))); + let sent = poll_all( + &mut sender, + much_later, + BUDGET, + (Some(shape(2)), Some(at(1))), + ); assert_eq!(shapes(&sent), [2]); assert_eq!(states(&sent), [1]); sender.serve(None); @@ -450,4 +565,69 @@ mod tests { .poll(much_later, BUDGET, (Some(shape(3)), Some(at(2)))) .is_empty()); } + + /// Review r2: over USB the tunnel's control queue takes 64 datagrams and + /// refuses the rest. An image of more chunks than that resumes where the + /// link refused it, also when it is due again meanwhile, and counts as + /// sent only once every chunk went. + #[test] + fn a_refused_image_resumes_where_it_stopped_and_is_sent_only_when_whole() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + let total = 194; + let big = shape(1); + let offered = |sent: &[Send]| { + sent.iter().find_map(|send| match send { + Send::Shape(shape, next) => Some((shape.id, *next)), + Send::State(_) => None, + }) + }; + assert_eq!( + offered(&sender.poll(start, BUDGET, (Some(Arc::clone(&big)), None))), + Some((1, 0)) + ); + // The queue takes 64, then refuses. + sender.shape_progress(1, 64, total, true, start); + assert_eq!(sender.repeat_due(BUDGET), Some(start + SHAPE_RETRY)); + let soon = start + SHAPE_RETRY / 2; + assert_eq!( + offered(&sender.poll(soon, BUDGET, (Some(Arc::clone(&big)), None))), + None, + "waits" + ); + // A keyframe asks for it again: it carries on, it doesn't restart. + sender.keyframe(soon); + let retry = start + SHAPE_RETRY; + assert_eq!( + offered(&sender.poll(retry, BUDGET, (Some(Arc::clone(&big)), None))), + Some((1, 64)) + ); + sender.shape_progress(1, 128, total, true, retry); + // Not sent yet: a report naming nothing doesn't resend from 0. + sender.report(0, retry + SHAPE_RESEND_AFTER); + let later = retry + SHAPE_RETRY; + assert_eq!( + offered(&sender.poll(later, BUDGET, (Some(Arc::clone(&big)), None))), + Some((1, 128)) + ); + sender.shape_progress(1, total, total, false, later); + assert_eq!( + offered(&sender.poll(later, BUDGET, (Some(Arc::clone(&big)), None))), + None, + "whole" + ); + assert_eq!(sender.repeat_due(BUDGET), None); + // A newer image replaces one still going. + let next = later + SHAPE_RETRY; + assert_eq!( + offered(&sender.poll(next, BUDGET, (Some(shape(2)), None))), + Some((2, 0)) + ); + sender.shape_progress(2, 10, total, true, next); + assert_eq!( + offered(&sender.poll(next + SHAPE_RETRY, BUDGET, (Some(shape(3)), None))), + Some((3, 0)) + ); + } } diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 88536110..164c9912 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -721,14 +721,19 @@ impl Session { .map(|session| session.session_id) } - /// The session's id while its client draws the pointer itself. A - /// client keeps its HELLO2's answer for the whole session, so capture - /// never switches mid-session. + /// The session's id while its client draws the pointer itself, from + /// when it has answered: until then nothing confirms it received its + /// HELLO_ACK, so it gets no pointer messages (an image can be hundreds + /// of kilobytes), and capture keeps the pointer in the video for it, as + /// for any other client. Pointer messages start once the iPad answers; + /// capture switches then, once, and never again in the session. pub fn pointer_session_id(&self) -> Option { self.active .as_ref() .filter(|session| { - session.info.feature_caps & eternal_wire::v2::control::FEATURE_DRAWS_POINTER != 0 + session.answered + && session.info.feature_caps & eternal_wire::v2::control::FEATURE_DRAWS_POINTER + != 0 }) .map(|session| session.session_id) } @@ -2237,18 +2242,30 @@ mod tests { unreachable!() }; draws.feature_caps = FEATURE_DRAWS_POINTER; - let actions = - session.handle_control_authed(peer, ControlMessage::Hello2(draws), &TestConfig, now); - let (ack, _) = parse_control(&actions.replies[0].1).unwrap(); - let id = session.pointer_session_id().expect("this iPad draws it"); - assert_eq!(Some(id), session.session_id()); + session.handle_control_authed(peer, ControlMessage::Hello2(draws), &TestConfig, now); + // Review r2: nothing has confirmed it got its HELLO_ACK, so no + // pointer messages yet, and capture keeps drawing the pointer. + assert_eq!( + session.pointer_session_id(), + None, + "a HELLO that only claims the bit gets no pointer messages" + ); + let id = session.session_id().unwrap(); + let answer = session.handle_control( + peer, + id, + ControlMessage::ReceiverReport(ReceiverReport::default()), + &TestConfig, + now, + ); + assert!(answer.replies.is_empty()); + assert_eq!(session.pointer_session_id(), Some(id), "it answered"); let state = ControlMessage::PointerState(PointerState::default()); let (to, datagram) = session.encode_for_client(&state).unwrap(); assert_eq!(to, peer); let (header, parsed) = parse_control(&datagram).unwrap(); assert_eq!(parsed, state); assert_eq!(header.session_id, id); - assert_eq!(header.msg_seq, ack.msg_seq + 1); session.handle_control_authed( peer, diff --git a/host/src/transport/usb.rs b/host/src/transport/usb.rs index b3a3e94c..779cfe07 100644 --- a/host/src/transport/usb.rs +++ b/host/src/transport/usb.rs @@ -263,6 +263,12 @@ impl TransportLinks { links } + /// Tests: a link over a stream they hold, as if its cable connected. + #[cfg(test)] + pub(crate) fn adopt_for_test(&mut self, link: Arc) { + self.adopt(0, link); + } + fn adopt(&mut self, generation: u64, link: Arc) { // Drops already counted by an earlier generation are not new. let reported_drops = link.stats().frames_dropped.load(Ordering::Relaxed); diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index 13bb02aa..8a832a74 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -3880,3 +3880,99 @@ fn the_pointer_moves_without_video_frames_for_an_ipad_that_draws_it() { .recv_timeout(Duration::from_secs(5)) .expect("supervisor must shut down within 5s"); } + +/// Review r2: a HELLO2 that only claims `FEATURE_DRAWS_POINTER` must not get +/// pointer messages (an image can be hundreds of kilobytes) before it has +/// answered, as audio waits for it; the pointer stays in its video until +/// then. Once it answers, pointer messages start. +#[test] +fn pointer_messages_start_once_the_ipad_has_answered() { + use eternal_wire::v2::control::FEATURE_DRAWS_POINTER; + let _env = test_env(); + init_test_tracing(); + let _ = ffmpeg_next::init(); + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_POINTER", "1"); + for name in ["ETERNAL_DROP", "ETERNAL_REORDER", "ETERNAL_JITTER_MS"] { + std::env::remove_var(name); + } + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".to_string()); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor = { + let shared = shared.clone(); + let tx = supervisor_tx.clone(); + std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + shared, + GpuInfo::software_fallback(), + tx, + supervisor_rx, + ); + }) + }; + + // The handshake alone, no ping: nothing confirms it got its HELLO_ACK. + let mut receiver = FakeReceiver::handshake( + listen_port, + CAP_DECODE_H264, + FEATURE_DRAWS_POINTER, + 60, + 0xE2E0_0185, + ); + let mut datagram = [0u8; 2048]; + let count = |receiver: &mut FakeReceiver, datagram: &mut [u8; 2048], for_: Duration| { + let (mut pointer, mut media) = (0, 0); + let until = Instant::now() + for_; + while Instant::now() < until { + let Ok((len, _)) = receiver.socket.recv_from(datagram) else { + continue; + }; + match classify(&datagram[..len]) { + Classified::Media { .. } => media += 1, + Classified::Control(_) => { + if matches!( + parse_control(&datagram[..len]).map(|(_, m)| m), + Ok(ControlMessage::PointerShape(_) | ControlMessage::PointerState(_)) + ) { + pointer += 1; + } + } + _ => {} + } + } + (pointer, media) + }; + // Silent after its HELLO2: the pointer moves every frame meanwhile. + let (pointer, media) = count(&mut receiver, &mut datagram, Duration::from_millis(1500)); + assert_eq!(pointer, 0, "pointer messages before the iPad answered"); + assert!(media > 0, "its first picture still came"); + assert!( + !shared.client_draws_pointer(), + "capture keeps the pointer in its video" + ); + + receiver.send(&ControlMessage::ReceiverReport(ReceiverReport::default())); + let (pointer, _) = count(&mut receiver, &mut datagram, Duration::from_millis(1500)); + assert!( + pointer > 10, + "only {pointer} pointer messages after it answered" + ); + assert!(shared.client_draws_pointer()); + + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + shared.stop(); + supervisor_tx.send(SupervisorCommand::Shutdown).unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = supervisor.join(); + let _ = done_tx.send(()); + }); + done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("supervisor must shut down within 5s"); +} From b1e5c7d6373fa1bf9b91a4404094f3232b2a08eb Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 22:06:18 -0500 Subject: [PATCH 122/226] Start each audio session's wait for a taken output afresh The wait for an output another app has to itself belonged to the stage, not to one session. A new session, or audio turned off and on, could inherit a 2 s delay, skip its "unavailable" line and later log a recovery for audio it never lost. The wait now ends with its session, when audio is turned off, and when another session wants audio. The stage's retries now run on an injected clock, so the tests step through each deadline instead of sleeping. --- host/src/audio/stage.rs | 432 +++++++++++++++++++++++++++++----------- 1 file changed, 315 insertions(+), 117 deletions(-) diff --git a/host/src/audio/stage.rs b/host/src/audio/stage.rs index 6775b7de..9367cb24 100644 --- a/host/src/audio/stage.rs +++ b/host/src/audio/stage.rs @@ -31,6 +31,8 @@ enum Source { Wasapi(Box), #[cfg(test)] Scripted(std::collections::VecDeque), + #[cfg(test)] + Fake(Box AudioResult>>), } impl Source { @@ -65,6 +67,8 @@ impl Source { Self::Wasapi(source) => source.name(), #[cfg(test)] Self::Scripted(_) => "Scripted test source", + #[cfg(test)] + Self::Fake(_) => "Fake test source", } } @@ -98,6 +102,8 @@ impl Source { Self::Wasapi(source) => source.read(), #[cfg(test)] Self::Scripted(blocks) => Ok(blocks.pop_front()), + #[cfg(test)] + Self::Fake(read) => read(), } } } @@ -341,6 +347,32 @@ impl Retry { } } +/// An output another app has to itself, for one audio session: when to try +/// it again. Its "unavailable" line was logged when it began; its "back" +/// line is logged only if that session's audio starts. +struct OutputWait { + session_id: u32, + retry: Retry, +} + +/// The stage loop's time, so tests can step through retry deadlines. +trait Clock { + fn now(&self) -> Instant; + fn sleep(&self, wait: Duration); +} + +struct SystemClock; + +impl Clock for SystemClock { + fn now(&self) -> Instant { + Instant::now() + } + + fn sleep(&self, wait: Duration) { + std::thread::sleep(wait); + } +} + /// An endpoint is opened only after both peers opt in. A failed capture /// attempt stays disabled for that session; reconnecting or toggling the /// host audio setting off and on retries without restarting video. An output @@ -351,7 +383,7 @@ pub fn run_audio_stage( generation: u64, reporter: HealthReporter, ) { - run_stage(tx, shared, generation, reporter, Source::open); + run_stage(tx, shared, generation, reporter, Source::open, &SystemClock); } fn run_stage( @@ -360,19 +392,26 @@ fn run_stage( generation: u64, reporter: HealthReporter, mut open: impl FnMut() -> AudioResult, + clock: &impl Clock, ) { let mut active: Option = None; let mut failed_session = None; let mut stalled = Retry::STALL; - let mut in_use = Retry::DEVICE_IN_USE; - // Logged once when an app takes the output, once when audio is back. - let mut waiting_for_output = false; - let mut next_log = Instant::now() + Duration::from_secs(1); + // Belongs to one session: it ends with it, when audio is turned off, + // or when another session wants audio. + let mut in_use: Option = None; + let mut next_log = clock.now() + Duration::from_secs(1); while shared.running.load(Ordering::SeqCst) && generation_is_current(generation) && !tx.is_closed() { let wanted = wanted_session(&shared); + if in_use + .as_ref() + .is_some_and(|wait| Some(wait.session_id) != wanted) + { + in_use = None; + } if wanted.is_none() { log_final(active.take()); if !shared.audio_enabled.load(Ordering::SeqCst) { @@ -388,13 +427,16 @@ fn run_stage( audio.error = None; } }); - std::thread::sleep(Duration::from_millis(20)); + clock.sleep(Duration::from_millis(20)); continue; } let session_id = wanted.unwrap(); - let now = Instant::now(); - if failed_session == Some(session_id) || !stalled.due(now) || !in_use.due(now) { - std::thread::sleep(Duration::from_millis(20)); + let now = clock.now(); + if failed_session == Some(session_id) + || !stalled.due(now) + || in_use.as_ref().is_some_and(|wait| !wait.retry.due(now)) + { + clock.sleep(Duration::from_millis(20)); continue; } let result = (|| -> AudioResult { @@ -424,7 +466,7 @@ fn run_stage( bitrate = super::BITRATE, "PC audio session started (Opus, 48 kHz stereo, 20 ms)" ); - if std::mem::take(&mut waiting_for_output) { + if in_use.take().is_some() { tracing::info!( session_id, "PC audio is back: the other app let go of the output" @@ -438,17 +480,14 @@ fn run_stage( break; } match result { - Ok(true) => { - stalled = Retry::STALL; - in_use = Retry::DEVICE_IN_USE; - } + Ok(true) => stalled = Retry::STALL, Ok(false) => break, // Video's encoder holds FFmpeg's lock in a stuck driver call: // not this session's failure. Video goes on; the open is tried // again later, once the call has returned. Err(error) if super::encoder::is_stall(error.as_ref()) => { log_final(active.take()); - let wait = stalled.failed(Instant::now()); + let wait = stalled.failed(clock.now()); tracing::warn!( session_id, retry_ms = wait.as_millis() as u64, @@ -464,14 +503,18 @@ fn run_stage( // the app lets go, and the window says why meanwhile. Err(error) if super::is_device_in_use(error.as_ref()) => { log_final(active.take()); - in_use.failed(Instant::now()); - if !std::mem::replace(&mut waiting_for_output, true) { + let wait = in_use.get_or_insert_with(|| { tracing::warn!( session_id, reason = %error, "PC audio unavailable while another app has the output to itself; video continues, and audio starts when it lets go" ); - } + OutputWait { + session_id, + retry: Retry::DEVICE_IN_USE, + } + }); + wait.retry.failed(clock.now()); update_current(generation, |audio| { audio.stopped(DEVICE_IN_USE_STATUS); audio.error = None; @@ -488,8 +531,8 @@ fn run_stage( reporter.stage_exited(Stage::Audio, StageOutcome::Failed(reason)); } } - if Instant::now() >= next_log { - next_log = Instant::now() + Duration::from_secs(1); + if clock.now() >= next_log { + next_log = clock.now() + Duration::from_secs(1); let stats = PIPELINE_STATS.lock().audio.clone(); // Log only this stage's capture, never a successor's stats. if active @@ -809,44 +852,133 @@ mod tests { assert_eq!(logged(|| restart_stats(&target, |_| {})), ""); } - /// Runs the stage on this thread, opening its source with `open`, until - /// `control` returns on another. Gives the stage's log lines, its health - /// reports and what `control` returned. - fn run_with( - open: impl FnMut() -> AudioResult, - control: impl FnOnce(&mut tokio::sync::mpsc::Receiver) -> T + Send + 'static, - ) -> (String, Vec, T) { - let (shared, _) = + /// Virtual time for the stage: a sleep moves it on at once, then runs + /// `on_sleep` with the time since the start, so a test changes the + /// session, the setting or `running` at exact points. + struct StepClock<'a> { + start: Instant, + elapsed: std::cell::Cell, + on_sleep: std::cell::RefCell>, + } + + impl<'a> StepClock<'a> { + fn new(on_sleep: impl FnMut(Duration) + 'a) -> Self { + Self { + start: Instant::now(), + elapsed: Default::default(), + on_sleep: std::cell::RefCell::new(Box::new(on_sleep)), + } + } + + fn ms(&self) -> u64 { + self.elapsed.get().as_millis() as u64 + } + } + + impl Clock for StepClock<'_> { + fn now(&self) -> Instant { + self.start + self.elapsed.get() + } + + fn sleep(&self, wait: Duration) { + self.elapsed.set(self.elapsed.get() + wait); + (self.on_sleep.borrow_mut())(self.elapsed.get()); + } + } + + fn at(elapsed: Duration, ms: u64) -> bool { + elapsed == Duration::from_millis(ms) + } + + /// A running stream with an iPad session that wants audio. + fn audio_session() -> (SharedControl, u32) { + let (shared, id) = crate::transport::test_session(eternal_wire::v2::control::FEATURE_WANTS_AUDIO); shared.running.store(true, Ordering::SeqCst); + (shared, id) + } + + fn busy() -> AudioResult { + Err(crate::audio::DeviceInUse.into()) + } + + /// The audio status and error the window shows. + type Status = (String, Option); + + fn audio_status() -> Status { + let audio = PIPELINE_STATS.lock().audio.clone(); + (audio.status, audio.error) + } + + /// Ten 20 ms blocks of tone, then it notes the audio status in `seen` + /// and stops the stream. + fn tone_then_stop( + shared: &SharedControl, + seen: &std::rc::Rc>>, + ) -> Source { + let (shared, seen) = (shared.clone(), seen.clone()); + let start = crate::clock::host_now_us() + 50_000; + let mut index = 0; + Source::Fake(Box::new(move || { + if index == 10 { + *seen.borrow_mut() = Some(audio_status()); + shared.running.store(false, Ordering::SeqCst); + return Ok(None); + } + index += 1; + Ok(Some(PcmBlock { + format: PcmFormat::STEREO_48K, + samples: vec![0.25; 2 * super::super::FRAME_SAMPLES], + capture_ts_us: start + (index - 1) * FRAME_US, + discontinuity: index == 1, + })) + })) + } + + fn names_session(line: &str, id: u32) -> bool { + line.split_whitespace() + .any(|field| field == format!("session_id={id}")) + } + + const UNAVAILABLE: &str = "PC audio unavailable while another app has the output to itself"; + const BACK: &str = "PC audio is back"; + + /// Runs the stage on this thread until it stops, opening its source with + /// `open`, on `clock`. Gives its log lines and the packets it queued. + fn run_stepped( + shared: &SharedControl, + open: impl FnMut() -> AudioResult, + clock: &StepClock, + ) -> (String, Vec) { let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); let (tx, mut rx) = tokio::sync::mpsc::channel(super::super::QUEUE_CAPACITY); let (health_tx, health_rx) = std::sync::mpsc::channel(); - let controller = { - let shared = shared.clone(); - std::thread::spawn(move || { - let out = control(&mut rx); - shared.running.store(false, Ordering::SeqCst); - out - }) - }; let reporter = HealthReporter::new(health_tx, generation); - let lines = logged(|| run_stage(tx, shared.clone(), generation, reporter, open)); - let out = controller.join().unwrap(); - // Video goes on: nothing the stage reported stops or restarts it. + let lines = logged(|| run_stage(tx, shared.clone(), generation, reporter, open, clock)); + let mut packets = Vec::new(); + while let Ok(packet) = rx.try_recv() { + packets.push(packet); + } + // Video goes on: the stage reported nothing but its own end, which + // neither stops nor restarts the pipeline. let health: Vec<_> = health_rx.try_iter().collect(); let mut machine = crate::supervisor::Machine::new(generation); for message in &health { let crate::supervisor::SupMsg::Health { generation, - stage, - outcome, + stage: Stage::Audio, + outcome: StageOutcome::Completed, } = message else { - unreachable!() + panic!("{message:?}") }; assert_eq!( - machine.on_stage_exit(*generation, *stage, outcome, Instant::now()), + machine.on_stage_exit( + *generation, + Stage::Audio, + &StageOutcome::Completed, + clock.now() + ), crate::supervisor::Effect::None ); } @@ -854,78 +986,54 @@ mod tests { machine.state, crate::supervisor::SupervisorState::Running { generation } ); - assert!(shared.audio_enabled.load(Ordering::SeqCst)); - (lines, health, out) - } - - fn only_completed(health: &[crate::supervisor::SupMsg]) -> bool { - health.iter().all(|message| { - matches!( - message, - crate::supervisor::SupMsg::Health { - stage: Stage::Audio, - outcome: StageOutcome::Completed, - .. - } - ) - }) + // The lines this change adds name no device. + for line in lines + .lines() + .filter(|line| line.contains(UNAVAILABLE) || line.contains(BACK)) + { + assert!( + !line.contains("device=") && !line.contains("Fake test source"), + "{line}" + ); + } + (lines, packets) } - const UNAVAILABLE: &str = "PC audio unavailable while another app has the output to itself"; - const BACK: &str = "PC audio is back"; - #[test] fn audio_starts_once_another_app_lets_go_of_the_output() { let _stats = crate::audio::STATS_TEST_LOCK.lock(); ffmpeg_next::init().unwrap(); - const BUSY: usize = 3; - // The audio status each time the stage opens its source. - let seen = std::cell::RefCell::new(Vec::new()); + let (shared, id) = audio_session(); + let clock = StepClock::new(|_| {}); + let seen = Default::default(); + // When the stage opened its source, and the audio status then. + let opens = std::cell::RefCell::new(Vec::new()); let open = || { - let mut seen = seen.borrow_mut(); - seen.push(PIPELINE_STATS.lock().audio.status.clone()); - if seen.len() <= BUSY { - return Err(crate::audio::DeviceInUse.into()); + opens.borrow_mut().push((clock.ms(), audio_status().0)); + if opens.borrow().len() <= 3 { + busy() + } else { + Ok(tone_then_stop(&shared, &seen)) } - let start = crate::clock::host_now_us() + 50_000; - Ok(Source::Scripted( - (0..10u64) - .map(|index| PcmBlock { - format: PcmFormat::STEREO_48K, - samples: vec![0.25; 2 * super::super::FRAME_SAMPLES], - capture_ts_us: start + index * FRAME_US, - discontinuity: index == 0, - }) - .collect(), - )) }; - let (lines, health, (packet, status)) = run_with(open, |rx| { - let until = Instant::now() + Duration::from_secs(10); - let mut packet = None; - while packet.is_none() && Instant::now() < until { - packet = rx.try_recv().ok(); - std::thread::sleep(Duration::from_millis(5)); - } - let audio = PIPELINE_STATS.lock().audio.clone(); - (packet, (audio.status, audio.error)) - }); - let seen = seen.into_inner(); - assert_eq!(seen.len(), BUSY + 1, "opens: {seen:?}"); - assert!( - seen[1..] - .iter() - .all(|status| status == DEVICE_IN_USE_STATUS), - "the window said why while it waited: {seen:?}" + let (lines, packets) = run_stepped(&shared, open, &clock); + let opens = opens.into_inner(); + assert_eq!( + opens.iter().map(|(ms, _)| *ms).collect::>(), + [0, 260, 760, 1760], + "retries after 250, 500 and 1000 ms, on the 20 ms poll" ); assert!( - packet.is_some(), - "audio never started -{lines}" + opens[1..] + .iter() + .all(|(_, status)| status == DEVICE_IN_USE_STATUS), + "the window said why while it waited: {opens:?}" ); - assert_eq!(status, ("Streaming".to_string(), None)); + assert!(!packets.is_empty(), "audio never started\n{lines}"); + assert!(packets.iter().all(|packet| packet.session_id == id)); + assert_eq!(*seen.borrow(), Some(("Streaming".to_string(), None))); assert_eq!(lines.matches(UNAVAILABLE).count(), 1, "{lines}"); assert_eq!(lines.matches(BACK).count(), 1, "{lines}"); - assert!(only_completed(&health), "{health:?}"); } #[test] @@ -938,25 +1046,115 @@ mod tests { .collect(); assert_eq!(waits, [250, 500, 1000, 2000, 2000, 2000]); - let opens = std::cell::Cell::new(0); + for (stop_ms, tries) in [ + (4000, &[0, 260, 760, 1760, 3760][..]), + (1000, &[0, 260, 760][..]), + ] { + let (shared, _) = audio_session(); + let clock = StepClock::new(|elapsed| { + if at(elapsed, stop_ms) { + shared.running.store(false, Ordering::SeqCst); + } + }); + let opens = std::cell::RefCell::new(Vec::new()); + let open = || { + opens.borrow_mut().push((clock.ms(), audio_status().0)); + busy() + }; + let (lines, _) = run_stepped(&shared, open, &clock); + let opens = opens.into_inner(); + assert_eq!(opens.iter().map(|(ms, _)| *ms).collect::>(), tries); + assert!(opens[1..] + .iter() + .all(|(_, status)| status == DEVICE_IN_USE_STATUS)); + // Stopped while a retry is pending: the stage ends at its next + // 20 ms poll and tries nothing more. + assert_eq!(clock.ms(), stop_ms); + assert_eq!(lines.matches(UNAVAILABLE).count(), 1, "{lines}"); + assert_eq!(lines.matches(BACK).count(), 0, "{lines}"); + } + } + + #[test] + fn a_new_session_starts_with_no_wait_left_over_from_the_last() { + let _stats = crate::audio::STATS_TEST_LOCK.lock(); + ffmpeg_next::init().unwrap(); + let (shared, first) = audio_session(); + let (next, second) = audio_session(); + assert_ne!(first, second); + let clock = StepClock::new(|elapsed| { + // The iPad goes away while a retry waits until 1760 ms, and + // another session wants audio 100 ms later. + if at(elapsed, 800) { + *shared.session.lock() = crate::transport::session::Session::new(0); + } + if at(elapsed, 900) { + std::mem::swap(&mut *shared.session.lock(), &mut *next.session.lock()); + } + if at(elapsed, 880) { + assert_eq!(audio_status().0, "Waiting for client"); + } + }); + let seen = Default::default(); + let opens = std::cell::RefCell::new(Vec::new()); let open = || { - opens.set(opens.get() + 1); - Err(crate::audio::DeviceInUse.into()) + opens.borrow_mut().push(clock.ms()); + // The new session finds the output still taken once. + if opens.borrow().len() <= 4 { + busy() + } else { + Ok(tone_then_stop(&shared, &seen)) + } }; - // Tries at 0, 0.25, 0.75 and 1.75 s. - let (lines, health, status) = run_with(open, |_| { - std::thread::sleep(Duration::from_millis(2200)); - PIPELINE_STATS.lock().audio.status.clone() + let (lines, packets) = run_stepped(&shared, open, &clock); + assert_eq!(opens.into_inner(), [0, 260, 760, 900, 1160]); + assert!(packets.iter().all(|packet| packet.session_id == second)); + assert!(!packets.is_empty()); + let unavailable: Vec<_> = lines + .lines() + .filter(|line| line.contains(UNAVAILABLE)) + .collect(); + assert_eq!(unavailable.len(), 2, "{lines}"); + assert!(names_session(unavailable[0], first), "{lines}"); + assert!(names_session(unavailable[1], second), "{lines}"); + let back: Vec<_> = lines.lines().filter(|line| line.contains(BACK)).collect(); + assert_eq!(back.len(), 1, "{lines}"); + assert!(names_session(back[0], second), "{lines}"); + } + + #[test] + fn turning_audio_off_and_on_starts_the_wait_over() { + let _stats = crate::audio::STATS_TEST_LOCK.lock(); + ffmpeg_next::init().unwrap(); + let (shared, _) = audio_session(); + let clock = StepClock::new(|elapsed| { + if at(elapsed, 800) { + shared.audio_enabled.store(false, Ordering::SeqCst); + } + if at(elapsed, 880) { + assert_eq!(audio_status(), ("Off".to_string(), None)); + } + if at(elapsed, 900) { + shared.audio_enabled.store(true, Ordering::SeqCst); + } }); - assert!( - (3..=5).contains(&opens.get()), - "{} opens in 2.2 s", - opens.get() - ); - assert_eq!(status, DEVICE_IN_USE_STATUS); - assert_eq!(lines.matches(UNAVAILABLE).count(), 1, "{lines}"); - assert_eq!(lines.matches(BACK).count(), 0, "{lines}"); - assert!(only_completed(&health), "{health:?}"); + let seen = Default::default(); + let opens = std::cell::RefCell::new(Vec::new()); + let open = || { + opens.borrow_mut().push(clock.ms()); + if opens.borrow().len() <= 4 { + busy() + } else { + Ok(tone_then_stop(&shared, &seen)) + } + }; + let (lines, packets) = run_stepped(&shared, open, &clock); + // Back on: tried at once, not when the old wait ran out (1760 ms). + assert_eq!(opens.into_inner(), [0, 260, 760, 900, 1160]); + assert!(!packets.is_empty()); + assert_eq!(*seen.borrow(), Some(("Streaming".to_string(), None))); + assert_eq!(lines.matches(UNAVAILABLE).count(), 2, "{lines}"); + assert_eq!(lines.matches(BACK).count(), 1, "{lines}"); } /// Native check of the same startup path with the PC's real loopback. From cb523e20e6c70e8835d3bcd0ae1a5f2e85653973 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 22:26:02 -0500 Subject: [PATCH 123/226] Prefer the adapter that drew, and make the renderer off the window's thread With no surface to check against, a replacement could spend every try on an adapter that can't draw on the window, as on a hybrid PC. It now prefers the adapter that last drew there (at first the one from start) while it is listed, and leaves out each one the window can't draw on, so the next try moves on (replacement_candidates, tested on a fake adapter list). egui's renderer (buffers, shaders, the pipeline) was made on the window's thread, where a driver stall froze the window and a panic ended the app. It is now a second polled stage on the replacement thread, given the device, format and options only, timed out and counted like the first. The loss handlers go on the new device before anything is made on it. --- host/tests/window_device_loss.rs | 74 ++++++++- vendor/README.md | 36 +++-- vendor/egui-wgpu/src/lib.rs | 119 ++++++++++---- vendor/egui-wgpu/src/winit.rs | 258 +++++++++++++++++++++---------- 4 files changed, 363 insertions(+), 124 deletions(-) diff --git a/host/tests/window_device_loss.rs b/host/tests/window_device_loss.rs index cacf3fd5..f52e47e2 100644 --- a/host/tests/window_device_loss.rs +++ b/host/tests/window_device_loss.rs @@ -9,7 +9,8 @@ use std::time::{Duration, Instant}; use eframe::egui; use eframe::egui_wgpu::winit::{Answer, Asked, Replacement, REPLACEMENT_TIMEOUT}; use eframe::egui_wgpu::{ - RenderState, Renderer, RendererOptions, ScreenDescriptor, WgpuConfiguration, + replacement_candidates, AdapterKey, RenderState, Renderer, RendererOptions, ScreenDescriptor, + WgpuConfiguration, }; use eframe::wgpu; @@ -233,12 +234,26 @@ fn a_replacement_that_never_answers_ends_in_giving_up() { #[test] fn a_replacement_device_is_made_without_any_window_or_surface() { - // What the replacement thread runs gets the configuration and the - // instance only, so a thread stuck in a driver can't hold the window or - // touch it after it is gone. + // What the replacement thread runs gets the configuration, the instance and + // adapter identities only, so a thread stuck in a driver can't hold the + // window or touch it after it is gone. let config = WgpuConfiguration::default(); let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); - let new = block_on(RenderState::new_device(&config, &instance)).expect("a device"); + let drew_on = block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())) + .expect("a graphics adapter"); + let preferred = AdapterKey::of(&drew_on.get_info()); + let mut watched = false; + let new = block_on(RenderState::new_device( + &config, + &instance, + Some(&preferred), + &[], + |_| watched = true, + )) + .expect("a device"); + // The loss handlers went on before anything was made on the device. + assert!(watched); + assert_eq!(new.adapter_key(), preferred); let buffer = new.device().create_buffer(&wgpu::BufferDescriptor { label: None, size: 256, @@ -247,3 +262,52 @@ fn a_replacement_device_is_made_without_any_window_or_surface() { }); assert_eq!(buffer.size(), 256); } + +fn adapter(name: &str, backend: wgpu::Backend, vendor: u32) -> AdapterKey { + AdapterKey { + backend, + vendor, + device: 1, + device_type: wgpu::DeviceType::DiscreteGpu, + name: name.into(), + pci_bus: String::new(), + } +} + +#[test] +fn a_replacement_prefers_the_adapter_that_drew_and_moves_past_ones_that_cannot() { + use wgpu::Backend::{Dx12, Vulkan}; + // A hybrid PC: the window started on the dGPU with DX12. + let igpu = adapter("iGPU", Dx12, 0x1002); + let dgpu = adapter("dGPU", Dx12, 0x10de); + let dgpu_vulkan = adapter("dGPU", Vulkan, 0x10de); + let listed = [igpu.clone(), dgpu.clone(), dgpu_vulkan.clone()]; + + // The adapter that drew is the only choice while it is listed. + assert_eq!(replacement_candidates(&listed, Some(&dgpu), &[]), [1]); + // Gone from the listing, as across a driver reset: any of the others. + assert_eq!( + replacement_candidates(&[igpu.clone(), dgpu_vulkan.clone()], Some(&dgpu), &[]), + [0, 1] + ); + // Nothing left to try. + assert!(replacement_candidates(&listed, Some(&dgpu), &listed).is_empty()); + + // Each one the window can't draw on is left out, and the next try moves on, + // with the selector taking the first candidate. + let can_draw = |key: &AdapterKey| *key == dgpu_vulkan; + let mut excluded = Vec::new(); + let mut tried = Vec::new(); + let chosen = loop { + let candidates = replacement_candidates(&listed, Some(&dgpu), &excluded); + let key = listed[candidates[0]].clone(); + tried.push(key.clone()); + if can_draw(&key) { + break key; + } + excluded.push(key); + assert!(tried.len() <= listed.len()); + }; + assert_eq!(chosen, dgpu_vulkan); + assert_eq!(tried, [dgpu, igpu, dgpu_vulkan]); +} diff --git a/vendor/README.md b/vendor/README.md index 6f6bbd86..69ed2577 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -30,19 +30,29 @@ The patch: its own devices and is not affected. The new device's first frame uploads the font atlas again. Other textures are the app's to load again (`host/src/gui/renderer.rs` counts devices for that). -- A replacement never blocks the window's thread. Listing adapters and - making the device, where a driver can stall, run on their own thread - (`winit::Answer`, `RenderState::new_device`), which the window polls each - frame. That thread gets no window or surface: the adapter is chosen as at - start but with no surface to check. So a thread stuck in a driver neither - keeps the window alive (eframe ends its loop only once the window is - destroyed) nor touches it after it is gone; a late answer after quitting is - dropped with the closed channel. One that hasn't answered in 10 s counts as - a failed try, and no other starts while it is out; a late device is still - taken. The window's thread then makes the surface, checks the new adapter - can draw on it (a failed try if not), makes the renderer for its format - (`RenderState::with_new_device`) and configures it. The swapchain stays on - that thread because DXGI sends the window messages while it makes one. +- A replacement never blocks the window's thread. Two stages run on other + threads, which the window polls each frame: + - Listing adapters and making the device (`RenderState::new_device`). The + loss handlers go on the device before anything is made on it. + - egui's renderer (buffers, shaders, the pipeline), from the device, the + format and the options only. + - A driver can stall in either. Neither thread gets the window or a surface, + so a stuck one neither keeps the window alive (eframe ends its loop only + once the window is destroyed) nor touches it after it is gone. A late + answer after quitting is dropped with the closed channel. + - A stage that hasn't answered in 10 s, or that panicked, is a failed try, + and no other starts while one is out; a late answer is still taken. + - Between the stages, the window's thread makes the surface and checks the + new adapter can draw on it (`RenderState::surface_format`). After them it + configures the surface: DXGI sends the window messages while it makes the + swapchain. +- With no surface to check against, a replacement could keep choosing an + adapter that can't draw on the window, as on a hybrid PC. + `replacement_candidates` gives the host's adapter choice only the adapter + that last drew on the window, starting with the one from start, while it is + listed. An adapter that can't draw on the window is left out of later tries + until a replacement works. Adapters are told apart by `AdapterKey` (backend, + PCI ids, type, name, PCI bus), because wgpu reports no LUID. The thread is named `winit::REPLACEMENT_THREAD`, so the host's panic hook treats a panic there as one failed try: it doesn't turn the virtual display off as it does for other background threads. diff --git a/vendor/egui-wgpu/src/lib.rs b/vendor/egui-wgpu/src/lib.rs index da2731e8..9aefdce1 100644 --- a/vendor/egui-wgpu/src/lib.rs +++ b/vendor/egui-wgpu/src/lib.rs @@ -296,9 +296,54 @@ impl RenderState { } } +/// EternalMonitor patch: which adapter one is, across listings: wgpu's `AdapterInfo` +/// carries no LUID, so the backend, the PCI ids, the type, the name and the PCI bus (when +/// the backend reports one). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdapterKey { + pub backend: wgpu::Backend, + pub vendor: u32, + pub device: u32, + pub device_type: wgpu::DeviceType, + pub name: String, + pub pci_bus: String, +} + +impl AdapterKey { + pub fn of(info: &wgpu::AdapterInfo) -> Self { + Self { + backend: info.backend, + vendor: info.vendor, + device: info.device, + device_type: info.device_type, + name: info.name.clone(), + pci_bus: info.device_pci_bus_id.clone(), + } + } +} + +/// EternalMonitor patch: the adapters, by index into `keys`, that a replacement may choose +/// from: only `preferred` (the last one that drew on the window) while it is listed and +/// not `excluded`, else every one not `excluded` (ones that couldn't draw on the window). +pub fn replacement_candidates( + keys: &[AdapterKey], + preferred: Option<&AdapterKey>, + excluded: &[AdapterKey], +) -> Vec { + let allowed = |key: &AdapterKey| !excluded.contains(key); + if let Some(preferred) = preferred.filter(|preferred| allowed(preferred)) + && let Some(index) = keys.iter().position(|key| key == preferred) + { + return vec![index]; + } + (0..keys.len()) + .filter(|&index| allowed(&keys[index])) + .collect() +} + /// EternalMonitor patch: an adapter and device to replace a lost one, made without the /// window's surface: the thread that makes them, where a driver can stall, never touches -/// the window. The window's thread finishes it with [`RenderState::with_new_device`]. +/// the window. The window's thread then checks the surface ([`RenderState::surface_format`]). #[cfg(not(target_arch = "wasm32"))] pub struct NewDevice { adapter: wgpu::Adapter, @@ -312,16 +357,24 @@ impl NewDevice { pub fn device(&self) -> &wgpu::Device { &self.device } + + pub fn adapter_key(&self) -> AdapterKey { + AdapterKey::of(&self.adapter.get_info()) + } } #[cfg(not(target_arch = "wasm32"))] impl RenderState { - /// EternalMonitor patch: list adapters, choose one as [`Self::create`] does but with no - /// surface to check against, and make its device. Takes no window or surface, so it - /// can run on a thread that may outlive the window. + /// EternalMonitor patch: list adapters, choose among [`replacement_candidates`] as + /// [`Self::create`] does but with no surface to check against, and make the device. + /// `watch` gets the device before anything is made on it. Takes no window or surface, + /// so it can run on a thread that may outlive the window. pub async fn new_device( config: &WgpuConfiguration, instance: &wgpu::Instance, + preferred: Option<&AdapterKey>, + excluded: &[AdapterKey], + watch: impl FnOnce(&wgpu::Device), ) -> Result { let WgpuSetup::CreateNew(create_new) = &config.wgpu_setup else { return Err(WgpuError::CustomNativeAdapterSelectionError( @@ -331,21 +384,28 @@ impl RenderState { let available_adapters = instance .enumerate_adapters(create_new.instance_descriptor.backends) .await; + let keys: Vec = available_adapters + .iter() + .map(|adapter| AdapterKey::of(&adapter.get_info())) + .collect(); + let candidates: Vec = replacement_candidates(&keys, preferred, excluded) + .into_iter() + .map(|index| available_adapters[index].clone()) + .collect(); + if candidates.is_empty() { + return Err(WgpuError::CustomNativeAdapterSelectionError( + "no adapter left that could draw on the window".to_owned(), + )); + } let adapter = if let Some(selector) = &create_new.native_adapter_selector { - selector(&available_adapters, None) - .map_err(WgpuError::CustomNativeAdapterSelectionError)? + selector(&candidates, None).map_err(WgpuError::CustomNativeAdapterSelectionError)? } else { - request_adapter( - instance, - create_new.power_preference, - None, - &available_adapters, - ) - .await? + request_adapter(instance, create_new.power_preference, None, &candidates).await? }; let (device, queue) = adapter .request_device(&(*create_new.device_descriptor)(&adapter)) .await?; + watch(&device); Ok(NewDevice { adapter, available_adapters, @@ -354,26 +414,31 @@ impl RenderState { }) } - /// EternalMonitor patch: finish `new` for the window's `surface`, on the window's - /// thread: the adapter must be able to draw on it, and the renderer is made for its - /// format. - pub fn with_new_device( - new: NewDevice, - config: &WgpuConfiguration, - instance: &wgpu::Instance, + /// EternalMonitor patch: on the window's thread, the format to draw on `surface` with + /// `new`'s adapter, or an error when that adapter can't draw on it. + pub fn surface_format( + new: &NewDevice, surface: &wgpu::Surface<'_>, - options: RendererOptions, - ) -> Result { + ) -> Result { if !new.adapter.is_surface_supported(surface) { return Err(WgpuError::CustomNativeAdapterSelectionError( "the new adapter can't draw on the window".to_owned(), )); } + preferred_framebuffer_format(&surface.get_capabilities(&new.adapter).formats) + } + + /// EternalMonitor patch: the render state of a replacement, from its device and a + /// renderer made for `target_format`. + pub fn from_new_device( + new: NewDevice, + config: &WgpuConfiguration, + instance: &wgpu::Instance, + target_format: wgpu::TextureFormat, + renderer: Renderer, + ) -> Self { log_adapter_info(&new.adapter.get_info()); - let target_format = - preferred_framebuffer_format(&surface.get_capabilities(&new.adapter).formats)?; - let renderer = Renderer::new(&new.device, target_format, options); - Ok(Self { + Self { instance: instance.clone(), adapter: new.adapter, available_adapters: new.available_adapters, @@ -382,7 +447,7 @@ impl RenderState { target_format, renderer: Arc::new(RwLock::new(renderer)), surface_config: config.surface, - }) + } } } diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs index e928ddbf..a91af2c5 100644 --- a/vendor/egui-wgpu/src/winit.rs +++ b/vendor/egui-wgpu/src/winit.rs @@ -109,8 +109,43 @@ fn block_on(future: F) -> F::Output { } } -/// EternalMonitor patch: a replacement device being made on another thread. -type Replacing = Answer>; +/// EternalMonitor patch: a replacement being made on other threads, in two stages, each +/// polled and timed out on its own. Neither thread gets the window or a surface. +enum Replacing { + /// The adapter and the device, with the flag its handlers set when it is lost. + Device { + answer: Answer>, + lost: Arc, + }, + /// egui's renderer for the device, while the window's thread keeps the device and the + /// surface it checked it against. + Renderer { + answer: Answer, + new: crate::NewDevice, + surface: wgpu::Surface<'static>, + format: wgpu::TextureFormat, + lost: Arc, + }, +} + +/// EternalMonitor patch: flag the loss of `device` in `lost`. An error wgpu reports outside +/// an error scope flags it too, instead of panicking as wgpu's default handler does: a +/// window whose device or surface fails is drawn again on a new one. +fn watch(device: &wgpu::Device, lost: &Arc) { + let flag = Arc::clone(lost); + device.set_device_lost_callback(move |reason, message| { + log::warn!("The wgpu device was lost ({reason:?}): {message}"); + flag.store(true, Ordering::SeqCst); + }); + let flag = Arc::clone(lost); + device.on_uncaptured_error(Arc::new(move |error| { + if flag.swap(true, Ordering::SeqCst) { + log::debug!("wgpu error on the lost device: {error}"); + } else { + log::error!("wgpu error; replacing the window's device: {error}"); + } + })); +} /// EternalMonitor patch: when the painter tries to replace a lost device, and when it /// stops trying. @@ -227,6 +262,10 @@ pub struct Painter { /// The replacement being made, if any. Never more than one: a stuck one is waited /// for, not joined by another. replacing: Option, + /// The adapter that last drew on the window, which a replacement prefers. + preferred: Option, + /// Adapters that couldn't draw on the window since the last replacement that could. + excluded: Vec, } impl Painter { @@ -273,6 +312,8 @@ impl Painter { restore_fonts: false, replacement: Replacement::default(), replacing: None, + preferred: None, + excluded: Vec::new(), } } @@ -449,34 +490,16 @@ impl Painter { let render_state = RenderState::create(&self.config, &self.instance, Some(&surface), self.options) .await?; - self.watch_device(&render_state.device); + let lost = Arc::new(AtomicBool::new(false)); + watch(&render_state.device, &lost); + self.device_lost = lost; + self.preferred = Some(crate::AdapterKey::of(&render_state.adapter.get_info())); self.render_state = Some(render_state); } self.install_surface(surface, viewport_id, size.width, size.height, false); Ok(()) } - /// EternalMonitor patch: flag the loss of `device`. An error wgpu reports outside an - /// error scope flags it too, instead of panicking as wgpu's default handler does: a - /// window whose device or surface fails is drawn again on a new one. - fn watch_device(&mut self, device: &wgpu::Device) { - let lost = Arc::new(AtomicBool::new(false)); - let flag = Arc::clone(&lost); - device.set_device_lost_callback(move |reason, message| { - log::warn!("The wgpu device was lost ({reason:?}): {message}"); - flag.store(true, Ordering::SeqCst); - }); - let flag = Arc::clone(&lost); - device.on_uncaptured_error(Arc::new(move |error| { - if flag.swap(true, Ordering::SeqCst) { - log::debug!("wgpu error on the lost device: {error}"); - } else { - log::error!("wgpu error; replacing the window's device: {error}"); - } - })); - self.device_lost = lost; - } - /// EternalMonitor patch: forget a lost device and everything made on it. eframe calls /// [`Self::set_window`] before each paint, which builds a new device and surfaces. fn drop_lost_device(&mut self) { @@ -501,86 +524,163 @@ impl Painter { /// EternalMonitor patch: work towards a new device for a lost one, without blocking. /// - /// Listing adapters and making the device, where a driver can stall, run on another - /// thread, which this polls. That thread gets no window or surface, so one stuck in a - /// driver neither keeps the window alive (eframe ends its loop only once the window is - /// destroyed) nor touches it after it is gone. One that hasn't answered in - /// [`REPLACEMENT_TIMEOUT`] counts as a failed try, and no other starts while it is out; - /// its late device is still taken. On this thread the window's surface is then made, - /// checked against the new adapter and configured (DXGI sends the window messages while - /// it makes the swapchain), and the renderer is made; a mismatch is a failed try. + /// Two stages run on other threads, which this polls: listing adapters and making the + /// device ([`RenderState::new_device`], which installs the loss handlers before anything + /// is made on the device), then egui's renderer (buffers, shaders, the pipeline). A + /// driver can stall in either. Neither thread gets the window or a surface, so one stuck + /// in a driver neither keeps the window alive (eframe ends its loop only once the window + /// is destroyed) nor touches it after it is gone. A stage that hasn't answered in + /// [`REPLACEMENT_TIMEOUT`], or that panicked, is a failed try, and no other starts while + /// one is out; a late answer is still taken. Between the stages, this thread makes the + /// window's surface and checks the new adapter can draw on it (if not, the adapter is + /// left out of later tries); after them it configures the surface, as DXGI sends the + /// window messages while it makes the swapchain. fn replace_device(&mut self, viewport_id: ViewportId, window: Arc) { let now = Instant::now(); - if let Some(replacing) = &mut self.replacing { - match replacing.poll(now) { - Asked::Waiting => {} - Asked::TimedOut => self.replacement_failed(now, &"no new device within 10 s"), - Asked::Ended => { - self.replacing = None; - self.replacement_failed(now, &"the thread making it ended"); + match self.replacing.take() { + None => self.start_replacement(now), + Some(Replacing::Device { mut answer, lost }) => match answer.poll(now) { + Asked::Waiting => self.replacing = Some(Replacing::Device { answer, lost }), + Asked::TimedOut => { + self.replacing = Some(Replacing::Device { answer, lost }); + self.replacement_failed(now, &"no new device within 10 s"); } - Asked::Answered(Ok(new)) => { - self.replacing = None; - match self.adopt(new, viewport_id, window) { - Ok(()) => return, - Err(err) => self.replacement_failed(now, &err), - } + Asked::Ended => self.replacement_failed(now, &"the thread making the device ended"), + Asked::Answered(Err(err)) => self.replacement_failed(now, &err), + Asked::Answered(Ok(new)) => self.make_renderer(new, lost, &window, now), + }, + Some(Replacing::Renderer { + mut answer, + new, + surface, + format, + lost, + }) => match answer.poll(now) { + Asked::Answered(renderer) => { + self.adopt(new, surface, format, renderer, lost, viewport_id, &window); + return; } - Asked::Answered(Err(err)) => { - self.replacing = None; - self.replacement_failed(now, &err); + Asked::Waiting => { + self.replacing = Some(Replacing::Renderer { + answer, + new, + surface, + format, + lost, + }); } - } - if let Some(wait) = self.next_check(now) { - self.context.request_repaint_after(wait); - } - return; + Asked::TimedOut => { + self.replacing = Some(Replacing::Renderer { + answer, + new, + surface, + format, + lost, + }); + self.replacement_failed(now, &"no renderer within 10 s"); + } + Asked::Ended => { + self.replacement_failed(now, &"the thread making the renderer ended"); + } + }, } - match self.replacement.wait(now) { - None => return, - Some(wait) if !wait.is_zero() => { - self.context.request_repaint_after(wait); - return; - } - Some(_) => {} + if let Some(wait) = self.next_check(now) { + self.context.request_repaint_after(wait); + } + } + + /// EternalMonitor patch: start the first stage of a replacement when it is due. + fn start_replacement(&mut self, now: Instant) { + if self.replacement.wait(now) != Some(Duration::ZERO) { + return; } + let lost = Arc::new(AtomicBool::new(false)); + let flag = Arc::clone(&lost); let (config, instance) = (self.config.clone(), self.instance.clone()); + let (preferred, excluded) = (self.preferred.clone(), self.excluded.clone()); let asked = Answer::ask(REPLACEMENT_THREAD, now, REPLACEMENT_TIMEOUT, move || { - block_on(RenderState::new_device(&config, &instance)) + block_on(RenderState::new_device( + &config, + &instance, + preferred.as_ref(), + &excluded, + |device| watch(device, &flag), + )) }); match asked { - Ok(replacing) => { - self.replacing = Some(replacing); - self.context.request_repaint_after(REPLACEMENT_POLL); + Ok(answer) => self.replacing = Some(Replacing::Device { answer, lost }), + Err(err) => self.replacement_failed(now, &err), + } + } + + /// EternalMonitor patch: on the window's thread, make the window's surface and check the + /// new adapter can draw on it, then start the renderer stage with the device, the format + /// and the options only. + fn make_renderer( + &mut self, + new: crate::NewDevice, + lost: Arc, + window: &Arc, + now: Instant, + ) { + let surface = match self.instance.create_surface(Arc::clone(window)) { + Ok(surface) => surface, + Err(err) => return self.replacement_failed(now, &err), + }; + let format = match RenderState::surface_format(&new, &surface) { + Ok(format) => format, + Err(err) => { + let key = new.adapter_key(); + log::warn!( + "Leaving out {} ({:?}) for the window: {err}", + key.name, + key.backend + ); + self.excluded.push(key); + return self.replacement_failed(now, &err); + } + }; + let (device, options) = (new.device().clone(), self.options); + let asked = Answer::ask(REPLACEMENT_THREAD, now, REPLACEMENT_TIMEOUT, move || { + renderer::Renderer::new(&device, format, options) + }); + match asked { + Ok(answer) => { + self.replacing = Some(Replacing::Renderer { + answer, + new, + surface, + format, + lost, + }); } Err(err) => self.replacement_failed(now, &err), } } - /// EternalMonitor patch: on the window's thread, make the window's surface for a device - /// from the replacement thread, check them together and draw with them. + /// EternalMonitor patch: draw with a finished replacement. + #[expect(clippy::too_many_arguments)] fn adopt( &mut self, new: crate::NewDevice, + surface: wgpu::Surface<'static>, + format: wgpu::TextureFormat, + renderer: renderer::Renderer, + lost: Arc, viewport_id: ViewportId, - window: Arc, - ) -> Result<(), crate::WgpuError> { - let size = window.inner_size(); - let surface = self.instance.create_surface(window)?; - let render_state = RenderState::with_new_device( - new, - &self.config, - &self.instance, - &surface, - self.options, - )?; - self.watch_device(&render_state.device); + window: &Arc, + ) { + self.preferred = Some(new.adapter_key()); + self.excluded.clear(); + let render_state = + RenderState::from_new_device(new, &self.config, &self.instance, format, renderer); + self.device_lost = lost; self.render_state = Some(render_state); + let size = window.inner_size(); self.install_surface(surface, viewport_id, size.width, size.height, false); self.rebuilding = false; self.restore_fonts = true; log::info!("Replaced the lost wgpu device"); - Ok(()) } /// EternalMonitor patch: a try at a new device failed at `now`. From bb92d402312ee8727dbfc24abf559451e79ff40f Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 22:29:04 -0500 Subject: [PATCH 124/226] Draw no pointer over Windows' own, and trust the second screen's cursor only when the driver read it When Windows draws the pointer into the picture (the second screen without the driver's hardware cursor, or an image larger than that cursor takes), capture no longer draws the pointer Desktop Duplication reports there, which stayed at one spot. The hardware cursor now counts only when an Enable started the driver from off with a settings file that turns it on, read as the driver reads it, from a driver that reads (24.10.27 on) or one this host installed; a driver already on keeps what it read when it started. An iPad that draws the pointer gets it shown, and capture stops drawing it, once the iPad's report names the image it holds whole. --- host/src/capture/dxgi.rs | 85 ++++++++++++------ host/src/capture/mod.rs | 111 ++++++++++++++++++----- host/src/capture/second_screen.rs | 29 +++++- host/src/control.rs | 40 ++++++++- host/src/transport/mod.rs | 22 ++++- host/src/vdd_settings.rs | 141 ++++++++++++++++++++++++++---- host/tests/e2e_synthetic.rs | 34 +++++-- 7 files changed, 383 insertions(+), 79 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 35166c03..36f7af15 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -16,10 +16,10 @@ use super::recovery::{ LossStep, OpenStep, LOSS_GIVE_UP, }; use super::{ - acquired, client_connected, frame_budget_for, frame_buffer, heartbeat, idle_heartbeat, - keepalive_allowed, keepalive_due, pointer_apart, pointer_in_picture, read_back_image, - reconcile_virtual_display, recovery_poll, resolve_target, Acquired, FrameSlot, FrameUse, - IdleGate, OutputInfo, RawFrame, ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, + acquired, client_connected, composite_pointer, frame_budget_for, frame_buffer, heartbeat, + idle_heartbeat, keepalive_allowed, keepalive_due, pointer_apart, pointer_in_picture, + read_back_image, reconcile_virtual_display, recovery_poll, resolve_target, Acquired, FrameSlot, + FrameUse, IdleGate, OutputInfo, RawFrame, ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, }; use crate::control::{CaptureTarget, SharedControl}; use crate::gpu_preference::Outcome; @@ -319,10 +319,17 @@ pub fn run_capture_loop( // Whether `last_frame` has the pointer drawn in: it has unless the // client draws the pointer itself (`SharedControl::client_draws_pointer`). let mut last_frame_has_pointer = true; - let pointer_in_image = pointer_in_picture( - second_screen, - shared.vdd_hardware_cursor.load(Ordering::Relaxed), - ); + // Windows draws the pointer into the picture itself: on the second + // screen without the driver's hardware cursor, or for an image too + // large for it. Then nothing else draws one (`composite_pointer`). + let in_picture = |cursor: &CursorState| { + pointer_in_picture( + second_screen, + shared.vdd_hardware_cursor.load(Ordering::Relaxed), + cursor.size(), + ) + }; + let mut pointer_in_image = in_picture(&cursor); if pointer_in_image { // A position from a mirror before this one would stay drawn. shared.pointer.set_position(super::pointer::Position { @@ -366,6 +373,7 @@ pub fn run_capture_loop( // other client, or none, one with it. The staging texture still // holds the last desktop image, so the cached frame is redrawn from // it when the connected client's answer differs from the cache's. + pointer_in_image = in_picture(&cursor); let draws_pointer = pointer_apart(shared.client_draws_pointer(), pointer_in_image); if seen_image && last_frame_has_pointer == draws_pointer { let mut pixel_buf = frame_buffer(frame_bytes); @@ -376,7 +384,12 @@ pub fn run_capture_loop( row_bytes, &mut pixel_buf, )?; - if !draws_pointer && cursor_visible && cursor.has_shape { + if composite_pointer( + draws_pointer, + pointer_in_image, + cursor_visible, + cursor.has_shape, + ) { draw_cursor( &mut pixel_buf, tex_width, @@ -448,23 +461,13 @@ pub fn run_capture_loop( super::second_screen::note_image(&shared, generation); } - // --- Track cursor position/shape (must read shape before ReleaseFrame) --- - if frame_info.LastMouseUpdateTime != 0 { - cursor_visible = frame_info.PointerPosition.Visible.as_bool(); - cursor_x = frame_info.PointerPosition.Position.x; - cursor_y = frame_info.PointerPosition.Position.y; - shared.pointer.set_position(super::pointer::Position { - // A picture with the pointer in it has none apart. - visible: cursor_visible && !pointer_in_image, - x: cursor_x, - y: cursor_y, - host_time_us: crate::clock::host_now_us(), - }); - } - if frame_info.PointerShapeBufferSize > 0 - && cursor.update_shape(dupl, frame_info.PointerShapeBufferSize) - && !pointer_in_image - { + // --- Track cursor shape/position (must read shape before ReleaseFrame) --- + // The shape is always read: its size decides whether Windows + // draws it into the picture. + let new_shape = frame_info.PointerShapeBufferSize > 0 + && cursor.update_shape(dupl, frame_info.PointerShapeBufferSize); + pointer_in_image = in_picture(&cursor); + if new_shape && !pointer_in_image { if let Some(image) = super::pointer::convert_shape( cursor.shape_type, cursor.width, @@ -476,6 +479,20 @@ pub fn run_capture_loop( shared.pointer.set_shape(image); } } + if frame_info.LastMouseUpdateTime != 0 || new_shape { + if frame_info.LastMouseUpdateTime != 0 { + cursor_visible = frame_info.PointerPosition.Visible.as_bool(); + cursor_x = frame_info.PointerPosition.Position.x; + cursor_y = frame_info.PointerPosition.Position.y; + } + shared.pointer.set_position(super::pointer::Position { + // A picture with the pointer in it has none apart. + visible: cursor_visible && !pointer_in_image, + x: cursor_x, + y: cursor_y, + host_time_us: crate::clock::host_now_us(), + }); + } // --- Get dirty rect count --- let dirty_count = get_dirty_rect_count(dupl, &frame_info, &mut dirty_rect_buf); @@ -572,7 +589,12 @@ pub fn run_capture_loop( )?; // Composite the mouse cursor into the BGRA frame. - if !draws_pointer && cursor_visible && cursor.has_shape { + if composite_pointer( + draws_pointer, + pointer_in_image, + cursor_visible, + cursor.has_shape, + ) { draw_cursor( &mut pixel_buf, tex_width, @@ -1075,6 +1097,15 @@ struct CursorState { } impl CursorState { + /// The pointer image's size in pixels, once there is one. A monochrome + /// shape's buffer stacks two masks. + fn size(&self) -> Option<(u32, u32)> { + self.has_shape.then(|| match self.shape_type { + DXGI_POINTER_SHAPE_MONOCHROME => (self.width, self.height / 2), + _ => (self.width, self.height), + }) + } + /// Fetch the current pointer shape. Must be called while a frame is acquired /// (before `ReleaseFrame`). On failure the previous shape is kept and /// this returns false. diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index d080fab7..8b7d20c2 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -109,13 +109,34 @@ pub(crate) fn frame_has_image(last_present_time: i64, seen_image: bool) -> bool } /// Whether Windows draws the pointer into the captured picture: on the -/// second screen, unless the virtual display driver was given a hardware -/// cursor (`vdd_settings::hardware_cursor`). Without one, measured on the -/// reference PC, every pointer move there was a new image and Desktop -/// Duplication kept reporting the pointer at one stale position. +/// second screen, unless the virtual display driver runs with a hardware +/// cursor (`vdd_settings::cursor_in_effect`), and then too for an image +/// larger than that cursor takes (`vdd_settings::CURSOR_MAX`). Without +/// one, measured on the reference PC, every pointer move there was a new +/// image and Desktop Duplication kept reporting the pointer at one stale +/// position. #[cfg(any(windows, test))] -pub(crate) fn pointer_in_picture(second_screen: bool, driver_cursor: bool) -> bool { - second_screen && !driver_cursor +pub(crate) fn pointer_in_picture( + second_screen: bool, + driver_cursor: bool, + shape: Option<(u32, u32)>, +) -> bool { + let max = crate::vdd_settings::CURSOR_MAX; + second_screen && (!driver_cursor || shape.is_some_and(|(w, h)| w > max || h > max)) +} + +/// Whether capture draws the pointer into the video: for a client that +/// doesn't draw it, unless Windows already drew it there. Drawing the +/// reported pointer over Windows' own put a second, frozen pointer at the +/// stale position. +#[cfg(any(windows, test))] +pub(crate) fn composite_pointer( + client_draws_pointer: bool, + pointer_in_picture: bool, + visible: bool, + has_shape: bool, +) -> bool { + !client_draws_pointer && !pointer_in_picture && visible && has_shape } /// Whether the pointer goes apart from the video for this capture: to an @@ -474,6 +495,11 @@ struct VddRequest { mode: Option, /// The adapters' LUIDs before the Enable, to find the driver's own. before: Vec, + /// The driver was off, so the Enable that follows starts it with this + /// file (see `vdd_settings::cursor_in_effect`). + device_off: bool, + /// And it will give Windows a hardware cursor when it starts. + hardware_cursor: bool, } /// Before each enable: name the main screen's graphics card for the driver @@ -559,19 +585,29 @@ fn prepare_vdd_settings(shared: &SharedControl, outputs: &[OutputInfo]) -> VddRe } Err(_) => {} } - // Whatever wrote it, the file the driver reads next decides whether it - // gives Windows a hardware cursor, and so whether the second screen's - // pointer is reported apart from its picture. - let hardware_cursor = + // The driver reads this file when it starts. A driver already on keeps + // what it read then; one found off forgets anything this host knew. + let device_off = crate::vdd_device::query() == crate::vdd_device::DeviceState::Problem(22); + if device_off { + shared + .vdd_hardware_cursor + .store(false, std::sync::atomic::Ordering::Relaxed); + } + let reads_cursor = owned + || crate::vdd_settings::driver_version() + .is_some_and(|version| crate::vdd_settings::driver_reads_cursor(&version)); + let file_cursor = std::fs::read_to_string(path).is_ok_and(|xml| crate::vdd_settings::hardware_cursor(&xml)); - shared - .vdd_hardware_cursor - .store(hardware_cursor, std::sync::atomic::Ordering::Relaxed); - info!(hardware_cursor, "The virtual display's pointer"); + info!( + device_off, + reads_cursor, file_cursor, "The virtual display's pointer settings" + ); VddRequest { pin: crate::vdd_settings::pin_result(wanted, &applied), mode, before, + device_off, + hardware_cursor: reads_cursor && file_cursor, } } @@ -1199,26 +1235,55 @@ mod tests { } /// #183 is a second screen report: with the driver's hardware cursor - /// the second screen sends the pointer apart, as Mirror does. A driver - /// file without it keeps the pointer in the picture. + /// the second screen sends the pointer apart, as Mirror does. Without + /// it, or for an image over the cursor's size, Windows draws the + /// pointer into the picture. #[test] fn the_second_screen_sends_the_pointer_apart_with_the_drivers_cursor() { - for (second_screen, driver_cursor, in_picture) in [ - (false, false, false), - (false, true, false), - (true, true, false), - (true, false, true), + let small = Some((32, 32)); + let large = Some((crate::vdd_settings::CURSOR_MAX + 1, 32)); + for (second_screen, driver_cursor, shape, in_picture) in [ + (false, false, small, false), + (false, true, large, false), + (true, true, small, false), + (true, true, None, false), + (true, true, large, true), + (true, false, small, true), ] { assert_eq!( - pointer_in_picture(second_screen, driver_cursor), + pointer_in_picture(second_screen, driver_cursor, shape), in_picture, - "second_screen={second_screen} driver_cursor={driver_cursor}" + "second_screen={second_screen} driver_cursor={driver_cursor} shape={shape:?}" ); assert_eq!(pointer_apart(true, in_picture), !in_picture); assert!(!pointer_apart(false, in_picture), "an older iPad"); } } + /// Review r3: with the pointer in the picture Desktop Duplication still + /// reports it visible, with a shape, at a stale spot. Drawing that over + /// Windows' own pointer left a second, frozen one in every frame. + #[test] + fn capture_draws_the_pointer_only_when_nobody_else_does() { + assert!( + composite_pointer(false, false, true, true), + "Mirror, an older iPad" + ); + assert!( + !composite_pointer(true, false, true, true), + "the iPad draws it" + ); + assert!( + !composite_pointer(false, true, true, true), + "Windows drew it into the picture" + ); + assert!(!composite_pointer(false, false, false, true), "hidden"); + assert!( + !composite_pointer(false, false, true, false), + "no shape yet" + ); + } + pub(super) fn connect_viewer(shared: &SharedControl) { use crate::transport::link::PeerId; use crate::transport::session::ConfigSource; diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 09e5a2f2..65672f9b 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -1668,6 +1668,27 @@ struct WindowsDriver<'a> { request: std::cell::RefCell>, } +#[cfg(windows)] +impl WindowsDriver<'_> { + /// Whether the running driver gives Windows a hardware cursor, after a + /// driver change (`vdd_settings::cursor_in_effect`). + fn note_cursor(&self, enabled: bool, done: bool) { + let request = self.request.borrow(); + let request = request.as_ref(); + let flag = &self.shared.vdd_hardware_cursor; + let now = flag.load(std::sync::atomic::Ordering::Relaxed); + let in_effect = crate::vdd_settings::cursor_in_effect( + now, + enabled, + done, + request.is_some_and(|request| request.device_off), + request.is_some_and(|request| request.hardware_cursor), + ); + flag.store(in_effect, std::sync::atomic::Ordering::Relaxed); + info!(hardware_cursor = in_effect, "The virtual display's pointer"); + } +} + #[cfg(windows)] impl Driver for WindowsDriver<'_> { fn outputs(&self) -> Vec { @@ -1679,11 +1700,15 @@ impl Driver for WindowsDriver<'_> { } fn set_enabled(&self, enabled: bool) -> bool { - crate::vdd::set_enabled(enabled, || {}) + let done = crate::vdd::set_enabled(enabled, || {}); + self.note_cursor(enabled, done); + done } fn enable(&self) -> Result<(), EnableReport> { - crate::vdd::enable(|| {}) + let result = crate::vdd::enable(|| {}); + self.note_cursor(true, result.is_ok()); + result } fn settle(&self, chosen: OutputInfo) -> OutputInfo { diff --git a/host/src/control.rs b/host/src/control.rs index a65d12bc..6c377975 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -114,6 +114,9 @@ pub struct SharedControl { pub listen_on: Arc>, /// The pointer for an iPad that draws it (see `capture::pointer`). pub pointer: Arc, + /// The session whose iPad reported it holds the pointer image whole + /// (see [`SharedControl::client_draws_pointer`]). + pub(crate) pointer_confirmed: Arc, } /// Runtime state of the managed virtual extended display. @@ -177,6 +180,7 @@ impl SharedControl { )), listen_on: Arc::new(Mutex::new(Default::default())), pointer: Arc::new(Default::default()), + pointer_confirmed: Arc::new(AtomicU32::new(0)), running: Arc::new(AtomicBool::new(false)), bitrate_bps: Arc::new(AtomicU32::new(initial_bitrate_bps)), abr_current_bps: Arc::new(AtomicU32::new(initial_bitrate_bps)), @@ -247,9 +251,23 @@ impl SharedControl { /// after a silence counts as gone for the second screen, so its display /// still goes the same time after the iPad's last packet. /// The connected iPad draws the pointer, so capture leaves it out of - /// the video and sends it apart. + /// the video. From when its reports say it holds the current pointer + /// image whole: until then it has nothing to draw, so the pointer stays + /// in the video and the positions sent say hidden. pub fn client_draws_pointer(&self) -> bool { - self.session.lock().pointer_session_id().is_some() + let session = self.session.lock().pointer_session_id(); + session.is_some_and(|id| self.pointer_confirmed.load(Ordering::Relaxed) == id) + } + + /// A receiver report named the pointer image its iPad holds. + pub(crate) fn note_pointer_report(&self, shape_id: u32) { + let Some(session) = self.session.lock().pointer_session_id() else { + return; + }; + let current = self.pointer.latest().0.map(|shape| shape.id); + if shape_id != 0 && current == Some(shape_id) { + self.pointer_confirmed.store(session, Ordering::Relaxed); + } } pub(crate) fn client_heard(&self) -> bool { @@ -391,6 +409,24 @@ mod tests { &Config, std::time::Instant::now(), ); + assert!( + !shared.client_draws_pointer(), + "not before it holds the image" + ); + shared + .pointer + .set_shape(crate::capture::pointer::PointerImage { + source_type: 2, + flags: 0, + width: 1, + height: 1, + hot: (0, 0), + bytes: vec![nonce as u8; 4], + }); + let current = shared.pointer.latest().0.unwrap().id; + shared.note_pointer_report(current + 1); + assert!(!shared.client_draws_pointer(), "another image"); + shared.note_pointer_report(current); assert_eq!(shared.client_draws_pointer(), draws, "caps {caps:#x}"); } } diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 38d7439b..49c48587 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -340,7 +340,7 @@ pub async fn start_sender( // Tokio's timer can wake 15 ms late on Windows, longer than a frame // budget: the budget's end waits on a high-resolution timer. let pointer_repeat = { - let (shape, position) = shared.pointer.latest(); + let (shape, position) = pointer_latest(&shared); let budget = crate::capture::frame_budget_for(shared.effective_fps()); pointer_budget.arm(state.pointer.budget_end( budget, @@ -888,6 +888,7 @@ async fn act( } if let Some(report) = actions.report.as_ref() { state.pointer.report(report.pointer_shape_id, now()); + shared.note_pointer_report(report.pointer_shape_id); } if let Some(report) = actions.report.take() { // A slow log sink must not hold the capture/encode stats lock. @@ -938,6 +939,23 @@ async fn act( send_pointer(&mut state.pointer, links, shared).await; } +/// The pointer to send: hidden until the iPad holds its image, while +/// capture still draws it into the video (`client_draws_pointer`). +fn pointer_latest( + shared: &SharedControl, +) -> ( + Option>, + Option, +) { + let (shape, position) = shared.pointer.latest(); + let shown = shared.client_draws_pointer(); + let position = position.map(|position| crate::capture::pointer::Position { + visible: position.visible && shown, + ..position + }); + (shape, position) +} + /// Sends what [`pointer::PointerSender`] says is due, to a client that /// draws the pointer. async fn send_pointer( @@ -957,7 +975,7 @@ async fn send_pointer( return; } let budget = crate::capture::frame_budget_for(shared.effective_fps()); - for send in sender.poll(now(), budget, shared.pointer.latest()) { + for send in sender.poll(now(), budget, pointer_latest(shared)) { match send { pointer::Send::Shape(shape, from) => { let image = &shape.image; diff --git a/host/src/vdd_settings.rs b/host/src/vdd_settings.rs index 9c904f27..18f2d381 100644 --- a/host/src/vdd_settings.rs +++ b/host/src/vdd_settings.rs @@ -142,17 +142,73 @@ const CURSOR: &str = " \n truetrue\n \ 2\n \n"; -/// Whether a settings document turns the driver's hardware cursor on, as -/// the driver reads it: the text of ``, `true` or `1`. +/// The largest pointer image, in pixels a side, the cursor block lets the +/// driver take as a hardware cursor. Windows draws a larger one into the +/// picture itself. +pub const CURSOR_MAX: u32 = 128; + +/// Whether a settings document turns the driver's hardware cursor on, read +/// as the driver reads it (`EnabledQuery` in its `Driver.cpp`): the first +/// `` element whose first child is text decides, and only +/// the exact text `true` turns it on. Comments are not elements. Stricter +/// than the driver, a document that is not well-formed counts as off, so +/// the pointer is taken to be in the picture. pub fn hardware_cursor(xml: &str) -> bool { - let Some(start) = xml.find("") else { - return false; - }; - let rest = &xml[start + "".len()..]; - let Some(end) = rest.find("") else { - return false; - }; - matches!(rest[..end].trim(), "true" | "1") + use quick_xml::events::Event; + let mut reader = quick_xml::Reader::from_str(xml); + reader.config_mut().trim_text(false); + let (mut in_element, mut on) = (false, None); + loop { + match reader.read_event() { + Ok(Event::Start(tag)) => in_element = tag.local_name().as_ref() == b"HardwareCursor", + Ok(Event::Text(text)) if in_element && on.is_none() => { + on = Some(text.as_ref() == b"true"); + in_element = false; + } + // The whole document must be well-formed for the answer to count. + Ok(Event::Eof) => return on.unwrap_or(false), + Err(_) => return false, + Ok(_) => in_element = false, + } + } +} + +/// Whether the running driver gives Windows a hardware cursor, after this +/// host turned the driver on (`enabled`) or off, and the change is `done`. +/// Only an Enable that started the driver from off (`device_off`) made it +/// read the file just written, which turns the cursor on +/// (`file_cursor`: the file says so and the driver reads ``). An +/// Enable that found it on changes nothing it read, and a driver turned off +/// has no cursor. When unsure, the answer is no: the pointer is then taken +/// to be in the picture and left there. +pub fn cursor_in_effect( + before: bool, + enabled: bool, + done: bool, + device_off: bool, + file_cursor: bool, +) -> bool { + match (enabled, done) { + (false, true) => false, + (true, true) if device_off => file_cursor, + (true, true) => before, + (_, false) => before && !device_off, + } +} + +/// Whether the installed driver reads `` at all: releases from +/// 24.10.27 on do. `version` is its installer's display version, such as +/// `25.05.03`. +pub fn driver_reads_cursor(version: &str) -> bool { + let parts: Option> = version + .trim() + .split('.') + .map(|part| part.parse().ok()) + .collect(); + match parts.as_deref() { + Some([year, month, day, ..]) => (*year, *month, *day) >= (24, 10, 27), + _ => false, + } } fn escape(text: &str) -> String { @@ -831,7 +887,8 @@ mod tests { /// The driver turns its hardware cursor on only when the file says so; /// a file from before this version, or the vendor's with it off, keeps - /// the pointer in the second screen's picture. + /// the pointer in the second screen's picture. Read as the driver reads + /// it: comments don't count, and only the text `true` does. #[test] fn the_hardware_cursor_is_read_as_the_driver_reads_it() { assert!(!hardware_cursor(VENDOR)); @@ -839,17 +896,67 @@ mod tests { assert!(!hardware_cursor(&v032)); for (text, on) in [ ("true", true), - (" 1 ", true), + (" true", false), + ("1", false), ("false", false), - ("0", false), - ("", false), + ("TRUE", false), ] { let xml = format!("{text}"); assert_eq!(hardware_cursor(&xml), on, "{text:?}"); } - assert!(!hardware_cursor( - "true" - )); + let commented = "false"; + assert!( + !hardware_cursor(commented), + "a commented-out line is not a setting" + ); + let first_wins = "falsetrue"; + assert!(!hardware_cursor(first_wins)); + assert!( + !hardware_cursor("true"), + "not well-formed" + ); + } + + /// Review r3: the flag comes from what the driver read when it last + /// started, not from the file alone. + #[test] + fn the_cursor_counts_only_when_an_enable_started_the_driver_with_it() { + // An Enable that started the driver from off: the file decides. + assert!(cursor_in_effect(false, true, true, true, true)); + assert!(!cursor_in_effect(true, true, true, true, false)); + // The driver was already on (AlreadyDone, or the task found it on): + // it keeps what it read when it started. + assert!( + !cursor_in_effect(false, true, true, false, true), + "never started with it" + ); + assert!( + cursor_in_effect(true, true, true, false, true), + "a quick reconnect" + ); + // Turned off, or a failed Enable from off: no cursor. + assert!(!cursor_in_effect(true, false, true, false, true)); + assert!(!cursor_in_effect(true, true, false, true, true)); + assert!( + cursor_in_effect(true, true, false, false, true), + "a failed Enable that found it on" + ); + } + + #[test] + fn only_drivers_that_read_the_cursor_block_are_trusted_with_it() { + for (version, reads) in [ + ("25.05.03", true), + ("24.10.27", true), + ("24.12.24", true), + ("24.10.16", false), + ("23.12.2", false), + ("24.10", false), + ("", false), + ("beta", false), + ] { + assert_eq!(driver_reads_cursor(version), reads, "{version:?}"); + } } #[test] diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index 8a832a74..57b09b05 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -3794,17 +3794,26 @@ fn the_pointer_moves_without_video_frames_for_an_ipad_that_draws_it() { Instant::now(), ); if matches!(done, AddOutcome::Completed(_)) { - // The window starts after the session's first frame. + // The window starts after the session's first frame, + // or, for an iPad that draws the pointer, its first + // shown position: until its report names the image, + // the pointer stays in the video. if window_start.is_some() { frames += 1; + } else if !draws { + window_start = Some(Instant::now()); } - window_start.get_or_insert_with(Instant::now); } } Classified::Control(_) => match parse_control(bytes).map(|(_, m)| m) { - Ok(ControlMessage::PointerState(state)) if window_start.is_some() => { - assert_ne!(state.flags & POINTER_VISIBLE, 0); - states.push((state.x, state.y)); + Ok(ControlMessage::PointerState(state)) + if state.flags & POINTER_VISIBLE != 0 => + { + assert!(held_shape != 0, "shown before the iPad held the image"); + if window_start.is_some() { + states.push((state.x, state.y)); + } + window_start.get_or_insert_with(Instant::now); } Ok(ControlMessage::PointerState(_)) => {} Ok(ControlMessage::PointerShape(chunk)) => { @@ -3956,11 +3965,24 @@ fn pointer_messages_start_once_the_ipad_has_answered() { "capture keeps the pointer in its video" ); + // Answered, without the image yet: messages start, and the pointer + // stays in the video until a report names the image. receiver.send(&ControlMessage::ReceiverReport(ReceiverReport::default())); let (pointer, _) = count(&mut receiver, &mut datagram, Duration::from_millis(1500)); + assert!(pointer > 0, "no pointer messages after it answered"); + assert!( + !shared.client_draws_pointer(), + "not before it holds the image" + ); + let shape_id = shared.pointer.latest().0.unwrap().id; + receiver.send(&ControlMessage::ReceiverReport(ReceiverReport { + pointer_shape_id: shape_id, + ..Default::default() + })); + let (pointer, _) = count(&mut receiver, &mut datagram, Duration::from_millis(1500)); assert!( pointer > 10, - "only {pointer} pointer messages after it answered" + "only {pointer} pointer messages once it held the image" ); assert!(shared.client_draws_pointer()); From feaa4eaebe1f643660c3cf52171d9044f2f6bcce Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 22:29:43 -0500 Subject: [PATCH 125/226] Wait for NVENC's session before the stalled-reopen test's ABR step The native test took the stage's first open as NVENC's. When that open is slower than the first picture's wait, libx264 streams first and NVENC's session follows from the background, so the test now waits for NVENC's own open. --- host/src/encoder/mod.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index be23a719..3a6b19a8 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -6636,7 +6636,22 @@ mod tests { fn a_bitrate_change_after_a_stalled_nvenc_reopen_clears_is_prompt() { use std::time::Duration; let mut stage = NativeStage::start(); - let (_, opened) = stage.publish_until("Encoder opened", 0, Duration::from_secs(20)); + // NVENC streams before the ABR step. A first open slower than the + // first picture's wait streams libx264 until NVENC's session, opened + // in the background, takes over. + let mut from = 0; + let opened = loop { + let (_, at) = stage.publish_until("Encoder opened", from, Duration::from_secs(20)); + if stage.text()[at..] + .lines() + .next() + .unwrap() + .contains("h264_nvenc") + { + break at; + } + from = at + 1; + }; for _ in 0..30 { stage.publish(); } From c51be8038a4e1f648b7bb04fa5e38f59b2910cdc Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 22:41:36 -0500 Subject: [PATCH 126/226] Translate the out-of-date PC card and the locked-PC notice --- .../Resources/Localizable.xcstrings | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/ios/EternalMonitor/Resources/Localizable.xcstrings b/ios/EternalMonitor/Resources/Localizable.xcstrings index 75d48faa..6688e590 100644 --- a/ios/EternalMonitor/Resources/Localizable.xcstrings +++ b/ios/EternalMonitor/Resources/Localizable.xcstrings @@ -8548,11 +8548,71 @@ "comment" : "Connect screen card under \"Your PC app is out of date\". The value is the PC's name. Keep eternalmonitor.dev as is", "extractionState" : "manual", "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "حدّث EternalMonitor على %@ لبث أسرع وأكثر استقرارًا. نزّله على PC من eternalmonitor.dev." + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "Aktualisiere EternalMonitor auf %@ für schnelleres, stabileres Streaming. Lade es auf dem PC von eternalmonitor.dev herunter." + } + }, "en" : { "stringUnit" : { "state" : "translated", "value" : "Update EternalMonitor on %@ for faster, steadier streaming. Download it on the PC from eternalmonitor.dev." } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "Actualiza EternalMonitor en %@ para una transmisión más rápida y estable. Descárgalo en el PC desde eternalmonitor.dev." + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "Mettez à jour EternalMonitor sur %@ pour une diffusion plus rapide et stable. Téléchargez-le sur le PC depuis eternalmonitor.dev." + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "तेज़ और स्थिर स्ट्रीमिंग के लिए %@ पर EternalMonitor अपडेट करें। इसे PC पर eternalmonitor.dev से डाउनलोड करें।" + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "より速く安定したストリーミングのために、%@ の EternalMonitor をアップデートしてください。PC で eternalmonitor.dev からダウンロードできます。" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "더 빠르고 안정적인 스트리밍을 위해 %@에서 EternalMonitor를 업데이트하세요. PC에서 eternalmonitor.dev를 통해 다운로드하세요." + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "Atualize o EternalMonitor em %@ para uma transmissão mais rápida e estável. Baixe-o no PC em eternalmonitor.dev." + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "Обновите EternalMonitor на %@ для более быстрой и стабильной трансляции. Скачайте его на PC с eternalmonitor.dev." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "更新 %@ 上的 EternalMonitor，让串流更快更稳定。在 PC 上从 eternalmonitor.dev 下载。" + } } } }, @@ -9059,11 +9119,71 @@ "comment" : "Title of the Connect screen card shown when the PC runs an EternalMonitor older than 0.3.4", "extractionState" : "manual", "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "تطبيق PC لديك قديم" + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "Deine PC-App ist veraltet" + } + }, "en" : { "stringUnit" : { "state" : "translated", "value" : "Your PC app is out of date" } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "La app de tu PC está desactualizada" + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "L’application sur votre PC n’est pas à jour" + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "आपके PC का ऐप पुराना है" + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC アプリが古くなっています" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC 앱이 오래되었습니다" + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "O app do seu PC está desatualizado" + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "Приложение на PC устарело" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC 应用已过期" + } } } }, @@ -9071,11 +9191,71 @@ "comment" : "Over the held picture while the PC shows its lock screen or a permission prompt", "extractionState" : "manual", "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC مقفل. افتح قفله للمتابعة." + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "Dein PC ist gesperrt. Entsperre ihn, um fortzufahren." + } + }, "en" : { "stringUnit" : { "state" : "translated", "value" : "Your PC is locked. Unlock it to continue." } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "Tu PC está bloqueado. Desbloquéalo para continuar." + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "Votre PC est verrouillé. Déverrouillez-le pour continuer." + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "आपका PC लॉक है। जारी रखने के लिए उसे अनलॉक करें।" + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC がロックされています。続行するにはロックを解除してください。" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC가 잠겨 있습니다. 계속하려면 잠금을 해제하세요." + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "Seu PC está bloqueado. Desbloqueie-o para continuar." + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC заблокирован. Разблокируйте его, чтобы продолжить." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC 已锁定。解锁后即可继续。" + } } } }, From 1a6a8845e2b93cd51b06ee63d14f4a82ecea7727 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 22:42:58 -0500 Subject: [PATCH 127/226] Expect every string translated again --- ios/EternalMonitorTests/LocalizationTests.swift | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ios/EternalMonitorTests/LocalizationTests.swift b/ios/EternalMonitorTests/LocalizationTests.swift index d17d441c..0db3ce8d 100644 --- a/ios/EternalMonitorTests/LocalizationTests.swift +++ b/ios/EternalMonitorTests/LocalizationTests.swift @@ -22,11 +22,7 @@ final class LocalizationTests: XCTestCase { /// English strings whose translations come in a separate pass. Until /// then every language shows them in English. - private let awaitingTranslation: Set = [ - "Your PC is locked. Unlock it to continue.", - "Your PC app is out of date", - "Update EternalMonitor on %@ for faster, steadier streaming. Download it on the PC from eternalmonitor.dev.", - ] + private let awaitingTranslation: Set = [] /// Every language has exactly English's strings, none empty, each with /// the same placeholders. From 36d2e91acb81e943b374e94535b3124ec4024f96 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 22:53:09 -0500 Subject: [PATCH 128/226] Clear a second screen's proof when capture's listing misses its output Review r4 of #205. When the duplicated output still described itself as attached but a re-creation failed, capture listed the outputs and could find the virtual display gone, yet kept its proof. The same display back as the same output before the next reconciliation then inherited it, and a dark one would have cost a watchdog restart. The listing is now judged in one place: a listing without the output as it was (absent, or its name under another adapter) clears the proof, and only a listing that still shows the same output keeps it. --- host/src/capture/dxgi.rs | 10 ++++---- host/src/capture/mod.rs | 21 ++++++++++++++++ host/src/capture/second_screen.rs | 41 +++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 020bc3f7..03e2ca4d 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -972,9 +972,9 @@ fn access_lost_or( /// primary for a missing display, or when `list` asks after a failed /// re-creation. Both run under the stall limits while `progress` keeps the /// picture alive; one that does not return in time counts as no move, and -/// the re-creation goes on. An output seen off the desktop or renamed is a -/// detach for a second screen's proof (see -/// [`super::second_screen::note_detached`]). +/// the re-creation goes on. An output seen off the desktop or renamed, or a +/// listing without it as it was, is a detach for a second screen's proof +/// (see [`super::second_screen::note_detached`]). fn target_moved( shared: &SharedControl, output: &IDXGIOutput1, @@ -1000,11 +1000,11 @@ fn target_moved( } super::second_screen::note_detached(shared); } - let (target, chosen) = (target.clone(), chosen.clone()); + let (shared, target, chosen) = (shared.clone(), target.clone(), chosen.clone()); let started = Instant::now(); let moved = crate::stall::run( Call::ListOutputs, - move || super::target_moved(&enumerate_outputs(), &target, &chosen), + move || super::judge_listing(&shared, &enumerate_outputs(), &target, &chosen), progress, ) .unwrap_or(false); diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 45c89a6d..c400e2c7 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -357,6 +357,27 @@ pub(crate) fn target_moved( }) } +/// Capture's listing after a lost duplication, judged: whether the target +/// moved off `current` (see [`target_moved`]). A listing without `current` +/// as it was (absent, or its name under another adapter) is also a detach +/// for a second screen's proof: kept, a display back as the same output +/// before the next reconciliation would inherit it. +#[cfg(any(windows, test))] +pub(crate) fn judge_listing( + shared: &SharedControl, + outputs: &[OutputInfo], + target: &crate::control::CaptureTarget, + current: &OutputInfo, +) -> bool { + let still_attached = outputs.iter().any(|output| { + output.device_name == current.device_name && output.adapter_id == current.adapter_id + }); + if !still_attached { + second_screen::note_detached(shared); + } + target_moved(outputs, target, current) +} + /// How long to wait for the virtual display to attach after enabling its driver. Overridable /// via `ETERNAL_VDD_TIMEOUT_SECS`, or within 30–60 s by the update manifest. The iPad mirrors /// meanwhile, so a longer wait costs the user nothing; it was 10 s while the iPad gave up 10 s diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 041b460f..4a0a688e 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -2616,6 +2616,47 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// Review r4 of #205: capture's own description of the output still said + /// attached, a re-creation failed, and the listing that followed found + /// the virtual display gone. The proof was kept, so the same display back + /// before the next reconciliation inherited it. A listing without the + /// output as it was clears the proof; one with it keeps the proof. + #[test] + fn a_listing_without_the_output_makes_it_prove_itself_again() { + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, "listing"); + let target = CaptureTarget::Output("virtual".into()); + let streamed = Some(target.clone()); + let current = super::super::outputs(&shared) + .into_iter() + .find(|output| output.device_name == "virtual") + .unwrap(); + // The output is still there as it was: no move, and the proof stays. + let listed = super::super::outputs(&shared); + assert!(!super::super::judge_listing( + &shared, &listed, &target, ¤t + )); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + streamed + ); + assert!(!proving(&shared), "the same output, still attached"); + // The listing finds it gone; it is back, the same, before the stage + // reconciles again. + assert!(Driver::set_enabled(&display, false)); + let listed = super::super::outputs(&shared); + assert!(super::super::judge_listing( + &shared, &listed, &target, ¤t + )); + assert!(Driver::set_enabled(&display, true)); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + streamed + ); + assert!(proving(&shared), "back after the listing saw it gone"); + let _ = std::fs::remove_dir_all(&dir); + } + /// Review r3 of #205: the same output reopened in the generation that /// proved it keeps its proof, so a display gone dark reaches the /// watchdog (see the test above for the restart itself). From 0b7e302b6486564e71c7b305f14deb0f712cb097 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 23:07:05 -0500 Subject: [PATCH 129/226] Choose a replacement only among the candidates without a selector too With no native adapter selector, new_device took wgpu's own choice, which looks at every adapter, so the preferred or remaining candidates were ignored. The test caught it on a hybrid PC. Now wgpu's choice counts only if it is a candidate; otherwise the first candidate is taken. --- vendor/egui-wgpu/src/lib.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/vendor/egui-wgpu/src/lib.rs b/vendor/egui-wgpu/src/lib.rs index 9aefdce1..6128cfa8 100644 --- a/vendor/egui-wgpu/src/lib.rs +++ b/vendor/egui-wgpu/src/lib.rs @@ -400,7 +400,16 @@ impl RenderState { let adapter = if let Some(selector) = &create_new.native_adapter_selector { selector(&candidates, None).map_err(WgpuError::CustomNativeAdapterSelectionError)? } else { - request_adapter(instance, create_new.power_preference, None, &candidates).await? + // wgpu chooses among all adapters, not these: keep its choice only if it is one. + let keys: Vec = candidates + .iter() + .map(|adapter| AdapterKey::of(&adapter.get_info())) + .collect(); + request_adapter(instance, create_new.power_preference, None, &candidates) + .await + .ok() + .filter(|adapter| keys.contains(&AdapterKey::of(&adapter.get_info()))) + .unwrap_or_else(|| candidates[0].clone()) }; let (device, queue) = adapter .request_device(&(*create_new.device_descriptor)(&adapter)) From e5c53892bd5943b3300a6dcce059c8d707559640 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 23:21:58 -0500 Subject: [PATCH 130/226] Count refused repairs, wait the downgrade cooldown, and log a failed retry A refused send's step down now waits out the 3 s downgrade cooldown, as a loss report's does, so episodes a second apart can't walk the bitrate to the floor, and a report's step isn't followed by another for the same loss. NACK repairs go through the same sends as everything else, so a refused repair counts in the episode. A video retry that fails for any other reason is logged like any failed send, and the frame goes on. The test now checks the order of sends and the back-off deadline where the transport sent them, through the test seam, not where the fake iPad read them. --- host/src/transport/abr.rs | 50 +++++++- host/src/transport/link.rs | 4 +- host/src/transport/mod.rs | 147 +++++++++++++++++----- host/src/transport/pacer.rs | 2 + host/tests/send_congestion.rs | 221 ++++++++++++++++++++++++++-------- 5 files changed, 337 insertions(+), 87 deletions(-) diff --git a/host/src/transport/abr.rs b/host/src/transport/abr.rs index 85188523..1827e614 100644 --- a/host/src/transport/abr.rs +++ b/host/src/transport/abr.rs @@ -248,8 +248,10 @@ impl AbrController { } /// The system had no buffer space for a datagram: one rung down at - /// once, without waiting for the iPad to report the loss. It counts as a - /// downgrade, so the loss reports that follow wait out the cooldown. + /// once, without waiting for the iPad to report the loss. Like any + /// downgrade it waits out the cooldown after the last one, and starts + /// it, so refusals and loss reports together step at most once per + /// [`DOWN_COOLDOWN`]. pub fn on_send_refused(&mut self, now: Instant) -> AbrDecision { let unchanged = AbrDecision { target_bps: self.current_bps, @@ -259,6 +261,12 @@ impl AbrController { return unchanged; } self.clean_since = None; + if matches!( + self.last_change, + Some((when, true)) if now.duration_since(when) < DOWN_COOLDOWN + ) { + return unchanged; + } let target = step(self.current_bps, -1, self.ceiling_bps); if target == self.current_bps { return unchanged; @@ -623,10 +631,48 @@ mod tests { assert!(!d.changed, "the same loss, inside the cooldown"); assert_eq!(abr.current_bps(), 12_000_000); + let d = abr.on_send_refused(t0 + Duration::from_secs(2)); + assert!(!d.changed, "a refusal waits out the cooldown too"); + assert_eq!(abr.current_bps(), 12_000_000); + let mut floor = AbrController::new(4_000_000, true); assert!(!floor.on_send_refused(t0).changed); let mut disabled = AbrController::new(20_000_000, false); assert!(!disabled.on_send_refused(t0).changed); assert_eq!(disabled.current_bps(), 20_000_000); } + + /// Episodes of refused sends a little over a second apart each ask for + /// a step, but steps come at most once per cooldown: not 15 Mbps down to + /// the floor in five seconds. + #[test] + fn refusal_episodes_step_at_most_once_per_cooldown() { + let mut abr = AbrController::new(20_000_000, true); + let t0 = Instant::now(); + let mut steps = Vec::new(); + for n in 0..5u64 { + let at = Duration::from_millis(1_100 * n); + if abr.on_send_refused(t0 + at).changed { + steps.push((at.as_millis(), abr.current_bps())); + } + } + assert_eq!(steps, [(0, 12_000_000), (3_300, 10_000_000)]); + } + + /// A receiver report that just stepped down starts the cooldown: a + /// refusal right after it doesn't step again for the same congestion. + #[test] + fn a_refusal_right_after_a_report_downgrade_waits_the_cooldown() { + let mut abr = AbrController::new(20_000_000, true); + let t0 = Instant::now(); + let d = abr.on_report(&report(950, 50, 0), None, t0); + assert_eq!((d.target_bps, d.changed), (12_000_000, true)); + assert!(!abr.on_send_refused(t0 + Duration::from_millis(500)).changed); + assert!( + !abr.on_send_refused(t0 + Duration::from_millis(2_900)) + .changed + ); + let d = abr.on_send_refused(t0 + Duration::from_millis(3_100)); + assert_eq!((d.target_bps, d.changed), (10_000_000, true)); + } } diff --git a/host/src/transport/link.rs b/host/src/transport/link.rs index ae0f53e8..221c397f 100644 --- a/host/src/transport/link.rs +++ b/host/src/transport/link.rs @@ -104,8 +104,8 @@ impl Link for UdpLink { }; check_length(datagram.len())?; #[cfg(feature = "test-hooks")] - if super::probe::refuse(addr, datagram) { - return Err(super::congestion::no_buffer_space()); + if let Some(error) = super::probe::fail(addr, datagram) { + return Err(error); } self.socket.send_to(datagram, addr).await?; #[cfg(feature = "test-hooks")] diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index d9993f61..980bf455 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -57,14 +57,14 @@ fn now() -> Instant { /// is in flight before moving the clock on. #[cfg(feature = "test-hooks")] pub mod probe { - use std::collections::BTreeMap; + use std::collections::{BTreeMap, VecDeque}; use std::net::SocketAddr; + use std::time::Instant; use parking_lot::Mutex; static SENT: Mutex> = Mutex::new(BTreeMap::new()); static TAKEN: Mutex> = Mutex::new(BTreeMap::new()); - static REFUSE: Mutex> = Mutex::new(BTreeMap::new()); pub(crate) fn sent(addr: SocketAddr) { *SENT.lock().entry(addr).or_default() += 1; @@ -89,37 +89,116 @@ pub mod probe { Control, } + /// How a failed send fails. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum Fail { + /// As if the system had no buffer space (WSAENOBUFS, ENOBUFS). + NoBufferSpace, + /// Any other error: connection reset. + Other, + } + + struct Failing { + kind: Kind, + after: u64, + fails: VecDeque, + } + + static FAIL: Mutex> = Mutex::new(BTreeMap::new()); + /// Of the UDP sends of `kind` to `addr`, the next `after` go out and - /// the `count` after them fail as if the system had no buffer space for - /// them (WSAENOBUFS, ENOBUFS). - pub fn refuse_sends(addr: SocketAddr, kind: Kind, after: u64, count: u64) { - REFUSE.lock().insert(addr, (kind, after, count)); + /// the ones after them fail as `fails` says, in order. + pub fn fail_sends(addr: SocketAddr, kind: Kind, after: u64, fails: &[Fail]) { + let fails = fails.iter().copied().collect(); + FAIL.lock().insert(addr, Failing { kind, after, fails }); + } + + /// As [`fail_sends`], with `count` sends refused for lack of buffer space. + pub fn refuse_sends(addr: SocketAddr, kind: Kind, after: u64, count: usize) { + fail_sends(addr, kind, after, &vec![Fail::NoBufferSpace; count]); } - /// Sends of the kind given to [`refuse_sends`] still to be refused. - pub fn refusals_left(addr: SocketAddr) -> u64 { - REFUSE.lock().get(&addr).map_or(0, |&(_, _, count)| count) + /// Failures [`fail_sends`] still has to give. + pub fn refusals_left(addr: SocketAddr) -> usize { + FAIL.lock() + .get(&addr) + .map_or(0, |failing| failing.fails.len()) } - /// Whether this send of `datagram` to `addr` is refused, counting it. - pub(crate) fn refuse(addr: SocketAddr, datagram: &[u8]) -> bool { + /// How this send of `datagram` to `addr` fails, if it does. + pub(crate) fn fail(addr: SocketAddr, datagram: &[u8]) -> Option { use eternal_wire::v2::{classify, Classified}; let kind = match classify(datagram) { Classified::Media { .. } => Kind::Media, Classified::Control(_) => Kind::Control, - _ => return false, + _ => return None, }; - let mut refuse = REFUSE.lock(); - match refuse.get_mut(&addr) { - Some((refused, after, _)) if *refused == kind && *after > 0 => { - *after -= 1; - false - } - Some((refused, _, count)) if *refused == kind && *count > 0 => { - *count -= 1; - true + let fail = match FAIL.lock().get_mut(&addr).filter(|f| f.kind == kind) { + Some(failing) if failing.after > 0 => { + failing.after -= 1; + None } - _ => false, + Some(failing) => failing.fails.pop_front(), + None => None, + }; + attempted(addr, datagram, fail); + fail.map(|fail| match fail { + Fail::NoBufferSpace => super::congestion::no_buffer_space(), + Fail::Other => std::io::Error::from(std::io::ErrorKind::ConnectionReset), + }) + } + + /// One video datagram the transport tried to send, as the send saw it. + #[derive(Clone, Copy, Debug)] + pub struct Attempt { + pub at: Instant, + pub frame_seq: u32, + pub frag_index: u16, + pub retransmit: bool, + pub failed: Option, + } + + static RECORDED: Mutex> = Mutex::new(None); + static ATTEMPTS: Mutex> = Mutex::new(Vec::new()); + /// When each pacing wait was asked for, and its deadline. + static WAITS: Mutex> = Mutex::new(Vec::new()); + + /// Records every video send to `addr` from now on, and every pacing + /// wait, clearing what was recorded. + pub fn record(addr: SocketAddr) { + *RECORDED.lock() = Some(addr); + ATTEMPTS.lock().clear(); + WAITS.lock().clear(); + } + + /// The video sends recorded so far, in order. + pub fn attempts() -> Vec { + ATTEMPTS.lock().clone() + } + + /// The pacing waits recorded so far: when each was asked for, and until when. + pub fn waits() -> Vec<(Instant, Instant)> { + WAITS.lock().clone() + } + + fn attempted(addr: SocketAddr, datagram: &[u8], failed: Option) { + if *RECORDED.lock() != Some(addr) { + return; + } + if let Ok((header, _)) = eternal_wire::v2::media::MediaHeader::decode(datagram) { + ATTEMPTS.lock().push(Attempt { + at: Instant::now(), + frame_seq: header.frame_seq, + frag_index: header.frag_index, + retransmit: header.is_retransmit, + failed, + }); + } + } + + pub(crate) fn waited(deadline: Instant) { + if RECORDED.lock().is_some() { + WAITS.lock().push((Instant::now(), deadline)); } } } @@ -583,8 +662,15 @@ pub async fn start_sender( } } } - if refused && links.send_to(datagram, target_addr).await.is_err() { - break; + if refused { + match links.send_to(datagram, target_addr).await { + Err(e) if congestion::is_no_buffer_space(&e) => break, + Err(e) if !send_failed => { + log_send_failed(target_addr.link, &e); + send_failed = true; + } + _ => {} + } } } @@ -777,15 +863,15 @@ async fn receive_control( } async fn send_repairs( - socket: &UdpSocket, + links: &TransportLinks, ring: &mut RetransmitRing, shared: &SharedControl, nack: &eternal_wire::v2::control::Nack, ) { let target = *shared.target_addr.lock(); - let Some(destination) = target.addr.filter(|_| target.link == LinkId::Udp) else { + if target.link != LinkId::Udp || target.addr.is_none() { return; - }; + } let mut sent = 0; for datagram in ring.resend( nack.stream_epoch, @@ -794,9 +880,8 @@ async fn send_repairs( &nack.missing, Instant::now(), ) { - if socket.send_to(&datagram, destination).await.is_ok() { - #[cfg(feature = "test-hooks")] - probe::sent(destination); + // Through the links, so a refused repair counts in the episode. + if links.send_to(&datagram, target).await.is_ok() { sent += 1; } } @@ -877,7 +962,7 @@ async fn act( state.input.reset(); } if let Some(nack) = actions.retransmit.take() { - send_repairs(&links.udp.socket, &mut state.retransmit_ring, shared, &nack).await; + send_repairs(links, &mut state.retransmit_ring, shared, &nack).await; } if let Some((session_id, event)) = actions.input.take() { shared.transport_beat.mark(Phase::Input); diff --git a/host/src/transport/pacer.rs b/host/src/transport/pacer.rs index 43467f0a..7f28d7ba 100644 --- a/host/src/transport/pacer.rs +++ b/host/src/transport/pacer.rs @@ -39,6 +39,8 @@ impl PacingClock { /// Waits until `deadline`, or not at all if a wait is already queued. pub async fn until(&self, deadline: Instant) { + #[cfg(feature = "test-hooks")] + super::probe::waited(deadline); let (done, ready) = tokio::sync::oneshot::channel(); if self.tx.try_send((deadline, done)).is_ok() { let _ = ready.await; diff --git a/host/tests/send_congestion.rs b/host/tests/send_congestion.rs index 77714344..f56980d7 100644 --- a/host/tests/send_congestion.rs +++ b/host/tests/send_congestion.rs @@ -1,8 +1,11 @@ //! When the system has no buffer space for a datagram (WSAENOBUFS 10055 on //! Windows, ENOBUFS elsewhere), the real transport loop backs off, steps the -//! bitrate down once per episode, gives up a frame it can't send rather than -//! wait on it, retries a control reply once, and logs one line per episode -//! plus its end. The sends are refused through the test-hooks seam. +//! bitrate down at most once per episode, gives up a frame it can't send +//! rather than wait on it, retries a control reply once, and logs one line +//! per episode plus its end. The sends are refused through the test-hooks +//! seam, which also records each video send and pacing wait, so the order +//! and deadlines are checked where the transport sent, not where the fake +//! iPad happened to read. use std::net::{SocketAddr, UdpSocket}; use std::sync::atomic::Ordering; @@ -12,7 +15,7 @@ use std::time::{Duration, Instant}; use eternal_host::control::SharedControl; use eternal_host::encoder::NALUnit; use eternal_host::transport::congestion::{BACKOFF, EPISODE_QUIET}; -use eternal_host::transport::probe::{self, Kind}; +use eternal_host::transport::probe::{self, Attempt, Fail, Kind}; use eternal_wire::v2::control::{ encode_control, parse_control, ControlMessage, Hello2, Nack, Ping, CAP_DECODE_H264, FEATURE_SUPPORTS_NACK, @@ -94,18 +97,46 @@ impl Ipad { received } - /// Frame `frame_seq`'s fragments among `received`, with when each came. - fn fragments(received: &[(Instant, Vec)], frame_seq: u32) -> Vec<(Instant, u16)> { + /// Frame `frame_seq`'s original fragments among `received`, in order. + fn fragments(received: &[(Instant, Vec)], frame_seq: u32) -> Vec { received .iter() - .filter_map(|(at, bytes)| { + .filter_map(|(_, bytes)| { let (header, _) = MediaHeader::decode(bytes).ok()?; (header.frame_seq == frame_seq && !header.is_retransmit) - .then_some((*at, header.frag_index)) + .then_some(header.frag_index) }) .collect() } + /// Waits until the transport has tried at least `count` sends of frame + /// `frame_seq`'s originals, and returns them. + fn attempts(&mut self, frame_seq: u32, count: usize) -> Vec { + self.attempted(frame_seq, false, count) + } + + /// As [`Self::attempts`], for the frame's repairs. + fn retransmits(&mut self, frame_seq: u32, count: usize) -> Vec { + self.attempted(frame_seq, true, count) + } + + fn attempted(&mut self, frame_seq: u32, retransmit: bool, count: usize) -> Vec { + let until = Instant::now() + Duration::from_secs(10); + loop { + let attempts = attempts_of(frame_seq, retransmit); + if attempts.len() >= count { + return attempts; + } + assert!( + Instant::now() < until, + "only {} sends: {attempts:?}", + attempts.len() + ); + self.keep_alive(); + std::thread::sleep(Duration::from_millis(1)); + } + } + /// Waits for a heartbeat, so the next second has none. fn after_heartbeat(&mut self) { let mut buffer = [0; 2048]; @@ -122,6 +153,34 @@ impl Ipad { } } +fn attempts_of(frame_seq: u32, retransmit: bool) -> Vec { + probe::attempts() + .into_iter() + .filter(|a| a.frame_seq == frame_seq && a.retransmit == retransmit) + .collect() +} + +fn indexes(attempts: &[Attempt]) -> Vec { + attempts.iter().map(|a| a.frag_index).collect() +} + +/// Which attempts failed, by position, and how. +fn failures(attempts: &[Attempt]) -> Vec<(usize, Fail)> { + attempts + .iter() + .enumerate() + .filter_map(|(n, a)| a.failed.map(|fail| (n, fail))) + .collect() +} + +/// The pacing waits asked for after `from` and before `to`. +fn waits_between(from: Instant, to: Instant) -> Vec<(Instant, Instant)> { + probe::waits() + .into_iter() + .filter(|&(asked, _)| asked > from && asked < to) + .collect() +} + /// A keyframe of exactly `fragments` datagrams. fn keyframe(shared: &SharedControl, fragments: usize) -> NALUnit { let payload = media_payload_size(shared.max_dgram.load(Ordering::SeqCst) as usize).unwrap(); @@ -212,47 +271,72 @@ fn refused_sends_back_off_step_down_once_and_log_one_episode() { }; let addr = ipad.addr(); assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 15_000_000); + probe::record(addr); - // Frame 1: fragment 10 is refused once. The frame backs off and sends it - // again, and the bitrate steps down without waiting for a loss report. + // Frame 1: fragment 10 is refused once. The frame asks the pacing clock + // for one back-off, sends it again once that deadline has passed, and + // the bitrate steps down without waiting for a loss report. probe::refuse_sends(addr, Kind::Media, 10, 1); nal_tx.blocking_send(keyframe(&shared, 40)).unwrap(); - let received = ipad.receive(Duration::from_millis(300)); - let frame = Ipad::fragments(&received, 1); - let indexes: Vec = frame.iter().map(|&(_, index)| index).collect(); + let frame = ipad.attempts(1, 41); + let expected: Vec = (0..=10).chain(10..40).collect(); assert_eq!( - indexes, - (0..40).collect::>(), - "the refused fragment is sent again, not lost" + indexes(&frame), + expected, + "fragment 10 is tried twice, in order" ); - let paused = frame[10].0.duration_since(frame[9].0); + assert_eq!(failures(&frame), [(10, Fail::NoBufferSpace)]); + let (refused, retried) = (frame[10].at, frame[11].at); + let backoffs = waits_between(refused, retried); + assert_eq!(backoffs.len(), 1, "one back-off: {backoffs:?}"); + let (_, deadline) = backoffs[0]; assert!( - paused >= BACKOFF - Duration::from_millis(1), - "it waited {paused:?} before sending again" + deadline >= refused + BACKOFF, + "the back-off lasts {BACKOFF:?}" ); + assert!(retried >= deadline, "the retry waits for the deadline"); + let stepped = Instant::now(); assert_eq!( shared.abr_current_bps.load(Ordering::SeqCst), 12_000_000, "one rung down at once" ); + let received = ipad.receive(Duration::from_millis(100)); + assert_eq!( + Ipad::fragments(&received, 1), + (0..40).collect::>(), + "the refused fragment is sent again, not lost" + ); // Frame 2, in the same episode: fragment 5 is refused, and so is its // retry. The frame gives up its rest, which repair recovers, and the // bitrate stays: one step per episode. probe::refuse_sends(addr, Kind::Media, 5, 2); nal_tx.blocking_send(keyframe(&shared, 40)).unwrap(); - let received = ipad.receive(Duration::from_millis(300)); - let indexes: Vec = Ipad::fragments(&received, 2) - .iter() - .map(|&(_, index)| index) - .collect(); + let frame = ipad.attempts(2, 7); + let received = ipad.receive(Duration::from_millis(100)); + let frame_after = attempts_of(2, false); assert_eq!( - indexes, + frame_after.len(), + frame.len(), + "nothing after the second refusal" + ); + assert_eq!(indexes(&frame), [0, 1, 2, 3, 4, 5, 5]); + assert_eq!( + failures(&frame), + [(5, Fail::NoBufferSpace), (6, Fail::NoBufferSpace)] + ); + assert_eq!(waits_between(frame[5].at, frame[6].at).len(), 1); + assert_eq!( + Ipad::fragments(&received, 2), (0..5).collect::>(), "the rest is given up" ); - assert_eq!(probe::refusals_left(addr), 0); assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 12_000_000); + + // Repairs go through the same sends: five of them are refused, count in + // the episode, and don't arrive. + probe::refuse_sends(addr, Kind::Media, 10, 5); let epoch = received .iter() .find_map(|(_, bytes)| MediaHeader::decode(bytes).ok()) @@ -265,43 +349,71 @@ fn refused_sends_back_off_step_down_once_and_log_one_episode() { frag_count: 40, missing: (5..40).collect(), })); + let repairs = ipad.retransmits(2, 35); + assert_eq!( + failures(&repairs), + (10..15) + .map(|n| (n, Fail::NoBufferSpace)) + .collect::>() + ); let repaired = ipad - .receive(Duration::from_millis(300)) + .receive(Duration::from_millis(100)) .iter() .filter(|(_, bytes)| { MediaHeader::decode(bytes) .is_ok_and(|(header, _)| header.frame_seq == 2 && header.is_retransmit) }) .count(); - assert_eq!(repaired, 35, "repair recovers what was given up"); + assert_eq!( + repaired, 30, + "repair recovers what was given up, less what was refused" + ); + + // Frame 3: fragment 3 is refused, and its retry fails another way. That + // is logged as any failed send is, and the frame goes on. + probe::fail_sends(addr, Kind::Media, 3, &[Fail::NoBufferSpace, Fail::Other]); + nal_tx.blocking_send(keyframe(&shared, 40)).unwrap(); + let frame = ipad.attempts(3, 41); + let expected: Vec = (0..=3).chain(3..40).collect(); + assert_eq!(indexes(&frame), expected); + assert_eq!( + failures(&frame), + [(3, Fail::NoBufferSpace), (4, Fail::Other)] + ); + let failed = log_lines("UDP send failed link="); + assert_eq!( + failed, + ["WARN UDP send failed link=wifi error=connection reset"] + ); // Five frames of 100 fragments while every send is refused: each frame - // backs off once and gives up, so none holds the loop for more than one - // back-off. - probe::refuse_sends(addr, Kind::Media, 0, 1_000_000); - let pushed = Instant::now(); + // asks for one back-off, no longer, and gives up after its retry, so no + // frame holds the loop for more than one back-off. + probe::refuse_sends(addr, Kind::Media, 0, 1000); for _ in 0..5 { nal_tx.blocking_send(keyframe(&shared, 100)).unwrap(); } - while 1_000_000 - probe::refusals_left(addr) < 10 { + let last = ipad.attempts(8, 2); + ipad.receive(Duration::from_millis(100)); + assert_eq!(probe::refusals_left(addr), 990, "two refusals per frame"); + let first = attempts_of(4, false)[0].at; + let backoffs = waits_between(first, last[1].at); + assert_eq!(backoffs.len(), 5, "{backoffs:?}"); + for (asked, until) in &backoffs { assert!( - pushed.elapsed() < Duration::from_secs(2), - "refused {} sends", - 1_000_000 - probe::refusals_left(addr) + *until <= *asked + BACKOFF, + "a pause of {:?}", + *until - *asked + ); + } + for frame_seq in 4..=8 { + let frame = attempts_of(frame_seq, false); + assert_eq!(indexes(&frame), [0, 0], "frame {frame_seq}"); + assert_eq!( + failures(&frame), + [(0, Fail::NoBufferSpace), (1, Fail::NoBufferSpace)] ); - std::thread::sleep(Duration::from_millis(1)); } - let took = pushed.elapsed(); - ipad.receive(Duration::from_millis(100)); - assert_eq!( - 1_000_000 - probe::refusals_left(addr), - 10, - "two refusals per frame, the first and its retry" - ); - assert!( - took < 5 * (BACKOFF + Duration::from_millis(30)), - "five frames took {took:?}" - ); probe::refuse_sends(addr, Kind::Media, 0, 0); assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 12_000_000); @@ -320,7 +432,7 @@ fn refused_sends_back_off_step_down_once_and_log_one_episode() { assert_eq!(ended.len(), 1, "{ended:#?}"); assert!( ended[0] - .starts_with("INFO UDP sends have buffer space again link=wifi refused=13 took_ms="), + .starts_with("INFO UDP sends have buffer space again link=wifi refused=19 took_ms="), "{ended:#?}" ); let steps = log_lines("ABR stepping bitrate down"); @@ -331,7 +443,11 @@ fn refused_sends_back_off_step_down_once_and_log_one_episode() { ); // A refused control reply is sent again once. This is a new episode, - // with its own step down. + // and the downgrade cooldown (3 s) since the first step is over, so it + // steps down again. + while stepped.elapsed() < Duration::from_millis(3_200) { + ipad.receive(Duration::from_millis(50)); + } ipad.after_heartbeat(); // No keep-alive Pong still on its way to take the refusal. ipad.pinged = Instant::now(); @@ -357,8 +473,9 @@ fn refused_sends_back_off_step_down_once_and_log_one_episode() { assert_eq!(shared.abr_current_bps.load(Ordering::SeqCst), 10_000_000); assert_eq!(log_lines("no buffer space").len(), 2); assert!(log_lines("Failed to send control reply").is_empty()); - assert!( - log_lines("UDP send failed link=").is_empty(), + assert_eq!( + log_lines("UDP send failed link=").len(), + 1, "no line per send" ); From 75acc7e98b44bf31b378ef8b98b910badced524d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 23:44:05 -0500 Subject: [PATCH 131/226] Keep a stopped PC stopped when its chosen network changes A stream that noticed its chosen network change asked the supervisor to restart, and the supervisor restarted from any state. When the check ran while Stop was stopping that stream, the PC started streaming again by itself. The same held for the virtual display's restart request. The stream now asks to restart only itself, by its generation. The supervisor restarts it only while that stream still runs: after Stop, the storm brake or a newer stream, nothing starts. Restart from the window is unchanged. --- host/src/control.rs | 7 +++++ host/src/supervisor.rs | 45 ++++++++++++++++++++++++++++++++ host/src/transport/mod.rs | 10 +++++-- host/tests/hello_standby.rs | 52 +++++++++++++++++++++++++++++++++++++ host/tests/listen_on.rs | 3 ++- 5 files changed, 114 insertions(+), 3 deletions(-) diff --git a/host/src/control.rs b/host/src/control.rs index 070047c4..c69b042e 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -266,6 +266,13 @@ pub fn merge_fps(host: u32, client: u32) -> u32 { pub enum SupervisorCommand { /// Tear the current pipeline down and start a fresh generation. Restart, + /// The stream of `generation` asks to start again: its chosen network + /// changed, or its virtual display has to go. Only while that stream + /// still runs: one the user stopped, or one a newer stream replaced, + /// stays as it is. + RestartStream { + generation: u64, + }, /// Stop streaming (pipeline down, GUI stays). `Start` resumes. Stop, /// Start a fresh pipeline from Stopped/Failed. diff --git a/host/src/supervisor.rs b/host/src/supervisor.rs index bf2bd173..0e8c3325 100644 --- a/host/src/supervisor.rs +++ b/host/src/supervisor.rs @@ -155,6 +155,22 @@ impl Machine { Effect::Restart } }, + SupervisorCommand::RestartStream { generation } => match self.state { + SupervisorState::Running { + generation: live_generation, + } if live_generation == *generation => { + self.attempt = 0; + self.restarts.clear(); + Effect::Restart + } + _ => { + info!( + generation, + "A stream that no longer runs asked to start again: left as it is" + ); + Effect::None + } + }, SupervisorCommand::Stop => match self.state { SupervisorState::Running { .. } | SupervisorState::BackingOff { .. } @@ -916,6 +932,35 @@ mod tests { assert!(matches!(machine.state, SupervisorState::Running { .. })); } + /// A stream asks to start again on its own only while it runs: after + /// Stop, the storm brake or a newer stream, nothing starts. + #[test] + fn a_stream_restarts_itself_only_while_it_runs() { + let restart = |generation| SupervisorCommand::RestartStream { generation }; + let mut machine = Machine::new(5); + assert_eq!(machine.on_command(&restart(4)), Effect::None); + assert_eq!(machine.on_command(&restart(5)), Effect::Restart); + machine.note_started(6); + + assert_eq!( + machine.on_command(&SupervisorCommand::Stop), + Effect::StopPipeline + ); + assert_eq!(machine.on_command(&restart(6)), Effect::None); + assert_eq!(machine.state, SupervisorState::Stopped); + + machine.state = SupervisorState::Failed { + reason: "encoder".into(), + stage: Stage::Encoder, + }; + assert_eq!(machine.on_command(&restart(6)), Effect::None); + let t0 = Instant::now(); + machine.note_started(7); + machine.on_stage_exit(7, Stage::Capture, &failed("dxgi died"), t0); + assert_eq!(machine.on_command(&restart(7)), Effect::None); + assert!(matches!(machine.state, SupervisorState::BackingOff { .. })); + } + #[test] fn restart_storm_lands_in_failed_until_manual_start() { let mut machine = Machine::new(1); diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index effb9a61..70f6dcd2 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -192,6 +192,9 @@ pub async fn start_sender( shared: SharedControl, supervisor_tx: std_mpsc::Sender, ) -> Result<(), Box> { + // The generation this stream belongs to: it asks the supervisor to + // restart only itself, never a stream the user stopped. + let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); let listen_on = shared.listen_on.lock().clone(); let bind_ip = crate::network::listen_ip_now(&listen_on); if bind_ip.is_loopback() { @@ -340,7 +343,7 @@ pub async fn start_sender( if !shared.client_connected() && crate::network::listen_ip_now(&listen_on) != bind_ip { info!(listen_on = listen_on.label(), "The chosen network changed: restarting the stream to listen on it"); shared.stop(); - if let Err(error) = supervisor_tx.send(SupervisorCommand::Restart) { + if let Err(error) = supervisor_tx.send(SupervisorCommand::RestartStream { generation }) { warn!(error = %error, "Failed to request a pipeline restart for the chosen network"); } } @@ -991,7 +994,10 @@ async fn execute_actions( } if restart { shared.stop(); - if let Err(error) = supervisor_tx.send(SupervisorCommand::Restart) { + // Only this stream: one the user stopped keeps its generation until + // Start, so the supervisor leaves it stopped. + let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + if let Err(error) = supervisor_tx.send(SupervisorCommand::RestartStream { generation }) { warn!(error = %error, "Failed to request pipeline restart for the virtual display"); } } diff --git a/host/tests/hello_standby.rs b/host/tests/hello_standby.rs index b79632b4..15f287af 100644 --- a/host/tests/hello_standby.rs +++ b/host/tests/hello_standby.rs @@ -468,6 +468,58 @@ fn the_storm_brake_says_why_and_waits_for_start() { assert_eq!(ack.status, HelloStatus::Failed); } +/// A stream that finds its chosen network changed as Stop stops it asks to +/// start again on the new address. That request comes from a stream the +/// user already stopped: the PC stays stopped, and only Start streams. +/// +/// The session lock holds the stream between its bind and its first look +/// at the network, so it always looks after Stop, on the new address. +#[cfg(debug_assertions)] +#[test] +fn a_stream_stopping_as_its_network_changes_stays_stopped() { + use eternal_host::network::{set_test_adapters, Adapter, Kind, ListenOn}; + let wifi = |up| { + vec![Adapter { + name: "Wi-Fi".into(), + kind: Kind::Wifi, + up, + gateway: true, + ipv4: vec![std::net::Ipv4Addr::new(192, 168, 1, 20)], + }] + }; + let host = Host::start_with(GpuInfo::software_fallback(), 500, |shared| { + shared.pairing.lock().required = false; + *shared.listen_on.lock() = ListenOn::Adapter("Wi-Fi".into()); + set_test_adapters(Some(wifi(false))); + std::mem::forget(shared.session.lock()); + }); + // The stream binds this PC only, the network being down, then waits + // for the session. + let shown = format!("unknown:{}", host.port); + let deadline = Instant::now() + Duration::from_secs(20); + while PIPELINE_STATS.lock().listen_addr != shown { + assert!(Instant::now() < deadline, "the stream never bound"); + std::thread::sleep(Duration::from_millis(10)); + } + host.send(SupervisorCommand::Stop); + set_test_adapters(Some(wifi(true))); + // SAFETY: the guard forgotten above was taken on this lock, and nothing + // else holds it. + unsafe { host.shared.session.force_unlock() }; + + host.wait_until_stopped(); + let deadline = Instant::now() + Duration::from_secs(3); + while Instant::now() < deadline { + assert_eq!( + host.shared.session.lock().not_streaming(), + Some(NotStreaming::Stopped), + "the stopped stream restarted itself" + ); + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!host.streaming(), "the stopped stream restarted itself"); +} + /// A stopped PC listens where iPads can reach it, as the stream does: on /// this PC only while the chosen network is down, on the network's address /// once it is back, on this PC again when it goes, and on every network diff --git a/host/tests/listen_on.rs b/host/tests/listen_on.rs index d7a40034..4f60faa9 100644 --- a/host/tests/listen_on.rs +++ b/host/tests/listen_on.rs @@ -91,7 +91,8 @@ fn the_address_row_shows_a_chosen_network_that_came_back() { let command = supervisor_rx .recv_timeout(Duration::from_secs(30)) .expect("the network check asks for a restart"); - assert!(matches!(command, SupervisorCommand::Restart)); + let generation = eternal_host::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + assert_eq!(command, SupervisorCommand::RestartStream { generation }); transport.join().unwrap(); // The supervisor starts the stream again, now on the network's address. From b546298cc288cec14d1ecf96273e8caa13287101 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 23:44:05 -0500 Subject: [PATCH 132/226] Time the iPad's retries to a stopped PC by when they arrive The test measured the gaps between its own answers by the clock after each answer, starting from a moment before the first retry was due. A test thread that ran late, or a quick retry already on its way, made a gap look short. The fake PC now keeps each datagram's kernel arrival time, and the test measures the gaps between the HELLO2s that arrived after the iPad showed it heard the answer. Retries are still about 2 s apart, with the same HELLO2. --- ios/EternalMonitorTests/FakeUDPHost.swift | 57 +++++++++++++++++-- .../PCNotStreamingTests.swift | 15 +++-- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index ee97164a..261b56f8 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -19,6 +19,11 @@ final class FakeUDPHost: @unchecked Sendable { private(set) var lastHelloSize = 0 /// The pairing token the HELLO2 answered last carried. private(set) var lastAuthToken = Data() + /// When the HELLO2 answered last arrived, by the kernel's receive + /// time: how fast this fake read it doesn't move it. + private(set) var lastHelloArrival: Date? + /// When the datagram read last arrived, by the kernel's receive time. + private var arrival = Date() /// A HELLO2 came before the identify request was answered. private(set) var helloBeforeIdentify = false @@ -38,20 +43,63 @@ final class FakeUDPHost: @unchecked Sendable { } var timeout = timeval(tv_sec: 0, tv_usec: 300_000) setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, socklen_t(MemoryLayout.size)) + var on: Int32 = 1 + setsockopt(fd, SOL_SOCKET, SO_TIMESTAMP, &on, socklen_t(MemoryLayout.size)) } deinit { Darwin.close(fd) } private func receive() -> (Data, sockaddr_storage, socklen_t)? { var buffer = [UInt8](repeating: 0, count: 2048) + var control = [UInt8](repeating: 0, count: 64) var from = sockaddr_storage() var length = socklen_t(MemoryLayout.size) - let count = withUnsafeMutablePointer(to: &from) { pointer in - pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { - recvfrom(fd, &buffer, buffer.count, 0, $0, &length) + var controlLength = 0 + let count = buffer.withUnsafeMutableBytes { bytes in + control.withUnsafeMutableBytes { controlBytes in + withUnsafeMutablePointer(to: &from) { name in + var vector = iovec(iov_base: bytes.baseAddress, iov_len: bytes.count) + return withUnsafeMutablePointer(to: &vector) { vectors in + var message = msghdr( + msg_name: UnsafeMutableRawPointer(name), msg_namelen: length, + msg_iov: vectors, msg_iovlen: 1, + msg_control: controlBytes.baseAddress, msg_controllen: socklen_t(controlBytes.count), + msg_flags: 0) + let count = recvmsg(fd, &message, 0) + length = message.msg_namelen + controlLength = Int(message.msg_controllen) + return count + } + } + } + } + guard count > 0 else { return nil } + arrival = Self.kernelTimestamp(control.prefix(controlLength)) ?? Date() + return (Data(buffer.prefix(count)), from, length) + } + + /// The SCM_TIMESTAMP in a datagram's control messages. Darwin aligns + /// each message and its data to 4 bytes after a 12-byte header. + private static func kernelTimestamp(_ control: ArraySlice) -> Date? { + let bytes = Array(control) + var offset = 0 + while offset + 12 <= bytes.count { + let (length, level, type) = bytes.withUnsafeBytes { + ($0.loadUnaligned(fromByteOffset: offset, as: UInt32.self), + $0.loadUnaligned(fromByteOffset: offset + 4, as: Int32.self), + $0.loadUnaligned(fromByteOffset: offset + 8, as: Int32.self)) + } + guard length >= 12 else { return nil } + if level == SOL_SOCKET, type == SCM_TIMESTAMP, + offset + 12 + MemoryLayout.size <= bytes.count { + let time = bytes.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset + 12, as: timeval.self) + } + return Date(timeIntervalSince1970: TimeInterval(time.tv_sec) + TimeInterval(time.tv_usec) / 1_000_000) } + offset += (Int(length) + 3) & ~3 } - return count > 0 ? (Data(buffer.prefix(count)), from, length) : nil + return nil } /// Answers the next HELLO2 attempt, with session 41 unless `status` @@ -70,6 +118,7 @@ final class FakeUDPHost: @unchecked Sendable { lastHello = hello lastHelloSize = data.count lastAuthToken = hello.authToken + lastHelloArrival = arrival send(Self.ack(nonce: hello.clientNonce, status: status, acceptedVersion: acceptedVersion, hostName: hostName, authToken: authToken, hostCaps: hostCaps, reason: reason)) return true diff --git a/ios/EternalMonitorTests/PCNotStreamingTests.swift b/ios/EternalMonitorTests/PCNotStreamingTests.swift index f0ff7b04..2ed41b9f 100644 --- a/ios/EternalMonitorTests/PCNotStreamingTests.swift +++ b/ios/EternalMonitorTests/PCNotStreamingTests.swift @@ -36,15 +36,20 @@ final class PCNotStreamingTests: XCTestCase { XCTAssertEqual(manager.state, .connecting) // It keeps asking, every two seconds rather than four times a - // second, with the same HELLO2, past the connect timeout. - var answeredAt = [Date()] - for _ in 0..<3 { + // second, with the same HELLO2, past the connect timeout. The gaps + // are between the HELLO2s the PC received, by their arrival times, + // from those sent after the iPad showed it heard the answer: a quick + // retry sent before that may still have been on its way. + let told = Date() + var askedAt: [Date] = [] + for _ in 0..<6 where askedAt.count < 3 { let again = await Task.detached { host.acceptSession(status: .stopped) }.value XCTAssertTrue(again, "the iPad stopped asking") - answeredAt.append(Date()) XCTAssertEqual(host.lastNonce, firstNonce) + if let arrival = host.lastHelloArrival, arrival > told { askedAt.append(arrival) } } - for (earlier, later) in zip(answeredAt.dropLast(), answeredAt.dropFirst()) { + XCTAssertEqual(askedAt.count, 3, "the iPad stopped asking") + for (earlier, later) in zip(askedAt.dropLast(), askedAt.dropFirst()) { XCTAssertGreaterThan(later.timeIntervalSince(earlier), 1.5, "asking too often") } XCTAssertEqual(manager.state, .connecting) From e83f7905d058813db576cd94d95f0fbb8f808939 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sat, 7 Nov 2026 23:30:12 -0500 Subject: [PATCH 133/226] Start each capture with the pointer hidden The pointer feed outlives a capture, and a second screen starts from a mirror of the main screen. A position from the output captured before must not carry over to the new one if Desktop Duplication reports no pointer on its first frame. Each capture now starts with the pointer hidden; Desktop Duplication reports the output's own pointer once it is on it. --- host/src/capture/dxgi.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 36f7af15..19d7baec 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -329,9 +329,12 @@ pub fn run_capture_loop( cursor.size(), ) }; - let mut pointer_in_image = in_picture(&cursor); - if pointer_in_image { - // A position from a mirror before this one would stay drawn. + let mut pointer_in_image; + { + // The feed outlives this capture: a position from the output + // captured before (the mirror a second screen starts from) would be + // drawn on this one until Desktop Duplication reports its own, + // which it does once the pointer is on this output. shared.pointer.set_position(super::pointer::Position { visible: false, x: 0, @@ -1100,7 +1103,7 @@ impl CursorState { /// The pointer image's size in pixels, once there is one. A monochrome /// shape's buffer stacks two masks. fn size(&self) -> Option<(u32, u32)> { - self.has_shape.then(|| match self.shape_type { + self.has_shape.then_some(match self.shape_type { DXGI_POINTER_SHAPE_MONOCHROME => (self.width, self.height / 2), _ => (self.width, self.height), }) From c0c7f0471fb01d282800e76cf528cacb433bdf86 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 00:21:45 -0500 Subject: [PATCH 134/226] Tell identical graphics cards apart by their LUID Two cards of the same model had the same AdapterKey: wgpu's DX12 PCI bus comes from the vendor and device ids, so it names the first card for both. A replacement could then prefer the wrong card, and leaving one out left out both. The key now carries the adapter's LUID, from DXGI on DX12 and from VkPhysicalDeviceIDProperties on Vulkan 1.1+ on Windows (ash at wgpu-hal's version); the other fields remain the fallback. The device test now checks the loss handlers are installed: a wgpu error marks the new device lost instead of hitting wgpu's panicking default. --- Cargo.lock | 1 + host/tests/window_device_loss.rs | 65 +++++++++++++++++++++++++++++--- vendor/README.md | 7 +++- vendor/egui-wgpu/Cargo.toml | 6 +++ vendor/egui-wgpu/src/lib.rs | 64 +++++++++++++++++++++++-------- vendor/egui-wgpu/src/winit.rs | 4 +- 6 files changed, 121 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5427b912..63450620 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1000,6 +1000,7 @@ name = "egui-wgpu" version = "0.36.2" dependencies = [ "ahash", + "ash", "bytemuck", "document-features", "egui", diff --git a/host/tests/window_device_loss.rs b/host/tests/window_device_loss.rs index f52e47e2..8090de2f 100644 --- a/host/tests/window_device_loss.rs +++ b/host/tests/window_device_loss.rs @@ -4,10 +4,12 @@ //! patched copy in `vendor/egui-wgpu` skips the frame and replaces the device //! on a schedule (`Replacement`), giving up after repeated failures. +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use std::time::{Duration, Instant}; use eframe::egui; -use eframe::egui_wgpu::winit::{Answer, Asked, Replacement, REPLACEMENT_TIMEOUT}; +use eframe::egui_wgpu::winit::{watch, Answer, Asked, Replacement, REPLACEMENT_TIMEOUT}; use eframe::egui_wgpu::{ replacement_candidates, AdapterKey, RenderState, Renderer, RendererOptions, ScreenDescriptor, WgpuConfiguration, @@ -241,19 +243,27 @@ fn a_replacement_device_is_made_without_any_window_or_surface() { let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); let drew_on = block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())) .expect("a graphics adapter"); - let preferred = AdapterKey::of(&drew_on.get_info()); - let mut watched = false; + let preferred = AdapterKey::of(&drew_on); + let lost = Arc::new(AtomicBool::new(false)); let new = block_on(RenderState::new_device( &config, &instance, Some(&preferred), &[], - |_| watched = true, + |device| watch(device, &lost), )) .expect("a device"); - // The loss handlers went on before anything was made on the device. - assert!(watched); assert_eq!(new.adapter_key(), preferred); + // The loss handlers are on the device: a wgpu error marks it lost, where + // wgpu's default handler would panic. + assert!(!lost.load(Ordering::SeqCst)); + let _misaligned = new.device().create_buffer(&wgpu::BufferDescriptor { + label: None, + size: 3, + usage: wgpu::BufferUsages::COPY_DST, + mapped_at_creation: true, + }); + assert!(lost.load(Ordering::SeqCst)); let buffer = new.device().create_buffer(&wgpu::BufferDescriptor { label: None, size: 256, @@ -265,6 +275,7 @@ fn a_replacement_device_is_made_without_any_window_or_surface() { fn adapter(name: &str, backend: wgpu::Backend, vendor: u32) -> AdapterKey { AdapterKey { + luid: None, backend, vendor, device: 1, @@ -311,3 +322,45 @@ fn a_replacement_prefers_the_adapter_that_drew_and_moves_past_ones_that_cannot() assert_eq!(chosen, dgpu_vulkan); assert_eq!(tried, [dgpu, igpu, dgpu_vulkan]); } + +#[test] +fn two_identical_cards_are_told_apart_by_their_luid() { + use wgpu::Backend::Dx12; + // Two of the same card on different buses: wgpu's DX12 bus comes from the + // PCI ids, so everything but the LUID matches. + let card = |luid| AdapterKey { + luid: Some(luid), + pci_bus: "0000:01:00.0".into(), + ..adapter("RTX 5080", Dx12, 0x10de) + }; + let (first, second) = (card(0x0000_0000_0001_a2b3), card(0x0000_0000_0001_c4d5)); + let listed = [first.clone(), second.clone()]; + + // The one that drew is chosen, not its twin listed first. + assert_eq!(replacement_candidates(&listed, Some(&second), &[]), [1]); + // Leaving one out leaves the other. + assert_eq!( + replacement_candidates(&listed, Some(&first), std::slice::from_ref(&first)), + [1] + ); + assert_eq!(replacement_candidates(&listed, None, &[second]), [0]); +} + +#[cfg(windows)] +#[test] +fn every_dx12_adapter_has_its_own_luid() { + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { + backends: wgpu::Backends::DX12, + ..wgpu::InstanceDescriptor::new_without_display_handle() + }); + let adapters = block_on(instance.enumerate_adapters(wgpu::Backends::DX12)); + assert!(!adapters.is_empty()); + let luids: Vec = adapters + .iter() + .map(|adapter| AdapterKey::of(adapter).luid.expect("a DX12 LUID")) + .collect(); + let mut unique = luids.clone(); + unique.sort_unstable(); + unique.dedup(); + assert_eq!(unique.len(), luids.len(), "{luids:x?}"); +} diff --git a/vendor/README.md b/vendor/README.md index 69ed2577..de1d5a6c 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -51,8 +51,11 @@ The patch: `replacement_candidates` gives the host's adapter choice only the adapter that last drew on the window, starting with the one from start, while it is listed. An adapter that can't draw on the window is left out of later tries - until a replacement works. Adapters are told apart by `AdapterKey` (backend, - PCI ids, type, name, PCI bus), because wgpu reports no LUID. + until a replacement works. Adapters are told apart by `AdapterKey`: the + LUID (from DXGI on DX12, and `VkPhysicalDeviceIDProperties` on Vulkan 1.1+ + on Windows, through the `ash` dependency at wgpu-hal's version), else the + backend, PCI ids, type, name and PCI bus. wgpu's DX12 bus comes from the + PCI ids alone, so identical cards differ only by LUID. The thread is named `winit::REPLACEMENT_THREAD`, so the host's panic hook treats a panic there as one failed try: it doesn't turn the virtual display off as it does for other background threads. diff --git a/vendor/egui-wgpu/Cargo.toml b/vendor/egui-wgpu/Cargo.toml index cd9495a2..86d7f87d 100644 --- a/vendor/egui-wgpu/Cargo.toml +++ b/vendor/egui-wgpu/Cargo.toml @@ -338,3 +338,9 @@ priority = -1 [lints.rustdoc] all = "warn" + +# EternalMonitor patch: an adapter's Vulkan LUID (`AdapterKey`), at wgpu-hal's version. +[target.'cfg(windows)'.dependencies.ash] +version = "0.38" +default-features = false + diff --git a/vendor/egui-wgpu/src/lib.rs b/vendor/egui-wgpu/src/lib.rs index 6128cfa8..bac52760 100644 --- a/vendor/egui-wgpu/src/lib.rs +++ b/vendor/egui-wgpu/src/lib.rs @@ -296,11 +296,13 @@ impl RenderState { } } -/// EternalMonitor patch: which adapter one is, across listings: wgpu's `AdapterInfo` -/// carries no LUID, so the backend, the PCI ids, the type, the name and the PCI bus (when -/// the backend reports one). +/// EternalMonitor patch: which adapter one is, across listings. Its LUID where the +/// backend gives one (DX12 always, Vulkan 1.1+ on Windows), which tells identical cards +/// apart. Without one, the backend, the PCI ids, the type, the name and the PCI bus: wgpu's +/// DX12 bus comes from the PCI ids alone, so identical cards there would share it. #[derive(Clone, Debug, PartialEq, Eq)] pub struct AdapterKey { + pub luid: Option, pub backend: wgpu::Backend, pub vendor: u32, pub device: u32, @@ -310,18 +312,54 @@ pub struct AdapterKey { } impl AdapterKey { - pub fn of(info: &wgpu::AdapterInfo) -> Self { + pub fn of(adapter: &wgpu::Adapter) -> Self { + let info = adapter.get_info(); Self { + luid: luid(adapter), backend: info.backend, vendor: info.vendor, device: info.device, device_type: info.device_type, - name: info.name.clone(), - pci_bus: info.device_pci_bus_id.clone(), + name: info.name, + pci_bus: info.device_pci_bus_id, } } } +/// EternalMonitor patch: the adapter's LUID, from DXGI on DX12, or from +/// `VkPhysicalDeviceIDProperties` on Vulkan 1.1+. +#[cfg(windows)] +fn luid(adapter: &wgpu::Adapter) -> Option { + // SAFETY: the backend objects are only read, while `adapter` holds them. + if let Some(hal) = unsafe { adapter.as_hal::() } { + let desc = unsafe { hal.raw_adapter().GetDesc1() }.ok()?; + let luid = desc.AdapterLuid; + return Some((u64::from(luid.HighPart as u32) << 32) | u64::from(luid.LowPart)); + } + if let Some(hal) = unsafe { adapter.as_hal::() } { + let shared = hal.shared_instance(); + if shared.instance_api_version() < ash::vk::API_VERSION_1_1 { + return None; + } + let mut id = ash::vk::PhysicalDeviceIDProperties::default(); + { + let mut properties = ash::vk::PhysicalDeviceProperties2::default().push_next(&mut id); + unsafe { + shared + .raw_instance() + .get_physical_device_properties2(hal.raw_physical_device(), &mut properties); + } + } + return (id.device_luid_valid == ash::vk::TRUE).then(|| u64::from_le_bytes(id.device_luid)); + } + None +} + +#[cfg(not(windows))] +fn luid(_adapter: &wgpu::Adapter) -> Option { + None +} + /// EternalMonitor patch: the adapters, by index into `keys`, that a replacement may choose /// from: only `preferred` (the last one that drew on the window) while it is listed and /// not `excluded`, else every one not `excluded` (ones that couldn't draw on the window). @@ -359,7 +397,7 @@ impl NewDevice { } pub fn adapter_key(&self) -> AdapterKey { - AdapterKey::of(&self.adapter.get_info()) + AdapterKey::of(&self.adapter) } } @@ -384,10 +422,7 @@ impl RenderState { let available_adapters = instance .enumerate_adapters(create_new.instance_descriptor.backends) .await; - let keys: Vec = available_adapters - .iter() - .map(|adapter| AdapterKey::of(&adapter.get_info())) - .collect(); + let keys: Vec = available_adapters.iter().map(AdapterKey::of).collect(); let candidates: Vec = replacement_candidates(&keys, preferred, excluded) .into_iter() .map(|index| available_adapters[index].clone()) @@ -401,14 +436,11 @@ impl RenderState { selector(&candidates, None).map_err(WgpuError::CustomNativeAdapterSelectionError)? } else { // wgpu chooses among all adapters, not these: keep its choice only if it is one. - let keys: Vec = candidates - .iter() - .map(|adapter| AdapterKey::of(&adapter.get_info())) - .collect(); + let keys: Vec = candidates.iter().map(AdapterKey::of).collect(); request_adapter(instance, create_new.power_preference, None, &candidates) .await .ok() - .filter(|adapter| keys.contains(&AdapterKey::of(&adapter.get_info()))) + .filter(|adapter| keys.contains(&AdapterKey::of(adapter))) .unwrap_or_else(|| candidates[0].clone()) }; let (device, queue) = adapter diff --git a/vendor/egui-wgpu/src/winit.rs b/vendor/egui-wgpu/src/winit.rs index a91af2c5..707321fb 100644 --- a/vendor/egui-wgpu/src/winit.rs +++ b/vendor/egui-wgpu/src/winit.rs @@ -131,7 +131,7 @@ enum Replacing { /// EternalMonitor patch: flag the loss of `device` in `lost`. An error wgpu reports outside /// an error scope flags it too, instead of panicking as wgpu's default handler does: a /// window whose device or surface fails is drawn again on a new one. -fn watch(device: &wgpu::Device, lost: &Arc) { +pub fn watch(device: &wgpu::Device, lost: &Arc) { let flag = Arc::clone(lost); device.set_device_lost_callback(move |reason, message| { log::warn!("The wgpu device was lost ({reason:?}): {message}"); @@ -493,7 +493,7 @@ impl Painter { let lost = Arc::new(AtomicBool::new(false)); watch(&render_state.device, &lost); self.device_lost = lost; - self.preferred = Some(crate::AdapterKey::of(&render_state.adapter.get_info())); + self.preferred = Some(crate::AdapterKey::of(&render_state.adapter)); self.render_state = Some(render_state); } self.install_surface(surface, viewport_id, size.width, size.height, false); From 4508997910c11e5892cb4b8db218195d840b11d6 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 00:29:40 -0500 Subject: [PATCH 135/226] Give the startup and quit device read a 2 s limit The Disable at startup, at quit and in a panic reads the virtual display's device through Configuration Manager before deciding whether to run the task. That read had no limit: one that hung held the Disable, and every request queued behind it, for as long as it hung. The read now runs on its own thread and is waited on for 2 s. If it doesn't answer in time, the host logs it once and runs the Disable task under the task's own limit; a late answer is dropped. A device that reads disabled in time still runs no task. Tests: a read that blocks runs the Disable after the limit; a read that answers disabled at once keeps the skip. --- host/src/vdd.rs | 87 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/host/src/vdd.rs b/host/src/vdd.rs index 5bba26d1..140bc949 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -595,16 +595,48 @@ pub fn disable_in_background() { } } +/// How long the Disable at startup, at quit or in a panic waits for +/// Configuration Manager's read of the device before it runs the task. +#[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. +const DEVICE_READ_LIMIT: Duration = Duration::from_secs(2); + /// Whether Configuration Manager reads the device disabled, said in the log. #[cfg(windows)] fn device_off() -> bool { - let off = reports_disabled(crate::vdd_device::query()); + let off = off_within(crate::vdd_device::query, DEVICE_READ_LIMIT); if off { info!("Virtual display already off — not running the Disable task"); } off } +/// Whether `query`, run on its own thread, reads the device disabled within +/// `limit`. A read that doesn't answer in time, or whose thread can't start, +/// counts as not disabled, so the Disable task runs under its own limit; a +/// late answer is dropped. +#[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. +fn off_within( + query: impl FnOnce() -> crate::vdd_device::DeviceState + Send + 'static, + limit: Duration, +) -> bool { + let (answer, answered) = std::sync::mpsc::channel(); + let spawned = std::thread::Builder::new() + .name("vdd-device-read".into()) + .spawn(move || { + let _ = answer.send(query()); + }); + match spawned.ok().map(|_| answered.recv_timeout(limit)) { + Some(Ok(device)) => reports_disabled(device), + _ => { + tracing::warn!( + limit_ms = limit.as_millis() as u64, + "Could not read the virtual display's state in time — running the Disable task" + ); + false + } + } +} + /// Whether the device reads disabled (problem 22): a Disable has nothing to /// turn off. #[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. @@ -845,6 +877,59 @@ mod tests { } } + /// Configuration Manager's read of the device had no limit: a read that + /// hung held the startup, quit and panic Disables, and every request + /// behind them, for as long as it hung. + #[test] + fn a_device_read_that_hangs_runs_the_disable_task_after_its_limit() { + use crate::vdd_device::DeviceState; + use std::cell::Cell; + let (release, held) = std::sync::mpsc::channel::<()>(); + let (done, finished) = std::sync::mpsc::channel(); + let started = Instant::now(); + std::thread::spawn(move || { + let tasks = DriverTasks::new(); + let ran = Cell::new(false); + let hung = move || { + let _ = held.recv(); + DeviceState::Problem(22) + }; + tasks.disable_unless_off( + || {}, + || off_within(hung, DEVICE_READ_LIMIT), + |enabled, _| { + assert!(!enabled); + ran.set(true); + TaskOutcome::Done + }, + ); + let _ = done.send(ran.get()); + }); + let ran = finished + .recv_timeout(DEVICE_READ_LIMIT + Duration::from_secs(5)) + .expect("the Disable waited on a hung device read past its limit"); + assert!(ran, "a hung device read skipped the Disable task"); + assert!(started.elapsed() >= DEVICE_READ_LIMIT); + // The read answers late; nothing waits for it. + let _ = release.send(()); + } + + #[test] + fn a_device_read_that_answers_disabled_at_once_keeps_the_skip() { + use crate::vdd_device::DeviceState; + let tasks = DriverTasks::new(); + let started = Instant::now(); + assert!(tasks.disable_unless_off( + || {}, + || off_within(|| DeviceState::Problem(22), DEVICE_READ_LIMIT), + |_, _| panic!("ran the Disable task for a display that reads disabled") + )); + assert!(started.elapsed() < DEVICE_READ_LIMIT); + for device in [DeviceState::Ready, DeviceState::Missing] { + assert!(!off_within(move || device, DEVICE_READ_LIMIT), "{device:?}"); + } + } + #[test] fn a_hung_task_is_not_waited_out_again_until_the_cooldown_ends() { let mut gate = TaskGate::new(); From d9e5a791e45cc3da71b2a584e232eed69814de62 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 00:31:25 -0500 Subject: [PATCH 136/226] Tie the second screen's pointer to the running driver's start The host now records what each driver start gave the pointer, and capture trusts that record only for the same start. - An Enable that started the virtual display driver records it on disk, in vdd-cursor.json in the app data folder: - the boot id; - the device's last arrival (DEVPKEY_Device_LastArrivalDate), which changes at every driver start; - the SHA-256 of the settings file it started with; - the cursor that file gives it, read as the driver reads it. - Each second screen capture compares the running driver's boot and arrival with the record. A record of another start, or none, means Unknown. This covers AlreadyDone, a relaunch with the driver left on, and a restart by another program. - With a hardware cursor, each pointer image is classed against the caps that start used, such as the user's 64 or 256, not a fixed 128. - Unknown is handled as in 0.3.4: - capture draws the pointer Desktop Duplication reports; - no image and no shown position go to the iPad; - an iPad that was drawing the pointer before gets hidden positions. - A registry override, a file the driver can't read as a full hardware cursor, or a driver version that doesn't tell also count as Unknown. - The pointer e2e now fails on a hidden state after the pointer went apart, and on any pointer message to an older client. --- ARCHITECTURE.md | 2 +- host/Cargo.toml | 1 + host/src/capture/dxgi.rs | 49 ++- host/src/capture/mod.rs | 242 ++++++++++---- host/src/capture/pointer.rs | 14 + host/src/capture/second_screen.rs | 47 ++- host/src/control.rs | 5 - host/src/transport/mod.rs | 10 +- host/src/transport/pointer.rs | 48 +++ host/src/vdd_device.rs | 209 ++++++++---- host/src/vdd_settings.rs | 531 +++++++++++++++++++++++++----- host/tests/e2e_synthetic.rs | 21 +- 12 files changed, 901 insertions(+), 278 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9159a5be..9461197c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -75,7 +75,7 @@ Rust and Swift share golden vectors and stable packet-type registry tests. | KEYFRAME_REQUEST | Recovery after repair expiry or decoder error | | PING / PONG | Min-RTT clock estimate for end-to-end latency | | INPUT_EVENT | Normalized mouse/touch events, HID keys and UTF-16 text | -| POINTER_SHAPE / POINTER_STATE | The Mirror's pointer image (chunked premultiplied BGRA and an invert mask) and its position, at most once a frame budget, for an iPad that draws the pointer itself (feature bit 5, host capability bit 6); the video then leaves the pointer out. The Second screen does the same when the virtual display driver's settings give Windows a hardware cursor, as the host writes them; otherwise its pointer stays in the video. | +| POINTER_SHAPE / POINTER_STATE | The Mirror's pointer image (chunked premultiplied BGRA and an invert mask) and its position, at most once a frame budget, for an iPad that draws the pointer itself (feature bit 5, host capability bit 6); the video then leaves the pointer out. The Second screen does the same when the running virtual display driver started with a hardware cursor (as the host's settings file sets it), which the host knows from its record of that start (boot and device arrival); a pointer image larger than that cursor's caps stays in the video. When the host can't tell, it does as 0.3.4: no pointer messages, and the pointer in the video. | | Audio | Sequenced Opus packets with discontinuity and capture timing | | BYE | User, background or shutdown teardown; the host ending a silent session | diff --git a/host/Cargo.toml b/host/Cargo.toml index d8ece94d..eaa7a406 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -94,6 +94,7 @@ features = [ "Win32_System_ProcessStatus", "Win32_Storage_FileSystem", "Win32_Devices_FunctionDiscovery", + "Win32_Devices_Properties", "Win32_UI_Shell_PropertiesSystem", "Win32_Graphics_DXCore", "Win32_Graphics_Dxgi", diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 19d7baec..e8335758 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -17,9 +17,10 @@ use super::recovery::{ }; use super::{ acquired, client_connected, composite_pointer, frame_budget_for, frame_buffer, heartbeat, - idle_heartbeat, keepalive_allowed, keepalive_due, pointer_apart, pointer_in_picture, + idle_heartbeat, keepalive_allowed, keepalive_due, pointer_apart, pointer_place, read_back_image, reconcile_virtual_display, recovery_poll, resolve_target, Acquired, FrameSlot, - FrameUse, IdleGate, OutputInfo, RawFrame, ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, + FrameUse, IdleGate, OutputInfo, PointerPlace, RawFrame, ACQUIRE_TIMEOUT_MS, FPS_WINDOW, + FRAME_PADDING, }; use crate::control::{CaptureTarget, SharedControl}; use crate::gpu_preference::Outcome; @@ -319,17 +320,20 @@ pub fn run_capture_loop( // Whether `last_frame` has the pointer drawn in: it has unless the // client draws the pointer itself (`SharedControl::client_draws_pointer`). let mut last_frame_has_pointer = true; - // Windows draws the pointer into the picture itself: on the second - // screen without the driver's hardware cursor, or for an image too - // large for it. Then nothing else draws one (`composite_pointer`). - let in_picture = |cursor: &CursorState| { - pointer_in_picture( - second_screen, - shared.vdd_hardware_cursor.load(Ordering::Relaxed), - cursor.size(), - ) + // Where Windows puts the pointer (`pointer_place`): into the picture on + // the second screen without the driver's hardware cursor, or for an + // image too large for it, and then nothing else draws one. Where the + // running driver can't be told, as 0.3.4: no pointer messages. + let driver = if second_screen { + super::running_driver_cursor() + } else { + crate::vdd_settings::DriverCursor::Unknown }; - let mut pointer_in_image; + shared + .pointer + .set_withheld(second_screen && driver == crate::vdd_settings::DriverCursor::Unknown); + let place_of = |cursor: &CursorState| pointer_place(second_screen, driver, cursor.size()); + let mut place; { // The feed outlives this capture: a position from the output // captured before (the mirror a second screen starts from) would be @@ -376,8 +380,8 @@ pub fn run_capture_loop( // other client, or none, one with it. The staging texture still // holds the last desktop image, so the cached frame is redrawn from // it when the connected client's answer differs from the cache's. - pointer_in_image = in_picture(&cursor); - let draws_pointer = pointer_apart(shared.client_draws_pointer(), pointer_in_image); + place = place_of(&cursor); + let draws_pointer = pointer_apart(shared.client_draws_pointer(), place); if seen_image && last_frame_has_pointer == draws_pointer { let mut pixel_buf = frame_buffer(frame_bytes); copy_staging( @@ -387,12 +391,7 @@ pub fn run_capture_loop( row_bytes, &mut pixel_buf, )?; - if composite_pointer( - draws_pointer, - pointer_in_image, - cursor_visible, - cursor.has_shape, - ) { + if composite_pointer(draws_pointer, place, cursor_visible, cursor.has_shape) { draw_cursor( &mut pixel_buf, tex_width, @@ -469,8 +468,8 @@ pub fn run_capture_loop( // draws it into the picture. let new_shape = frame_info.PointerShapeBufferSize > 0 && cursor.update_shape(dupl, frame_info.PointerShapeBufferSize); - pointer_in_image = in_picture(&cursor); - if new_shape && !pointer_in_image { + place = place_of(&cursor); + if new_shape && place == PointerPlace::Apart { if let Some(image) = super::pointer::convert_shape( cursor.shape_type, cursor.width, @@ -489,8 +488,8 @@ pub fn run_capture_loop( cursor_y = frame_info.PointerPosition.Position.y; } shared.pointer.set_position(super::pointer::Position { - // A picture with the pointer in it has none apart. - visible: cursor_visible && !pointer_in_image, + // Only a pointer Windows keeps apart is shown apart. + visible: cursor_visible && place == PointerPlace::Apart, x: cursor_x, y: cursor_y, host_time_us: crate::clock::host_now_us(), @@ -594,7 +593,7 @@ pub fn run_capture_loop( // Composite the mouse cursor into the BGRA frame. if composite_pointer( draws_pointer, - pointer_in_image, + place, cursor_visible, cursor.has_shape, ) { diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 8b7d20c2..16220286 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -108,43 +108,72 @@ pub(crate) fn frame_has_image(last_present_time: i64, seen_image: bool) -> bool last_present_time != 0 || seen_image } -/// Whether Windows draws the pointer into the captured picture: on the -/// second screen, unless the virtual display driver runs with a hardware -/// cursor (`vdd_settings::cursor_in_effect`), and then too for an image -/// larger than that cursor takes (`vdd_settings::CURSOR_MAX`). Without -/// one, measured on the reference PC, every pointer move there was a new -/// image and Desktop Duplication kept reporting the pointer at one stale -/// position. +/// Where Windows puts the pointer on a captured output. #[cfg(any(windows, test))] -pub(crate) fn pointer_in_picture( +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PointerPlace { + /// Apart from the picture, as Desktop Duplication reports it. + Apart, + /// Windows draws it into the picture. + InPicture, + /// The second screen's driver can't be told (`DriverCursor::Unknown`). + /// The pointer is handled as in 0.3.4: no pointer messages, and capture + /// draws the reported pointer when Desktop Duplication reports it + /// visible with an image. + Unknown, +} + +/// Where Windows puts the pointer: apart, except on the second screen, +/// where the running driver decides (`vdd_settings::running_cursor`). +/// With its hardware cursor, an image larger than its caps goes into the +/// picture; without one, every image does. Measured on the reference PC, +/// a pointer in the picture made every move a new image, and Desktop +/// Duplication kept reporting a stale position. +#[cfg(any(windows, test))] +pub(crate) fn pointer_place( second_screen: bool, - driver_cursor: bool, + driver: crate::vdd_settings::DriverCursor, shape: Option<(u32, u32)>, -) -> bool { - let max = crate::vdd_settings::CURSOR_MAX; - second_screen && (!driver_cursor || shape.is_some_and(|(w, h)| w > max || h > max)) +) -> PointerPlace { + use crate::vdd_settings::DriverCursor; + match driver { + _ if !second_screen => PointerPlace::Apart, + DriverCursor::Hardware(caps) if shape.is_none_or(|size| caps.takes(size)) => { + PointerPlace::Apart + } + DriverCursor::Hardware(_) | DriverCursor::Software => PointerPlace::InPicture, + DriverCursor::Unknown => PointerPlace::Unknown, + } } /// Whether capture draws the pointer into the video: for a client that /// doesn't draw it, unless Windows already drew it there. Drawing the /// reported pointer over Windows' own put a second, frozen pointer at the -/// stale position. +/// stale position. Where that can't be told, as 0.3.4 did: whenever +/// Desktop Duplication reports it visible with an image, which it doesn't +/// for a pointer Windows drew in (on the reference PC's second screen +/// without the driver's cursor it reported it hidden from the first frame). #[cfg(any(windows, test))] pub(crate) fn composite_pointer( client_draws_pointer: bool, - pointer_in_picture: bool, + place: PointerPlace, visible: bool, has_shape: bool, ) -> bool { - !client_draws_pointer && !pointer_in_picture && visible && has_shape + let drawn_elsewhere = match place { + PointerPlace::Apart => client_draws_pointer, + PointerPlace::InPicture => true, + PointerPlace::Unknown => false, + }; + !drawn_elsewhere && visible && has_shape } /// Whether the pointer goes apart from the video for this capture: to an -/// iPad that draws it, unless Windows draws it into the picture. Then it -/// stays in the video, as before, and the iPad is told it is hidden. +/// iPad that draws it, when Windows keeps it apart. Otherwise it stays in +/// the video, as before, and the iPad is told it is hidden. #[cfg(any(windows, test))] -pub(crate) fn pointer_apart(client_draws_pointer: bool, pointer_in_picture: bool) -> bool { - client_draws_pointer && !pointer_in_picture +pub(crate) fn pointer_apart(client_draws_pointer: bool, place: PointerPlace) -> bool { + client_draws_pointer && place == PointerPlace::Apart } /// What the capture loop does with an acquired frame. @@ -495,11 +524,12 @@ struct VddRequest { mode: Option, /// The adapters' LUIDs before the Enable, to find the driver's own. before: Vec, - /// The driver was off, so the Enable that follows starts it with this - /// file (see `vdd_settings::cursor_in_effect`). - device_off: bool, - /// And it will give Windows a hardware cursor when it starts. - hardware_cursor: bool, + /// The driver's start before the Enable, if it was running, and what + /// an Enable that starts it gives Windows for the pointer, from the + /// file it reads (see `vdd_settings::record_after_enable`). + instance: Option, + settings: Option, + cursor: crate::vdd_settings::DriverCursor, } /// Before each enable: name the main screen's graphics card for the driver @@ -585,32 +615,64 @@ fn prepare_vdd_settings(shared: &SharedControl, outputs: &[OutputInfo]) -> VddRe } Err(_) => {} } - // The driver reads this file when it starts. A driver already on keeps - // what it read then; one found off forgets anything this host knew. - let device_off = crate::vdd_device::query() == crate::vdd_device::DeviceState::Problem(22); - if device_off { - shared - .vdd_hardware_cursor - .store(false, std::sync::atomic::Ordering::Relaxed); - } - let reads_cursor = owned - || crate::vdd_settings::driver_version() - .is_some_and(|version| crate::vdd_settings::driver_reads_cursor(&version)); - let file_cursor = - std::fs::read_to_string(path).is_ok_and(|xml| crate::vdd_settings::hardware_cursor(&xml)); + // The driver reads this file when it starts; a driver already running + // keeps what it read then. + let settings = std::fs::read_to_string(path).ok(); + let reads_cursor = if owned { + Some(true) + } else { + crate::vdd_settings::driver_version() + .and_then(|version| crate::vdd_settings::driver_reads_cursor(&version)) + }; + let overridden = crate::vdd_settings::registry_overrides_cursor(); + let cursor = + crate::vdd_settings::starting_cursor(settings.as_deref(), reads_cursor, overridden); + let instance = crate::vdd_device::instance(); info!( - device_off, - reads_cursor, file_cursor, "The virtual display's pointer settings" + running = instance.is_some(), + ?reads_cursor, + overridden, + ?cursor, + "The virtual display's pointer settings" ); VddRequest { pin: crate::vdd_settings::pin_result(wanted, &applied), mode, before, - device_off, - hardware_cursor: reads_cursor && file_cursor, + instance, + settings, + cursor, } } +/// Where the record of the driver's last start is kept. +#[cfg(windows)] +pub(crate) fn cursor_record_path() -> Option { + crate::settings::app_data_dir().map(|dir| dir.join(crate::vdd_settings::CURSOR_RECORD)) +} + +/// What the running driver does with the pointer, from the record of its +/// start (`vdd_settings::running_cursor`). +#[cfg(windows)] +pub(crate) fn running_driver_cursor() -> crate::vdd_settings::DriverCursor { + let record = cursor_record_path().and_then(|path| crate::vdd_settings::load_record(&path)); + let running = crate::vdd_device::instance(); + let cursor = crate::vdd_settings::running_cursor(record.as_ref(), running); + info!( + recorded = record.is_some(), + same_start = record + .as_ref() + .zip(running) + .is_some_and(|(record, running)| record.instance == running), + settings_sha256 = record + .as_ref() + .map_or("", |record| record.settings_sha256.as_str()), + ?cursor, + "The running virtual display's pointer" + ); + cursor +} + /// Bring a freshly attached second screen to the mode asked for, when /// Windows restored another one for this set of displays, and log the mode /// it ends up in, once per attach. Only the virtual display changes. @@ -1235,53 +1297,87 @@ mod tests { } /// #183 is a second screen report: with the driver's hardware cursor - /// the second screen sends the pointer apart, as Mirror does. Without - /// it, or for an image over the cursor's size, Windows draws the - /// pointer into the picture. + /// the second screen sends the pointer apart, as Mirror does. Review + /// r4: an image goes into the picture when it is larger than the caps + /// the running driver started with, the user's own (64, 256) as well as + /// this host's 128. #[test] - fn the_second_screen_sends_the_pointer_apart_with_the_drivers_cursor() { - let small = Some((32, 32)); - let large = Some((crate::vdd_settings::CURSOR_MAX + 1, 32)); - for (second_screen, driver_cursor, shape, in_picture) in [ - (false, false, small, false), - (false, true, large, false), - (true, true, small, false), - (true, true, None, false), - (true, true, large, true), - (true, false, small, true), + fn the_second_screen_places_the_pointer_by_the_running_drivers_caps() { + use crate::vdd_settings::{CursorCaps, DriverCursor}; + use PointerPlace::{Apart, InPicture, Unknown}; + let caps = |max| { + DriverCursor::Hardware(CursorCaps { + max_x: max, + max_y: max, + alpha: true, + xor: 2, + }) + }; + for (second_screen, driver, shape, place) in [ + (false, DriverCursor::Software, Some((300, 300)), Apart), + (false, DriverCursor::Unknown, Some((32, 32)), Apart), + (true, caps(128), Some((32, 32)), Apart), + (true, caps(128), None, Apart), + (true, caps(128), Some((128, 128)), Apart), + (true, caps(128), Some((129, 32)), InPicture), + (true, caps(128), Some((32, 129)), InPicture), + (true, caps(64), Some((64, 64)), Apart), + (true, caps(64), Some((65, 64)), InPicture), + (true, caps(64), Some((96, 96)), InPicture), + (true, caps(256), Some((160, 160)), Apart), + (true, caps(256), Some((256, 256)), Apart), + (true, caps(256), Some((257, 32)), InPicture), + (true, DriverCursor::Software, Some((32, 32)), InPicture), + (true, DriverCursor::Software, None, InPicture), + (true, DriverCursor::Unknown, Some((32, 32)), Unknown), ] { assert_eq!( - pointer_in_picture(second_screen, driver_cursor, shape), - in_picture, - "second_screen={second_screen} driver_cursor={driver_cursor} shape={shape:?}" + pointer_place(second_screen, driver, shape), + place, + "second_screen={second_screen} driver={driver:?} shape={shape:?}" ); - assert_eq!(pointer_apart(true, in_picture), !in_picture); - assert!(!pointer_apart(false, in_picture), "an older iPad"); + assert_eq!(pointer_apart(true, place), place == Apart); + assert!(!pointer_apart(false, place), "an older iPad"); } } - /// Review r3: with the pointer in the picture Desktop Duplication still - /// reports it visible, with a shape, at a stale spot. Drawing that over - /// Windows' own pointer left a second, frozen one in every frame. + /// One pointer, never none and never two. Review r3: with the pointer + /// in the picture Desktop Duplication can still report it visible, + /// with a shape, at a stale spot, and drawing that over Windows' own + /// left a second, frozen one. Review r4: where the driver can't be + /// told, capture draws what Desktop Duplication reports, as 0.3.4 did, + /// whoever the client is. #[test] fn capture_draws_the_pointer_only_when_nobody_else_does() { + use PointerPlace::{Apart, InPicture, Unknown}; assert!( - composite_pointer(false, false, true, true), + composite_pointer(false, Apart, true, true), "Mirror, an older iPad" ); assert!( - !composite_pointer(true, false, true, true), + !composite_pointer(true, Apart, true, true), "the iPad draws it" ); - assert!( - !composite_pointer(false, true, true, true), - "Windows drew it into the picture" - ); - assert!(!composite_pointer(false, false, false, true), "hidden"); - assert!( - !composite_pointer(false, false, true, false), - "no shape yet" - ); + for client_draws in [false, true] { + assert!( + !composite_pointer(client_draws, InPicture, true, true), + "Windows drew it into the picture" + ); + assert!( + composite_pointer(client_draws, Unknown, true, true), + "a driver that can't be told: as 0.3.4" + ); + for place in [Apart, InPicture, Unknown] { + assert!( + !composite_pointer(client_draws, place, false, true), + "hidden" + ); + assert!( + !composite_pointer(client_draws, place, true, false), + "no shape yet" + ); + } + } } pub(super) fn connect_viewer(shared: &SharedControl) { diff --git a/host/src/capture/pointer.rs b/host/src/capture/pointer.rs index 469545aa..25747084 100644 --- a/host/src/capture/pointer.rs +++ b/host/src/capture/pointer.rs @@ -164,6 +164,7 @@ struct FeedState { shape: Option>, position: Option, last_id: u32, + withheld: bool, } impl PointerFeed { @@ -192,6 +193,19 @@ impl PointerFeed { self.changed.notify_one(); } + /// Capture can't tell where Windows puts the pointer + /// (`capture::PointerPlace::Unknown`): it stays in the video, and no + /// pointer messages go to the iPad but a hidden state to clear one it + /// was drawing. + pub fn set_withheld(&self, withheld: bool) { + self.state.lock().withheld = withheld; + self.changed.notify_one(); + } + + pub fn withheld(&self) -> bool { + self.state.lock().withheld + } + pub fn latest(&self) -> (Option>, Option) { let state = self.state.lock(); (state.shape.clone(), state.position) diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 65672f9b..54078804 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -1670,22 +1670,33 @@ struct WindowsDriver<'a> { #[cfg(windows)] impl WindowsDriver<'_> { - /// Whether the running driver gives Windows a hardware cursor, after a - /// driver change (`vdd_settings::cursor_in_effect`). - fn note_cursor(&self, enabled: bool, done: bool) { + /// After an Enable that succeeded: a driver that started in it read + /// the file `prepare` wrote, so keep what that file gives the pointer + /// (`vdd_settings::record_after_enable`). One already running keeps the + /// record of its own start. + fn note_start(&self) { let request = self.request.borrow(); - let request = request.as_ref(); - let flag = &self.shared.vdd_hardware_cursor; - let now = flag.load(std::sync::atomic::Ordering::Relaxed); - let in_effect = crate::vdd_settings::cursor_in_effect( - now, - enabled, - done, - request.is_some_and(|request| request.device_off), - request.is_some_and(|request| request.hardware_cursor), + let Some(request) = request.as_ref() else { + return; + }; + let after = crate::vdd_device::instance(); + let record = crate::vdd_settings::record_after_enable( + request.instance, + after, + request.settings.as_deref(), + request.cursor, ); - flag.store(in_effect, std::sync::atomic::Ordering::Relaxed); - info!(hardware_cursor = in_effect, "The virtual display's pointer"); + info!( + started = record.is_some(), + running = after.is_some(), + "The virtual display's driver after the Enable" + ); + let (Some(record), Some(path)) = (record, super::cursor_record_path()) else { + return; + }; + if let Err(error) = crate::vdd_settings::save_record(&path, &record) { + warn!(%error, "Could not keep the virtual display's pointer record"); + } } } @@ -1701,13 +1712,17 @@ impl Driver for WindowsDriver<'_> { fn set_enabled(&self, enabled: bool) -> bool { let done = crate::vdd::set_enabled(enabled, || {}); - self.note_cursor(enabled, done); + if enabled && done { + self.note_start(); + } done } fn enable(&self) -> Result<(), EnableReport> { let result = crate::vdd::enable(|| {}); - self.note_cursor(true, result.is_ok()); + if result.is_ok() { + self.note_start(); + } result } diff --git a/host/src/control.rs b/host/src/control.rs index 0c3ce654..1f50e850 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -72,10 +72,6 @@ pub struct SharedControl { pub(crate) vdd_worker: Arc, /// Output owned by the enabled driver, retained across pipeline restarts. pub(crate) vdd_output: Arc>>, - /// The virtual display driver was given a hardware cursor, so the - /// second screen's pointer is reported apart from its picture. - #[cfg_attr(not(windows), allow(dead_code))] - pub(crate) vdd_hardware_cursor: Arc, /// Second screen timing across pipeline restarts: the pending switch onto /// it, the grace period after its viewer left, the last driver change. pub(crate) second_screen: Arc>, @@ -210,7 +206,6 @@ impl SharedControl { handoff: Default::default(), vdd_worker: Arc::new(Default::default()), vdd_output: Arc::new(Mutex::new(None)), - vdd_hardware_cursor: Arc::new(AtomicBool::new(false)), second_screen: Arc::new(Mutex::new(Default::default())), hb_capture_loop_ms: Arc::new(AtomicU64::new(0)), hb_capture_frame_ms: Arc::new(AtomicU64::new(0)), diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 705d62ef..ac8d4436 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -915,7 +915,8 @@ async fn act( } /// The pointer to send: hidden until the iPad holds its image, while -/// capture still draws it into the video (`client_draws_pointer`). +/// capture still draws it into the video (`client_draws_pointer`). No image +/// and hidden while capture withholds it (`PointerFeed::set_withheld`). fn pointer_latest( shared: &SharedControl, ) -> ( @@ -923,12 +924,13 @@ fn pointer_latest( Option, ) { let (shape, position) = shared.pointer.latest(); - let shown = shared.client_draws_pointer(); + let withheld = shared.pointer.withheld(); + let shown = shared.client_draws_pointer() && !withheld; let position = position.map(|position| crate::capture::pointer::Position { visible: position.visible && shown, ..position }); - (shape, position) + (shape.filter(|_| !withheld), position) } /// Sends what [`pointer::PointerSender`] says is due, to a client that @@ -946,7 +948,7 @@ async fn send_pointer( ); } sender.serve(session_id); - if session_id.is_none() { + if session_id.is_none() || (shared.pointer.withheld() && !sender.withhold()) { return; } let budget = crate::capture::frame_budget_for(shared.effective_fps()); diff --git a/host/src/transport/pointer.rs b/host/src/transport/pointer.rs index 3bb8a61f..8002e847 100644 --- a/host/src/transport/pointer.rs +++ b/host/src/transport/pointer.rs @@ -55,6 +55,8 @@ pub struct PointerSender { shape_due: bool, /// The image the iPad's last report said it holds, and when it came. client_shape: Option<(u32, Instant)>, + /// A visible position went this session. + shown: bool, } impl PointerSender { @@ -85,6 +87,15 @@ impl PointerSender { self.repeat_at = Some(now); } + /// Capture withholds the pointer (`PointerFeed::set_withheld`): no + /// image goes, and only hidden positions to an iPad that was shown + /// one this session. Whether to poll at all. + pub fn withhold(&mut self) -> bool { + self.pending = None; + self.retry_at = None; + self.shown + } + pub fn report(&mut self, pointer_shape_id: u32, now: Instant) { self.client_shape = Some((pointer_shape_id, now)); } @@ -149,6 +160,7 @@ impl PointerSender { let changed = self.sent.is_none_or(|(sent, _)| sent != state); self.repeat_at = changed.then_some(now + REPEAT_AFTER); self.sent = Some((state, now)); + self.shown |= position.visible; out.push(Send::State(PointerState { host_time_us: position.host_time_us, ..state @@ -453,6 +465,42 @@ mod tests { assert_eq!(states(&poll_all(&mut sender, now, BUDGET, latest)), [5]); } + /// Review r4: where capture can't tell where Windows puts the pointer, + /// an iPad that was never shown one gets no pointer messages, and one + /// that was (the mirror before a second screen) only hidden positions. + #[test] + fn a_withheld_pointer_sends_nothing_but_a_hidden_state_to_clear_one() { + let start = Instant::now(); + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + assert!(!sender.withhold(), "never shown: nothing to send"); + let hidden = Position { + visible: false, + ..at(5) + }; + poll_all(&mut sender, start, BUDGET, (None, Some(hidden))); + assert!(!sender.withhold(), "a hidden position shows nothing"); + + let mut sender = PointerSender::default(); + sender.serve(Some(1)); + let sent = sender.poll(start, BUDGET, (Some(shape(1)), Some(at(5)))); + assert!(matches!(sent[0], Send::Shape(_, 0)), "{sent:?}"); + assert!(sender.withhold(), "shown: the hidden state is owed"); + let later = start + BUDGET; + let sent = poll_all(&mut sender, later, BUDGET, (None, Some(hidden))); + assert_eq!( + sent, + [Send::State(PointerState { + flags: 0, + shape_id: 0, + x: 5, + y: 0, + host_time_us: 5, + })], + "no image, not even the one under way" + ); + } + #[test] fn hiding_the_pointer_is_a_change_that_goes() { let start = Instant::now(); diff --git a/host/src/vdd_device.rs b/host/src/vdd_device.rs index 75c207da..8ab69460 100644 --- a/host/src/vdd_device.rs +++ b/host/src/vdd_device.rs @@ -140,83 +140,164 @@ pub fn from_task_stderr(stderr: &str) -> Option { /// Read the driver's device from Configuration Manager. #[cfg(windows)] pub fn query() -> DeviceState { - use windows::core::{w, PCWSTR}; use windows::Win32::Devices::DeviceAndDriverInstallation::{ - CM_Get_Child, CM_Get_DevNode_Registry_PropertyW, CM_Get_DevNode_Status, - CM_Get_Device_ID_ListW, CM_Get_Device_ID_List_SizeW, CM_Locate_DevNodeW, - CM_DEVNODE_STATUS_FLAGS, CM_DRP_DRIVER, CM_DRP_HARDWAREID, CM_GETIDLIST_FILTER_ENUMERATOR, - CM_GETIDLIST_FILTER_PRESENT, CM_LOCATE_DEVNODE_NORMAL, CM_PROB, CR_BUFFER_SMALL, - CR_SUCCESS, + CM_Get_Child, CM_DRP_DRIVER, CR_SUCCESS, }; - let status = |devinst: u32| { - let (mut status, mut problem) = (CM_DEVNODE_STATUS_FLAGS(0), CM_PROB(0)); - (unsafe { CM_Get_DevNode_Status(&mut status, &mut problem, devinst, 0) } == CR_SUCCESS) - .then_some((status.0, problem.0)) + let Some(devnodes) = driver_devnodes() else { + return DeviceState::Missing; }; - let property = |devinst: u32, which: u32| -> Option> { - let mut bytes = 0u32; - let result = - unsafe { CM_Get_DevNode_Registry_PropertyW(devinst, which, None, None, &mut bytes, 0) }; - if result != CR_BUFFER_SMALL || bytes == 0 { - return None; - } - let mut buffer = vec![0u16; bytes.div_ceil(2) as usize]; - (unsafe { - CM_Get_DevNode_Registry_PropertyW( - devinst, - which, - None, - Some(buffer.as_mut_ptr().cast()), - &mut bytes, - 0, - ) - } == CR_SUCCESS) - .then_some(buffer) + let states = devnodes.into_iter().filter_map(|devinst| { + let (flags, problem) = devnode_status(devinst)?; + let mut child = 0u32; + let monitor = (unsafe { CM_Get_Child(&mut child, devinst, 0) } == CR_SUCCESS).then(|| { + devnode_status(child) + .is_some_and(|(flags, _)| flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0) + }); + let has_driver = registry_property(devinst, CM_DRP_DRIVER).is_some(); + Some(DeviceState::from_devnode( + flags, problem, has_driver, monitor, + )) + }); + DeviceState::best(states) +} + +/// The running driver's start (`vdd_settings::DriverInstance`): this boot +/// and its device's last arrival. `None` when no one device of the driver +/// runs, or Windows doesn't say. +#[cfg(windows)] +pub fn instance() -> Option { + use windows::Win32::Devices::DeviceAndDriverInstallation::{ + CM_Get_DevNode_PropertyW, CR_SUCCESS, + }; + use windows::Win32::Devices::Properties::{ + DEVPKEY_Device_LastArrivalDate, DEVPROPTYPE, DEVPROP_TYPE_FILETIME, + }; + + let mut running = driver_devnodes()?.into_iter().filter(|&devinst| { + devnode_status(devinst) + .is_some_and(|(flags, _)| flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0) + }); + let devinst = match (running.next(), running.next()) { + (Some(devinst), None) => devinst, + _ => return None, + }; + let (mut arrival, mut kind, mut size) = (0u64, DEVPROPTYPE(0), 8u32); + let result = unsafe { + CM_Get_DevNode_PropertyW( + devinst, + &DEVPKEY_Device_LastArrivalDate, + &mut kind, + Some((&mut arrival as *mut u64).cast()), + &mut size, + 0, + ) + }; + if result != CR_SUCCESS || kind != DEVPROP_TYPE_FILETIME || size != 8 || arrival == 0 { + return None; + } + Some(crate::vdd_settings::DriverInstance { + boot_id: boot_id()?, + arrival, + }) +} + +/// Windows' count of boots, which the kernel raises at each one. +#[cfg(windows)] +fn boot_id() -> Option { + use windows::core::w; + use windows::Win32::System::Registry::{RegGetValueW, HKEY_LOCAL_MACHINE, RRF_RT_REG_DWORD}; + let (mut value, mut bytes) = (0u32, 4u32); + unsafe { + RegGetValueW( + HKEY_LOCAL_MACHINE, + w!(r"SYSTEM\CurrentControlSet\Control\Session Manager\Memory Management\PrefetchParameters"), + w!("BootId"), + RRF_RT_REG_DWORD, + None, + Some((&mut value as *mut u32).cast()), + Some(&mut bytes), + ) + } + .is_ok() + .then_some(value) +} + +/// The present root devices with the driver's hardware ID, or `None` when +/// Windows can't list them. +#[cfg(windows)] +fn driver_devnodes() -> Option> { + use windows::core::{w, PCWSTR}; + use windows::Win32::Devices::DeviceAndDriverInstallation::{ + CM_Get_Device_ID_ListW, CM_Get_Device_ID_List_SizeW, CM_Locate_DevNodeW, CM_DRP_HARDWAREID, + CM_GETIDLIST_FILTER_ENUMERATOR, CM_GETIDLIST_FILTER_PRESENT, CM_LOCATE_DEVNODE_NORMAL, + CR_SUCCESS, }; let flags = CM_GETIDLIST_FILTER_ENUMERATOR | CM_GETIDLIST_FILTER_PRESENT; let mut len = 0u32; if unsafe { CM_Get_Device_ID_List_SizeW(&mut len, w!("ROOT"), flags) } != CR_SUCCESS { - return DeviceState::Missing; + return None; } let mut ids = vec![0u16; len as usize]; if unsafe { CM_Get_Device_ID_ListW(w!("ROOT"), &mut ids, flags) } != CR_SUCCESS { - return DeviceState::Missing; + return None; } - let states = ids - .split(|&unit| unit == 0) - .filter(|id| !id.is_empty()) - .filter_map(|id| { - let id: Vec = id.iter().copied().chain(Some(0)).collect(); - let mut devinst = 0u32; - if unsafe { - CM_Locate_DevNodeW(&mut devinst, PCWSTR(id.as_ptr()), CM_LOCATE_DEVNODE_NORMAL) - } != CR_SUCCESS - { - return None; - } - let hardware_ids = property(devinst, CM_DRP_HARDWAREID)?; - hardware_ids - .split(|&unit| unit == 0) - .any(|hwid| String::from_utf16_lossy(hwid).eq_ignore_ascii_case(HARDWARE_ID)) - .then_some(devinst) - }) - .filter_map(|devinst| { - let (flags, problem) = status(devinst)?; - let mut child = 0u32; - let monitor = - (unsafe { CM_Get_Child(&mut child, devinst, 0) } == CR_SUCCESS).then(|| { - status(child).is_some_and(|(flags, _)| { - flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0 - }) - }); - let has_driver = property(devinst, CM_DRP_DRIVER).is_some(); - Some(DeviceState::from_devnode( - flags, problem, has_driver, monitor, - )) - }); - DeviceState::best(states) + Some( + ids.split(|&unit| unit == 0) + .filter(|id| !id.is_empty()) + .filter_map(|id| { + let id: Vec = id.iter().copied().chain(Some(0)).collect(); + let mut devinst = 0u32; + if unsafe { + CM_Locate_DevNodeW(&mut devinst, PCWSTR(id.as_ptr()), CM_LOCATE_DEVNODE_NORMAL) + } != CR_SUCCESS + { + return None; + } + let hardware_ids = registry_property(devinst, CM_DRP_HARDWAREID)?; + hardware_ids + .split(|&unit| unit == 0) + .any(|hwid| String::from_utf16_lossy(hwid).eq_ignore_ascii_case(HARDWARE_ID)) + .then_some(devinst) + }) + .collect(), + ) +} + +#[cfg(windows)] +fn devnode_status(devinst: u32) -> Option<(u32, u32)> { + use windows::Win32::Devices::DeviceAndDriverInstallation::{ + CM_Get_DevNode_Status, CM_DEVNODE_STATUS_FLAGS, CM_PROB, CR_SUCCESS, + }; + let (mut status, mut problem) = (CM_DEVNODE_STATUS_FLAGS(0), CM_PROB(0)); + (unsafe { CM_Get_DevNode_Status(&mut status, &mut problem, devinst, 0) } == CR_SUCCESS) + .then_some((status.0, problem.0)) +} + +#[cfg(windows)] +fn registry_property(devinst: u32, which: u32) -> Option> { + use windows::Win32::Devices::DeviceAndDriverInstallation::{ + CM_Get_DevNode_Registry_PropertyW, CR_BUFFER_SMALL, CR_SUCCESS, + }; + let mut bytes = 0u32; + let result = + unsafe { CM_Get_DevNode_Registry_PropertyW(devinst, which, None, None, &mut bytes, 0) }; + if result != CR_BUFFER_SMALL || bytes == 0 { + return None; + } + let mut buffer = vec![0u16; bytes.div_ceil(2) as usize]; + (unsafe { + CM_Get_DevNode_Registry_PropertyW( + devinst, + which, + None, + Some(buffer.as_mut_ptr().cast()), + &mut bytes, + 0, + ) + } == CR_SUCCESS) + .then_some(buffer) } /// Set the mode of the virtual display `device` (a GDI name like diff --git a/host/src/vdd_settings.rs b/host/src/vdd_settings.rs index 18f2d381..b5abb56a 100644 --- a/host/src/vdd_settings.rs +++ b/host/src/vdd_settings.rs @@ -17,6 +17,8 @@ use std::io; use std::ops::Range; use std::path::Path; +use serde::{Deserialize, Serialize}; + /// A display mode for the virtual display's mode list. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct VddMode { @@ -142,75 +144,261 @@ const CURSOR: &str = " \n truetrue\n \ 2\n \n"; -/// The largest pointer image, in pixels a side, the cursor block lets the -/// driver take as a hardware cursor. Windows draws a larger one into the -/// picture itself. -pub const CURSOR_MAX: u32 = 128; - -/// Whether a settings document turns the driver's hardware cursor on, read -/// as the driver reads it (`EnabledQuery` in its `Driver.cpp`): the first -/// `` element whose first child is text decides, and only -/// the exact text `true` turns it on. Comments are not elements. Stricter -/// than the driver, a document that is not well-formed counts as off, so -/// the pointer is taken to be in the picture. -pub fn hardware_cursor(xml: &str) -> bool { +/// The hardware cursor a driver gives Windows when it starts, as its +/// settings set it. Windows draws a pointer image larger than this into +/// the picture itself (measured on the reference PC with a 160 px pointer +/// and a 128 px cap). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct CursorCaps { + pub max_x: u32, + pub max_y: u32, + /// `AlphaCursorSupport`, and the XOR level as the driver reads it. + pub alpha: bool, + pub xor: i32, +} + +impl CursorCaps { + /// Whether a pointer image of this size goes to the hardware cursor. + pub fn takes(&self, (width, height): (u32, u32)) -> bool { + width <= self.max_x && height <= self.max_y + } +} + +/// What the running driver does with the pointer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum DriverCursor { + /// It gives Windows a hardware cursor with these caps, which takes + /// every pointer type up to their size. + Hardware(CursorCaps), + /// It has none: Windows draws the pointer into the picture. + Software, + /// This host can't tell. Then the pointer is handled as in 0.3.4 + /// (`capture::PointerPlace::Unknown`). + Unknown, +} + +/// The cursor a settings document gives the driver, read as the driver +/// reads it (`EnabledQuery` and `GetIntegerSetting` in its `Driver.cpp`): +/// for each setting, the first element of that name whose first child is +/// text decides. Only the text `true` turns a switch on, and numbers are +/// read as `std::stoi` reads them, a missing or unreadable one as -1. A +/// document that is not well-formed is `Unknown`. +/// +/// Only a cursor that takes every pointer type counts as `Hardware`: with +/// alpha off, or XOR support off (the driver reads the level but, in +/// 25.5.2, always gives full support), Windows' choice for the types it +/// doesn't take isn't documented, and so is a size the driver can't use. +pub fn cursor_set_by(xml: &str) -> DriverCursor { use quick_xml::events::Event; + const NAMES: [&[u8]; 5] = [ + b"HardwareCursor", + b"AlphaCursorSupport", + b"CursorMaxX", + b"CursorMaxY", + b"XorCursorSupportLevel", + ]; let mut reader = quick_xml::Reader::from_str(xml); reader.config_mut().trim_text(false); - let (mut in_element, mut on) = (false, None); + let mut values: [Option; 5] = Default::default(); + let mut current = None; loop { match reader.read_event() { - Ok(Event::Start(tag)) => in_element = tag.local_name().as_ref() == b"HardwareCursor", - Ok(Event::Text(text)) if in_element && on.is_none() => { - on = Some(text.as_ref() == b"true"); - in_element = false; + Ok(Event::Start(tag)) => { + current = NAMES + .iter() + .position(|name| tag.local_name().as_ref() == *name); + } + // XmlLite reads white space alone as another kind of node. + Ok(Event::Text(text)) if text.iter().all(u8::is_ascii_whitespace) => current = None, + Ok(Event::Text(text)) => { + if let Some(slot) = current.take().filter(|&slot| values[slot].is_none()) { + values[slot] = Some(String::from_utf8_lossy(text.as_ref()).into_owned()); + } } - // The whole document must be well-formed for the answer to count. - Ok(Event::Eof) => return on.unwrap_or(false), - Err(_) => return false, - Ok(_) => in_element = false, + Ok(Event::Eof) => break, + Err(_) => return DriverCursor::Unknown, + Ok(_) => current = None, + } + } + let switch = |slot: usize| values[slot].as_deref() == Some("true"); + let number = |slot: usize| values[slot].as_deref().map_or(-1, stoi); + if !switch(0) { + return DriverCursor::Software; + } + let xor = match number(4) { + level @ 0..=3 => level, + // The driver's default for a level it doesn't know: full support. + _ => 2, + }; + match (u32::try_from(number(2)), u32::try_from(number(3))) { + (Ok(max_x @ 1..), Ok(max_y @ 1..)) if switch(1) && matches!(xor, 2 | 3) => { + DriverCursor::Hardware(CursorCaps { + max_x, + max_y, + alpha: true, + xor, + }) } + _ => DriverCursor::Unknown, } } -/// Whether the running driver gives Windows a hardware cursor, after this -/// host turned the driver on (`enabled`) or off, and the change is `done`. -/// Only an Enable that started the driver from off (`device_off`) made it -/// read the file just written, which turns the cursor on -/// (`file_cursor`: the file says so and the driver reads ``). An -/// Enable that found it on changes nothing it read, and a driver turned off -/// has no cursor. When unsure, the answer is no: the pointer is then taken -/// to be in the picture and left there. -pub fn cursor_in_effect( - before: bool, - enabled: bool, - done: bool, - device_off: bool, - file_cursor: bool, -) -> bool { - match (enabled, done) { - (false, true) => false, - (true, true) if device_off => file_cursor, - (true, true) => before, - (_, false) => before && !device_off, +/// `std::stoi` as the driver uses it: leading white space, a sign and +/// digits, ignoring what follows them; -1 (the driver's "unset") when +/// there are no digits or the number doesn't fit. +fn stoi(text: &str) -> i32 { + let text = text.trim_start(); + let (sign, digits) = match text.as_bytes().first() { + Some(b'-') => (-1, &text[1..]), + Some(b'+') => (1, &text[1..]), + _ => (1, text), + }; + let end = digits + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(digits.len()); + digits[..end] + .parse::() + .ok() + .and_then(|value| i32::try_from(sign * value).ok()) + .unwrap_or(-1) +} + +/// One start of the driver: the boot, and the device's last arrival +/// (`DEVPKEY_Device_LastArrivalDate`, a FILETIME). On the reference PC each +/// Enable changed the arrival and a driver turned off kept it; an arrival +/// only counts with the boot it came in. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct DriverInstance { + pub boot_id: u32, + pub arrival: u64, +} + +/// What a driver start read, kept on disk so a relaunched host knows it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CursorRecord { + pub instance: DriverInstance, + /// SHA-256 of the settings file it started with, for the logs. + pub settings_sha256: String, + pub cursor: DriverCursor, +} + +/// What the driver starts with when this host's Enable starts it: the +/// file's cursor (`cursor_set_by`), if the driver reads `` +/// (`reads_cursor`, `None` when its version can't tell) and no registry +/// value overrides the file. +pub fn starting_cursor( + xml: Option<&str>, + reads_cursor: Option, + overridden: bool, +) -> DriverCursor { + match (xml, reads_cursor) { + _ if overridden => DriverCursor::Unknown, + (_, Some(false)) => DriverCursor::Software, + (Some(xml), Some(true)) => cursor_set_by(xml), + _ => DriverCursor::Unknown, } } +/// The record to keep after an Enable that succeeded: a driver running +/// after it (`after`) that was not running before it (`before`) started +/// in this Enable, with the file this host just wrote (`xml`, read as +/// `starting_cursor`). An Enable that found it running (AlreadyDone, or a +/// relaunch with the driver left on) changes nothing it read: no record, +/// and the one on disk decides (`running_cursor`). +pub fn record_after_enable( + before: Option, + after: Option, + xml: Option<&str>, + cursor: DriverCursor, +) -> Option { + let instance = after.filter(|after| before != Some(*after))?; + Some(CursorRecord { + instance, + settings_sha256: xml.map_or_else(String::new, |xml| { + let digest = ring::digest::digest(&ring::digest::SHA256, xml.as_bytes()); + digest.as_ref().iter().map(|b| format!("{b:02x}")).collect() + }), + cursor, + }) +} + +/// What the running driver does with the pointer: what the record says, +/// when it is of this very start of the driver (same boot, same arrival); +/// `Unknown` for any other, and when the driver is not running. +pub fn running_cursor( + record: Option<&CursorRecord>, + running: Option, +) -> DriverCursor { + match (record, running) { + (Some(record), Some(running)) if record.instance == running => record.cursor, + _ => DriverCursor::Unknown, + } +} + +/// Where the record is kept, beside the app's other files. +pub const CURSOR_RECORD: &str = "vdd-cursor.json"; + +pub fn load_record(path: &Path) -> Option { + serde_json::from_str(&std::fs::read_to_string(path).ok()?).ok() +} + +pub fn save_record(path: &Path, record: &CursorRecord) -> io::Result<()> { + replace( + path, + &serde_json::to_string(record).map_err(io::Error::other)?, + ) +} + /// Whether the installed driver reads `` at all: releases from /// 24.10.27 on do. `version` is its installer's display version, such as -/// `25.05.03`. -pub fn driver_reads_cursor(version: &str) -> bool { +/// `25.05.03`; `None` when it doesn't read as one. +pub fn driver_reads_cursor(version: &str) -> Option { let parts: Option> = version .trim() .split('.') .map(|part| part.parse().ok()) .collect(); match parts.as_deref() { - Some([year, month, day, ..]) => (*year, *month, *day) >= (24, 10, 27), - _ => false, + Some([year, month, day, ..]) => Some((*year, *month, *day) >= (24, 10, 27)), + _ => None, } } +/// Whether a value under the driver's registry key can override the +/// file's cursor settings: the driver reads that key first. +#[cfg(windows)] +pub fn registry_overrides_cursor() -> bool { + use windows::core::PCWSTR; + use windows::Win32::System::Registry::{RegGetValueW, HKEY_LOCAL_MACHINE, RRF_RT_ANY}; + let key: Vec = r"SOFTWARE\MikeTheTech\VirtualDisplayDriver" + .encode_utf16() + .chain(Some(0)) + .collect(); + [ + "HARDWARECURSOR", + "ALPHACURSORSUPPORT", + "CURSORMAXX", + "CURSORMAXY", + "XORCURSORSUPPORTLEVEL", + ] + .iter() + .any(|name| { + let name: Vec = name.encode_utf16().chain(Some(0)).collect(); + unsafe { + RegGetValueW( + HKEY_LOCAL_MACHINE, + PCWSTR(key.as_ptr()), + PCWSTR(name.as_ptr()), + RRF_RT_ANY, + None, + None, + None, + ) + } + .is_ok() + }) +} + fn escape(text: &str) -> String { text.replace('&', "&") .replace('<', "<") @@ -873,8 +1061,9 @@ mod tests { 120" )); assert!(xml.ends_with("\n")); - assert!( - hardware_cursor(&xml), + assert_ne!( + cursor_set_by(&xml), + DriverCursor::Unknown, "the second screen's pointer is reported apart" ); assert!(xml.contains( @@ -885,15 +1074,31 @@ mod tests { )); } + /// A settings file with this cursor block. + fn with_cursor(block: &str) -> String { + format!("\n1{block}") + } + + fn hardware(max_x: u32, max_y: u32) -> DriverCursor { + DriverCursor::Hardware(CursorCaps { + max_x, + max_y, + alpha: true, + xor: 2, + }) + } + /// The driver turns its hardware cursor on only when the file says so; /// a file from before this version, or the vendor's with it off, keeps /// the pointer in the second screen's picture. Read as the driver reads - /// it: comments don't count, and only the text `true` does. + /// it: comments don't count, only the text `true` does, and numbers as + /// `std::stoi` reads them. #[test] - fn the_hardware_cursor_is_read_as_the_driver_reads_it() { - assert!(!hardware_cursor(VENDOR)); + fn the_cursor_is_read_as_the_driver_reads_it() { + assert_eq!(cursor_set_by(VENDOR), DriverCursor::Software); let v032 = render(&modes_for_client(1668, 2420, 60), None).replace(CURSOR, ""); - assert!(!hardware_cursor(&v032)); + assert_eq!(cursor_set_by(&v032), DriverCursor::Software); + let caps = "128128true"; for (text, on) in [ ("true", true), (" true", false), @@ -901,59 +1106,209 @@ mod tests { ("false", false), ("TRUE", false), ] { - let xml = format!("{text}"); - assert_eq!(hardware_cursor(&xml), on, "{text:?}"); + let xml = with_cursor(&format!("{text}{caps}")); + assert_eq!(cursor_set_by(&xml) == hardware(128, 128), on, "{text:?}"); } - let commented = "false"; - assert!( - !hardware_cursor(commented), + let commented = format!("false{caps}"); + assert_eq!( + cursor_set_by(&commented), + DriverCursor::Software, "a commented-out line is not a setting" ); - let first_wins = "falsetrue"; - assert!(!hardware_cursor(first_wins)); - assert!( - !hardware_cursor("true"), + let first_wins = format!("falsetrue{caps}"); + assert_eq!(cursor_set_by(&first_wins), DriverCursor::Software); + let blank_skipped = format!(" true{caps}"); + assert_eq!( + cursor_set_by(&blank_skipped), + hardware(128, 128), + "white space alone is not text to XmlLite" + ); + assert_eq!( + cursor_set_by("true"), + DriverCursor::Unknown, "not well-formed" ); + for (x, y, cursor) in [ + ("64", "64", hardware(64, 64)), + ("256", "256", hardware(256, 256)), + (" 96px", "+48", hardware(96, 48)), + ("0", "128", DriverCursor::Unknown), + ("-64", "128", DriverCursor::Unknown), + ("big", "128", DriverCursor::Unknown), + ("99999999999", "128", DriverCursor::Unknown), + ] { + let xml = with_cursor(&format!("true{x}{y}true")); + assert_eq!(cursor_set_by(&xml), cursor, "{x:?} x {y:?}"); + } + let missing = with_cursor("true64true"); + assert_eq!( + cursor_set_by(&missing), + DriverCursor::Unknown, + "a missing size reads as -1" + ); + let no_alpha = with_cursor("true6464false"); + assert_eq!( + cursor_set_by(&no_alpha), + DriverCursor::Unknown, + "not every pointer type" + ); + for (level, cursor) in [ + ("1", DriverCursor::Unknown), + ("0", DriverCursor::Unknown), + ( + "3", + DriverCursor::Hardware(CursorCaps { + max_x: 64, + max_y: 64, + alpha: true, + xor: 3, + }), + ), + ("7", hardware(64, 64)), + ] { + let xml = with_cursor(&format!("true6464true{level}")); + assert_eq!(cursor_set_by(&xml), cursor, "XOR level {level}"); + } } - /// Review r3: the flag comes from what the driver read when it last - /// started, not from the file alone. #[test] - fn the_cursor_counts_only_when_an_enable_started_the_driver_with_it() { - // An Enable that started the driver from off: the file decides. - assert!(cursor_in_effect(false, true, true, true, true)); - assert!(!cursor_in_effect(true, true, true, true, false)); - // The driver was already on (AlreadyDone, or the task found it on): - // it keeps what it read when it started. - assert!( - !cursor_in_effect(false, true, true, false, true), - "never started with it" + fn this_hosts_file_gives_a_128_px_cursor_that_takes_every_pointer_type() { + let xml = render(&modes_for_client(1668, 2420, 60), None); + assert_eq!(cursor_set_by(&xml), hardware(128, 128)); + assert_eq!( + starting_cursor(Some(&xml), Some(true), false), + hardware(128, 128) ); - assert!( - cursor_in_effect(true, true, true, false, true), - "a quick reconnect" + assert_eq!( + starting_cursor(Some(&xml), Some(false), false), + DriverCursor::Software, + "a driver that doesn't read " ); - // Turned off, or a failed Enable from off: no cursor. - assert!(!cursor_in_effect(true, false, true, false, true)); - assert!(!cursor_in_effect(true, true, false, true, true)); - assert!( - cursor_in_effect(true, true, false, false, true), - "a failed Enable that found it on" + assert_eq!( + starting_cursor(Some(&xml), None, false), + DriverCursor::Unknown, + "a driver version that doesn't tell" + ); + assert_eq!( + starting_cursor(Some(&xml), Some(true), true), + DriverCursor::Unknown, + "the driver's registry key overrides the file" + ); + assert_eq!( + starting_cursor(None, Some(true), false), + DriverCursor::Unknown, + "no file read" + ); + } + + /// Review r4: what the running driver does with the pointer is tied to + /// its start (boot and device arrival), kept on disk. + #[test] + fn the_cursor_record_belongs_to_one_start_of_the_driver() { + let dir = std::env::temp_dir().join(format!("vdd-cursor-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(CURSOR_RECORD); + let start = |arrival| DriverInstance { + boot_id: 205, + arrival, + }; + let ours = render(&modes_for_client(1668, 2420, 60), None); + let cursor = starting_cursor(Some(&ours), Some(true), false); + + // Enable from off: the driver started with the file just written. + let record = record_after_enable(None, Some(start(10)), Some(&ours), cursor).unwrap(); + assert_eq!(record.cursor, hardware(128, 128)); + assert_eq!(record.settings_sha256.len(), 64); + save_record(&path, &record).unwrap(); + assert_eq!( + running_cursor(load_record(&path).as_ref(), Some(start(10))), + hardware(128, 128) + ); + + // AlreadyDone with a matching record: the driver kept what it read, + // whatever the file says now. + let vendor = starting_cursor(Some(VENDOR), Some(true), false); + assert_eq!( + record_after_enable(Some(start(10)), Some(start(10)), Some(VENDOR), vendor), + None ); + assert_eq!( + running_cursor(load_record(&path).as_ref(), Some(start(10))), + hardware(128, 128) + ); + + // AlreadyDone with a changed arrival: someone else restarted it. + assert_eq!( + running_cursor(load_record(&path).as_ref(), Some(start(11))), + DriverCursor::Unknown + ); + + // Relaunch: a new host reads the record from disk. Same start, same + // answer; after a reboot the same arrival counts for nothing; with + // no record, or the driver off, nothing is known. + let relaunched = load_record(&path); + assert_eq!( + running_cursor(relaunched.as_ref(), Some(start(10))), + hardware(128, 128) + ); + let rebooted = DriverInstance { + boot_id: 206, + arrival: 10, + }; + assert_eq!( + running_cursor(relaunched.as_ref(), Some(rebooted)), + DriverCursor::Unknown + ); + assert_eq!(running_cursor(None, Some(start(10))), DriverCursor::Unknown); + assert_eq!( + running_cursor(relaunched.as_ref(), None), + DriverCursor::Unknown + ); + + // An Enable that restarted it, with the user's own caps of 64 and + // then 256: each start keeps its own. + for (arrival, max) in [(12, 64), (13, 256)] { + let users = with_cursor(&format!("true{max}{max}true")); + let cursor = starting_cursor(Some(&users), Some(true), false); + let record = record_after_enable( + Some(start(arrival - 1)), + Some(start(arrival)), + Some(&users), + cursor, + ) + .unwrap(); + save_record(&path, &record).unwrap(); + let running = running_cursor(load_record(&path).as_ref(), Some(start(arrival))); + assert_eq!(running, hardware(max, max)); + let DriverCursor::Hardware(caps) = running else { + unreachable!() + }; + assert!(caps.takes((max, max))); + assert!(!caps.takes((max + 1, 1))); + } + + // A start with a file that leaves the cursor off is kept as such. + let record = record_after_enable(None, Some(start(14)), Some(VENDOR), vendor).unwrap(); + assert_eq!(record.cursor, DriverCursor::Software); + // A driver that is not running after the Enable keeps no record. + assert_eq!(record_after_enable(None, None, Some(&ours), cursor), None); + // A damaged record is no record. + std::fs::write(&path, "{").unwrap(); + assert_eq!(load_record(&path), None); + std::fs::remove_dir_all(&dir).unwrap(); } #[test] fn only_drivers_that_read_the_cursor_block_are_trusted_with_it() { for (version, reads) in [ - ("25.05.03", true), - ("24.10.27", true), - ("24.12.24", true), - ("24.10.16", false), - ("23.12.2", false), - ("24.10", false), - ("", false), - ("beta", false), + ("25.05.03", Some(true)), + ("24.10.27", Some(true)), + ("24.12.24", Some(true)), + ("24.10.16", Some(false)), + ("23.12.2", Some(false)), + ("24.10", None), + ("", None), + ("beta", None), ] { assert_eq!(driver_reads_cursor(version), reads, "{version:?}"); } diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index 57b09b05..43946e06 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -3764,6 +3764,7 @@ fn the_pointer_moves_without_video_frames_for_an_ipad_that_draws_it() { let (mut frames, mut highest, mut states) = (0u32, 0u32, Vec::new()); let mut shape: Option<(u32, Vec, usize)> = None; let mut held_shape = 0; + let mut hidden_at_start = 0u32; let mut window_start: Option = None; let deadline = Instant::now() + DEADLINE; while window_start.is_none_or(|at| at.elapsed() < WINDOW) { @@ -3806,6 +3807,12 @@ fn the_pointer_moves_without_video_frames_for_an_ipad_that_draws_it() { } } Classified::Control(_) => match parse_control(bytes).map(|(_, m)| m) { + Ok( + message @ (ControlMessage::PointerState(_) + | ControlMessage::PointerShape(_)), + ) if !draws => { + panic!("an older iPad got a pointer message: {message:?}") + } Ok(ControlMessage::PointerState(state)) if state.flags & POINTER_VISIBLE != 0 => { @@ -3815,7 +3822,16 @@ fn the_pointer_moves_without_video_frames_for_an_ipad_that_draws_it() { } window_start.get_or_insert_with(Instant::now); } - Ok(ControlMessage::PointerState(_)) => {} + // Hidden only while the session starts: until the iPad's + // report names the image, the pointer stays in the video. + // The synthetic pointer never hides. + Ok(ControlMessage::PointerState(state)) => { + assert!( + window_start.is_none(), + "a hidden state after the pointer went apart: {state:?}" + ); + hidden_at_start += 1; + } Ok(ControlMessage::PointerShape(chunk)) => { let (id, bytes, got) = shape.get_or_insert_with(|| { (chunk.shape_id, vec![0; chunk.total_len as usize], 0) @@ -3838,7 +3854,8 @@ fn the_pointer_moves_without_video_frames_for_an_ipad_that_draws_it() { let distinct: std::collections::HashSet<_> = states.iter().collect(); let load = std::fs::read_to_string("/proc/loadavg").unwrap_or_default(); eprintln!( - "draws={draws}: {frames} frames, {} pointer states ({} distinct) in {WINDOW:?} {load}", + "draws={draws}: {frames} frames, {} pointer states ({} distinct) in {WINDOW:?}, \ + {hidden_at_start} hidden before {load}", states.len(), distinct.len() ); From db9ec41508e6fe40ca37fe56860e4b2c46c5caf4 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 00:16:12 -0500 Subject: [PATCH 137/226] Tag a stream's restart request with its own generation The virtual display's restart request read the supervisor's current generation when it was sent, and the network check read it when the stream started. A stream that outlived its generation would have asked in the name of the newer stream, and restarted it. The pipeline now hands the stream its generation, and every restart the stream asks for carries it. A unit test drives the virtual display's release and checks the request names the stream's own generation; the listen-on test checks the network check's does. --- host/src/pipeline.rs | 20 +++-- host/src/transport/mod.rs | 153 +++++++++++++++++++++++----------- host/tests/frame_numbering.rs | 1 + host/tests/goodbye.rs | 6 +- host/tests/input_restart.rs | 1 + host/tests/input_stall.rs | 1 + host/tests/listen_on.rs | 12 ++- host/tests/liveness_hold.rs | 2 +- host/tests/send_budget.rs | 2 +- host/tests/send_congestion.rs | 1 + host/tests/transport_stall.rs | 1 + 11 files changed, 136 insertions(+), 64 deletions(-) diff --git a/host/src/pipeline.rs b/host/src/pipeline.rs index 2633a12e..694189d7 100644 --- a/host/src/pipeline.rs +++ b/host/src/pipeline.rs @@ -103,13 +103,19 @@ pub async fn run_pipeline_supervised( return; } - let transport_outcome = - match transport::start_sender(nal_rx, audio_rx, listen_port, shared.clone(), supervisor_tx) - .await - { - Ok(()) => StageOutcome::Completed, - Err(e) => transport_failure(&*e), - }; + let transport_outcome = match transport::start_sender( + nal_rx, + audio_rx, + listen_port, + shared.clone(), + supervisor_tx, + generation, + ) + .await + { + Ok(()) => StageOutcome::Completed, + Err(e) => transport_failure(&*e), + }; reporter.stage_exited(Stage::Transport, transport_outcome); stats::PIPELINE_STATS.lock().mark_pipeline_stopped(); diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 486d67f0..c2f7b1da 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -305,6 +305,24 @@ impl ConfigSource for SharedConfigSource<'_> { } } +/// The supervisor a stream answers to, and the pipeline generation the +/// stream belongs to. +struct StreamSupervisor { + tx: std_mpsc::Sender, + generation: u64, +} + +impl StreamSupervisor { + /// Asks the supervisor to restart this stream. It does only while this + /// stream's generation still runs: never one the user stopped, or one a + /// newer stream replaced. + fn restart(&self) -> Result<(), std_mpsc::SendError> { + self.tx.send(SupervisorCommand::RestartStream { + generation: self.generation, + }) + } +} + /// Consumes NAL units from the encoder, fragments each access unit into v2 media /// datagrams (raw Annex B — no FlatBuffer wrapper), and runs the protocol-v2 /// control plane (hello/ack, heartbeats, keyframe requests, liveness) on the @@ -315,10 +333,12 @@ pub async fn start_sender( listen_port: u16, shared: SharedControl, supervisor_tx: std_mpsc::Sender, + generation: u64, ) -> Result<(), Box> { - // The generation this stream belongs to: it asks the supervisor to - // restart only itself, never a stream the user stopped. - let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation, + }; let listen_on = shared.listen_on.lock().clone(); let bind_ip = crate::network::listen_ip_now(&listen_on); if bind_ip.is_loopback() { @@ -460,14 +480,14 @@ pub async fn start_sender( _ = heartbeat.tick() => { beat.mark(Phase::Tick); let actions = shared.session.lock().tick(&config, true, now()); - act(actions, &mut state, &links, &shared, &config, &supervisor_tx).await; + act(actions, &mut state, &links, &shared, &config, &supervisor).await; } _ = network_check.tick(), if listen_on != crate::network::ListenOn::All => { beat.mark(Phase::Tick); if !shared.client_connected() && crate::network::listen_ip_now(&listen_on) != bind_ip { info!(listen_on = listen_on.label(), "The chosen network changed: restarting the stream to listen on it"); shared.stop(); - if let Err(error) = supervisor_tx.send(SupervisorCommand::RestartStream { generation }) { + if let Err(error) = supervisor.restart() { warn!(error = %error, "Failed to request a pipeline restart for the chosen network"); } } @@ -482,7 +502,7 @@ pub async fn start_sender( break; } let actions = shared.session.lock().tick(&config, false, now()); - act(actions, &mut state, &links, &shared, &config, &supervisor_tx).await; + act(actions, &mut state, &links, &shared, &config, &supervisor).await; } result = receive_control(&mut links, &mut deferred_control) => { @@ -490,7 +510,7 @@ pub async fn start_sender( match result { Ok(LinkEvent::Closed(id)) => { let actions = shared.session.lock().link_lost(id); - act(actions, &mut state, &links, &shared, &config, &supervisor_tx).await; + act(actions, &mut state, &links, &shared, &config, &supervisor).await; } Ok(LinkEvent::Datagram(src, datagram)) => { #[cfg(feature = "test-hooks")] @@ -510,7 +530,7 @@ pub async fn start_sender( &config, now(), ); - act(actions, &mut state, &links, &shared, &config, &supervisor_tx).await; + act(actions, &mut state, &links, &shared, &config, &supervisor).await; } Err(error) => { debug!(peer = %src, %error, "Dropped malformed control datagram"); @@ -654,7 +674,7 @@ pub async fn start_sender( let paced = frame_pacer.after_send(Instant::now()); let pause = if refused { congestion::BACKOFF } else { paced }; if target_addr.link == LinkId::Udp && pause > Duration::ZERO { - let repairs = repair_while_pacing(&links, &mut state, &shared, &config, &supervisor_tx, + let repairs = repair_while_pacing(&links, &mut state, &shared, &config, &supervisor, pacing_clock.until(Instant::now() + pause), &mut deferred_control); tokio::pin!(repairs); loop { @@ -910,7 +930,7 @@ async fn repair_while_pacing( state: &mut LoopState, shared: &SharedControl, config: &impl ConfigSource, - supervisor_tx: &std_mpsc::Sender, + supervisor: &StreamSupervisor, timer: impl std::future::Future, deferred: &mut DeferredControl, ) { @@ -928,7 +948,7 @@ async fn repair_while_pacing( probe::taken(peer); let actions = shared.session.lock().handle_datagram(PeerId::udp(peer), &header, message, len, config, now()); - act(actions, state, links, shared, config, supervisor_tx).await; + act(actions, state, links, shared, config, supervisor).await; continue; } if deferred.len() < 64 { deferred.push_back(LinkEvent::Datagram(PeerId::udp(peer), bytes[..len].to_vec())); } @@ -957,7 +977,7 @@ async fn act( links: &TransportLinks, shared: &SharedControl, config: &impl ConfigSource, - supervisor_tx: &std_mpsc::Sender, + supervisor: &StreamSupervisor, ) { if actions.new_target.is_some() || actions.client_lost { state.retransmit_ring.clear(); @@ -1011,7 +1031,7 @@ async fn act( }; apply_bitrate(decision, links, shared, config).await; } - execute_actions(actions, links, shared, supervisor_tx).await; + execute_actions(actions, links, shared, supervisor).await; } /// Steps the bitrate down once for each episode of sends the system had no @@ -1052,7 +1072,7 @@ async fn execute_actions( mut actions: Actions, links: &TransportLinks, shared: &SharedControl, - supervisor_tx: &std_mpsc::Sender, + supervisor: &StreamSupervisor, ) { shared.transport_beat.mark(Phase::Actions); // An ended session's summary waits for this pass's second screen step, @@ -1175,10 +1195,7 @@ async fn execute_actions( } if restart { shared.stop(); - // Only this stream: one the user stopped keeps its generation until - // Start, so the supervisor leaves it stopped. - let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); - if let Err(error) = supervisor_tx.send(SupervisorCommand::RestartStream { generation }) { + if let Err(error) = supervisor.restart() { warn!(error = %error, "Failed to request pipeline restart for the virtual display"); } } @@ -1211,6 +1228,43 @@ mod tests { CAP_DECODE_H264, FEATURE_SUPPORTS_NACK, }; + /// No iPad came back to the virtual display within the grace period: + /// the stream asks to restart itself, by its own generation, whichever + /// generation the supervisor counts now. + #[test] + fn the_second_screen_release_restarts_only_this_stream() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let shared = SharedControl::new(socket.local_addr().unwrap().port(), 15_000_000); + shared.running.store(true, Ordering::SeqCst); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + *shared.vdd_status.lock() = VddStatus::Active; + // The viewer left a grace period ago. + assert_eq!( + second_screen::next_step(&shared, Instant::now() - second_screen::RELEASE_GRACE), + second_screen::Step::Nothing + ); + let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); + let (supervisor_tx, supervisor_rx) = std_mpsc::channel(); + let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst) + 1000; + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation, + }; + execute_actions(Actions::default(), &links, &shared, &supervisor).await; + assert_eq!( + supervisor_rx.try_recv(), + Ok(SupervisorCommand::RestartStream { generation }) + ); + assert!(!shared.running.load(Ordering::SeqCst)); + assert!(supervisor_rx.try_recv().is_err()); + }); + } + /// The second screen gives up for a session in the same pass that ends /// it: the session's summary names that fallback. #[test] @@ -1238,6 +1292,10 @@ mod tests { *shared.vdd_status.lock() = VddStatus::Active; let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation: 1, + }; let config = SharedConfigSource { shared: &shared, stream_epoch: 1, @@ -1268,7 +1326,7 @@ mod tests { .session .lock() .handle_control(peer, 0, hello, &config, now); - execute_actions(accepted, &links, &shared, &supervisor_tx).await; + execute_actions(accepted, &links, &shared, &supervisor).await; // The virtual display attached and has shown no picture for too long. second_screen::note_attached(&shared, "virtual".into()); second_screen::start_proving(&shared, 1, now - second_screen::PROVE_TIMEOUT); @@ -1281,7 +1339,7 @@ mod tests { Instant::now(), ); assert!(bye.ended.is_some()); - execute_actions(bye, &links, &shared, &supervisor_tx).await; + execute_actions(bye, &links, &shared, &supervisor).await; }); let summaries = summaries.lock(); assert_eq!(summaries.len(), 1, "{summaries:?}"); @@ -1383,13 +1441,17 @@ mod tests { }; let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation: 1, + }; let mut deferred = DeferredControl::new(); repair_while_pacing( &links, &mut state, &shared, &config, - &supervisor_tx, + &supervisor, tokio::time::sleep(Duration::from_secs(1)), &mut deferred, ) @@ -1484,6 +1546,10 @@ mod tests { }; let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation: 1, + }; let mut state = loop_state(); let cable = PeerId::usb(5); let hello = ControlMessage::Hello2(Hello2 { @@ -1508,15 +1574,7 @@ mod tests { .session .lock() .handle_control(cable, 0, hello, &config, now()); - act( - accepted, - &mut state, - &links, - &shared, - &config, - &supervisor_tx, - ) - .await; + act(accepted, &mut state, &links, &shared, &config, &supervisor).await; let session_id = shared.session.lock().session_id().unwrap(); let press = |event_id, kind, keycode| { ControlMessage::InputEvent(InputEvent { @@ -1543,15 +1601,7 @@ mod tests { .session .lock() .handle_control(cable, session_id, event, &config, now()); - act( - pressed, - &mut state, - &links, - &shared, - &config, - &supervisor_tx, - ) - .await; + act(pressed, &mut state, &links, &shared, &config, &supervisor).await; } // Windows takes input on the input thread. let injected = |count: usize| { @@ -1572,15 +1622,7 @@ mod tests { let unplugged = shared.session.lock().link_lost(cable.link); assert!(unplugged.client_lost); - act( - unplugged, - &mut state, - &links, - &shared, - &config, - &supervisor_tx, - ) - .await; + act(unplugged, &mut state, &links, &shared, &config, &supervisor).await; assert!(shared.session.lock().session_id().is_none()); assert!(matches!( injected(2).as_slice(), @@ -1676,13 +1718,17 @@ mod tests { let links = TransportLinks::new(socket, std::sync::Arc::clone(&shared.force_next_idr)); let (supervisor_tx, _supervisor_rx) = std_mpsc::channel(); + let supervisor = StreamSupervisor { + tx: supervisor_tx, + generation: 1, + }; let mut deferred = DeferredControl::new(); repair_while_pacing( &links, &mut state, &shared, &config, - &supervisor_tx, + &supervisor, async { let _ = paced.await; }, @@ -1924,7 +1970,14 @@ mod tests { .enable_all() .build() .unwrap() - .block_on(start_sender(nal_rx, audio_rx, port, control, supervisor_tx)) + .block_on(start_sender( + nal_rx, + audio_rx, + port, + control, + supervisor_tx, + 1, + )) .unwrap(); }); // The refresh ticks from the stream's start; Windows holds it. diff --git a/host/tests/frame_numbering.rs b/host/tests/frame_numbering.rs index abe5d7cc..9a1ec519 100644 --- a/host/tests/frame_numbering.rs +++ b/host/tests/frame_numbering.rs @@ -47,6 +47,7 @@ fn skipped_capture_frames_leave_no_gap_on_the_wire() { port, sender_shared, supervisor_tx, + 1, )) .unwrap(); }); diff --git a/host/tests/goodbye.rs b/host/tests/goodbye.rs index b01c53df..546fab36 100644 --- a/host/tests/goodbye.rs +++ b/host/tests/goodbye.rs @@ -124,7 +124,7 @@ fn udp_goodbyes(end: End, stop: Stop, reason: ByeReason) -> usize { let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); let control = shared.clone(); let transport = tokio::spawn(async move { - transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx) + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx, 1) .await .unwrap() }); @@ -257,7 +257,7 @@ fn a_stop_during_startup_still_says_goodbye() { let control = shared.clone(); tokio::time::timeout( Duration::from_secs(30), - transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx), + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx, 1), ) .await .expect("the transport stops") @@ -326,7 +326,7 @@ fn usb_goodbye(end: End, reason: ByeReason) { let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); let control = shared.clone(); let transport = tokio::spawn(async move { - transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx) + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx, 1) .await .unwrap() }); diff --git a/host/tests/input_restart.rs b/host/tests/input_restart.rs index 14e5c5a8..5e44a3d0 100644 --- a/host/tests/input_restart.rs +++ b/host/tests/input_restart.rs @@ -50,6 +50,7 @@ fn start(shared: &SharedControl, port: u16) -> Generation { port, control, supervisor_tx, + 1, )) .unwrap(); }); diff --git a/host/tests/input_stall.rs b/host/tests/input_stall.rs index bfba0f50..0e795f94 100644 --- a/host/tests/input_stall.rs +++ b/host/tests/input_stall.rs @@ -153,6 +153,7 @@ fn input_windows_holds_never_stops_the_transport_loop() { port, control, supervisor_tx, + 1, )) .unwrap(); }); diff --git a/host/tests/listen_on.rs b/host/tests/listen_on.rs index 4f60faa9..af594008 100644 --- a/host/tests/listen_on.rs +++ b/host/tests/listen_on.rs @@ -29,6 +29,9 @@ fn wifi(up: bool, ip: Ipv4Addr) -> Adapter { } } +/// The pipeline generation of the streams this test starts. +const GENERATION: u64 = 7; + /// Runs the transport until it stops, on its own thread. fn start( shared: &SharedControl, @@ -55,6 +58,7 @@ fn start( port, shared, supervisor_tx, + GENERATION, )) .unwrap(); }); @@ -91,8 +95,12 @@ fn the_address_row_shows_a_chosen_network_that_came_back() { let command = supervisor_rx .recv_timeout(Duration::from_secs(30)) .expect("the network check asks for a restart"); - let generation = eternal_host::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); - assert_eq!(command, SupervisorCommand::RestartStream { generation }); + assert_eq!( + command, + SupervisorCommand::RestartStream { + generation: GENERATION + } + ); transport.join().unwrap(); // The supervisor starts the stream again, now on the network's address. diff --git a/host/tests/liveness_hold.rs b/host/tests/liveness_hold.rs index d92e078b..dbea1b48 100644 --- a/host/tests/liveness_hold.rs +++ b/host/tests/liveness_hold.rs @@ -381,7 +381,7 @@ impl Sim { let control = shared.clone(); let transport = tokio::spawn(async move { let _audio = audio_tx; - transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx).await + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx, 1).await }); // The transport binds before its first wait. for _ in 0..10 { diff --git a/host/tests/send_budget.rs b/host/tests/send_budget.rs index c097aed1..842d7723 100644 --- a/host/tests/send_budget.rs +++ b/host/tests/send_budget.rs @@ -336,7 +336,7 @@ impl Sim { let control = shared.clone(); let transport = tokio::spawn(async move { let _audio = audio_tx; - transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx).await + transport::start_sender(nal_rx, audio_rx, port, control, supervisor_tx, 1).await }); // The transport binds before its first wait. for _ in 0..10 { diff --git a/host/tests/send_congestion.rs b/host/tests/send_congestion.rs index f56980d7..985eb2c2 100644 --- a/host/tests/send_congestion.rs +++ b/host/tests/send_congestion.rs @@ -225,6 +225,7 @@ fn refused_sends_back_off_step_down_once_and_log_one_episode() { port, sender_shared, supervisor_tx, + 1, )) .unwrap(); }); diff --git a/host/tests/transport_stall.rs b/host/tests/transport_stall.rs index 4bafae14..38446046 100644 --- a/host/tests/transport_stall.rs +++ b/host/tests/transport_stall.rs @@ -64,6 +64,7 @@ fn a_stuck_transport_loop_is_named_with_its_step() { port, control, supervisor_tx, + 1, )) .unwrap(); }); From 486447f341448e9062e546b1fb2c980cc600d092 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 00:16:12 -0500 Subject: [PATCH 138/226] Wait for the supervisor's answer, and require kernel receive times The stopped-stream test waited a fixed 3 s to see that nothing restarted. It now waits for the supervisor to log that it left the stopped stream as it is, then checks the PC is still stopped. The iPad's fake PC no longer falls back to the time it read a datagram when the kernel gives no receive time, and the retry test requires one. --- host/tests/hello_standby.rs | 79 ++++++++++++++++--- ios/EternalMonitorTests/FakeUDPHost.swift | 7 +- .../PCNotStreamingTests.swift | 3 +- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/host/tests/hello_standby.rs b/host/tests/hello_standby.rs index 15f287af..e76f2458 100644 --- a/host/tests/hello_standby.rs +++ b/host/tests/hello_standby.rs @@ -123,6 +123,68 @@ impl Drop for Host { } } +/// The supervisor's log lines since [`capture_supervisor_log`], from every +/// thread. +#[cfg(debug_assertions)] +static SUPERVISOR_LOG: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + +#[cfg(debug_assertions)] +struct SupervisorLog; + +#[cfg(debug_assertions)] +impl tracing_subscriber::Layer for SupervisorLog { + fn on_event(&self, event: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { + struct Message(Option); + impl tracing::field::Visit for Message { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + if field.name() == "message" { + self.0 = Some(format!("{value:?}")); + } + } + } + let mut message = Message(None); + event.record(&mut message); + if let Some(line) = message.0 { + SUPERVISOR_LOG.lock().unwrap().push(line); + } + } +} + +/// Starts capturing the supervisor's log lines afresh. The capture is the +/// process's subscriber, and sees nothing else; `ENV_LOCK` runs one test at +/// a time. +#[cfg(debug_assertions)] +fn capture_supervisor_log() { + use tracing_subscriber::prelude::*; + static INSTALLED: std::sync::Once = std::sync::Once::new(); + INSTALLED.call_once(|| { + let only_the_supervisor = tracing_subscriber::filter::filter_fn(|metadata| { + metadata.target() == "eternal_host::supervisor" + }); + let _ = tracing_subscriber::registry() + .with(SupervisorLog.with_filter(only_the_supervisor)) + .try_init(); + }); + SUPERVISOR_LOG.lock().unwrap().clear(); +} + +/// Waits for the supervisor to log `line`. +#[cfg(debug_assertions)] +fn wait_for_supervisor_line(line: &str) { + let deadline = Instant::now() + Duration::from_secs(20); + loop { + let logged = SUPERVISOR_LOG.lock().unwrap().clone(); + if logged.iter().any(|logged| logged == line) { + return; + } + assert!( + Instant::now() < deadline, + "the supervisor never logged {line:?}: {logged:?}" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + /// A GPU whose encoder FFmpeg doesn't have: every generation fails as it /// starts, as a broken driver does. fn broken_gpu() -> GpuInfo { @@ -493,6 +555,7 @@ fn a_stream_stopping_as_its_network_changes_stays_stopped() { set_test_adapters(Some(wifi(false))); std::mem::forget(shared.session.lock()); }); + capture_supervisor_log(); // The stream binds this PC only, the network being down, then waits // for the session. let shown = format!("unknown:{}", host.port); @@ -508,15 +571,13 @@ fn a_stream_stopping_as_its_network_changes_stays_stopped() { unsafe { host.shared.session.force_unlock() }; host.wait_until_stopped(); - let deadline = Instant::now() + Duration::from_secs(3); - while Instant::now() < deadline { - assert_eq!( - host.shared.session.lock().not_streaming(), - Some(NotStreaming::Stopped), - "the stopped stream restarted itself" - ); - std::thread::sleep(Duration::from_millis(10)); - } + // The stream asked after Stop, and the supervisor turned it down. + wait_for_supervisor_line("A stream that no longer runs asked to start again: left as it is"); + assert_eq!( + host.shared.session.lock().not_streaming(), + Some(NotStreaming::Stopped), + "the stopped stream restarted itself" + ); assert!(!host.streaming(), "the stopped stream restarted itself"); } diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index 261b56f8..fdb71a7a 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -20,10 +20,11 @@ final class FakeUDPHost: @unchecked Sendable { /// The pairing token the HELLO2 answered last carried. private(set) var lastAuthToken = Data() /// When the HELLO2 answered last arrived, by the kernel's receive - /// time: how fast this fake read it doesn't move it. + /// time: how fast this fake read it doesn't move it. Nil if the kernel + /// gave none. private(set) var lastHelloArrival: Date? /// When the datagram read last arrived, by the kernel's receive time. - private var arrival = Date() + private var arrival: Date? /// A HELLO2 came before the identify request was answered. private(set) var helloBeforeIdentify = false @@ -74,7 +75,7 @@ final class FakeUDPHost: @unchecked Sendable { } } guard count > 0 else { return nil } - arrival = Self.kernelTimestamp(control.prefix(controlLength)) ?? Date() + arrival = Self.kernelTimestamp(control.prefix(controlLength)) return (Data(buffer.prefix(count)), from, length) } diff --git a/ios/EternalMonitorTests/PCNotStreamingTests.swift b/ios/EternalMonitorTests/PCNotStreamingTests.swift index 2ed41b9f..3d5a6013 100644 --- a/ios/EternalMonitorTests/PCNotStreamingTests.swift +++ b/ios/EternalMonitorTests/PCNotStreamingTests.swift @@ -46,7 +46,8 @@ final class PCNotStreamingTests: XCTestCase { let again = await Task.detached { host.acceptSession(status: .stopped) }.value XCTAssertTrue(again, "the iPad stopped asking") XCTAssertEqual(host.lastNonce, firstNonce) - if let arrival = host.lastHelloArrival, arrival > told { askedAt.append(arrival) } + let arrival = try XCTUnwrap(host.lastHelloArrival, "no kernel receive time for the HELLO2") + if arrival > told { askedAt.append(arrival) } } XCTAssertEqual(askedAt.count, 3, "the iPad stopped asking") for (earlier, later) in zip(askedAt.dropLast(), askedAt.dropFirst()) { From f50860dbdef9aa8ea307da6d5d3385aee3ca03a4 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 00:48:37 -0500 Subject: [PATCH 139/226] Allow the LUID reading's unsafe calls in the vendored crate egui-wgpu denies unsafe code outside the winit painter, so the Windows build stopped on the LUID reading. The function now opts in, with a safety note on each call. --- vendor/egui-wgpu/src/lib.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/vendor/egui-wgpu/src/lib.rs b/vendor/egui-wgpu/src/lib.rs index bac52760..0ea047cd 100644 --- a/vendor/egui-wgpu/src/lib.rs +++ b/vendor/egui-wgpu/src/lib.rs @@ -329,14 +329,19 @@ impl AdapterKey { /// EternalMonitor patch: the adapter's LUID, from DXGI on DX12, or from /// `VkPhysicalDeviceIDProperties` on Vulkan 1.1+. #[cfg(windows)] +#[expect(unsafe_code)] // EternalMonitor patch: reads the adapter's backend objects. fn luid(adapter: &wgpu::Adapter) -> Option { - // SAFETY: the backend objects are only read, while `adapter` holds them. - if let Some(hal) = unsafe { adapter.as_hal::() } { + // SAFETY: the DX12 adapter is only read, while `adapter` holds it. + let dx12 = unsafe { adapter.as_hal::() }; + if let Some(hal) = dx12 { + // SAFETY: `GetDesc1` only fills in a description of a live adapter. let desc = unsafe { hal.raw_adapter().GetDesc1() }.ok()?; let luid = desc.AdapterLuid; - return Some((u64::from(luid.HighPart as u32) << 32) | u64::from(luid.LowPart)); + return Some((u64::from(luid.HighPart.cast_unsigned()) << 32) | u64::from(luid.LowPart)); } - if let Some(hal) = unsafe { adapter.as_hal::() } { + // SAFETY: the Vulkan adapter is only read, while `adapter` holds it. + let vulkan = unsafe { adapter.as_hal::() }; + if let Some(hal) = vulkan { let shared = hal.shared_instance(); if shared.instance_api_version() < ash::vk::API_VERSION_1_1 { return None; @@ -344,13 +349,15 @@ fn luid(adapter: &wgpu::Adapter) -> Option { let mut id = ash::vk::PhysicalDeviceIDProperties::default(); { let mut properties = ash::vk::PhysicalDeviceProperties2::default().push_next(&mut id); + // SAFETY: a query on the instance's own physical device, core in Vulkan 1.1. unsafe { shared .raw_instance() .get_physical_device_properties2(hal.raw_physical_device(), &mut properties); } } - return (id.device_luid_valid == ash::vk::TRUE).then(|| u64::from_le_bytes(id.device_luid)); + return (id.device_luid_valid == ash::vk::TRUE) + .then_some(u64::from_le_bytes(id.device_luid)); } None } From fa808171a5ab16fb05c2be14201947208667bf32 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 01:14:01 -0500 Subject: [PATCH 140/226] Watch the window until it is confirmed drawing, and key it by a list that returned Second review of #206 found five more gaps; this closes them. - The first-paint watcher stopped at the second UI pass, which egui counts whether or not a frame was presented: egui-wgpu returns from a paint without presenting after a failed acquire, and recreates the surface at the start of the next one. The watch now ends only when the renderer confirms the window drawing (frames for a while with no surface error, visible). The notice goes up when the window's thread has begun no UI pass for 4 s; the window repaints at least every 200 ms, and a minimized one is left alone. - Configure could still commit an empty GPU key when its own DXGI list timed out. The list now runs before configure, in the startup lane, until one returns in time; configure gets that list, and the window keys its adapters by it until it is confirmed drawing. - A Cancel from the watcher left the renderer's marker, so the next launch counted a deliberate quit as a failed start. Cancel now clears it, and the marker and that quit share one lock, so a worker still in the driver cannot save it again. - The end of the wait, the user's answer and raising the notice now change one state under one lock: a Cancel that comes as the window draws still quits, and no notice goes up after the wait ended. - The first DXGI list had no marker. It now leaves one before it starts and names its thread once it runs, until a later step replaces it. A launch that dies there makes the next ones key the GPUs by wgpu's driver text instead, until clean launches retry the list. --- host/src/gui/driver_wait.rs | 592 +++++++++++++++++++++++++----------- host/src/gui/mod.rs | 38 ++- host/src/gui/renderer.rs | 305 +++++++++++++++---- 3 files changed, 685 insertions(+), 250 deletions(-) diff --git a/host/src/gui/driver_wait.rs b/host/src/gui/driver_wait.rs index 5b5185d4..126de88f 100644 --- a/host/src/gui/driver_wait.rs +++ b/host/src/gui/driver_wait.rs @@ -3,23 +3,24 @@ //! The window's thread lists adapters, creates its device and draws through //! the graphics driver, and Windows cannot stop a thread stuck inside it. //! So before that thread makes its first graphics call ([`before_window`]): -//! - the startup graphics lane ([`Call::GpuDetect`]) must be free: a GPU -//! detection or screen list given up on may still be stuck there, and the -//! renderer's crash avoidance reads the GPU list through the same lane; +//! - the graphics cards are listed through DXGI on a thread of their own, +//! in the startup graphics lane ([`Call::GpuDetect`]), once nothing else +//! is stuck there. The renderer's crash avoidance keys its history by that +//! list, so it is configured only with a list that returned; //! - then the same wgpu calls the window's thread makes first are made once -//! on a thread of their own ([`Call::GpuProbe`]), under its limit. +//! on a thread of their own ([`Call::GpuProbe`]). //! //! Past that, the window's thread can still stall in calls that cannot be -//! moved off it (its own device and swap chain), so [`watch_first_paint`] -//! watches from another thread until its first frame is presented. +//! moved off it (its own device and swap chain, a surface it recreates), so +//! [`watch_first_paint`] watches from another thread until the window is +//! confirmed drawing. //! //! Whenever the window waits on the driver, a notice drawn by Windows -//! without the graphics card says why ([`Prompt`]). It stays until the -//! first frame. OK hides it for a while; Cancel quits. +//! without the graphics card says why ([`Prompt`]). OK hides it for a +//! while; Cancel quits. It goes for good when the wait is over. // Only Windows waits; the logic is tested everywhere. #![cfg_attr(not(windows), allow(dead_code))] -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -52,9 +53,21 @@ pub(super) trait Notice: Send { fn close(&mut self); } +/// Where the wait stands. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Phase { + Waiting, + /// The wait is over and the user did not quit. + Over, + /// The user chose Cancel, also one that came as the wait ended. + Quit, +} + /// The notice while startup waits on the driver, shared by the steps on -/// the window's thread and the first-paint watcher: up from the first wait, -/// back [`REMIND_AFTER`] after OK, down for good at the first frame. +/// the window's thread, the first-paint watcher and the renderer: up from +/// the first wait, back [`REMIND_AFTER`] after OK, down for good once the +/// wait is over. Every change happens under one lock, so the end of the +/// wait, the user's answer and showing the notice never cross. #[derive(Clone)] pub(super) struct Prompt(Arc>); @@ -63,8 +76,7 @@ struct PromptState { up: bool, raised: bool, hidden_at: Option, - quit: bool, - done: bool, + phase: Phase, } impl Prompt { @@ -74,8 +86,7 @@ impl Prompt { up: false, raised: false, hidden_at: None, - quit: false, - done: false, + phase: Phase::Waiting, }))) } @@ -84,7 +95,7 @@ impl Prompt { pub(super) fn raise(&self, now: Instant) { let mut state = self.0.lock(); state.read_answer(now); - if state.up || state.quit || state.done { + if state.up || state.phase != Phase::Waiting { return; } if state @@ -93,17 +104,20 @@ impl Prompt { { return; } + if !state.raised { + warn!("The graphics driver is not answering; the window opens once it does"); + } state.notice.show(); state.up = true; state.raised = true; state.hidden_at = None; } - /// The user chose Cancel, by `now`. - pub(super) fn quit(&self, now: Instant) -> bool { + /// Where the wait stands at `now`, with the user's answer read. + pub(super) fn phase(&self, now: Instant) -> Phase { let mut state = self.0.lock(); state.read_answer(now); - state.quit + state.phase } /// The notice was ever shown. @@ -111,12 +125,17 @@ impl Prompt { self.0.lock().raised } - /// The wait is over: take the notice down and keep it down. - pub(super) fn close(&self) { + /// The wait is over: the answer given by now is kept, and the notice + /// goes down and stays down. Where it stands after. + pub(super) fn finish(&self) -> Phase { let mut state = self.0.lock(); - state.done = true; + state.read_answer(Instant::now()); + if state.phase == Phase::Waiting { + state.phase = Phase::Over; + } state.up = false; state.notice.close(); + state.phase } } @@ -132,7 +151,9 @@ impl PromptState { } Some(Answer::Quit) => { self.up = false; - self.quit = true; + if self.phase == Phase::Waiting { + self.phase = Phase::Quit; + } } None => {} } @@ -152,7 +173,7 @@ fn wait_until( let started = Instant::now(); loop { if free.iter().all(|&call| lanes.abandoned(call) == 0) && attempt() { - if prompt.quit(Instant::now()) { + if prompt.phase(Instant::now()) == Phase::Quit { break; } if prompt.raised() { @@ -163,11 +184,8 @@ fn wait_until( } return true; } - if !prompt.raised() { - warn!("The graphics driver is not answering; the window opens once it does"); - } prompt.raise(Instant::now()); - if prompt.quit(Instant::now()) { + if prompt.phase(Instant::now()) == Phase::Quit { break; } std::thread::sleep(poll); @@ -179,94 +197,116 @@ fn wait_until( false } -/// The steps before the window's thread makes its first graphics call. -/// First the startup lane frees, so `configure` reads the GPU list crash -/// avoidance keys its history by. Then fresh probes from `probe`, over what -/// `configure` returned, run in [`Call::GpuProbe`] under `limits` until one -/// returns in time. False when the user quit from the notice. -pub(super) fn before_window( +/// The steps before the window's thread makes its first graphics call: +/// fresh lists from `list` run in [`Call::GpuDetect`] under `list_limits` +/// until one returns in time, which `configure` gets; then fresh probes from +/// `probe`, over what `configure` returned, run in [`Call::GpuProbe`] until +/// one returns in time. False when the user quit from the notice. +#[allow(clippy::too_many_arguments)] +pub(super) fn before_window( lanes: &'static Lanes, - limits: Limits, + list_limits: Limits, + probe_limits: Limits, prompt: &Prompt, poll: Duration, - configure: impl FnOnce() -> S, + mut list: impl FnMut() -> L, + configure: impl FnOnce(T) -> S, probe: impl Fn(&S) -> F, ) -> bool where + T: Send + 'static, + L: FnOnce() -> T + Send + 'static, F: FnOnce() + Send + 'static, { - if !wait_until(lanes, &[Call::GpuDetect], || true, prompt, poll) { + let mut listed = None; + let got_list = wait_until( + lanes, + &[Call::GpuDetect], + || match crate::stall::run_on(lanes, Call::GpuDetect, list_limits, list(), || {}) { + Ok(adapters) => { + listed = Some(adapters); + true + } + Err(_) => false, + }, + prompt, + poll, + ); + let Some(adapters) = listed.filter(|_| got_list) else { return false; - } - let spec = configure(); + }; + let spec = configure(adapters); wait_until( lanes, &[Call::GpuDetect, Call::GpuProbe], - || crate::stall::run_on(lanes, Call::GpuProbe, limits, probe(&spec), || {}).is_ok(), + || crate::stall::run_on(lanes, Call::GpuProbe, probe_limits, probe(&spec), || {}).is_ok(), prompt, poll, ) } -/// Watches the window's first frame from another thread, from now: the -/// notice goes up if `painted` is still false after `limit` (or already -/// was), and down for good once it is true. Cancel runs `quit`, which ends -/// the process: the window's thread may never return. +/// Watches the window from another thread until the wait ends +/// ([`FirstPaintWatch::finisher`], called when the window is confirmed +/// drawing, or [`FirstPaintWatch::stop`]). The notice goes up once the +/// window's thread has begun no UI pass (`last_pass`, with whether the +/// window was minimized then) for `limit`, counted from now before the +/// first; a minimized window draws nothing and is left alone. Cancel runs +/// `quit`, which ends the process: the window's thread may never return. pub(super) fn watch_first_paint( prompt: Prompt, limit: Duration, poll: Duration, now: impl Fn() -> Instant + Send + 'static, - painted: impl Fn() -> bool + Send + 'static, + last_pass: impl Fn() -> Option<(Instant, bool)> + Send + 'static, quit: impl FnOnce() + Send + 'static, ) -> FirstPaintWatch { - let stop = Arc::new(AtomicBool::new(false)); - let stopped = Arc::clone(&stop); let started = now(); - let watch = move || { - let mut warned = false; - loop { - if painted() || stopped.load(Ordering::SeqCst) { - prompt.close(); - return; - } - let at = now(); - if prompt.raised() || at.saturating_duration_since(started) >= limit { - if !warned && !prompt.raised() { - warn!( - limit_ms = limit.as_millis() as u64, - "The window has not drawn its first frame; the graphics driver may not be answering" - ); - } - warned = true; - prompt.raise(at); - } - if prompt.quit(at) { - info!("Quit while the window waited for its first frame"); + let watched = prompt.clone(); + let watch = move || loop { + match watched.phase(now()) { + Phase::Quit => { + info!("Quit while the window waited for the graphics driver"); quit(); return; } - std::thread::sleep(poll); + Phase::Over => return, + Phase::Waiting => {} + } + let at = now(); + let (since, minimized) = last_pass().unwrap_or((started, false)); + if !minimized && at.saturating_duration_since(since.max(started)) >= limit { + watched.raise(at); } + std::thread::sleep(poll); }; let thread = std::thread::Builder::new() .name("first-paint-watch".into()) .spawn(watch) - .map_err(|error| warn!(%error, "Could not watch the window's first frame")) + .map_err(|error| warn!(%error, "Could not watch the window's first frames")) .ok(); - FirstPaintWatch { stop, thread } + FirstPaintWatch { prompt, thread } } /// A running [`watch_first_paint`]. pub(super) struct FirstPaintWatch { - stop: Arc, + prompt: Prompt, thread: Option>, } impl FirstPaintWatch { - /// The window ended without a first frame: take the notice down. + /// Ends the wait, for the renderer to call when the window is + /// confirmed drawing. + pub(super) fn finisher(&self) -> Box { + let prompt = self.prompt.clone(); + Box::new(move || { + prompt.finish(); + }) + } + + /// The window ended: the wait is over, and the watcher is done. A + /// Cancel given by then still quits. pub(super) fn stop(self) { - self.stop.store(true, Ordering::SeqCst); + self.prompt.finish(); if let Some(thread) = self.thread { let _ = thread.join(); } @@ -293,7 +333,7 @@ const NOTICE: &str = "The graphics driver is not responding, so the EternalMonit #[cfg(windows)] impl Notice for MessageNotice { fn show(&mut self) { - use std::sync::atomic::AtomicU32; + use std::sync::atomic::{AtomicU32, Ordering}; use windows::core::HSTRING; use windows::Win32::UI::WindowsAndMessaging::{ MessageBoxW, MB_ICONWARNING, MB_OKCANCEL, MB_SETFOREGROUND, @@ -334,6 +374,7 @@ impl Notice for MessageNotice { } fn close(&mut self) { + use std::sync::atomic::Ordering; use windows::Win32::Foundation::{BOOL, HWND, LPARAM, WPARAM}; use windows::Win32::UI::WindowsAndMessaging::{EnumThreadWindows, PostMessageW, WM_CLOSE}; unsafe extern "system" fn close_window(hwnd: HWND, _: LPARAM) -> BOOL { @@ -361,7 +402,7 @@ impl Notice for MessageNotice { #[cfg(test)] mod tests { use super::*; - use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering::SeqCst}; + use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering::SeqCst}; use std::sync::mpsc; const LIMITS: Limits = Limits { @@ -376,22 +417,22 @@ mod tests { /// What happened, in order. #[derive(Clone, Default)] - struct Events(Arc>>); + struct Events(Arc>>); impl Events { - fn push(&self, event: &'static str) { - self.0.lock().push(event); + fn push(&self, event: impl Into) { + self.0.lock().push(event.into()); } - fn all(&self) -> Vec<&'static str> { + fn all(&self) -> Vec { self.0.lock().clone() } - fn has(&self, event: &str) -> bool { - self.0.lock().contains(&event) + fn count(&self, event: &str) -> usize { + self.0.lock().iter().filter(|e| *e == event).count() } /// Backstop only; each wait ends as soon as the event happens. - fn wait_for(&self, event: &str) { + fn wait_for(&self, event: &str, times: usize) { let deadline = Instant::now() + Duration::from_secs(60); - while !self.has(event) { + while self.count(event) < times { assert!(Instant::now() < deadline, "no {event}: {:?}", self.all()); std::thread::sleep(Duration::from_millis(1)); } @@ -447,12 +488,16 @@ mod tests { release } - /// [`before_window`] on its own thread, with `configure` and each probe - /// recorded. + type ListStep = Box String + Send>; + type ProbeStep = Box; + + /// [`before_window`] on its own thread: each list's result and each + /// probe come from the test, and `configure` records what it got. fn start_before_window( lanes: &'static Lanes, fake: &Fake, - probe: impl Fn() -> Box + Send + 'static, + list: impl FnMut() -> ListStep + Send + 'static, + probe: impl Fn() -> ProbeStep + Send + 'static, ) -> std::thread::JoinHandle { let prompt = Prompt::new(fake.clone()); let events = fake.events.clone(); @@ -461,48 +506,125 @@ mod tests { before_window( lanes, LIMITS, + LIMITS, &prompt, SLICE, - move || { - // Crash avoidance reads the GPU list only once the - // startup lane is free. + list, + move |gpus: String| { + // Crash avoidance is configured only with a list that + // returned, and nothing is stuck in the startup lane. assert_eq!(lanes.abandoned(Call::GpuDetect), 0); - configured.push("configure"); + configured.push(format!("configure {gpus}")); }, |_| probe(), ) }) } + fn listing(events: &Events, gpus: &'static str) -> ListStep { + let events = events.clone(); + Box::new(move || { + events.push("list"); + gpus.to_string() + }) + } + + fn probing(events: &Events) -> ProbeStep { + let events = events.clone(); + Box::new(move || events.push("probe")) + } + #[test] fn a_driver_that_answers_opens_the_window_without_a_notice() { let fake = Fake::default(); - let events = fake.events.clone(); - let opened = start_before_window(lanes(), &fake, move || { - let events = events.clone(); - Box::new(move || events.push("probe")) - }); + let (list_events, probe_events) = (fake.events.clone(), fake.events.clone()); + let opened = start_before_window( + lanes(), + &fake, + move || listing(&list_events, "10de:2c02 32.0"), + move || probing(&probe_events), + ); assert!(opened.join().unwrap()); - assert_eq!(fake.events.all(), ["configure", "probe"]); + assert_eq!( + fake.events.all(), + ["list", "configure 10de:2c02 32.0", "probe"] + ); } #[test] - fn while_detection_is_stuck_nothing_reads_the_gpus_and_the_notice_is_up() { + fn while_detection_is_stuck_nothing_lists_and_the_notice_is_up() { let lanes = lanes(); let release = stuck_detection(lanes); let fake = Fake::default(); - let events = fake.events.clone(); - let opened = start_before_window(lanes, &fake, move || { - let events = events.clone(); - Box::new(move || events.push("probe")) - }); - fake.events.wait_for("shown"); + let (list_events, probe_events) = (fake.events.clone(), fake.events.clone()); + let opened = start_before_window( + lanes, + &fake, + move || listing(&list_events, "gpus"), + move || probing(&probe_events), + ); + fake.events.wait_for("shown", 1); // Nothing went near the stuck driver. assert_eq!(fake.events.all(), ["shown"]); release.send(()).unwrap(); assert!(opened.join().unwrap()); // The notice stays up for the first-paint watcher to take down. - assert_eq!(fake.events.all(), ["shown", "configure", "probe"]); + assert_eq!( + fake.events.all(), + ["shown", "list", "configure gpus", "probe"] + ); + } + + #[test] + fn a_list_that_stalls_is_never_configured_and_the_next_one_is() { + let lanes = lanes(); + let fake = Fake::default(); + let (started, list_started) = mpsc::channel::<()>(); + let (release, held) = mpsc::channel::<()>(); + let held = Arc::new(Mutex::new(Some(held))); + let (list_events, probe_events) = (fake.events.clone(), fake.events.clone()); + let opened = start_before_window( + lanes, + &fake, + move || { + let events = list_events.clone(); + let started = started.clone(); + // The first list blocks in the driver; the next answers. + let held = held.lock().take(); + Box::new(move || { + let _ = started.send(()); + match held { + Some(held) => { + let _ = held.recv(); + events.push("stalled list returned"); + "late".to_string() + } + None => { + events.push("list"); + "gpus".to_string() + } + } + }) + }, + move || probing(&probe_events), + ); + list_started.recv().unwrap(); + fake.events.wait_for("shown", 1); + assert_eq!(fake.events.all(), ["shown"]); + release.send(()).unwrap(); + assert!(opened.join().unwrap()); + // The stalled list's result was dropped; the one that returned in + // time configured the renderer, once. + assert_eq!( + fake.events.all(), + [ + "shown", + "stalled list returned", + "list", + "configure gpus", + "probe" + ] + ); } #[test] @@ -512,32 +634,40 @@ mod tests { let (started, probe_started) = mpsc::channel::<()>(); let (release, held) = mpsc::channel::<()>(); let held = Arc::new(Mutex::new(Some(held))); - let events = fake.events.clone(); - let opened = start_before_window(lanes, &fake, move || { - let events = events.clone(); - let started = started.clone(); - // The first probe blocks in the driver; the next answers. - let held = held.lock().take(); - Box::new(move || { - events.push("probe"); - let _ = started.send(()); - if let Some(held) = held { - let _ = held.recv(); - events.push("probe returned"); - } - }) - }); + let (list_events, probe_events) = (fake.events.clone(), fake.events.clone()); + let opened = start_before_window( + lanes, + &fake, + move || listing(&list_events, "gpus"), + move || { + let events = probe_events.clone(); + let started = started.clone(); + let held = held.lock().take(); + Box::new(move || { + events.push("probe"); + let _ = started.send(()); + if let Some(held) = held { + let _ = held.recv(); + events.push("probe returned"); + } + }) + }, + ); probe_started.recv().unwrap(); - fake.events.wait_for("shown"); - assert!(!fake.events.has("probe returned")); + fake.events.wait_for("shown", 1); release.send(()).unwrap(); assert!(opened.join().unwrap()); - let events = fake.events.all(); - assert_eq!(events[..3], ["configure", "probe", "shown"], "{events:?}"); - // The second probe ran once the first had returned and freed its lane. - assert_eq!(events.iter().filter(|e| **e == "probe").count(), 2); - let returned = events.iter().position(|e| *e == "probe returned").unwrap(); - assert_eq!(events[returned + 1], "probe", "{events:?}"); + assert_eq!( + fake.events.all(), + [ + "list", + "configure gpus", + "probe", + "shown", + "probe returned", + "probe" + ] + ); } #[test] @@ -546,7 +676,13 @@ mod tests { let _release = stuck_detection(lanes); let fake = Fake::default(); *fake.answer.lock() = Some(Answer::Quit); - let opened = start_before_window(lanes, &fake, || Box::new(|| {})); + let (list_events, probe_events) = (fake.events.clone(), fake.events.clone()); + let opened = start_before_window( + lanes, + &fake, + move || listing(&list_events, "gpus"), + move || probing(&probe_events), + ); assert!(!opened.join().unwrap()); assert_eq!(fake.events.all(), ["shown", "cancel"]); } @@ -557,20 +693,29 @@ mod tests { let release = stuck_detection(lanes); let fake = Fake::default(); let answer = Arc::clone(&fake.answer); + let list_events = fake.events.clone(); // The user clicks Cancel while the probe runs, the moment before it // returns in time. - let opened = start_before_window(lanes, &fake, move || { - let answer = Arc::clone(&answer); - Box::new(move || *answer.lock() = Some(Answer::Quit)) - }); - fake.events.wait_for("shown"); + let opened = start_before_window( + lanes, + &fake, + move || listing(&list_events, "gpus"), + move || { + let answer = Arc::clone(&answer); + Box::new(move || *answer.lock() = Some(Answer::Quit)) + }, + ); + fake.events.wait_for("shown", 1); release.send(()).unwrap(); assert!(!opened.join().unwrap()); - assert_eq!(fake.events.all(), ["shown", "configure", "cancel"]); + assert_eq!( + fake.events.all(), + ["shown", "list", "configure gpus", "cancel"] + ); } #[test] - fn ok_hides_the_notice_until_the_reminder_and_closing_ends_it() { + fn ok_hides_the_notice_until_the_reminder_and_the_end_of_the_wait_ends_it() { let fake = Fake::default(); let prompt = Prompt::new(fake.clone()); let start = Instant::now(); @@ -579,19 +724,19 @@ mod tests { assert_eq!(fake.events.all(), ["shown"]); *fake.answer.lock() = Some(Answer::Wait); let ok_at = start + Duration::from_secs(2); - assert!(!prompt.quit(ok_at)); + assert_eq!(prompt.phase(ok_at), Phase::Waiting); prompt.raise(ok_at + REMIND_AFTER - Duration::from_millis(1)); assert_eq!(fake.events.all(), ["shown", "ok"]); prompt.raise(ok_at + REMIND_AFTER); assert_eq!(fake.events.all(), ["shown", "ok", "shown"]); // Cancel on the second showing quits. *fake.answer.lock() = Some(Answer::Quit); - assert!(prompt.quit(ok_at + REMIND_AFTER)); + assert_eq!(prompt.phase(ok_at + REMIND_AFTER), Phase::Quit); // Once the wait is over, nothing shows it again. let fake = Fake::default(); let prompt = Prompt::new(fake.clone()); prompt.raise(start); - prompt.close(); + assert_eq!(prompt.finish(), Phase::Over); prompt.raise(start + REMIND_AFTER * 2); assert_eq!(fake.events.all(), ["shown", "closed"]); } @@ -604,61 +749,114 @@ mod tests { (ms, move || start + Duration::from_millis(read.load(SeqCst))) } + /// The renderer's view of the window: when its thread last began a UI + /// pass, in the manual clock's time. + #[derive(Clone)] + struct Passes { + start: Instant, + last: Arc>>, + } + + impl Passes { + fn new(now: &impl Fn() -> Instant) -> Self { + Self { + start: now(), + last: Arc::default(), + } + } + fn pass_at(&self, ms: u64, minimized: bool) { + *self.last.lock() = Some((self.start + Duration::from_millis(ms), minimized)); + } + fn reader(&self) -> impl Fn() -> Option<(Instant, bool)> + Send + 'static { + let last = Arc::clone(&self.last); + move || *last.lock() + } + } + + fn never_quit() -> impl FnOnce() + Send + 'static { + || panic!("quit") + } + #[test] - fn a_window_that_paints_in_time_shows_no_notice() { + fn a_window_that_draws_in_time_shows_no_notice() { let fake = Fake::default(); - let (_clock, now) = manual_clock(); - let painted = Arc::new(AtomicBool::new(true)); - let seen = Arc::clone(&painted); + let (clock, now) = manual_clock(); + let passes = Passes::new(&now); let watch = watch_first_paint( Prompt::new(fake.clone()), LIMITS.limit, SLICE, now, - move || seen.load(SeqCst), - || panic!("quit"), + passes.reader(), + never_quit(), ); - watch.thread.unwrap().join().unwrap(); - assert!(fake.events.all().is_empty()); + // UI passes keep coming; then the window is confirmed drawing. + for ms in [100, 300, 500, 700] { + passes.pass_at(ms, false); + clock.store(ms + 50, SeqCst); + std::thread::sleep(Duration::from_millis(5)); + } + (watch.finisher())(); + watch.stop(); + assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); } #[test] - fn a_window_stuck_before_its_first_frame_gets_the_notice_until_it_paints() { + fn a_failed_first_acquire_then_a_stuck_recreation_gets_the_notice_until_the_window_draws() { let fake = Fake::default(); let (clock, now) = manual_clock(); - let painted = Arc::new(AtomicBool::new(false)); - let seen = Arc::clone(&painted); + let passes = Passes::new(&now); let watch = watch_first_paint( Prompt::new(fake.clone()), LIMITS.limit, SLICE, now, - move || seen.load(SeqCst), - || panic!("quit"), + passes.reader(), + never_quit(), ); - // Just under the limit: nothing yet. - clock.store(LIMITS.limit.as_millis() as u64 - 1, SeqCst); + // The first pass ran and its frame's surface acquire failed, so the + // renderer did not confirm the window; recreating the surface at the + // start of the next paint never returns: no more passes. + passes.pass_at(200, false); + let stuck_at = 200 + LIMITS.limit.as_millis() as u64; + clock.store(stuck_at - 1, SeqCst); std::thread::sleep(Duration::from_millis(20)); - assert!(fake.events.all().is_empty()); - clock.store(LIMITS.limit.as_millis() as u64, SeqCst); - fake.events.wait_for("shown"); - // OK, then the reminder. + assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); + clock.store(stuck_at, SeqCst); + fake.events.wait_for("shown", 1); + // OK, then the reminder while it is still stuck. *fake.answer.lock() = Some(Answer::Wait); - fake.events.wait_for("ok"); + fake.events.wait_for("ok", 1); clock.fetch_add(REMIND_AFTER.as_millis() as u64, SeqCst); - let deadline = Instant::now() + Duration::from_secs(60); - while fake.events.all().len() < 3 { - assert!(Instant::now() < deadline, "{:?}", fake.events.all()); - std::thread::sleep(Duration::from_millis(1)); - } - // The first frame takes it down. - painted.store(true, SeqCst); + fake.events.wait_for("shown", 2); + // It comes back and is confirmed drawing: the notice goes. + (watch.finisher())(); watch.thread.unwrap().join().unwrap(); assert_eq!(fake.events.all(), ["shown", "ok", "shown", "closed"]); } #[test] - fn cancel_while_the_first_frame_is_stuck_quits_from_the_watcher() { + fn a_minimized_window_gets_no_notice() { + let fake = Fake::default(); + let (clock, now) = manual_clock(); + let passes = Passes::new(&now); + let watch = watch_first_paint( + Prompt::new(fake.clone()), + LIMITS.limit, + SLICE, + now, + passes.reader(), + never_quit(), + ); + passes.pass_at(100, true); + clock.store(100 + LIMITS.limit.as_millis() as u64 * 10, SeqCst); + std::thread::sleep(Duration::from_millis(20)); + watch.stop(); + assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); + } + + #[test] + fn cancel_while_the_window_is_stuck_quits_from_the_watcher() { let fake = Fake::default(); let (clock, now) = manual_clock(); clock.store(LIMITS.limit.as_millis() as u64, SeqCst); @@ -670,7 +868,7 @@ mod tests { Duration::ZERO, SLICE, now, - || false, + || None, move || { count.fetch_add(1, SeqCst); }, @@ -681,25 +879,63 @@ mod tests { } #[test] - fn a_notice_raised_before_the_window_stays_until_its_first_frame() { + fn a_cancel_that_comes_with_the_window_drawing_still_quits() { let fake = Fake::default(); + let (_clock, now) = manual_clock(); + let quits = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&quits); let prompt = Prompt::new(fake.clone()); prompt.raise(Instant::now()); - let (_clock, now) = manual_clock(); - let painted = Arc::new(AtomicBool::new(false)); - let seen = Arc::clone(&painted); + // The user clicks Cancel; before the watcher reads it, the renderer + // confirms the window and ends the wait. + *fake.answer.lock() = Some(Answer::Quit); + assert_eq!(prompt.finish(), Phase::Quit); let watch = watch_first_paint( prompt, - REMIND_AFTER * 4, + REMIND_AFTER, SLICE, now, - move || seen.load(SeqCst), - || panic!("quit"), + || None, + move || { + count.fetch_add(1, SeqCst); + }, ); - std::thread::sleep(Duration::from_millis(20)); - assert_eq!(fake.events.all(), ["shown"]); - painted.store(true, SeqCst); watch.thread.unwrap().join().unwrap(); - assert_eq!(fake.events.all(), ["shown", "closed"]); + assert_eq!(quits.load(SeqCst), 1); + assert_eq!(fake.events.all(), ["shown", "cancel"]); + } + + #[test] + fn a_window_that_draws_while_the_watcher_decides_to_raise_shows_no_notice() { + let fake = Fake::default(); + let start = Instant::now(); + // The watcher reads the clock just before it raises the notice. The + // test holds that read until the renderer has ended the wait. + let (reading, clock_read) = mpsc::channel::<()>(); + let (go, proceed) = mpsc::channel::<()>(); + let proceed = std::sync::Mutex::new(proceed); + let reads = AtomicUsize::new(0); + let now = move || { + // Read 0 is the watcher's start, 1 its phase check; 2 is the + // time it raises at. + if reads.fetch_add(1, SeqCst) == 2 { + let _ = reading.send(()); + let _ = proceed.lock().unwrap().recv(); + } + start + LIMITS.limit * 2 + }; + let watch = watch_first_paint( + Prompt::new(fake.clone()), + LIMITS.limit, + SLICE, + now, + || None, + never_quit(), + ); + clock_read.recv().unwrap(); + (watch.finisher())(); + go.send(()).unwrap(); + watch.thread.unwrap().join().unwrap(); + assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); } } diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 07f05e07..c2127582 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -1798,40 +1798,48 @@ pub fn run_gui( ..Default::default() }; // The window's thread makes its first graphics call only once the - // driver has answered the same calls on another thread, and a stall - // after that is watched until the first frame. + // driver has answered the same calls on another thread, and is watched + // until it is confirmed drawing. + let prepared = renderer::prepare(); #[cfg(windows)] let watch = { - let lanes = crate::stall::lanes(); - let limits = crate::stall::Call::GpuProbe.limits(); let prompt = driver_wait::Prompt::new(driver_wait::MessageNotice::default()); + let skip_listing = prepared.listing_avoided(); + let probe_limits = crate::stall::Call::GpuProbe.limits(); let opened = driver_wait::before_window( - lanes, - limits, + crate::stall::lanes(), + crate::stall::Call::GpuDetect.limits(), + probe_limits, &prompt, driver_wait::POLL, - || { - renderer::configure(&mut options.wgpu_options); + || renderer::adapter_list(skip_listing), + |adapters| { + renderer::configure(&mut options.wgpu_options, prepared, adapters); renderer::probe_spec(&options.wgpu_options) }, |spec| renderer::adapter_probe(spec.clone()), ); if !opened { - renderer::window_closed(); + renderer::abandon(); return Ok(()); } - driver_wait::watch_first_paint( + let watch = driver_wait::watch_first_paint( prompt, - limits.limit, + probe_limits.limit, driver_wait::POLL, std::time::Instant::now, - renderer::first_frame_painted, - quit_while_stuck, - ) + renderer::last_pass, + move || { + renderer::abandon(); + quit_while_stuck(); + }, + ); + renderer::when_drawing(watch.finisher()); + watch }; #[cfg(not(windows))] { - renderer::configure(&mut options.wgpu_options); + renderer::configure(&mut options.wgpu_options, prepared, Vec::new()); drop(quit_while_stuck); } diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 260a56b0..aa562c0a 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -46,7 +46,7 @@ use std::fs; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; @@ -105,6 +105,10 @@ const DRIVER_MODULES: [&str; 18] = [ /// PCI vendor id of Intel GPUs. const INTEL: u32 = 0x8086; const HISTORY_FILE: &str = "history.json"; +/// The marker backend of a launch listing the graphics cards through DXGI, +/// before it knows their keys. A launch that dies there makes the next ones +/// key the GPUs by wgpu's driver text instead, until clean launches retry. +const LISTING: &str = "list dxgi"; /// Held while one instance reads, changes and saves the history. const HISTORY_LOCK: &str = "history.lock"; /// The window counts as working after this long and this many frames with no @@ -114,14 +118,21 @@ const CONFIRM_AFTER: Duration = Duration::from_secs(3); const CONFIRM_FRAMES: u64 = 10; static STATE_DIR: OnceLock = OnceLock::new(); -static STARTED: Mutex> = Mutex::new(None); +static LAUNCH: Mutex = Mutex::new(Launch::new()); static PRESENT: Mutex = Mutex::new(PresentTracker::new()); /// Frames whose surface texture could not be acquired, so nothing was presented. static FAILED_ACQUIRES: AtomicU64 = AtomicU64::new(0); static STATUS: Mutex> = Mutex::new(None); -/// The window's first frame was presented: its thread is past every -/// graphics call of startup. -static FIRST_PAINT: AtomicBool = AtomicBool::new(false); +/// The DXGI adapter list taken before the window, which the window's GPU +/// keys come from until it is confirmed drawing: no more DXGI calls on its +/// thread before then. +static ADAPTERS: Mutex>> = Mutex::new(None); +/// When the window's thread last began a UI pass, and whether the window +/// was minimized then. +static LAST_PASS: Mutex> = Mutex::new(None); +/// Run once, when the window is confirmed drawing. +type OnDrawing = Box; +static ON_DRAWING: Mutex> = Mutex::new(None); /// How this launch draws its window, for Settings > Troubleshooting. #[derive(Clone, Debug, Default, PartialEq)] @@ -275,9 +286,13 @@ fn driver_module(module: &str) -> bool { DRIVER_MODULES.iter().any(|name| module.starts_with(name)) } -/// Choose the instance backends and install the adapter selector, after -/// recording failed earlier starts. -pub(super) fn configure(config: &mut WgpuConfiguration) { +/// The failed starts this launch knows of: earlier launches' markers are +/// recorded first. Makes no graphics call. +pub(super) struct Prepared { + history: Vec, +} + +pub(super) fn prepare() -> Prepared { let history = match crate::settings::app_data_dir() { Some(base) => { let directory = base.join("renderer"); @@ -288,6 +303,57 @@ pub(super) fn configure(config: &mut WgpuConfiguration) { } None => Vec::new(), }; + Prepared { history } +} + +impl Prepared { + /// A launch died listing the graphics cards through DXGI, and that is + /// not retried yet. + #[cfg_attr(not(windows), allow(dead_code))] + pub(super) fn listing_avoided(&self) -> bool { + avoided(&self.history, "", LISTING) + } +} + +/// The DXGI adapter list, for another thread to take before the window +/// (`skip`: a launch died there; the GPUs are keyed by wgpu's driver text). +/// This launch's marker names the listing before it starts, and the thread +/// it runs on once it does, until a later step replaces it: a crash there, +/// even after the caller gave up on a list that did not return, is the +/// window's. +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn adapter_list(skip: bool) -> impl FnOnce() -> Vec<(u32, u32, String)> + Send { + if skip { + info!( + "A launch died listing the graphics cards; keying them by the renderer's driver text" + ); + } else { + let _ = start(Marker { + gpu: String::new(), + backend: LISTING.to_string(), + thread: None, + }); + } + move || { + if skip { + return Vec::new(); + } + mark_thread(current_thread()); + list_dxgi_adapters() + } +} + +/// Choose the instance backends and install the adapter selector, with the +/// graphics cards `adapters` listed before the window. +pub(super) fn configure( + config: &mut WgpuConfiguration, + prepared: Prepared, + adapters: Vec<(u32, u32, String)>, +) { + let Prepared { history } = prepared; + if let Ok(mut listed) = ADAPTERS.lock() { + *listed = Some(adapters); + } let choice = crate::settings::SettingsFile::load().window_drawing; // Saved by the last update check; this launch's choice only. let rules = if cfg!(windows) { @@ -375,15 +441,68 @@ pub(super) fn adapter_probe(spec: ProbeSpec) -> impl FnOnce() + Send + 'static { /// Name `thread` in this launch's marker and save it again. fn mark_thread(thread: Option) { - let Ok(mut started) = STARTED.lock() else { - return; - }; - let Some(marker) = started.as_mut() else { - return; - }; - marker.thread = thread; - if let Some(directory) = STATE_DIR.get() { - let _ = save_marker(directory, ProcessId::current(), marker); + if let Ok(mut launch) = LAUNCH.lock() { + launch.mark_thread( + STATE_DIR.get().map(PathBuf::as_path), + ProcessId::current(), + thread, + ); + } +} + +/// This launch's marker, and whether the user quit before the window drew. +/// One lock holds both, so a worker still in the driver cannot save the +/// marker again after the quit cleared it. +struct Launch { + started: Option, + abandoned: bool, +} + +impl Launch { + const fn new() -> Self { + Self { + started: None, + abandoned: false, + } + } + + /// Leave `marker` in `directory` until the window is confirmed working. + /// False when it could not be saved, or the launch was abandoned. + fn start(&mut self, directory: Option<&Path>, owner: ProcessId, marker: Marker) -> bool { + if self.abandoned { + return false; + } + let saved = directory.is_some_and(|directory| save_marker(directory, owner, &marker)); + self.started = Some(marker); + saved + } + + fn mark_thread(&mut self, directory: Option<&Path>, owner: ProcessId, thread: Option) { + if self.abandoned { + return; + } + let Some(marker) = self.started.as_mut() else { + return; + }; + marker.thread = thread; + if let Some(directory) = directory { + let _ = save_marker(directory, owner, marker); + } + } + + /// Clear the marker, after recording that its backend drew when `drew`. + fn clear(&mut self, directory: &Path, owner: ProcessId, drew: bool) { + if let Some(marker) = self.started.as_ref() { + if finish_launch(directory, marker, owner, drew) { + self.started = None; + } + } + } + + /// The user quit: the marker goes without counting, for good. + fn abandon(&mut self, directory: &Path, owner: ProcessId) { + self.abandoned = true; + self.clear(directory, owner, false); } } @@ -411,6 +530,9 @@ fn choose_backends( backend: plan.stage().to_string(), thread: current_thread(), }); + } else { + // The listing's marker: nothing is recorded under WGPU_BACKEND. + clear_marker(false); } descriptor.backends = instance_backends(forced, plan); } @@ -496,13 +618,13 @@ fn current_thread() -> Option { /// Leave this launch's marker until the window is confirmed working. False /// when it could not be saved. fn start(marker: Marker) -> bool { - let saved = STATE_DIR - .get() - .is_some_and(|directory| save_marker(directory, ProcessId::current(), &marker)); - if let Ok(mut started) = STARTED.lock() { - *started = Some(marker); - } - saved + LAUNCH.lock().is_ok_and(|mut launch| { + launch.start( + STATE_DIR.get().map(PathBuf::as_path), + ProcessId::current(), + marker, + ) + }) } fn save_marker(directory: &Path, owner: ProcessId, marker: &Marker) -> bool { @@ -513,12 +635,11 @@ fn save_marker(directory: &Path, owner: ProcessId, marker: &Marker) -> bool { /// Called at the start of each UI pass; clears this launch's marker once the /// window has kept drawing without a surface error. pub(super) fn frame_started(ctx: &egui::Context) { - // A pass after the first: the first frame was presented. - if ctx.cumulative_frame_nr() >= 1 { - FIRST_PAINT.store(true, Ordering::SeqCst); - } crate::startup::simulate_surface_error(&FAILED_ACQUIRES); let visible = ctx.input(|input| input.viewport().minimized != Some(true)); + if let Ok(mut last) = LAST_PASS.lock() { + *last = Some((Instant::now(), !visible)); + } let Ok(mut tracker) = PRESENT.lock() else { return; }; @@ -533,16 +654,40 @@ pub(super) fn frame_started(ctx: &egui::Context) { if now_drawing { crate::startup::mark_window_shown(); info!("Window renderer is drawing"); + let on_drawing = ON_DRAWING.lock().ok().and_then(|mut hook| hook.take()); + if let Some(on_drawing) = on_drawing { + on_drawing(); + } } if drawing { clear_marker(true); } } -/// The window's first frame was presented. +/// When the window's thread last began a UI pass, and whether the window +/// was minimized then. None before the first. +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn last_pass() -> Option<(Instant, bool)> { + LAST_PASS.lock().ok().and_then(|last| *last) +} + +/// Run `hook` once, when the window is confirmed drawing: frames presented +/// for a while with no surface error, visible ([`PresentTracker`]). #[cfg_attr(not(windows), allow(dead_code))] -pub(super) fn first_frame_painted() -> bool { - FIRST_PAINT.load(Ordering::SeqCst) +pub(super) fn when_drawing(hook: OnDrawing) { + if let Ok(mut on_drawing) = ON_DRAWING.lock() { + *on_drawing = Some(hook); + } +} + +/// The user quit before the window drew. A deliberate quit is no failed +/// start: the marker goes without counting, and no worker still in the +/// driver writes it again. +#[cfg_attr(not(windows), allow(dead_code))] +pub(super) fn abandon() { + if let (Some(directory), Ok(mut launch)) = (STATE_DIR.get(), LAUNCH.lock()) { + launch.abandon(directory, ProcessId::current()); + } } /// The window closed normally. A launch that quits before its window is @@ -555,13 +700,8 @@ pub(super) fn window_closed() { /// `drew`. The history may be busy with another instance; the next frame /// tries again, so the GUI never waits for it. fn clear_marker(drew: bool) { - let (Some(directory), Ok(mut started)) = (STATE_DIR.get(), STARTED.lock()) else { - return; - }; - if let Some(marker) = started.as_ref() { - if finish_launch(directory, marker, ProcessId::current(), drew) { - started.take(); - } + if let (Some(directory), Ok(mut launch)) = (STATE_DIR.get(), LAUNCH.lock()) { + launch.clear(directory, ProcessId::current(), drew); } } @@ -906,28 +1046,23 @@ fn system_gpus() -> String { } /// Vendor, device and driver version of each DXGI adapter. Creates no device. -/// Runs under a limit on a thread of its own: empty if it does not return -/// in time, or while an earlier call of its lane is still stuck. While it -/// lists, the marker names that thread, so a driver crash there is still -/// the window's. +/// Until the window is confirmed drawing, the list taken before it. After +/// that (a renderer rebuilt after a lost device), a fresh one under a limit +/// on a thread of its own: empty if it does not return in time. #[cfg(windows)] fn dxgi_adapters() -> Vec<(u32, u32, String)> { - let window = current_thread(); - let listed = crate::stall::run( - adapters_lane(first_frame_painted()), - || { - mark_thread(current_thread()); - list_dxgi_adapters() - }, - || {}, - ); - mark_thread(window); - listed.unwrap_or_default() + let shown = crate::startup::window_shown(); + if !shown { + if let Some(listed) = ADAPTERS.lock().ok().and_then(|listed| listed.clone()) { + return listed; + } + } + crate::stall::run(adapters_lane(shown), list_dxgi_adapters, || {}).unwrap_or_default() } -/// The lane of the window's DXGI adapter list: startup's until the first -/// frame, then the one the second screen's graphics reads use, so a later -/// renderer (after a lost device) neither waits behind nor holds startup's. +/// The lane of a DXGI adapter list the window takes itself: startup's until +/// it is confirmed drawing, then the one the second screen's graphics reads +/// use, so a later renderer neither waits behind nor holds startup's. #[cfg_attr(not(windows), allow(dead_code))] fn adapters_lane(painted: bool) -> crate::stall::Call { if painted { @@ -966,6 +1101,11 @@ fn dxgi_adapters() -> Vec<(u32, u32, String)> { Vec::new() } +#[cfg(not(windows))] +fn list_dxgi_adapters() -> Vec<(u32, u32, String)> { + Vec::new() +} + /// A crash inside a graphics driver stays avoided until the driver changes, /// which changes the GPU key. Every other failure, and one on a GPU whose /// driver version is unknown, is retried after clean launches. @@ -1740,10 +1880,61 @@ mod tests { fs::remove_dir_all(directory).unwrap(); } + #[test] + fn a_quit_from_the_driver_notice_leaves_no_marker_and_no_worker_writes_it_again() { + let directory = temp_dir("abandon"); + let owner = exited_process(); + let mut launch = Launch::new(); + assert!(launch.start( + Some(&directory), + owner, + marker("10de:2c02 32.0", "enumerate") + )); + assert!(marker_path(&directory, owner).exists()); + // Cancel while the window's thread is stuck in the driver. + launch.abandon(&directory, owner); + assert!(!marker_path(&directory, owner).exists()); + // A probe or list still in the driver returns later, and the + // window's thread may still reach its adapter selector. + launch.mark_thread(Some(&directory), owner, Some(WINDOW)); + assert!(!launch.start(Some(&directory), owner, marker("10de:2c02 32.0", "dx12"))); + assert!(!marker_path(&directory, owner).exists()); + // The next launch counts no failed start. + assert!(sweep(&directory, ProcessId::current()).is_empty()); + let _ = fs::remove_dir_all(&directory); + } + + #[test] + fn a_launch_that_dies_listing_the_graphics_cards_skips_the_list_next_time() { + let directory = temp_dir("listing"); + let owner = exited_process(); + let mut launch = Launch::new(); + // The marker is there before the list starts, and names the thread + // that lists once it runs. A crash there is the window's. + launch.start(Some(&directory), owner, marker("", LISTING)); + launch.mark_thread(Some(&directory), owner, Some(0x5150)); + let saved: Marker = + serde_json::from_slice(&fs::read(marker_path(&directory, owner)).unwrap()).unwrap(); + assert_eq!(saved.thread, Some(0x5150)); + // That launch died in the driver while listing. + let history = sweep(&directory, ProcessId::current()); + assert_eq!(history.len(), 1, "{history:?}"); + assert_eq!( + (history[0].gpu.as_str(), history[0].backend.as_str()), + ("", LISTING) + ); + assert!(Prepared { history }.listing_avoided()); + assert!(!Prepared { + history: Vec::new() + } + .listing_avoided()); + let _ = fs::remove_dir_all(&directory); + } + #[test] fn after_the_first_frame_the_adapter_list_leaves_the_startup_lane() { use crate::stall::Call; - // Before it, a stuck startup call refuses it at once. + // Before the window is confirmed, a stuck startup call refuses it. assert_eq!(adapters_lane(false), Call::GpuDetect); // A renderer started later (after a lost device) neither waits behind // nor holds startup's lane, nor logs its stalls as startup's. From 6365a06145ce7e210d60887dba3333884b98211f Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 01:28:18 -0500 Subject: [PATCH 141/226] Show the mirror before the moved-screen test's second screen attaches a_screen_moved_to_the_other_card_never_traps_the_ipad_on_a_blocked_mirror started with a display that attaches at once, and waited for 10 pictures after the first stream epoch. The PC sends from the HELLO's answer, and the second screen's first step runs in the same pass, so the switch could restart the stream before a mirror picture decoded. Every picture then was in the second screen's epoch, nothing came "after the first", and the test ran to its 40 s backstop at 60 fps. That happened once in CI. Its first part now works as the remembered-failure test's third run does: the display attaches 25 s after its Enable, and the fake clock moves on once a mirror picture has decoded. The test still requires a mirror, then the second screen, and ten pictures from it. With the iPad held off the stream until the switch, the old test failed the same way, 2387 pictures in 40 s. The new one passes with the iPad held off for 300 ms. --- host/tests/e2e_synthetic/second_screen.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/host/tests/e2e_synthetic/second_screen.rs b/host/tests/e2e_synthetic/second_screen.rs index e2437c49..02879b61 100644 --- a/host/tests/e2e_synthetic/second_screen.rs +++ b/host/tests/e2e_synthetic/second_screen.rs @@ -565,7 +565,10 @@ fn a_screen_moved_to_the_other_card_never_traps_the_ipad_on_a_blocked_mirror() { let _env = test_env(); synthetic_env(); let memory = memory_dir("moved-screen"); - let display = FakeDisplay::new(Some(Duration::ZERO), Duration::from_secs(10)); + // It attaches only once the mirror has shown, as below: an instant + // attach could switch the stream before the iPad decoded a mirror + // picture, and then no picture ever came after the first epoch. + let display = FakeDisplay::new(Some(Duration::from_secs(25)), Duration::from_secs(30)); let host = Host::start(&display, &memory, SystemTime::now()); // The fake clock skips the waits between display changes and the // attach wait; none of it saves a failure at the moved clock. @@ -577,8 +580,15 @@ fn a_screen_moved_to_the_other_card_never_traps_the_ipad_on_a_blocked_mirror() { // The second screen works, with the screen on the app's card. let mut receiver = host.connect(0x5707_2001); + let mut advanced = false; let pictures = watch(&mut receiver, Duration::from_secs(40), |pictures| { - skip_waits(); + if !advanced && !pictures.is_empty() && !display.toggles().is_empty() { + // Mirroring while the display attaches. Now its 25 s pass. + assert_eq!(epochs(pictures).len(), 1); + assert_eq!(host.status(), VddStatus::Attaching); + display.advance(Duration::from_secs(25)); + advanced = true; + } after_first_epoch(pictures) >= 10 }); assert_eq!(epochs(&pictures).len(), 2, "mirror, then the second screen"); From 5163125b80a36a0cac9f76c8a5185757bb8d5cfe Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 01:56:06 -0500 Subject: [PATCH 142/226] List display outputs only through one judge of the second screen's proof Review r5 of #205. The reopen's first listing could find no output at all and end capture, and nothing cleared the second screen's proof: the same display back before the next reconciliation inherited it, as after r3 and r4's listings. Each listing path had to remember to judge what it saw. A new capture::listing module is now the only way to list the outputs, and it judges every listing: one that does not show the proven output as it was (absent, or its name under another adapter) clears the proof. The DXGI enumerator takes a token only that module can make, so a new listing path cannot skip it. The capture loop's listing (which a reopen makes first), the check after a lost duplication, the no-display wait, reconciliation, the virtual display worker and its mode settling, and the Settings page's display list all go through it. Only the startup banner, before any host state exists, and a native test that prints the outputs list without one. A test holds every remaining way of seeing outputs to those places. --- host/src/capture/dxgi.rs | 28 +++--- host/src/capture/listing.rs | 149 ++++++++++++++++++++++++++++++ host/src/capture/mod.rs | 101 +++++--------------- host/src/capture/second_screen.rs | 136 ++++++++++++++++++++++++--- host/src/gui/mod.rs | 6 +- host/src/main.rs | 2 +- host/src/vdd_device.rs | 2 +- 7 files changed, 310 insertions(+), 114 deletions(-) create mode 100644 host/src/capture/listing.rs diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 03e2ca4d..2781dafa 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -11,6 +11,7 @@ use std::time::{Duration, Instant}; use tracing::{debug, error, info, warn}; use windows::core::Interface; +use super::listing::Judged; use super::recovery::{ note_capture_resumed, note_gpu_outcome, CaptureBlock, CaptureUnavailable, LossRecovery, LossStep, OpenStep, Reopen, LOSS_GIVE_UP, @@ -50,10 +51,10 @@ fn utf16_to_string(buf: &[u16]) -> String { ) } -/// Enumerate every desktop-attached display output across all adapters. Read-only and -/// reusable by both the GUI picker and the capture loop. Never panics — on any DXGI error -/// it returns whatever was discovered so far (possibly empty). -pub fn enumerate_outputs() -> Vec { +/// Every desktop-attached display output across all adapters, from a fresh +/// factory. Read-only. Never panics — on any DXGI error it returns whatever +/// was discovered so far (possibly empty). Through [`super::listing`]. +pub(super) fn outputs(judged: Judged) -> Vec { let factory: IDXGIFactory1 = match unsafe { CreateDXGIFactory1() } { Ok(f) => f, Err(error) => { @@ -61,30 +62,25 @@ pub fn enumerate_outputs() -> Vec { return Vec::new(); } }; - list_outputs(&factory) + list_outputs(&factory, judged) .0 .into_iter() .map(|listed| listed.info) .collect() } -/// Whether a fresh enumeration lists any display attached to the desktop. -pub fn display_attached() -> bool { - unsafe { CreateDXGIFactory1::() } - .is_ok_and(|factory| !list_outputs(&factory).0.is_empty()) -} - /// A desktop-attached output with the adapter and output objects it was /// listed from. Capture opens these objects rather than looking the /// output up again by its ordinals. -struct Listed { - info: OutputInfo, +pub(super) struct Listed { + pub(super) info: OutputInfo, adapter: IDXGIAdapter1, output: IDXGIOutput, } /// Every desktop-attached output, and how many adapters were listed. -fn list_outputs(factory: &IDXGIFactory1) -> (Vec, u32) { +/// Through [`super::listing`], which judges what it lists. +pub(super) fn list_outputs(factory: &IDXGIFactory1, _: Judged) -> (Vec, u32) { let mut outputs = Vec::new(); let mut adapter_index = 0u32; @@ -155,7 +151,7 @@ pub fn run_capture_loop( // DXGI factories snapshot adapters at creation. Enable the driver first, // then enumerate and open the chosen output from the same fresh factory. let factory: IDXGIFactory1 = unsafe { CreateDXGIFactory1()? }; - let (listed, adapters) = list_outputs(&factory); + let (listed, adapters) = super::listing::listed(&shared, &factory); let outputs: Vec = listed.iter().map(|listed| listed.info.clone()).collect(); for o in &outputs { info!( @@ -1004,7 +1000,7 @@ fn target_moved( let started = Instant::now(); let moved = crate::stall::run( Call::ListOutputs, - move || super::judge_listing(&shared, &enumerate_outputs(), &target, &chosen), + move || super::target_moved(&super::listing::outputs(&shared), &target, &chosen), progress, ) .unwrap_or(false); diff --git a/host/src/capture/listing.rs b/host/src/capture/listing.rs new file mode 100644 index 00000000..ab593c60 --- /dev/null +++ b/host/src/capture/listing.rs @@ -0,0 +1,149 @@ +//! The one way to list the PC's display outputs. Every listing is judged for +//! a second screen's proof ([`super::second_screen::observe_listing`]): one +//! that does not show the proven output as it was clears the proof. Kept, the +//! same display back before the next reconciliation inherited it, and a dark +//! one cost a watchdog restart. The DXGI enumerator takes a [`Judged`] that +//! only this module makes, so a new listing path cannot skip the judge. + +use super::OutputInfo; +use crate::control::SharedControl; + +/// Made only here: the DXGI enumerator takes one, so every listing it makes +/// passes through this module. +#[cfg(windows)] +pub(super) struct Judged(()); + +/// The outputs attached to the desktop, judged for `shared`'s second screen. +/// In debug builds with a fake virtual display installed, the fake's. +pub fn outputs(shared: &SharedControl) -> Vec { + let outputs = unjudged(shared); + super::second_screen::observe_listing(shared, &outputs); + outputs +} + +/// Whether any output is attached to the desktop, judged as [`outputs`]. +pub fn any_attached(shared: &SharedControl) -> bool { + !outputs(shared).is_empty() +} + +/// [`outputs`] with the adapter and output objects DXGI listed them from, +/// for capture to open, and how many adapters were listed. +#[cfg(windows)] +pub(super) fn listed( + shared: &SharedControl, + factory: &windows::Win32::Graphics::Dxgi::IDXGIFactory1, +) -> (Vec, u32) { + let (listed, adapters) = super::dxgi::list_outputs(factory, Judged(())); + let outputs: Vec = listed.iter().map(|listed| listed.info.clone()).collect(); + super::second_screen::observe_listing(shared, &outputs); + (listed, adapters) +} + +/// For the startup banner, before any host state exists: there is no proof +/// to judge yet. `every_listing_goes_through_the_judge` holds its callers to +/// that. +pub fn outputs_before_the_host_starts() -> Vec { + from_dxgi() +} + +fn unjudged(shared: &SharedControl) -> Vec { + #[cfg(debug_assertions)] + if let Some(outputs) = super::second_screen::fake::outputs(shared) { + return outputs; + } + // Only the debug-build fake reads it. + #[cfg(not(debug_assertions))] + let _ = shared; + from_dxgi() +} + +/// Windows-only in substance; other platforms have no selectable outputs +/// (the synthetic source ignores them). +fn from_dxgi() -> Vec { + #[cfg(windows)] + { + super::dxgi::outputs(Judged(())) + } + #[cfg(not(windows))] + { + Vec::new() + } +} + +#[cfg(test)] +mod tests { + /// Review r5 of #205: listings that bypassed the judge let a returning + /// display inherit an old proof, one path at a time. The DXGI enumerator + /// needs this module's token; this test holds the remaining ways to see + /// outputs to their known places, so a new one has to be routed here. + #[test] + fn every_listing_goes_through_the_judge() { + let src = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + // Each call, and the files allowed to make it. + let allowed: [(&str, &[&str]); 5] = [ + // The DXGI enumerator, which takes a `Judged`. + ("list_outputs(", &["capture/dxgi.rs"]), + ("dxgi::outputs(", &[]), + // The debug-build fake's outputs. + ("fake::outputs(", &["capture/second_screen/fake.rs"]), + // Before the host exists, and a native test that prints them. + ( + "outputs_before_the_host_starts(", + &["main.rs", "vdd_device.rs"], + ), + // DXGI's own enumeration: capture's, and a per-adapter output + // count in the virtual display's diagnostics snapshot, which + // names no output. + ("EnumOutputs(", &["capture/dxgi.rs", "vdd_snapshot.rs"]), + ]; + let mut files = Vec::new(); + let mut dirs = vec![src.clone()]; + while let Some(dir) = dirs.pop() { + for entry in std::fs::read_dir(&dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + dirs.push(path); + } else if path.extension().is_some_and(|ext| ext == "rs") { + files.push(path); + } + } + } + assert!(files.len() > 50, "{} source files", files.len()); + let mut strays = Vec::new(); + for file in &files { + let name = file + .strip_prefix(&src) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + // This module is the judge, and this test names the calls. + if name == "capture/listing.rs" { + continue; + } + // Code only: comments may name the calls. + let text: String = std::fs::read_to_string(file) + .unwrap() + .lines() + .filter(|line| !line.trim_start().starts_with("//")) + .collect::>() + .join("\n"); + for (call, places) in allowed { + if text.contains(call) && !places.contains(&name.as_str()) { + strays.push(format!("{name}: {call}")); + } + } + } + assert!( + strays.is_empty(), + "list outputs through capture::listing: {strays:?}" + ); + // The old unjudged entry points are gone. + for file in files.iter().filter(|file| !file.ends_with("listing.rs")) { + let text = std::fs::read_to_string(file).unwrap(); + assert!( + !text.contains(concat!("enumerate", "_outputs(")), + "{file:?}" + ); + } + } +} diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index c400e2c7..48b8bfe5 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -22,6 +22,7 @@ use crate::vdd_device::DeviceState; #[cfg(windows)] pub mod dxgi; +pub mod listing; pub mod recovery; pub mod second_screen; pub mod synthetic; @@ -282,19 +283,6 @@ pub struct OutputInfo { pub adapter_id: crate::gpu_preference::AdapterId, } -/// Enumerate every desktop-attached display output. Windows-only in substance; -/// other platforms have no selectable outputs (the synthetic source ignores them). -pub fn enumerate_outputs() -> Vec { - #[cfg(windows)] - { - dxgi::enumerate_outputs() - } - #[cfg(not(windows))] - { - Vec::new() - } -} - /// Resolve which output (index into `outputs`) the capture loop should duplicate for /// `target`. `PrimaryAuto` picks the (0,0) output, else the first attached. `Output(name)` /// matches the `DeviceName`, falling back to the primary (with a warning) when not found — @@ -357,27 +345,6 @@ pub(crate) fn target_moved( }) } -/// Capture's listing after a lost duplication, judged: whether the target -/// moved off `current` (see [`target_moved`]). A listing without `current` -/// as it was (absent, or its name under another adapter) is also a detach -/// for a second screen's proof: kept, a display back as the same output -/// before the next reconciliation would inherit it. -#[cfg(any(windows, test))] -pub(crate) fn judge_listing( - shared: &SharedControl, - outputs: &[OutputInfo], - target: &crate::control::CaptureTarget, - current: &OutputInfo, -) -> bool { - let still_attached = outputs.iter().any(|output| { - output.device_name == current.device_name && output.adapter_id == current.adapter_id - }); - if !still_attached { - second_screen::note_detached(shared); - } - target_moved(outputs, target, current) -} - /// How long to wait for the virtual display to attach after enabling its driver. Overridable /// via `ETERNAL_VDD_TIMEOUT_SECS`, or within 30–60 s by the update manifest. The iPad mirrors /// meanwhile, so a longer wait costs the user nothing; it was 10 s while the iPad gave up 10 s @@ -577,6 +544,7 @@ fn prepare_vdd_settings(shared: &SharedControl, outputs: &[OutputInfo]) -> VddRe /// it ends up in, once per attach. Only the virtual display changes. #[cfg(windows)] fn settle_vdd_mode( + shared: &SharedControl, chosen: OutputInfo, requested: Option, mut heartbeat: impl FnMut(), @@ -613,7 +581,7 @@ fn settle_vdd_mode( // Windows applies the change asynchronously to DXGI's view of the output. for _ in 0..30 { heartbeat(); - if let Some(output) = enumerate_outputs() + if let Some(output) = listing::outputs(shared) .into_iter() .find(|output| output.device_name == chosen.device_name) { @@ -797,20 +765,6 @@ pub(crate) fn client_connected(shared: &SharedControl) -> bool { shared.client_connected() } -/// The displays to capture from: [`enumerate_outputs`], or in end-to-end -/// tests the fake virtual display's. -#[cfg(any(windows, test, debug_assertions))] -fn outputs(shared: &SharedControl) -> Vec { - #[cfg(debug_assertions)] - if let Some(outputs) = second_screen::fake::outputs(shared) { - return outputs; - } - // Only the debug-build fake reads it. - #[cfg(not(debug_assertions))] - let _ = shared; - enumerate_outputs() -} - /// Reconcile the virtual display device to the requested target and return the concrete /// target to capture. `VirtualExtended` captures the virtual display once it is attached and /// a viewer is connected (or one left less than [`second_screen::RELEASE_GRACE`] ago), and @@ -853,28 +807,24 @@ pub(crate) fn reconcile_virtual_display( // managed monitor, and the stream moves onto it this way once it // attaches. Reuse that output rather than turning it on again. let attached = owned.as_ref().and_then(|name| { - outputs(shared) + listing::outputs(shared) .into_iter() .find(|output| !output.is_primary && &output.device_name == name) }); - match attached { - Some(output) => { - if status == VddStatus::Active - && (connected || shared.second_screen.lock().keep_without_viewer()) - { - let proof_output = (output.adapter_id, output.device_name.clone()); - second_screen::start_proving_on( - shared, - generation, - Instant::now(), - Some(proof_output), - ); - return Some(CaptureTarget::Output(output.device_name)); - } + // Gone from the desktop, the listing cleared its proof. + if let Some(output) = attached { + if status == VddStatus::Active + && (connected || shared.second_screen.lock().keep_without_viewer()) + { + let proof_output = (output.adapter_id, output.device_name.clone()); + second_screen::start_proving_on( + shared, + generation, + Instant::now(), + Some(proof_output), + ); + return Some(CaptureTarget::Output(output.device_name)); } - // Gone from the desktop: if it comes back, it proves itself. - None if owned.is_some() => second_screen::note_detached(shared), - None => {} } // The transport's next tick starts the display beside this mirror. // A viewer that arrived meanwhile may already have claimed it. @@ -1037,7 +987,11 @@ pub fn run_capture_stage( } result }, - |delay| wait_while_blocked(&shared, generation, delay, display_attached), + |delay| { + wait_while_blocked(&shared, generation, delay, || { + listing::any_attached(&shared) + }) + }, ); if let Err(ref e) = result { error!(error = %e, "Capture loop exited with error"); @@ -1068,17 +1022,6 @@ fn note_stage_error( /// While no display is on, how often the wait looks for one to come back. const DISPLAY_LOOK: Duration = Duration::from_millis(250); -/// Whether Windows lists a display attached to the desktop again. -#[cfg(windows)] -fn display_attached() -> bool { - dxgi::display_attached() -} - -#[cfg(not(windows))] -fn display_attached() -> bool { - false -} - /// Sleeps between blocked-capture retries. The watchdog has nothing to fix /// while Windows withholds the desktop, so the stage keeps its heartbeats. /// While no display is on, `display_back` is asked every [`DISPLAY_LOOK`] diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index aa315806..a4d248f1 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -818,6 +818,42 @@ pub fn note_detached(shared: &SharedControl) { shared.second_screen.lock().proven = None; } +/// A listing of the outputs (see [`super::listing`]): a second screen whose +/// output it does not show as it was (absent, or its name under another +/// adapter) proves itself again when it is back, as after +/// [`note_detached`]. +pub(crate) fn observe_listing(shared: &SharedControl, outputs: &[OutputInfo]) { + let shows = |(adapter, name): &ProofOutput| { + outputs + .iter() + .any(|output| output.adapter_id == *adapter && &output.device_name == name) + }; + let mut state = shared.second_screen.lock(); + if state + .proven + .as_ref() + .is_some_and(|(_, output)| !shows(output)) + { + state.proven = None; + } + if state + .proving_output + .as_ref() + .is_some_and(|output| !shows(output)) + { + state.proving_output = None; + } +} + +/// The driver's outputs, judged as every listing is (see +/// [`observe_listing`]). +#[cfg(any(windows, test, debug_assertions))] +fn listed(shared: &SharedControl, driver: &impl Driver) -> Vec { + let outputs = driver.outputs(); + observe_listing(shared, &outputs); + outputs +} + /// [`note_attached`], unless the mode changed or quitting began. The mode /// is held while the display is recorded, so a mode change either comes /// first and is seen here, or comes after, and its capture generation finds @@ -1517,7 +1553,7 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura } // Still on from before (its release saw this start coming): reuse it, // with no change to wait for. - let outputs = driver.outputs(); + let outputs = listed(shared, driver); let owned = shared.vdd_output.lock().clone(); if let Some(name) = owned.filter(|name| { outputs @@ -1543,7 +1579,7 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura } driver.sleep(WAIT_STEP); } - let outputs = driver.outputs(); + let outputs = listed(shared, driver); driver.prepare(&outputs); let before: Vec = outputs .into_iter() @@ -1585,7 +1621,7 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura wait_for_virtual_output( &before, timeout, - || selected().then(|| driver.outputs()), + || selected().then(|| listed(shared, driver)), || driver.now(), |duration| driver.sleep(duration), ) @@ -1769,7 +1805,7 @@ struct WindowsDriver<'a> { #[cfg(windows)] impl Driver for WindowsDriver<'_> { fn outputs(&self) -> Vec { - super::enumerate_outputs() + super::listing::outputs(self.shared) } fn prepare(&self, outputs: &[OutputInfo]) { @@ -1786,7 +1822,12 @@ impl Driver for WindowsDriver<'_> { fn settle(&self, chosen: OutputInfo) -> OutputInfo { let request = self.request.borrow(); - let chosen = super::settle_vdd_mode(chosen, request.as_ref().and_then(|r| r.mode), || {}); + let chosen = super::settle_vdd_mode( + self.shared, + chosen, + request.as_ref().and_then(|r| r.mode), + || {}, + ); let before = request.as_ref().map_or(&[][..], |r| &r.before[..]); let render = crate::vdd_snapshot::render_card_now(before); info!( @@ -2734,15 +2775,13 @@ mod tests { let (display, dir, generation) = proven_fake_second_screen(&shared, "listing"); let target = CaptureTarget::Output("virtual".into()); let streamed = Some(target.clone()); - let current = super::super::outputs(&shared) + let current = super::super::listing::outputs(&shared) .into_iter() .find(|output| output.device_name == "virtual") .unwrap(); // The output is still there as it was: no move, and the proof stays. - let listed = super::super::outputs(&shared); - assert!(!super::super::judge_listing( - &shared, &listed, &target, ¤t - )); + let listed = super::super::listing::outputs(&shared); + assert!(!super::super::target_moved(&listed, &target, ¤t)); assert_eq!( reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), streamed @@ -2751,10 +2790,8 @@ mod tests { // The listing finds it gone; it is back, the same, before the stage // reconciles again. assert!(Driver::set_enabled(&display, false)); - let listed = super::super::outputs(&shared); - assert!(super::super::judge_listing( - &shared, &listed, &target, ¤t - )); + let listed = super::super::listing::outputs(&shared); + assert!(super::super::target_moved(&listed, &target, ¤t)); assert!(Driver::set_enabled(&display, true)); assert_eq!( reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), @@ -2764,6 +2801,77 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// Review r5 of #205: reconciliation saw the proven display attached, the + /// reopen's first listing saw no output at all (no_output_listed), and + /// the same display was back before the next reconciliation. Neither + /// that path nor the stage's error handling cleared the proof. Every + /// listing goes through the judge now: the one capture opens from, the + /// no-display wait's, and the driver's. + #[test] + fn a_reopen_listing_with_no_outputs_makes_it_prove_itself_again() { + // What the reopen's DXGI listing hands the judge: nothing attached. + let shared = second_screen_selected(); + let (_display, dir, generation) = proven_fake_second_screen(&shared, "no-outputs"); + observe_listing(&shared, &[]); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(&shared), "back after a listing with no outputs"); + let _ = std::fs::remove_dir_all(&dir); + + // Each listing entry point, with the display gone while it lists. + type Look = fn(&SharedControl); + let looks: [(&str, Look); 2] = [ + ("outputs", |shared| { + super::super::listing::outputs(shared); + }), + ("any_attached", |shared| { + super::super::listing::any_attached(shared); + }), + ]; + for (name, look) in looks { + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, name); + assert!(Driver::set_enabled(&display, false)); + look(&shared); + assert!(Driver::set_enabled(&display, true)); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())), + "{name}" + ); + assert!(proving(&shared), "{name}: back before reconciling"); + let _ = std::fs::remove_dir_all(&dir); + } + + // The worker's listing through its driver. + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, "worker"); + assert!(Driver::set_enabled(&display, false)); + listed(&shared, &display); + assert!(Driver::set_enabled(&display, true)); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(proving(&shared), "worker: back before reconciling"); + let _ = std::fs::remove_dir_all(&dir); + + // A listing that still shows it keeps the proof, whichever lists. + let shared = second_screen_selected(); + let (display, dir, generation) = proven_fake_second_screen(&shared, "kept"); + super::super::listing::outputs(&shared); + super::super::listing::any_attached(&shared); + listed(&shared, &display); + assert_eq!( + reconcile_virtual_display(&CaptureTarget::VirtualExtended, &shared, generation), + Some(CaptureTarget::Output("virtual".into())) + ); + assert!(!proving(&shared), "still attached throughout"); + let _ = std::fs::remove_dir_all(&dir); + } + /// Review r3 of #205: the same output reopened in the generation that /// proved it keeps its proof, so a display gone dark reaches the /// watchdog (see the test above for the restart itself). diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 37f4db2e..fa42d8c2 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -44,7 +44,7 @@ use tracing::{info, warn}; use crate::autostart::{first_lan_ipv4, read_startup_registry, set_startup_registry}; use crate::capture::recovery::CaptureBlock; -use crate::capture::{enumerate_outputs, OutputInfo}; +use crate::capture::{listing, OutputInfo}; use crate::control::{CaptureTarget, GuiControl}; use crate::gpu_preference::{Change, Preference, SwitchRecord}; use crate::handoff::Handoff; @@ -444,7 +444,7 @@ impl AnalyzerApp { String::new() } }; - let available_outputs = enumerate_outputs(); + let available_outputs = listing::outputs(&control.shared); // Same model as encoder_override: push the persisted preference into // SharedControl so the encoder honours it without a restart. @@ -710,7 +710,7 @@ impl AnalyzerApp { } fn refresh_outputs(&mut self) { - self.available_outputs = enumerate_outputs(); + self.available_outputs = listing::outputs(&self.control.shared); info!( count = self.available_outputs.len(), "Re-enumerated display outputs" diff --git a/host/src/main.rs b/host/src/main.rs index 2d0c8d21..c710658f 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -133,7 +133,7 @@ fn main() -> Result<(), Box> { // authoritative per-run source (including a persisted/fallback selection) is logged // inside the capture loop when the stream starts. let capture_summary = { - let outputs = capture::enumerate_outputs(); + let outputs = capture::listing::outputs_before_the_host_starts(); outputs .iter() .find(|o| o.is_primary) diff --git a/host/src/vdd_device.rs b/host/src/vdd_device.rs index 75c207da..cee23672 100644 --- a/host/src/vdd_device.rs +++ b/host/src/vdd_device.rs @@ -420,7 +420,7 @@ mod tests { let state = query(); println!("vdd_device={state} problem={:?}", state.problem()); // DXGI lists a virtual display under the adapter that renders it. - for output in crate::capture::enumerate_outputs() { + for output in crate::capture::listing::outputs_before_the_host_starts() { println!( "output {} {}x{} at {},{} on {}", output.device_name, From f6a174f60d26a9217ebf21165349aee9bccdc840 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 02:01:49 -0500 Subject: [PATCH 143/226] Find an adapter again after a driver update gives it a new LUID A driver update or reset, the case this window recovery is for, can give an adapter a new LUID. The adapter that drew was then no longer found, so the choice fell back to any adapter, and adapters left out came back in. Now an adapter that isn't listed as it was is found as the only listed one that differs in the LUID alone; two such are no match. The Vulkan LUID is read only when the device, not just the instance, is Vulkan 1.1+. --- host/tests/window_device_loss.rs | 41 ++++++++++++++++++++++++++ vendor/README.md | 6 +++- vendor/egui-wgpu/src/lib.rs | 50 ++++++++++++++++++++++++++++---- 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/host/tests/window_device_loss.rs b/host/tests/window_device_loss.rs index 8090de2f..ad0db156 100644 --- a/host/tests/window_device_loss.rs +++ b/host/tests/window_device_loss.rs @@ -364,3 +364,44 @@ fn every_dx12_adapter_has_its_own_luid() { unique.dedup(); assert_eq!(unique.len(), luids.len(), "{luids:x?}"); } + +#[test] +fn an_adapter_with_a_new_luid_after_a_driver_update_is_still_itself() { + use wgpu::Backend::Vulkan; + // Two of the same card; Vulkan reports each one's own PCI bus. + let card = |luid, bus: &str| AdapterKey { + luid: Some(luid), + pci_bus: bus.into(), + ..adapter("RTX 5080", Vulkan, 0x10de) + }; + let drew = card(1, "0000:01:00.0"); + let twin = card(2, "0000:02:00.0"); + // A driver update gives the card that drew a new LUID; its twin is listed first. + let listed = [twin.clone(), card(3, "0000:01:00.0")]; + assert_eq!(replacement_candidates(&listed, Some(&drew), &[]), [1]); + // An adapter left out stays left out with its new LUID. + assert_eq!( + replacement_candidates(&listed, None, std::slice::from_ref(&drew)), + [0] + ); + assert_eq!( + replacement_candidates(&listed, Some(&twin), std::slice::from_ref(&drew)), + [0] + ); +} + +#[test] +fn two_cards_alike_but_for_their_new_luids_are_no_preference() { + use wgpu::Backend::Dx12; + // On DX12 identical cards share the bus too: only the LUID tells them apart, + // and after an update both have new ones. + let card = |luid| AdapterKey { + luid: Some(luid), + pci_bus: "0000:01:00.0".into(), + ..adapter("RTX 5080", Dx12, 0x10de) + }; + let listed = [card(3), card(4)]; + assert_eq!(replacement_candidates(&listed, Some(&card(1)), &[]), [0, 1]); + // Nor is either left out for an old LUID. + assert_eq!(replacement_candidates(&listed, None, &[card(2)]), [0, 1]); +} diff --git a/vendor/README.md b/vendor/README.md index de1d5a6c..6653a502 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -55,7 +55,11 @@ The patch: LUID (from DXGI on DX12, and `VkPhysicalDeviceIDProperties` on Vulkan 1.1+ on Windows, through the `ash` dependency at wgpu-hal's version), else the backend, PCI ids, type, name and PCI bus. wgpu's DX12 bus comes from the - PCI ids alone, so identical cards differ only by LUID. + PCI ids alone, so identical cards differ only by LUID. A driver update or reset can + give an adapter a new LUID, so an adapter is also found as the only listed + one that differs in the LUID alone; two such are no match (no preference, + nothing left out). The Vulkan LUID is read only when both the instance and + the device are Vulkan 1.1+. The thread is named `winit::REPLACEMENT_THREAD`, so the host's panic hook treats a panic there as one failed try: it doesn't turn the virtual display off as it does for other background threads. diff --git a/vendor/egui-wgpu/src/lib.rs b/vendor/egui-wgpu/src/lib.rs index 0ea047cd..d172d709 100644 --- a/vendor/egui-wgpu/src/lib.rs +++ b/vendor/egui-wgpu/src/lib.rs @@ -343,7 +343,10 @@ fn luid(adapter: &wgpu::Adapter) -> Option { let vulkan = unsafe { adapter.as_hal::() }; if let Some(hal) = vulkan { let shared = hal.shared_instance(); - if shared.instance_api_version() < ash::vk::API_VERSION_1_1 { + let device_version = hal.physical_device_capabilities().properties().api_version; + if shared.instance_api_version() < ash::vk::API_VERSION_1_1 + || device_version < ash::vk::API_VERSION_1_1 + { return None; } let mut id = ash::vk::PhysicalDeviceIDProperties::default(); @@ -367,22 +370,59 @@ fn luid(_adapter: &wgpu::Adapter) -> Option { None } +impl AdapterKey { + /// EternalMonitor patch: the same adapter as `other` but for the LUID, which a driver + /// update or reset can change. + fn same_but_luid(&self, other: &Self) -> bool { + ( + self.backend, + self.vendor, + self.device, + self.device_type, + &self.name, + &self.pci_bus, + ) == ( + other.backend, + other.vendor, + other.device, + other.device_type, + &other.name, + &other.pci_bus, + ) + } +} + +/// EternalMonitor patch: where `wanted` is in `keys`: itself, else the only one that +/// differs from it in the LUID alone. Two such (identical cards the key can't otherwise +/// tell apart) are no match. +fn find(keys: &[AdapterKey], wanted: &AdapterKey) -> Option { + keys.iter().position(|key| key == wanted).or_else(|| { + let mut alike = (0..keys.len()).filter(|&index| keys[index].same_but_luid(wanted)); + match (alike.next(), alike.next()) { + (Some(index), None) => Some(index), + _ => None, + } + }) +} + /// EternalMonitor patch: the adapters, by index into `keys`, that a replacement may choose /// from: only `preferred` (the last one that drew on the window) while it is listed and /// not `excluded`, else every one not `excluded` (ones that couldn't draw on the window). +/// An adapter is found as [`find`] says, so one a driver update gave a new LUID is still +/// itself. pub fn replacement_candidates( keys: &[AdapterKey], preferred: Option<&AdapterKey>, excluded: &[AdapterKey], ) -> Vec { - let allowed = |key: &AdapterKey| !excluded.contains(key); - if let Some(preferred) = preferred.filter(|preferred| allowed(preferred)) - && let Some(index) = keys.iter().position(|key| key == preferred) + let left_out: Vec = excluded.iter().filter_map(|key| find(keys, key)).collect(); + if let Some(index) = preferred.and_then(|preferred| find(keys, preferred)) + && !left_out.contains(&index) { return vec![index]; } (0..keys.len()) - .filter(|&index| allowed(&keys[index])) + .filter(|index| !left_out.contains(index)) .collect() } From 2157f653763dae22a553e1ed390a7f8b8dcf90fb Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 02:40:02 -0500 Subject: [PATCH 144/226] Leave no public listing without the judge, and scan for aliases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review r6 of #205. The startup banner's listing was a public function that made its own token, and the backstop scan matched calls, so a `use … as` alias of it listed outputs unseen. Visibility can't be narrowed to the banner: it is in the binary crate, and `pub(in …)` only names ancestor modules. So the unjudged listing is gone: the banner gets only its display label from `startup_display_label`, and the native test that prints the outputs lists them through the judge with its own host state. The debug-build fake display's outputs now take the same token as the DXGI enumerator, so the compiler keeps both to `capture::listing`. The scan matches identifiers outside comments and string literals, so aliases are seen, and has fixtures for an alias, a grouped import, a removed name, a comment, a string and a longer name. `observe_listing` names `OutputInfo` by path: the module imports it only for Windows, test and debug builds, and a macOS release build failed. --- host/src/capture/listing.rs | 178 +++++++++++++++++-------- host/src/capture/second_screen.rs | 2 +- host/src/capture/second_screen/fake.rs | 5 +- host/src/main.rs | 10 +- host/src/vdd_device.rs | 2 +- 5 files changed, 131 insertions(+), 66 deletions(-) diff --git a/host/src/capture/listing.rs b/host/src/capture/listing.rs index ab593c60..d72d6096 100644 --- a/host/src/capture/listing.rs +++ b/host/src/capture/listing.rs @@ -8,9 +8,9 @@ use super::OutputInfo; use crate::control::SharedControl; -/// Made only here: the DXGI enumerator takes one, so every listing it makes -/// passes through this module. -#[cfg(windows)] +/// Made only here: the DXGI enumerator and the debug-build fake display take +/// one, so every listing either makes passes through this module. +#[cfg(any(windows, debug_assertions))] pub(super) struct Judged(()); /// The outputs attached to the desktop, judged for `shared`'s second screen. @@ -39,16 +39,27 @@ pub(super) fn listed( (listed, adapters) } -/// For the startup banner, before any host state exists: there is no proof -/// to judge yet. `every_listing_goes_through_the_judge` holds its callers to -/// that. -pub fn outputs_before_the_host_starts() -> Vec { - from_dxgi() +/// The display the startup banner names: the primary, or the first one. +/// Before any host state exists there is no proof to judge; only this label +/// leaves the module, never the outputs. +pub fn startup_display_label() -> String { + let outputs = from_dxgi(); + outputs + .iter() + .find(|output| output.is_primary) + .or_else(|| outputs.first()) + .map(|output| { + format!( + "{} ({}x{})", + output.device_name, output.width, output.height + ) + }) + .unwrap_or_else(|| "unknown".to_string()) } fn unjudged(shared: &SharedControl) -> Vec { #[cfg(debug_assertions)] - if let Some(outputs) = super::second_screen::fake::outputs(shared) { + if let Some(outputs) = super::second_screen::fake::outputs(shared, Judged(())) { return outputs; } // Only the debug-build fake reads it. @@ -72,30 +83,77 @@ fn from_dxgi() -> Vec { #[cfg(test)] mod tests { + /// Each name that sees outputs without the judge, and the files that may + /// name it. The compiler holds the token-taking ones (`list_outputs`, + /// `dxgi::outputs`, `fake::outputs`) to this module; this is the backstop + /// for what it cannot hold: the Windows API itself, and names that must + /// not come back. + const UNJUDGED: [(&str, &[&str]); 6] = [ + // The DXGI enumerator. + ("list_outputs", &["capture/dxgi.rs"]), + // DXGI's own enumeration: capture's, and a per-adapter output count in + // the virtual display's diagnostics snapshot, which names no output. + ("EnumOutputs", &["capture/dxgi.rs", "vdd_snapshot.rs"]), + // The banner's label, which carries no outputs. + ("startup_display_label", &["main.rs"]), + // Removed unjudged entry points. + ("outputs_before_the_host_starts", &[]), + ("enumerate_outputs", &[]), + ("display_attached", &[]), + ]; + + /// Whether `text` uses the identifier `name` (aliases included: a `use … + /// as` names it too). + fn uses(text: &str, name: &str) -> bool { + let word = |c: char| c.is_alphanumeric() || c == '_'; + text.match_indices(name).any(|(at, _)| { + !text[..at].ends_with(word) && !text[at + name.len()..].starts_with(word) + }) + } + + /// `line` without its string literals. + fn code_only(line: &str) -> String { + let mut code = String::new(); + let mut in_string = false; + let mut escaped = false; + for c in line.chars() { + if in_string { + match c { + _ if escaped => escaped = false, + '\\' => escaped = true, + '"' => in_string = false, + _ => {} + } + } else if c == '"' { + in_string = true; + } else { + code.push(c); + } + } + code + } + + /// The unjudged names `file` uses where it may not. Comments and string + /// literals may name them. + fn strays(file: &str, text: &str) -> Vec { + let code: String = text + .lines() + .filter(|line| !line.trim_start().starts_with("//")) + .map(code_only) + .collect::>() + .join("\n"); + UNJUDGED + .iter() + .filter(|(name, places)| uses(&code, name) && !places.contains(&file)) + .map(|(name, _)| format!("{file}: {name}")) + .collect() + } + /// Review r5 of #205: listings that bypassed the judge let a returning - /// display inherit an old proof, one path at a time. The DXGI enumerator - /// needs this module's token; this test holds the remaining ways to see - /// outputs to their known places, so a new one has to be routed here. + /// display inherit an old proof, one path at a time. #[test] fn every_listing_goes_through_the_judge() { let src = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); - // Each call, and the files allowed to make it. - let allowed: [(&str, &[&str]); 5] = [ - // The DXGI enumerator, which takes a `Judged`. - ("list_outputs(", &["capture/dxgi.rs"]), - ("dxgi::outputs(", &[]), - // The debug-build fake's outputs. - ("fake::outputs(", &["capture/second_screen/fake.rs"]), - // Before the host exists, and a native test that prints them. - ( - "outputs_before_the_host_starts(", - &["main.rs", "vdd_device.rs"], - ), - // DXGI's own enumeration: capture's, and a per-adapter output - // count in the virtual display's diagnostics snapshot, which - // names no output. - ("EnumOutputs(", &["capture/dxgi.rs", "vdd_snapshot.rs"]), - ]; let mut files = Vec::new(); let mut dirs = vec![src.clone()]; while let Some(dir) = dirs.pop() { @@ -109,7 +167,7 @@ mod tests { } } assert!(files.len() > 50, "{} source files", files.len()); - let mut strays = Vec::new(); + let mut found = Vec::new(); for file in &files { let name = file .strip_prefix(&src) @@ -117,33 +175,45 @@ mod tests { .to_string_lossy() .replace('\\', "/"); // This module is the judge, and this test names the calls. - if name == "capture/listing.rs" { - continue; - } - // Code only: comments may name the calls. - let text: String = std::fs::read_to_string(file) - .unwrap() - .lines() - .filter(|line| !line.trim_start().starts_with("//")) - .collect::>() - .join("\n"); - for (call, places) in allowed { - if text.contains(call) && !places.contains(&name.as_str()) { - strays.push(format!("{name}: {call}")); - } + if name != "capture/listing.rs" { + found.extend(strays(&name, &std::fs::read_to_string(file).unwrap())); } } assert!( - strays.is_empty(), - "list outputs through capture::listing: {strays:?}" + found.is_empty(), + "list outputs through capture::listing: {found:?}" ); - // The old unjudged entry points are gone. - for file in files.iter().filter(|file| !file.ends_with("listing.rs")) { - let text = std::fs::read_to_string(file).unwrap(); - assert!( - !text.contains(concat!("enumerate", "_outputs(")), - "{file:?}" - ); - } + } + + /// Review r6 of #205: the scan matched calls, so an alias made through + /// `use … as` went unseen. + #[test] + fn the_listing_scan_catches_aliases_and_ignores_comments() { + let alias = "use crate::capture::listing::startup_display_label as raw;\nfn f() { raw(); }"; + assert_eq!( + strays("gui/mod.rs", alias), + ["gui/mod.rs: startup_display_label"] + ); + assert!(strays("main.rs", alias).is_empty(), "the banner may"); + let grouped = "use windows::Win32::Graphics::Dxgi::{IDXGIAdapter1, EnumOutputs as list};"; + assert_eq!(strays("gpu.rs", grouped), ["gpu.rs: EnumOutputs"]); + let back = "pub fn outputs_before_the_host_starts() {}"; + assert_eq!( + strays("capture/mod.rs", back), + ["capture/mod.rs: outputs_before_the_host_starts"] + ); + // A comment, a string, or a longer name that contains one, is not a + // use; code after a string still counts. + assert!(strays("gui/mod.rs", "// startup_display_label as raw").is_empty()); + assert!(strays( + "capture/recovery.rs", + r#"detail: "EnumOutputs: 0x887a0002".into(),"# + ) + .is_empty()); + assert_eq!( + strays("gpu.rs", r#"let a = "x\"y"; adapter.EnumOutputs(0);"#), + ["gpu.rs: EnumOutputs"] + ); + assert!(strays("gui/mod.rs", "fn startup_display_label_text() {}").is_empty()); } } diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index a4d248f1..e406fbed 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -822,7 +822,7 @@ pub fn note_detached(shared: &SharedControl) { /// output it does not show as it was (absent, or its name under another /// adapter) proves itself again when it is back, as after /// [`note_detached`]. -pub(crate) fn observe_listing(shared: &SharedControl, outputs: &[OutputInfo]) { +pub(crate) fn observe_listing(shared: &SharedControl, outputs: &[super::OutputInfo]) { let shows = |(adapter, name): &ProofOutput| { outputs .iter() diff --git a/host/src/capture/second_screen/fake.rs b/host/src/capture/second_screen/fake.rs index f407d4b1..c640cb66 100644 --- a/host/src/capture/second_screen/fake.rs +++ b/host/src/capture/second_screen/fake.rs @@ -288,7 +288,10 @@ pub(crate) fn mirror(shared: &SharedControl) -> Result<(), CaptureUnavailable> { } /// The installed fake display's outputs, beside the primary one. -pub(crate) fn outputs(shared: &SharedControl) -> Option> { +pub(in crate::capture) fn outputs( + shared: &SharedControl, + _: crate::capture::listing::Judged, +) -> Option> { installed(shared).map(|display| display.outputs()) } diff --git a/host/src/main.rs b/host/src/main.rs index c710658f..7942f442 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -132,15 +132,7 @@ fn main() -> Result<(), Box> { // Best-effort capture source for the banner — reflects the default (primary). The // authoritative per-run source (including a persisted/fallback selection) is logged // inside the capture loop when the stream starts. - let capture_summary = { - let outputs = capture::listing::outputs_before_the_host_starts(); - outputs - .iter() - .find(|o| o.is_primary) - .or_else(|| outputs.first()) - .map(|o| format!("{} ({}x{})", o.device_name, o.width, o.height)) - .unwrap_or_else(|| "unknown".to_string()) - }; + let capture_summary = capture::listing::startup_display_label(); // Load persisted settings BEFORE the first pipeline spawns, so generation // 0 already streams with the user's bitrate/fps/encoder/display/codec. diff --git a/host/src/vdd_device.rs b/host/src/vdd_device.rs index cee23672..62b3352d 100644 --- a/host/src/vdd_device.rs +++ b/host/src/vdd_device.rs @@ -420,7 +420,7 @@ mod tests { let state = query(); println!("vdd_device={state} problem={:?}", state.problem()); // DXGI lists a virtual display under the adapter that renders it. - for output in crate::capture::listing::outputs_before_the_host_starts() { + for output in crate::capture::listing::outputs(&crate::control::SharedControl::new(0, 1)) { println!( "output {} {}x{} at {},{} on {}", output.device_name, From 8ac64952319939c04e7b5c6fd91ce017f279cbb2 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 02:48:29 -0500 Subject: [PATCH 145/226] Count more of the second screen's driver as unknown Four more cases now count as Unknown, where the host can't be sure what the running driver read. - The driver reads its settings folder from VDDPATH under its registry key. Unless that value is unset or names C:\VirtualDisplayDriver (ignoring case and a trailing backslash), the file this host writes isn't the driver's, and the cursor is Unknown. - Registry reads count as "no override" only when they find no key or no value. Any other error means Unknown. - The driver's state before an Enable is read as running, off, or unreadable. A record is kept only for a driver seen off before the Enable and running after it. - Software now comes only from a file that a driver which reads started with. An older or unreadable installer version, which may not be the running driver's, is Unknown. - A character or entity reference inside a cursor setting is Unknown: XmlLite expands it into the text, while quick-xml reports it apart. --- host/src/capture/mod.rs | 24 +- host/src/capture/second_screen.rs | 14 +- host/src/vdd_device.rs | 47 ++-- host/src/vdd_settings.rs | 386 ++++++++++++++++++++++-------- 4 files changed, 334 insertions(+), 137 deletions(-) diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 16220286..9aa15508 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -527,7 +527,7 @@ struct VddRequest { /// The driver's start before the Enable, if it was running, and what /// an Enable that starts it gives Windows for the pointer, from the /// file it reads (see `vdd_settings::record_after_enable`). - instance: Option, + instance: crate::vdd_settings::DriverRun, settings: Option, cursor: crate::vdd_settings::DriverCursor, } @@ -618,19 +618,16 @@ fn prepare_vdd_settings(shared: &SharedControl, outputs: &[OutputInfo]) -> VddRe // The driver reads this file when it starts; a driver already running // keeps what it read then. let settings = std::fs::read_to_string(path).ok(); - let reads_cursor = if owned { - Some(true) - } else { - crate::vdd_settings::driver_version() - .and_then(|version| crate::vdd_settings::driver_reads_cursor(&version)) - }; + let reads_cursor = owned + || crate::vdd_settings::driver_version() + .is_some_and(|version| crate::vdd_settings::driver_reads_cursor(&version)); let overridden = crate::vdd_settings::registry_overrides_cursor(); let cursor = crate::vdd_settings::starting_cursor(settings.as_deref(), reads_cursor, overridden); - let instance = crate::vdd_device::instance(); + let instance = crate::vdd_device::run_state(); info!( - running = instance.is_some(), - ?reads_cursor, + ?instance, + reads_cursor, overridden, ?cursor, "The virtual display's pointer settings" @@ -656,14 +653,11 @@ pub(crate) fn cursor_record_path() -> Option { #[cfg(windows)] pub(crate) fn running_driver_cursor() -> crate::vdd_settings::DriverCursor { let record = cursor_record_path().and_then(|path| crate::vdd_settings::load_record(&path)); - let running = crate::vdd_device::instance(); + let running = crate::vdd_device::run_state(); let cursor = crate::vdd_settings::running_cursor(record.as_ref(), running); info!( recorded = record.is_some(), - same_start = record - .as_ref() - .zip(running) - .is_some_and(|(record, running)| record.instance == running), + ?running, settings_sha256 = record .as_ref() .map_or("", |record| record.settings_sha256.as_str()), diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 6d162205..e5d1f925 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -1718,16 +1718,17 @@ struct WindowsDriver<'a> { #[cfg(windows)] impl WindowsDriver<'_> { - /// After an Enable that succeeded: a driver that started in it read - /// the file `prepare` wrote, so keep what that file gives the pointer - /// (`vdd_settings::record_after_enable`). One already running keeps the - /// record of its own start. + /// After an Enable that succeeded: a driver seen off before it and + /// running after it started in it, with the file `prepare` wrote, so + /// keep what that file gives the pointer + /// (`vdd_settings::record_after_enable`). One already running, or one + /// that couldn't be read, keeps the record of its own start. fn note_start(&self) { let request = self.request.borrow(); let Some(request) = request.as_ref() else { return; }; - let after = crate::vdd_device::instance(); + let after = crate::vdd_device::run_state(); let record = crate::vdd_settings::record_after_enable( request.instance, after, @@ -1736,7 +1737,8 @@ impl WindowsDriver<'_> { ); info!( started = record.is_some(), - running = after.is_some(), + before = ?request.instance, + ?after, "The virtual display's driver after the Enable" ); let (Some(record), Some(path)) = (record, super::cursor_record_path()) else { diff --git a/host/src/vdd_device.rs b/host/src/vdd_device.rs index 8ab69460..1f403c15 100644 --- a/host/src/vdd_device.rs +++ b/host/src/vdd_device.rs @@ -162,11 +162,14 @@ pub fn query() -> DeviceState { DeviceState::best(states) } -/// The running driver's start (`vdd_settings::DriverInstance`): this boot -/// and its device's last arrival. `None` when no one device of the driver -/// runs, or Windows doesn't say. +/// The driver as Configuration Manager reads it now +/// (`vdd_settings::DriverRun`): its one running start (this boot and its +/// device's last arrival), off when every device of the driver reads as +/// not started (or there is none), and unreadable when a read fails or +/// more than one runs. #[cfg(windows)] -pub fn instance() -> Option { +pub fn run_state() -> crate::vdd_settings::DriverRun { + use crate::vdd_settings::{DriverInstance, DriverRun}; use windows::Win32::Devices::DeviceAndDriverInstallation::{ CM_Get_DevNode_PropertyW, CR_SUCCESS, }; @@ -174,13 +177,25 @@ pub fn instance() -> Option { DEVPKEY_Device_LastArrivalDate, DEVPROPTYPE, DEVPROP_TYPE_FILETIME, }; - let mut running = driver_devnodes()?.into_iter().filter(|&devinst| { - devnode_status(devinst) - .is_some_and(|(flags, _)| flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0) - }); - let devinst = match (running.next(), running.next()) { - (Some(devinst), None) => devinst, - _ => return None, + let Some(devnodes) = driver_devnodes() else { + return DriverRun::Unreadable; + }; + let mut running = Vec::new(); + for devinst in devnodes { + let Some((flags, _)) = devnode_status(devinst) else { + return DriverRun::Unreadable; + }; + if flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0 { + running.push(devinst); + } else if flags & DN_STARTED != 0 { + // Started with a problem: neither cleanly off nor running. + return DriverRun::Unreadable; + } + } + let devinst = match running[..] { + [] => return DriverRun::Off, + [devinst] => devinst, + _ => return DriverRun::Unreadable, }; let (mut arrival, mut kind, mut size) = (0u64, DEVPROPTYPE(0), 8u32); let result = unsafe { @@ -194,12 +209,12 @@ pub fn instance() -> Option { ) }; if result != CR_SUCCESS || kind != DEVPROP_TYPE_FILETIME || size != 8 || arrival == 0 { - return None; + return DriverRun::Unreadable; + } + match boot_id() { + Some(boot_id) => DriverRun::Running(DriverInstance { boot_id, arrival }), + None => DriverRun::Unreadable, } - Some(crate::vdd_settings::DriverInstance { - boot_id: boot_id()?, - arrival, - }) } /// Windows' count of boots, which the kernel raises at each one. diff --git a/host/src/vdd_settings.rs b/host/src/vdd_settings.rs index c4faeb7f..5c2e0f44 100644 --- a/host/src/vdd_settings.rs +++ b/host/src/vdd_settings.rs @@ -200,14 +200,22 @@ pub fn cursor_set_by(xml: &str) -> DriverCursor { let mut reader = quick_xml::Reader::from_str(xml); reader.config_mut().trim_text(false); let mut values: [Option; 5] = Default::default(); - let mut current = None; + // The setting whose first child is next, and whether a setting's + // element is open. + let (mut current, mut inside) = (None, false); loop { match reader.read_event() { Ok(Event::Start(tag)) => { current = NAMES .iter() .position(|name| tag.local_name().as_ref() == *name); + inside |= current.is_some(); } + Ok(Event::End(_)) => (current, inside) = (None, false), + // XmlLite expands `t` and `&` into the text; quick-xml + // reports them apart. Where that could change a setting, the + // answer is not known. + Ok(Event::GeneralRef(_)) if inside => return DriverCursor::Unknown, // XmlLite reads white space alone as another kind of node. Ok(Event::Text(text)) if text.iter().all(u8::is_ascii_whitespace) => current = None, Ok(Event::Text(text)) => { @@ -273,6 +281,15 @@ pub struct DriverInstance { pub arrival: u64, } +/// The driver as Configuration Manager reads it: one start running, none +/// running, or a read that failed or was ambiguous. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DriverRun { + Running(DriverInstance), + Off, + Unreadable, +} + /// What a driver start read, kept on disk so a relaunched host knows it. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CursorRecord { @@ -284,34 +301,32 @@ pub struct CursorRecord { /// What the driver starts with when this host's Enable starts it: the /// file's cursor (`cursor_set_by`), if the driver reads `` -/// (`reads_cursor`, `None` when its version can't tell) and no registry -/// value overrides the file. -pub fn starting_cursor( - xml: Option<&str>, - reads_cursor: Option, - overridden: bool, -) -> DriverCursor { - match (xml, reads_cursor) { - _ if overridden => DriverCursor::Unknown, - (_, Some(false)) => DriverCursor::Software, - (Some(xml), Some(true)) => cursor_set_by(xml), +/// (`reads_cursor`) and the registry leaves the file in charge +/// (`overridden` when it doesn't, see `file_in_charge`). The version that +/// says whether it reads `` is the installer's, which may not be +/// the running driver's, so an older one is not taken to mean no cursor. +pub fn starting_cursor(xml: Option<&str>, reads_cursor: bool, overridden: bool) -> DriverCursor { + match xml { + Some(xml) if reads_cursor && !overridden => cursor_set_by(xml), _ => DriverCursor::Unknown, } } /// The record to keep after an Enable that succeeded: a driver running -/// after it (`after`) that was not running before it (`before`) started -/// in this Enable, with the file this host just wrote (`xml`, read as +/// after it (`after`) that was seen off before it (`before`) started in +/// this Enable, with the file this host just wrote (`xml`, read as /// `starting_cursor`). An Enable that found it running (AlreadyDone, or a -/// relaunch with the driver left on) changes nothing it read: no record, -/// and the one on disk decides (`running_cursor`). +/// relaunch with the driver left on), or couldn't tell, keeps no record: +/// the one on disk decides (`running_cursor`). pub fn record_after_enable( - before: Option, - after: Option, + before: DriverRun, + after: DriverRun, xml: Option<&str>, cursor: DriverCursor, ) -> Option { - let instance = after.filter(|after| before != Some(*after))?; + let (DriverRun::Off, DriverRun::Running(instance)) = (before, after) else { + return None; + }; Some(CursorRecord { instance, settings_sha256: xml.map_or_else(String::new, |xml| { @@ -325,12 +340,9 @@ pub fn record_after_enable( /// What the running driver does with the pointer: what the record says, /// when it is of this very start of the driver (same boot, same arrival); /// `Unknown` for any other, and when the driver is not running. -pub fn running_cursor( - record: Option<&CursorRecord>, - running: Option, -) -> DriverCursor { +pub fn running_cursor(record: Option<&CursorRecord>, running: DriverRun) -> DriverCursor { match (record, running) { - (Some(record), Some(running)) if record.instance == running => record.cursor, + (Some(record), DriverRun::Running(running)) if record.instance == running => record.cursor, _ => DriverCursor::Unknown, } } @@ -351,43 +363,100 @@ pub fn save_record(path: &Path, record: &CursorRecord) -> io::Result<()> { /// Whether the installed driver reads `` at all: releases from /// 24.10.27 on do. `version` is its installer's display version, such as -/// `25.05.03`; `None` when it doesn't read as one. -pub fn driver_reads_cursor(version: &str) -> Option { +/// `25.05.03`; not one that doesn't read as a version. +pub fn driver_reads_cursor(version: &str) -> bool { let parts: Option> = version .trim() .split('.') .map(|part| part.parse().ok()) .collect(); match parts.as_deref() { - Some([year, month, day, ..]) => Some((*year, *month, *day) >= (24, 10, 27)), - _ => None, + Some([year, month, day, ..]) => (*year, *month, *day) >= (24, 10, 27), + _ => false, } } -/// Whether a value under the driver's registry key can override the -/// file's cursor settings: the driver reads that key first. +/// A value under the driver's registry key, as read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RegValue { + /// The key or the value doesn't exist. + Absent, + /// It exists; the text, for a string read as one. + Present(String), + /// Any other error. + Failed, +} + +/// The settings folder the driver uses without `VDDPATH`, and the one this +/// host writes to. +const DRIVER_FOLDER: &str = r"C:\VirtualDisplayDriver"; + +/// The cursor values the driver reads from its registry key before the +/// file (`EnabledQuery`, `GetIntegerSetting`). +pub const REGISTRY_CURSOR_VALUES: [&str; 5] = [ + "HARDWARECURSOR", + "ALPHACURSORSUPPORT", + "CURSORMAXX", + "CURSORMAXY", + "XORCURSORSUPPORTLEVEL", +]; + +/// Whether the driver reads its cursor from the file this host writes: its +/// `VDDPATH` (read at `DriverEntry`) is unset or names +/// `C:\VirtualDisplayDriver`, ignoring case and a trailing `\`, as +/// `vdd_snapshot::collect` compares it, and no cursor value is set under +/// its key. A value that couldn't be read may be either. +pub fn file_in_charge(vddpath: &RegValue, cursor_values: &[RegValue]) -> bool { + let folder = match vddpath { + RegValue::Absent => true, + RegValue::Present(path) => path + .trim_end_matches('\\') + .eq_ignore_ascii_case(DRIVER_FOLDER), + RegValue::Failed => false, + }; + folder && cursor_values.iter().all(|value| *value == RegValue::Absent) +} + +/// Whether the driver's registry key may take the cursor away from the +/// file this host writes (`file_in_charge`, read now). #[cfg(windows)] pub fn registry_overrides_cursor() -> bool { - use windows::core::PCWSTR; - use windows::Win32::System::Registry::{RegGetValueW, HKEY_LOCAL_MACHINE, RRF_RT_ANY}; - let key: Vec = r"SOFTWARE\MikeTheTech\VirtualDisplayDriver" - .encode_utf16() - .chain(Some(0)) + let vddpath = registry_value("VDDPATH", true); + let cursor: Vec = REGISTRY_CURSOR_VALUES + .iter() + .map(|name| registry_value(name, false)) .collect(); - [ - "HARDWARECURSOR", - "ALPHACURSORSUPPORT", - "CURSORMAXX", - "CURSORMAXY", - "XORCURSORSUPPORTLEVEL", - ] - .iter() - .any(|name| { - let name: Vec = name.encode_utf16().chain(Some(0)).collect(); - unsafe { + !file_in_charge(&vddpath, &cursor) +} + +/// One value under HKLM\SOFTWARE\MikeTheTech\VirtualDisplayDriver, with its +/// text when `text` (a REG_SZ; any other type fails). +#[cfg(windows)] +fn registry_value(name: &str, text: bool) -> RegValue { + use windows::core::{w, PCWSTR}; + use windows::Win32::Foundation::{ERROR_FILE_NOT_FOUND, ERROR_SUCCESS}; + use windows::Win32::System::Registry::{ + RegGetValueW, HKEY_LOCAL_MACHINE, RRF_RT_ANY, RRF_RT_REG_SZ, + }; + let key = w!(r"SOFTWARE\MikeTheTech\VirtualDisplayDriver"); + let name: Vec = name.encode_utf16().chain(Some(0)).collect(); + let mut buffer = [0u16; 1024]; + let mut bytes = std::mem::size_of_val(&buffer) as u32; + let result = unsafe { + if text { RegGetValueW( HKEY_LOCAL_MACHINE, - PCWSTR(key.as_ptr()), + key, + PCWSTR(name.as_ptr()), + RRF_RT_REG_SZ, + None, + Some(buffer.as_mut_ptr().cast()), + Some(&mut bytes), + ) + } else { + RegGetValueW( + HKEY_LOCAL_MACHINE, + key, PCWSTR(name.as_ptr()), RRF_RT_ANY, None, @@ -395,8 +464,18 @@ pub fn registry_overrides_cursor() -> bool { None, ) } - .is_ok() - }) + }; + match result { + ERROR_SUCCESS => { + let end = buffer + .iter() + .position(|&unit| unit == 0) + .unwrap_or(buffer.len()); + RegValue::Present(String::from_utf16_lossy(&buffer[..end])) + } + ERROR_FILE_NOT_FOUND => RegValue::Absent, + _ => RegValue::Failed, + } } fn escape(text: &str) -> String { @@ -1248,32 +1327,113 @@ mod tests { fn this_hosts_file_gives_a_128_px_cursor_that_takes_every_pointer_type() { let xml = render(&modes_for_client(1668, 2420, 60), None); assert_eq!(cursor_set_by(&xml), hardware(128, 128)); + assert_eq!(starting_cursor(Some(&xml), true, false), hardware(128, 128)); assert_eq!( - starting_cursor(Some(&xml), Some(true), false), - hardware(128, 128) - ); - assert_eq!( - starting_cursor(Some(&xml), Some(false), false), - DriverCursor::Software, - "a driver that doesn't read " + starting_cursor(Some(&xml), true, true), + DriverCursor::Unknown, + "the driver's registry key overrides the file" ); assert_eq!( - starting_cursor(Some(&xml), None, false), + starting_cursor(None, true, false), DriverCursor::Unknown, - "a driver version that doesn't tell" + "no file read" ); + } + + /// Review r5: whether the driver reads `` comes from the + /// installer's version, which may not be the running driver's. An old + /// or unreadable version is Unknown, never Software: the pointer is + /// then handled as in 0.3.4, which for a truly old driver gives the + /// same picture. + #[test] + fn a_driver_version_that_doesnt_read_the_cursor_is_unknown_not_software() { + let xml = render(&modes_for_client(1668, 2420, 60), None); + for version in ["24.10.16", "23.12.2", "24.10", "beta", ""] { + let reads = driver_reads_cursor(version); + assert!(!reads, "{version:?}"); + assert_eq!( + starting_cursor(Some(&xml), reads, false), + DriverCursor::Unknown, + "{version:?}" + ); + assert_eq!( + starting_cursor(Some(VENDOR), reads, false), + DriverCursor::Unknown, + "{version:?}" + ); + } assert_eq!( - starting_cursor(Some(&xml), Some(true), true), - DriverCursor::Unknown, - "the driver's registry key overrides the file" + starting_cursor(Some(VENDOR), true, false), + DriverCursor::Software, + "only the file a driver that reads it started with says Software" + ); + } + + /// Review r5: XmlLite expands character and entity references into a + /// setting's text; quick-xml reports them apart, so a setting with one + /// is not known. + #[test] + fn a_reference_inside_a_cursor_setting_is_unknown() { + let caps = "6464true"; + for setting in [ + "true", + "true", + "true", + ] { + let xml = with_cursor(&format!("{setting}{caps}")); + assert_eq!(cursor_set_by(&xml), DriverCursor::Unknown, "{setting}"); + } + let size = with_cursor("true6464true"); + assert_eq!(cursor_set_by(&size), DriverCursor::Unknown); + let elsewhere = format!( + "A & Btrue{caps}" ); assert_eq!( - starting_cursor(None, Some(true), false), - DriverCursor::Unknown, - "no file read" + cursor_set_by(&elsewhere), + hardware(64, 64), + "a reference in another setting" ); } + /// Review r5: the driver reads its settings folder from `VDDPATH`, and + /// its cursor values first from its registry key. Only a read that + /// finds nothing leaves the file this host writes in charge; a failed + /// read could be either. + #[test] + fn the_registry_leaves_the_file_in_charge_only_when_it_says_nothing() { + use RegValue::{Absent, Failed, Present}; + let none = [Absent, Absent, Absent, Absent, Absent]; + assert!(file_in_charge(&Absent, &none)); + for path in [ + r"C:\VirtualDisplayDriver", + r"C:\VirtualDisplayDriver\", + r"c:\virtualdisplaydriver", + r"C:\VIRTUALDISPLAYDRIVER\\", + ] { + assert!(file_in_charge(&Present(path.into()), &none), "{path:?}"); + } + for path in [ + r"D:\VirtualDisplayDriver", + r"C:\VirtualDisplayDriver\sub", + "", + r"C:\VirtualDisplayDriver2", + ] { + assert!(!file_in_charge(&Present(path.into()), &none), "{path:?}"); + } + assert!(!file_in_charge(&Failed, &none), "VDDPATH unreadable"); + for at in 0..none.len() { + for value in [Present(String::new()), Failed] { + let mut cursor = none.clone(); + cursor[at] = value.clone(); + assert!( + !file_in_charge(&Absent, &cursor), + "{} {value:?}", + REGISTRY_CURSOR_VALUES[at] + ); + } + } + } + /// Review r4: what the running driver does with the pointer is tied to /// its start (boot and device arrival), kept on disk. #[test] @@ -1285,43 +1445,44 @@ mod tests { boot_id: 205, arrival, }; + let run = |arrival| DriverRun::Running(start(arrival)); let ours = render(&modes_for_client(1668, 2420, 60), None); - let cursor = starting_cursor(Some(&ours), Some(true), false); + let cursor = starting_cursor(Some(&ours), true, false); // Enable from off: the driver started with the file just written. - let record = record_after_enable(None, Some(start(10)), Some(&ours), cursor).unwrap(); + let record = record_after_enable(DriverRun::Off, run(10), Some(&ours), cursor).unwrap(); assert_eq!(record.cursor, hardware(128, 128)); assert_eq!(record.settings_sha256.len(), 64); save_record(&path, &record).unwrap(); assert_eq!( - running_cursor(load_record(&path).as_ref(), Some(start(10))), + running_cursor(load_record(&path).as_ref(), run(10)), hardware(128, 128) ); // AlreadyDone with a matching record: the driver kept what it read, // whatever the file says now. - let vendor = starting_cursor(Some(VENDOR), Some(true), false); + let vendor = starting_cursor(Some(VENDOR), true, false); assert_eq!( - record_after_enable(Some(start(10)), Some(start(10)), Some(VENDOR), vendor), + record_after_enable(run(10), run(10), Some(VENDOR), vendor), None ); assert_eq!( - running_cursor(load_record(&path).as_ref(), Some(start(10))), + running_cursor(load_record(&path).as_ref(), run(10)), hardware(128, 128) ); // AlreadyDone with a changed arrival: someone else restarted it. assert_eq!( - running_cursor(load_record(&path).as_ref(), Some(start(11))), + running_cursor(load_record(&path).as_ref(), run(11)), DriverCursor::Unknown ); // Relaunch: a new host reads the record from disk. Same start, same // answer; after a reboot the same arrival counts for nothing; with - // no record, or the driver off, nothing is known. + // no record, the driver off, or a failed read, nothing is known. let relaunched = load_record(&path); assert_eq!( - running_cursor(relaunched.as_ref(), Some(start(10))), + running_cursor(relaunched.as_ref(), run(10)), hardware(128, 128) ); let rebooted = DriverInstance { @@ -1329,29 +1490,26 @@ mod tests { arrival: 10, }; assert_eq!( - running_cursor(relaunched.as_ref(), Some(rebooted)), - DriverCursor::Unknown - ); - assert_eq!(running_cursor(None, Some(start(10))), DriverCursor::Unknown); - assert_eq!( - running_cursor(relaunched.as_ref(), None), + running_cursor(relaunched.as_ref(), DriverRun::Running(rebooted)), DriverCursor::Unknown ); + assert_eq!(running_cursor(None, run(10)), DriverCursor::Unknown); + for state in [DriverRun::Off, DriverRun::Unreadable] { + assert_eq!( + running_cursor(relaunched.as_ref(), state), + DriverCursor::Unknown + ); + } - // An Enable that restarted it, with the user's own caps of 64 and - // then 256: each start keeps its own. + // Enables from off with the user's own caps of 64 and then 256: + // each start keeps its own. for (arrival, max) in [(12, 64), (13, 256)] { let users = with_cursor(&format!("true{max}{max}true")); - let cursor = starting_cursor(Some(&users), Some(true), false); - let record = record_after_enable( - Some(start(arrival - 1)), - Some(start(arrival)), - Some(&users), - cursor, - ) - .unwrap(); + let cursor = starting_cursor(Some(&users), true, false); + let record = + record_after_enable(DriverRun::Off, run(arrival), Some(&users), cursor).unwrap(); save_record(&path, &record).unwrap(); - let running = running_cursor(load_record(&path).as_ref(), Some(start(arrival))); + let running = running_cursor(load_record(&path).as_ref(), run(arrival)); assert_eq!(running, hardware(max, max)); let DriverCursor::Hardware(caps) = running else { unreachable!() @@ -1361,27 +1519,55 @@ mod tests { } // A start with a file that leaves the cursor off is kept as such. - let record = record_after_enable(None, Some(start(14)), Some(VENDOR), vendor).unwrap(); + let record = record_after_enable(DriverRun::Off, run(14), Some(VENDOR), vendor).unwrap(); assert_eq!(record.cursor, DriverCursor::Software); // A driver that is not running after the Enable keeps no record. - assert_eq!(record_after_enable(None, None, Some(&ours), cursor), None); + for after in [DriverRun::Off, DriverRun::Unreadable] { + assert_eq!( + record_after_enable(DriverRun::Off, after, Some(&ours), cursor), + None + ); + } // A damaged record is no record. std::fs::write(&path, "{").unwrap(); assert_eq!(load_record(&path), None); std::fs::remove_dir_all(&dir).unwrap(); } + /// Review r5: only a driver seen off before the Enable started in it. + /// A failed read before it, or one that was running (even another + /// start than after it), may have read another file. + #[test] + fn a_record_is_kept_only_for_a_driver_seen_off_before_the_enable() { + let run = |arrival| { + DriverRun::Running(DriverInstance { + boot_id: 205, + arrival, + }) + }; + let ours = render(&modes_for_client(1668, 2420, 60), None); + let cursor = starting_cursor(Some(&ours), true, false); + assert!(record_after_enable(DriverRun::Off, run(20), Some(&ours), cursor).is_some()); + for before in [DriverRun::Unreadable, run(20), run(19)] { + assert_eq!( + record_after_enable(before, run(20), Some(&ours), cursor), + None, + "{before:?}" + ); + } + } + #[test] fn only_drivers_that_read_the_cursor_block_are_trusted_with_it() { for (version, reads) in [ - ("25.05.03", Some(true)), - ("24.10.27", Some(true)), - ("24.12.24", Some(true)), - ("24.10.16", Some(false)), - ("23.12.2", Some(false)), - ("24.10", None), - ("", None), - ("beta", None), + ("25.05.03", true), + ("24.10.27", true), + ("24.12.24", true), + ("24.10.16", false), + ("23.12.2", false), + ("24.10", false), + ("", false), + ("beta", false), ] { assert_eq!(driver_reads_cursor(version), reads, "{version:?}"); } From 5ff998263dcd3bb040dead716ffacda2904c5b5c Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 02:53:40 -0500 Subject: [PATCH 146/226] Prove a presented frame, leave a minimized window alone, and wait out a slow driver Third review of #206 found five more gaps; this closes them. - The watcher counted a minimized window as stuck: eframe runs no UI pass for one, so the renderer never saw it minimized, and a healthy host got the driver notice 4 s after Win+D. The watcher now checks the window live (IsIconic, or the move/size and menu loops Windows runs while the title bar is held), and counts the 4 s from the latest of its start, the last UI pass and the last time it saw the window held. - Confirming the window could still pass with nothing presented: after a failed surface recreation egui-wgpu's paints return early with no surface error. The renderer now asks for a screenshot once the window has drawn long enough, and confirms only when it arrives; egui-wgpu takes one only from a frame it acquired, drew and submitted. - A driver that always answers after 4 s never opened the window: each try was given up on and started again. The list and the probe are now one call each, waited for as long as they run; the notice goes up at their limit, and a late answer opens the window. - Tests whose steps answer use 60 s limits, and the watcher's panics reach the test. - The notice box closes on a thread of its own, so the window's thread never waits on it. --- host/src/gui/driver_wait.rs | 518 +++++++++++++++++++++--------------- host/src/gui/mod.rs | 4 +- host/src/gui/renderer.rs | 281 ++++++++++++++++--- 3 files changed, 545 insertions(+), 258 deletions(-) diff --git a/host/src/gui/driver_wait.rs b/host/src/gui/driver_wait.rs index 126de88f..d71fa772 100644 --- a/host/src/gui/driver_wait.rs +++ b/host/src/gui/driver_wait.rs @@ -10,6 +10,9 @@ //! - then the same wgpu calls the window's thread makes first are made once //! on a thread of their own ([`Call::GpuProbe`]). //! +//! Each is one call, waited for as long as it runs: past its limit the +//! notice goes up, and a driver that answers late still opens the window. +//! //! Past that, the window's thread can still stall in calls that cannot be //! moved off it (its own device and swap chain, a surface it recreates), so //! [`watch_first_paint`] watches from another thread until the window is @@ -160,48 +163,90 @@ impl PromptState { } } -/// Waits until none of `free` is stuck in `lanes` and `attempt` succeeds, -/// with the notice up from the first time either fails. False when the -/// user quit, also by a Cancel that came as the driver answered. -fn wait_until( - lanes: &Lanes, +/// A limit no startup call reaches: it is waited for as long as it runs. +const NEVER: Duration = Duration::from_secs(365 * 24 * 60 * 60); + +/// Starts a call from `make` in `call`'s lane once none of `free` is stuck, +/// and waits for it as long as it runs, with the notice up while a lane is +/// stuck or once the call has run for `limits`' limit. Its result, also a +/// late one; None when the user quit, also by a Cancel that came as the +/// driver answered. +fn await_call( + lanes: &'static Lanes, + call: Call, + limits: Limits, free: &[Call], - mut attempt: impl FnMut() -> bool, + mut make: impl FnMut() -> F, prompt: &Prompt, poll: Duration, -) -> bool { +) -> Option +where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, +{ let started = Instant::now(); - loop { - if free.iter().all(|&call| lanes.abandoned(call) == 0) && attempt() { - if prompt.phase(Instant::now()) == Phase::Quit { - break; - } - if prompt.raised() { - info!( - waited_ms = started.elapsed().as_millis() as u64, - "The graphics driver answered; opening the window" - ); + let unlimited = Limits { + warn: limits.warn, + limit: NEVER, + }; + let quit = |prompt: &Prompt| { + let quit = prompt.phase(Instant::now()) == Phase::Quit; + if quit { + info!( + waited_ms = started.elapsed().as_millis() as u64, + "Quit while the window waited for the graphics driver" + ); + } + quit + }; + let mut pending = loop { + if free.iter().all(|&stuck| lanes.abandoned(stuck) == 0) { + if let Ok(pending) = crate::stall::start_on(lanes, call, unlimited, make()) { + break pending; } - return true; } prompt.raise(Instant::now()); - if prompt.phase(Instant::now()) == Phase::Quit { - break; + if quit(prompt) { + return None; } std::thread::sleep(poll); + }; + let mut up_to = limits.limit; + loop { + match pending.wait_up_to(up_to, &mut || {}) { + Some(Ok(value)) => { + if quit(prompt) { + return None; + } + if prompt.raised() { + info!( + call = call.name(), + waited_ms = started.elapsed().as_millis() as u64, + "The graphics driver answered" + ); + } + return Some(value); + } + // Only past a limit no call reaches. + Some(Err(_)) => return None, + None => { + prompt.raise(Instant::now()); + // Dropped, a call still in the driver keeps its lane taken + // until it returns. + if quit(prompt) { + return None; + } + up_to += poll; + } + } } - info!( - waited_ms = started.elapsed().as_millis() as u64, - "Quit while the window waited for the graphics driver" - ); - false } /// The steps before the window's thread makes its first graphics call: -/// fresh lists from `list` run in [`Call::GpuDetect`] under `list_limits` -/// until one returns in time, which `configure` gets; then fresh probes from -/// `probe`, over what `configure` returned, run in [`Call::GpuProbe`] until -/// one returns in time. False when the user quit from the notice. +/// a list from `list`, in [`Call::GpuDetect`] under `list_limits`, which +/// `configure` gets; then a probe from `probe`, over what `configure` +/// returned, in [`Call::GpuProbe`] under `probe_limits`. Each is waited for +/// as long as it runs. False when the user quit from the notice. #[allow(clippy::too_many_arguments)] pub(super) fn before_window( lanes: &'static Lanes, @@ -209,7 +254,7 @@ pub(super) fn before_window( probe_limits: Limits, prompt: &Prompt, poll: Duration, - mut list: impl FnMut() -> L, + list: impl FnMut() -> L, configure: impl FnOnce(T) -> S, probe: impl Fn(&S) -> F, ) -> bool @@ -218,50 +263,50 @@ where L: FnOnce() -> T + Send + 'static, F: FnOnce() + Send + 'static, { - let mut listed = None; - let got_list = wait_until( + let Some(adapters) = await_call( lanes, + Call::GpuDetect, + list_limits, &[Call::GpuDetect], - || match crate::stall::run_on(lanes, Call::GpuDetect, list_limits, list(), || {}) { - Ok(adapters) => { - listed = Some(adapters); - true - } - Err(_) => false, - }, + list, prompt, poll, - ); - let Some(adapters) = listed.filter(|_| got_list) else { + ) else { return false; }; let spec = configure(adapters); - wait_until( + await_call( lanes, + Call::GpuProbe, + probe_limits, &[Call::GpuDetect, Call::GpuProbe], - || crate::stall::run_on(lanes, Call::GpuProbe, probe_limits, probe(&spec), || {}).is_ok(), + || probe(&spec), prompt, poll, ) + .is_some() } /// Watches the window from another thread until the wait ends /// ([`FirstPaintWatch::finisher`], called when the window is confirmed -/// drawing, or [`FirstPaintWatch::stop`]). The notice goes up once the -/// window's thread has begun no UI pass (`last_pass`, with whether the -/// window was minimized then) for `limit`, counted from now before the -/// first; a minimized window draws nothing and is left alone. Cancel runs -/// `quit`, which ends the process: the window's thread may never return. +/// drawing, or [`FirstPaintWatch::stop`]). The notice goes up once `limit` +/// has passed since the latest of: the watch's start, the window thread's +/// last UI pass (`last_pass`), and the last time the watcher saw the window +/// held (`held`: minimized, or its title bar held, when a working window +/// runs no UI pass either). Cancel runs `quit`, which ends the process: the +/// window's thread may never return. pub(super) fn watch_first_paint( prompt: Prompt, limit: Duration, poll: Duration, now: impl Fn() -> Instant + Send + 'static, - last_pass: impl Fn() -> Option<(Instant, bool)> + Send + 'static, + last_pass: impl Fn() -> Option + Send + 'static, + held: impl Fn() -> bool + Send + 'static, quit: impl FnOnce() + Send + 'static, ) -> FirstPaintWatch { let started = now(); let watched = prompt.clone(); + let mut last_held = None; let watch = move || loop { match watched.phase(now()) { Phase::Quit => { @@ -273,8 +318,15 @@ pub(super) fn watch_first_paint( Phase::Waiting => {} } let at = now(); - let (since, minimized) = last_pass().unwrap_or((started, false)); - if !minimized && at.saturating_duration_since(since.max(started)) >= limit { + if held() { + last_held = Some(at); + } + let quiet_since = [Some(started), last_pass(), last_held] + .into_iter() + .flatten() + .max() + .unwrap_or(started); + if at.saturating_duration_since(quiet_since) >= limit { watched.raise(at); } std::thread::sleep(poll); @@ -304,12 +356,10 @@ impl FirstPaintWatch { } /// The window ended: the wait is over, and the watcher is done. A - /// Cancel given by then still quits. - pub(super) fn stop(self) { + /// Cancel given by then still quits. Err if the watcher panicked. + pub(super) fn stop(self) -> std::thread::Result<()> { self.prompt.finish(); - if let Some(thread) = self.thread { - let _ = thread.join(); - } + self.thread.map_or(Ok(()), std::thread::JoinHandle::join) } } @@ -384,17 +434,26 @@ impl Notice for MessageNotice { let Some(shown) = self.shown.take() else { return; }; - // The box may not be up yet when the driver answers at once, so ask - // again until its thread ends. Bounded: the window opens regardless. - let deadline = Instant::now() + Duration::from_secs(2); - while !shown.answer.is_finished() && Instant::now() < deadline { - let thread = shown.thread.load(Ordering::SeqCst); - if thread != 0 { - unsafe { - let _ = EnumThreadWindows(thread, Some(close_window), LPARAM(0)); + // On a thread of its own: this may run on the window's thread, under + // the prompt's lock. The box may not be up yet when the driver + // answers at once, so ask again until its thread ends, for 2 s. + let closing = move || { + let deadline = Instant::now() + Duration::from_secs(2); + while !shown.answer.is_finished() && Instant::now() < deadline { + let thread = shown.thread.load(Ordering::SeqCst); + if thread != 0 { + unsafe { + let _ = EnumThreadWindows(thread, Some(close_window), LPARAM(0)); + } } + std::thread::sleep(Duration::from_millis(50)); } - std::thread::sleep(Duration::from_millis(50)); + }; + if let Err(error) = std::thread::Builder::new() + .name("driver-wait-notice-close".into()) + .spawn(closing) + { + warn!(%error, "Could not close the graphics driver notice"); } } } @@ -405,10 +464,17 @@ mod tests { use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering::SeqCst}; use std::sync::mpsc; - const LIMITS: Limits = Limits { + /// For a call that hangs until the test releases it. + const SHORT: Limits = Limits { warn: Duration::from_millis(100), limit: Duration::from_millis(300), }; + /// For a call that answers: a slow machine never reaches it, so the + /// events stay exactly what the test lists. + const LONG: Limits = Limits { + warn: Duration::from_secs(30), + limit: Duration::from_secs(60), + }; const SLICE: Duration = Duration::from_millis(1); fn lanes() -> &'static Lanes { @@ -478,7 +544,7 @@ mod tests { let result = crate::stall::run_on( lanes, Call::GpuDetect, - LIMITS, + SHORT, move || { let _ = held.recv(); }, @@ -496,6 +562,7 @@ mod tests { fn start_before_window( lanes: &'static Lanes, fake: &Fake, + (list_limits, probe_limits): (Limits, Limits), list: impl FnMut() -> ListStep + Send + 'static, probe: impl Fn() -> ProbeStep + Send + 'static, ) -> std::thread::JoinHandle { @@ -505,8 +572,8 @@ mod tests { let configured = events.clone(); before_window( lanes, - LIMITS, - LIMITS, + list_limits, + probe_limits, &prompt, SLICE, list, @@ -534,6 +601,28 @@ mod tests { Box::new(move || events.push("probe")) } + /// A step that blocks until the test sends on the returned sender, and + /// says when it started. + fn held_step( + events: &Events, + name: &'static str, + ) -> ( + mpsc::Sender<()>, + mpsc::Receiver<()>, + Box, + ) { + let (release, held) = mpsc::channel::<()>(); + let (started, running) = mpsc::channel::<()>(); + let events = events.clone(); + let step = Box::new(move || { + events.push(name); + let _ = started.send(()); + let _ = held.recv(); + events.push(format!("{name} returned")); + }); + (release, running, step) + } + #[test] fn a_driver_that_answers_opens_the_window_without_a_notice() { let fake = Fake::default(); @@ -541,6 +630,7 @@ mod tests { let opened = start_before_window( lanes(), &fake, + (LONG, LONG), move || listing(&list_events, "10de:2c02 32.0"), move || probing(&probe_events), ); @@ -560,6 +650,7 @@ mod tests { let opened = start_before_window( lanes, &fake, + (LONG, LONG), move || listing(&list_events, "gpus"), move || probing(&probe_events), ); @@ -576,102 +667,97 @@ mod tests { } #[test] - fn a_list_that_stalls_is_never_configured_and_the_next_one_is() { + fn a_list_that_answers_after_its_limit_is_waited_for_and_configures_once() { let lanes = lanes(); let fake = Fake::default(); - let (started, list_started) = mpsc::channel::<()>(); - let (release, held) = mpsc::channel::<()>(); - let held = Arc::new(Mutex::new(Some(held))); - let (list_events, probe_events) = (fake.events.clone(), fake.events.clone()); + let (release, list_running, step) = held_step(&fake.events, "list"); + let step = Arc::new(Mutex::new(Some(step))); + let probe_events = fake.events.clone(); let opened = start_before_window( lanes, &fake, + (SHORT, LONG), move || { - let events = list_events.clone(); - let started = started.clone(); - // The first list blocks in the driver; the next answers. - let held = held.lock().take(); + let step = step.lock().take().expect("listed once"); Box::new(move || { - let _ = started.send(()); - match held { - Some(held) => { - let _ = held.recv(); - events.push("stalled list returned"); - "late".to_string() - } - None => { - events.push("list"); - "gpus".to_string() - } - } + step(); + "late".to_string() }) }, move || probing(&probe_events), ); - list_started.recv().unwrap(); + list_running.recv().unwrap(); fake.events.wait_for("shown", 1); - assert_eq!(fake.events.all(), ["shown"]); release.send(()).unwrap(); assert!(opened.join().unwrap()); - // The stalled list's result was dropped; the one that returned in - // time configured the renderer, once. + // One list, past its limit; its late result configured the renderer. assert_eq!( fake.events.all(), - [ - "shown", - "stalled list returned", - "list", - "configure gpus", - "probe" - ] + ["list", "shown", "list returned", "configure late", "probe"] ); + assert_eq!(lanes.abandoned(Call::GpuDetect), 0); } #[test] - fn a_probe_that_hangs_raises_the_notice_and_the_next_one_opens_the_window() { + fn a_probe_that_answers_after_its_limit_opens_the_window() { let lanes = lanes(); let fake = Fake::default(); - let (started, probe_started) = mpsc::channel::<()>(); - let (release, held) = mpsc::channel::<()>(); - let held = Arc::new(Mutex::new(Some(held))); - let (list_events, probe_events) = (fake.events.clone(), fake.events.clone()); + let (release, probe_running, step) = held_step(&fake.events, "probe"); + let step = Arc::new(Mutex::new(Some(step))); + let list_events = fake.events.clone(); let opened = start_before_window( lanes, &fake, + (LONG, SHORT), move || listing(&list_events, "gpus"), - move || { - let events = probe_events.clone(); - let started = started.clone(); - let held = held.lock().take(); - Box::new(move || { - events.push("probe"); - let _ = started.send(()); - if let Some(held) = held { - let _ = held.recv(); - events.push("probe returned"); - } - }) - }, + move || step.lock().take().expect("probed once"), ); - probe_started.recv().unwrap(); + probe_running.recv().unwrap(); fake.events.wait_for("shown", 1); release.send(()).unwrap(); assert!(opened.join().unwrap()); assert_eq!( fake.events.all(), - [ - "list", - "configure gpus", - "probe", - "shown", - "probe returned", - "probe" - ] + ["list", "configure gpus", "probe", "shown", "probe returned"] ); + assert_eq!(lanes.abandoned(Call::GpuProbe), 0); } #[test] - fn cancel_opens_no_window() { + fn cancel_while_a_call_hangs_opens_no_window_and_leaves_its_lane_taken() { + let lanes = lanes(); + let fake = Fake::default(); + let (release, probe_running, step) = held_step(&fake.events, "probe"); + let step = Arc::new(Mutex::new(Some(step))); + let list_events = fake.events.clone(); + let opened = start_before_window( + lanes, + &fake, + (LONG, SHORT), + move || listing(&list_events, "gpus"), + move || step.lock().take().expect("probed once"), + ); + probe_running.recv().unwrap(); + fake.events.wait_for("shown", 1); + *fake.answer.lock() = Some(Answer::Quit); + assert!(!opened.join().unwrap()); + assert_eq!( + fake.events.all(), + ["list", "configure gpus", "probe", "shown", "cancel"] + ); + // The probe still runs in the driver; its lane says so until it + // returns. + assert_eq!(lanes.abandoned(Call::GpuProbe), 1); + release.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(60); + while lanes.abandoned(Call::GpuProbe) != 0 { + assert!(Instant::now() < deadline, "the lane did not free"); + std::thread::sleep(Duration::from_millis(1)); + } + } + + #[test] + fn cancel_while_detection_is_stuck_opens_no_window() { let lanes = lanes(); let _release = stuck_detection(lanes); let fake = Fake::default(); @@ -680,6 +766,7 @@ mod tests { let opened = start_before_window( lanes, &fake, + (LONG, LONG), move || listing(&list_events, "gpus"), move || probing(&probe_events), ); @@ -695,10 +782,11 @@ mod tests { let answer = Arc::clone(&fake.answer); let list_events = fake.events.clone(); // The user clicks Cancel while the probe runs, the moment before it - // returns in time. + // returns. let opened = start_before_window( lanes, &fake, + (LONG, LONG), move || listing(&list_events, "gpus"), move || { let answer = Arc::clone(&answer); @@ -741,35 +829,48 @@ mod tests { assert_eq!(fake.events.all(), ["shown", "closed"]); } - /// A clock the test moves on by hand. - fn manual_clock() -> (Arc, impl Fn() -> Instant + Send + 'static) { - let start = Instant::now(); - let ms = Arc::new(AtomicU64::new(0)); - let read = Arc::clone(&ms); - (ms, move || start + Duration::from_millis(read.load(SeqCst))) - } - - /// The renderer's view of the window: when its thread last began a UI - /// pass, in the manual clock's time. - #[derive(Clone)] - struct Passes { + /// A clock the test moves on by hand, and what the renderer and the live + /// window check report, in its time. + struct World { start: Instant, - last: Arc>>, + ms: Arc, + last_pass: Arc>>, + held: Arc, } - impl Passes { - fn new(now: &impl Fn() -> Instant) -> Self { + impl World { + fn new() -> Self { Self { - start: now(), - last: Arc::default(), + start: Instant::now(), + ms: Arc::default(), + last_pass: Arc::default(), + held: Arc::default(), } } - fn pass_at(&self, ms: u64, minimized: bool) { - *self.last.lock() = Some((self.start + Duration::from_millis(ms), minimized)); + fn at(&self, ms: u64) { + self.ms.store(ms, SeqCst); } - fn reader(&self) -> impl Fn() -> Option<(Instant, bool)> + Send + 'static { - let last = Arc::clone(&self.last); - move || *last.lock() + fn pass_at(&self, ms: u64) { + *self.last_pass.lock() = Some(self.start + Duration::from_millis(ms)); + self.at(ms); + } + fn watch(&self, fake: &Fake, quit: impl FnOnce() + Send + 'static) -> FirstPaintWatch { + let (start, ms) = (self.start, Arc::clone(&self.ms)); + let last_pass = Arc::clone(&self.last_pass); + let held = Arc::clone(&self.held); + watch_first_paint( + Prompt::new(fake.clone()), + SHORT.limit, + SLICE, + move || start + Duration::from_millis(ms.load(SeqCst)), + move || *last_pass.lock(), + move || held.load(SeqCst), + quit, + ) + } + /// Lets the watcher look a few times. + fn settle(&self) { + std::thread::sleep(Duration::from_millis(20)); } } @@ -777,102 +878,88 @@ mod tests { || panic!("quit") } + const LIMIT_MS: u64 = SHORT.limit.as_millis() as u64; + #[test] fn a_window_that_draws_in_time_shows_no_notice() { let fake = Fake::default(); - let (clock, now) = manual_clock(); - let passes = Passes::new(&now); - let watch = watch_first_paint( - Prompt::new(fake.clone()), - LIMITS.limit, - SLICE, - now, - passes.reader(), - never_quit(), - ); + let world = World::new(); + let watch = world.watch(&fake, never_quit()); // UI passes keep coming; then the window is confirmed drawing. for ms in [100, 300, 500, 700] { - passes.pass_at(ms, false); - clock.store(ms + 50, SeqCst); - std::thread::sleep(Duration::from_millis(5)); + world.pass_at(ms); + world.settle(); } (watch.finisher())(); - watch.stop(); + watch.stop().unwrap(); assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); } #[test] fn a_failed_first_acquire_then_a_stuck_recreation_gets_the_notice_until_the_window_draws() { let fake = Fake::default(); - let (clock, now) = manual_clock(); - let passes = Passes::new(&now); - let watch = watch_first_paint( - Prompt::new(fake.clone()), - LIMITS.limit, - SLICE, - now, - passes.reader(), - never_quit(), - ); + let world = World::new(); + let watch = world.watch(&fake, never_quit()); // The first pass ran and its frame's surface acquire failed, so the // renderer did not confirm the window; recreating the surface at the // start of the next paint never returns: no more passes. - passes.pass_at(200, false); - let stuck_at = 200 + LIMITS.limit.as_millis() as u64; - clock.store(stuck_at - 1, SeqCst); - std::thread::sleep(Duration::from_millis(20)); + world.pass_at(200); + world.at(200 + LIMIT_MS - 1); + world.settle(); assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); - clock.store(stuck_at, SeqCst); + world.at(200 + LIMIT_MS); fake.events.wait_for("shown", 1); // OK, then the reminder while it is still stuck. *fake.answer.lock() = Some(Answer::Wait); fake.events.wait_for("ok", 1); - clock.fetch_add(REMIND_AFTER.as_millis() as u64, SeqCst); + world.at(200 + LIMIT_MS + REMIND_AFTER.as_millis() as u64 + 1); fake.events.wait_for("shown", 2); // It comes back and is confirmed drawing: the notice goes. (watch.finisher())(); - watch.thread.unwrap().join().unwrap(); + watch.stop().unwrap(); assert_eq!(fake.events.all(), ["shown", "ok", "shown", "closed"]); } #[test] - fn a_minimized_window_gets_no_notice() { + fn a_minimized_window_gets_no_notice_and_is_counted_from_when_it_is_restored() { let fake = Fake::default(); - let (clock, now) = manual_clock(); - let passes = Passes::new(&now); - let watch = watch_first_paint( - Prompt::new(fake.clone()), - LIMITS.limit, - SLICE, - now, - passes.reader(), - never_quit(), - ); - passes.pass_at(100, true); - clock.store(100 + LIMITS.limit.as_millis() as u64 * 10, SeqCst); - std::thread::sleep(Duration::from_millis(20)); - watch.stop(); + let world = World::new(); + let watch = world.watch(&fake, never_quit()); + // One pass, then minimized: eframe runs no pass for a minimized + // window, so nothing reaches the renderer; only the live check sees + // it. + world.pass_at(100); + world.held.store(true, SeqCst); + for ms in (200..=100 * LIMIT_MS).step_by(LIMIT_MS as usize) { + world.at(ms); + world.settle(); + } + assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); + // Restored at `back`, and its passes never come. + let back = 100 * LIMIT_MS + 1; + world.at(back); + world.settle(); + world.held.store(false, SeqCst); + world.at(back + LIMIT_MS - 1); + world.settle(); assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); + world.at(back + LIMIT_MS + 1); + fake.events.wait_for("shown", 1); + watch.stop().unwrap(); + assert_eq!(fake.events.all(), ["shown", "closed"]); } #[test] fn cancel_while_the_window_is_stuck_quits_from_the_watcher() { let fake = Fake::default(); - let (clock, now) = manual_clock(); - clock.store(LIMITS.limit.as_millis() as u64, SeqCst); + let world = World::new(); let quits = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&quits); *fake.answer.lock() = Some(Answer::Quit); - let watch = watch_first_paint( - Prompt::new(fake.clone()), - Duration::ZERO, - SLICE, - now, - || None, - move || { - count.fetch_add(1, SeqCst); - }, - ); + let watch = world.watch(&fake, move || { + count.fetch_add(1, SeqCst); + }); + world.at(LIMIT_MS); watch.thread.unwrap().join().unwrap(); assert_eq!(quits.load(SeqCst), 1); assert_eq!(fake.events.all(), ["shown", "cancel"]); @@ -881,7 +968,6 @@ mod tests { #[test] fn a_cancel_that_comes_with_the_window_drawing_still_quits() { let fake = Fake::default(); - let (_clock, now) = manual_clock(); let quits = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&quits); let prompt = Prompt::new(fake.clone()); @@ -894,8 +980,9 @@ mod tests { prompt, REMIND_AFTER, SLICE, - now, + Instant::now, || None, + || false, move || { count.fetch_add(1, SeqCst); }, @@ -922,20 +1009,21 @@ mod tests { let _ = reading.send(()); let _ = proceed.lock().unwrap().recv(); } - start + LIMITS.limit * 2 + start + SHORT.limit * 2 }; let watch = watch_first_paint( Prompt::new(fake.clone()), - LIMITS.limit, + SHORT.limit, SLICE, now, || None, + || false, never_quit(), ); clock_read.recv().unwrap(); (watch.finisher())(); go.send(()).unwrap(); - watch.thread.unwrap().join().unwrap(); + watch.stop().unwrap(); assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); } } diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index c2127582..abf92c34 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -381,6 +381,7 @@ pub struct AnalyzerApp { impl AnalyzerApp { pub fn new(cc: &eframe::CreationContext<'_>, control: GuiControl) -> Self { + renderer::remember_window(cc); // Load persisted settings first; values fall back to the live runtime state when the // file is missing or unreadable. let persisted = SettingsFile::load(); @@ -1829,6 +1830,7 @@ pub fn run_gui( driver_wait::POLL, std::time::Instant::now, renderer::last_pass, + renderer::window_held, move || { renderer::abandon(); quit_while_stuck(); @@ -1849,7 +1851,7 @@ pub fn run_gui( Box::new(|cc| Ok(Box::new(AnalyzerApp::new(cc, control)))), ); #[cfg(windows)] - watch.stop(); + let _ = watch.stop(); if result.is_ok() { renderer::window_closed(); } diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index aa562c0a..8024de2a 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -116,6 +116,9 @@ const HISTORY_LOCK: &str = "history.lock"; /// at the first present, within milliseconds. const CONFIRM_AFTER: Duration = Duration::from_secs(3); const CONFIRM_FRAMES: u64 = 10; +/// Frames after asking for the proof of a presented frame before asking +/// again: the ask is dropped with a frame whose surface was not acquired. +const PROOF_AGAIN_AFTER: u64 = 3 * CONFIRM_FRAMES; static STATE_DIR: OnceLock = OnceLock::new(); static LAUNCH: Mutex = Mutex::new(Launch::new()); @@ -127,9 +130,12 @@ static STATUS: Mutex> = Mutex::new(None); /// keys come from until it is confirmed drawing: no more DXGI calls on its /// thread before then. static ADAPTERS: Mutex>> = Mutex::new(None); -/// When the window's thread last began a UI pass, and whether the window -/// was minimized then. -static LAST_PASS: Mutex> = Mutex::new(None); +/// When the window's thread last began a UI pass. +static LAST_PASS: Mutex> = Mutex::new(None); +/// The window, once it exists (an HWND on Windows), for checks made from +/// other threads while its own may be stuck. +#[cfg_attr(not(windows), allow(dead_code))] +static WINDOW: std::sync::atomic::AtomicIsize = std::sync::atomic::AtomicIsize::new(0); /// Run once, when the window is confirmed drawing. type OnDrawing = Box; static ON_DRAWING: Mutex> = Mutex::new(None); @@ -632,26 +638,43 @@ fn save_marker(directory: &Path, owner: ProcessId, marker: &Marker) -> bool { .is_ok_and(|json| fs::write(marker_path(directory, owner), json).is_ok()) } +/// Marks the screenshot that proves a frame was presented. +struct PresentProof; + /// Called at the start of each UI pass; clears this launch's marker once the /// window has kept drawing without a surface error. pub(super) fn frame_started(ctx: &egui::Context) { crate::startup::simulate_surface_error(&FAILED_ACQUIRES); let visible = ctx.input(|input| input.viewport().minimized != Some(true)); if let Ok(mut last) = LAST_PASS.lock() { - *last = Some((Instant::now(), !visible)); + *last = Some(Instant::now()); } + let proof = ctx.input(|input| { + input.events.iter().any(|event| { + matches!(event, egui::Event::Screenshot { user_data, .. } + if user_data.data.as_ref().is_some_and(|data| data.is::())) + }) + }); let Ok(mut tracker) = PRESENT.lock() else { return; }; - let now_drawing = tracker.observe( + let step = tracker.observe( ctx.cumulative_frame_nr(), FAILED_ACQUIRES.load(Ordering::SeqCst), visible, Instant::now(), + proof, ); let drawing = tracker.confirmed; drop(tracker); - if now_drawing { + if step == Step::Prove { + // egui-wgpu answers only after it acquired the surface, drew the + // frame into it and submitted it, right before the present. + ctx.send_viewport_cmd(egui::ViewportCommand::Screenshot(egui::UserData::new( + PresentProof, + ))); + } + if step == Step::Confirmed { crate::startup::mark_window_shown(); info!("Window renderer is drawing"); let on_drawing = ON_DRAWING.lock().ok().and_then(|mut hook| hook.take()); @@ -664,13 +687,61 @@ pub(super) fn frame_started(ctx: &egui::Context) { } } -/// When the window's thread last began a UI pass, and whether the window -/// was minimized then. None before the first. +/// When the window's thread last began a UI pass. None before the first. #[cfg_attr(not(windows), allow(dead_code))] -pub(super) fn last_pass() -> Option<(Instant, bool)> { +pub(super) fn last_pass() -> Option { LAST_PASS.lock().ok().and_then(|last| *last) } +/// Remember the window, for [`window_held`]. +pub(super) fn remember_window(cc: &eframe::CreationContext<'_>) { + #[cfg(windows)] + { + use eframe::wgpu::rwh::{HasWindowHandle, RawWindowHandle}; + if let Ok(handle) = cc.window_handle() { + if let RawWindowHandle::Win32(window) = handle.as_raw() { + WINDOW.store(window.hwnd.get(), Ordering::SeqCst); + } + } + } + #[cfg(not(windows))] + let _ = cc; +} + +/// Whether the window is minimized, or its thread is in the move/size or +/// menu loop Windows runs while the title bar is held: no UI pass runs then +/// in a working window either. Read from another thread, live. +#[cfg(windows)] +pub(super) fn window_held() -> bool { + use windows::Win32::Foundation::HWND; + use windows::Win32::UI::WindowsAndMessaging::{ + GetGUIThreadInfo, GetWindowThreadProcessId, IsIconic, GUITHREADINFO, GUI_INMENUMODE, + GUI_INMOVESIZE, + }; + let hwnd = WINDOW.load(Ordering::SeqCst); + if hwnd == 0 { + return false; + } + let hwnd = HWND(hwnd as _); + if unsafe { IsIconic(hwnd) }.as_bool() { + return true; + } + let thread = unsafe { GetWindowThreadProcessId(hwnd, None) }; + let mut info = GUITHREADINFO { + cbSize: std::mem::size_of::() as u32, + ..Default::default() + }; + thread != 0 + && unsafe { GetGUIThreadInfo(thread, &mut info) }.is_ok() + && (info.flags & (GUI_INMOVESIZE | GUI_INMENUMODE)).0 != 0 +} + +#[cfg(not(windows))] +#[allow(dead_code)] +pub(super) fn window_held() -> bool { + false +} + /// Run `hook` once, when the window is confirmed drawing: frames presented /// for a while with no surface error, visible ([`PresentTracker`]). #[cfg_attr(not(windows), allow(dead_code))] @@ -713,12 +784,14 @@ fn finish_launch(directory: &Path, marker: &Marker, owner: ProcessId, drew: bool true } -/// Confirms a working window from the frame counter, the time and surface -/// errors. egui-wgpu presents inside its painter and reports nothing after a -/// present; a surface it fails to recreate is not reported at all. So the -/// window must keep drawing for a while after the last reported error, which -/// takes it past device creation and the first presents, where the drivers of -/// issue #59 fail. +/// Confirms a working window from the frame counter, the time, surface +/// errors and a proof of a presented frame. egui-wgpu presents inside its +/// painter and reports nothing after a present; a surface it fails to +/// recreate is not reported at all, and its later paints return before +/// drawing. So the window must keep drawing for a while after the last +/// reported error, which takes it past device creation and the first +/// presents, where the drivers of issue #59 fail, and then answer a +/// screenshot, which egui-wgpu takes only from a frame it acquired and drew. struct PresentTracker { span: Option, confirmed: bool, @@ -730,6 +803,18 @@ struct Span { since: Instant, first_frame: u64, failures: u64, + /// The frame that asked for the proof in this span. + asked: Option, +} + +/// What a UI pass does for the confirmation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Step { + Wait, + /// Ask for the proof: a screenshot of this frame. + Prove, + /// Confirmed, once. + Confirmed, } impl PresentTracker { @@ -740,20 +825,38 @@ impl PresentTracker { } } - /// True once: after [`CONFIRM_AFTER`] and [`CONFIRM_FRAMES`] of visible - /// frames with no new surface error. A surface error or a minimized - /// window starts the span again. - fn observe(&mut self, frame: u64, failures: u64, visible: bool, now: Instant) -> bool { + /// After [`CONFIRM_AFTER`] and [`CONFIRM_FRAMES`] of visible frames with + /// no new surface error, asks for the proof (again after + /// [`PROOF_AGAIN_AFTER`] frames without it), and confirms once it came + /// (`proof`) in the same span. A surface error or a minimized window + /// starts the span again. + fn observe( + &mut self, + frame: u64, + failures: u64, + visible: bool, + now: Instant, + proof: bool, + ) -> Step { if self.confirmed { - return false; + return Step::Wait; } - match self.span { + match self.span.as_mut() { Some(span) if visible && span.failures == failures => { - if frame >= span.first_frame + CONFIRM_FRAMES + if let Some(asked) = span.asked { + if proof { + self.confirmed = true; + return Step::Confirmed; + } + if frame >= asked + PROOF_AGAIN_AFTER { + span.asked = Some(frame); + return Step::Prove; + } + } else if frame >= span.first_frame + CONFIRM_FRAMES && now.duration_since(span.since) >= CONFIRM_AFTER { - self.confirmed = true; - return true; + span.asked = Some(frame); + return Step::Prove; } } _ => { @@ -761,10 +864,11 @@ impl PresentTracker { since: now, first_frame: frame, failures, + asked: None, }); } } - false + Step::Wait } } @@ -2277,49 +2381,117 @@ mod tests { } /// Visible frames every 200 ms, as the GUI repaints, counted from `start`. + /// Frames `frames` a 200 ms apart, the proof answering the frame after + /// each ask: whether the window was confirmed. fn run( tracker: &mut PresentTracker, start: Instant, frames: std::ops::Range, failures: u64, ) -> bool { + let mut asked = false; frames.into_iter().any(|frame| { - tracker.observe( + let step = tracker.observe( frame, failures, true, start + Duration::from_millis(200 * frame), - ) + std::mem::take(&mut asked), + ); + asked = step == Step::Prove; + step == Step::Confirmed }) } #[test] - fn the_window_works_after_frames_and_time_without_surface_errors() { + fn the_window_works_after_frames_and_time_without_surface_errors_and_a_proof() { let start = Instant::now(); let mut tracker = PresentTracker::new(); // Frames alone are not enough: egui may lay out one frame many times. for frame in 0..100 { - assert!(!tracker.observe(frame, 0, true, start)); + assert_eq!(tracker.observe(frame, 0, true, start, false), Step::Wait); } - assert!(tracker.observe(100, 0, true, start + CONFIRM_AFTER)); + assert_eq!( + tracker.observe(100, 0, true, start + CONFIRM_AFTER, false), + Step::Prove + ); + assert_eq!( + tracker.observe(101, 0, true, start + CONFIRM_AFTER, true), + Step::Confirmed + ); let mut tracker = PresentTracker::new(); // Nor is time alone, with one frame laid out repeatedly. - assert!(!tracker.observe(0, 0, true, start)); - assert!(!tracker.observe(0, 0, true, start + 10 * CONFIRM_AFTER)); - assert!(tracker.observe(CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER)); - assert!(!tracker.observe(CONFIRM_FRAMES + 1, 0, true, start + CONFIRM_AFTER)); + assert_eq!(tracker.observe(0, 0, true, start, false), Step::Wait); + assert_eq!( + tracker.observe(0, 0, true, start + 10 * CONFIRM_AFTER, false), + Step::Wait + ); + assert_eq!( + tracker.observe(CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER, false), + Step::Prove + ); + assert_eq!( + tracker.observe(CONFIRM_FRAMES + 1, 0, true, start + CONFIRM_AFTER, true), + Step::Confirmed + ); + // Once. + assert_eq!( + tracker.observe(CONFIRM_FRAMES + 2, 0, true, start + CONFIRM_AFTER, true), + Step::Wait + ); + } + + #[test] + fn frames_that_present_nothing_are_never_confirmed() { + let start = Instant::now(); + let later = start + CONFIRM_AFTER; + let mut tracker = PresentTracker::new(); + assert_eq!(tracker.observe(0, 0, true, start, false), Step::Wait); + assert_eq!( + tracker.observe(CONFIRM_FRAMES, 0, true, later, false), + Step::Prove + ); + // The surface was removed and its recreation failed: egui-wgpu's + // paints return before drawing, with no surface error, and the UI + // passes go on. No screenshot answers, so nothing is confirmed; the + // ask is repeated now and then. + for frame in CONFIRM_FRAMES + 1..CONFIRM_FRAMES + PROOF_AGAIN_AFTER { + assert_eq!(tracker.observe(frame, 0, true, later, false), Step::Wait); + } + assert_eq!( + tracker.observe(CONFIRM_FRAMES + PROOF_AGAIN_AFTER, 0, true, later, false), + Step::Prove + ); + // A proof after a surface error is not this span's. + let mut tracker = PresentTracker::new(); + tracker.observe(0, 0, true, start, false); + assert_eq!( + tracker.observe(CONFIRM_FRAMES, 0, true, later, false), + Step::Prove + ); + assert_eq!( + tracker.observe(CONFIRM_FRAMES + 1, 1, true, later, true), + Step::Wait + ); + assert_eq!( + tracker.observe(CONFIRM_FRAMES + 2, 1, true, later, true), + Step::Wait + ); } #[test] fn a_lost_surface_that_is_recreated_late_is_not_a_working_window() { let start = Instant::now(); let mut tracker = PresentTracker::new(); - assert!(!tracker.observe(0, 0, true, start)); + assert_eq!(tracker.observe(0, 0, true, start, false), Step::Wait); // Lost: egui reports it and recreates the surface on the next frame, // which can fail without a report and without a present. - assert!(!tracker.observe(1, 1, true, start + Duration::from_millis(16))); - assert!(!tracker.observe(2, 1, true, start + Duration::from_millis(32))); - assert!(!tracker.observe(3, 1, true, start + Duration::from_millis(48))); + for (frame, ms) in [(1, 16), (2, 32), (3, 48)] { + assert_eq!( + tracker.observe(frame, 1, true, start + Duration::from_millis(ms), false), + Step::Wait + ); + } // Confirmed only once the window kept drawing after the error. assert!(!run(&mut tracker, start, 4..15, 1)); let mut tracker = PresentTracker::new(); @@ -2330,10 +2502,35 @@ mod tests { fn a_minimized_window_is_not_confirmed() { let start = Instant::now(); let mut tracker = PresentTracker::new(); - assert!(!tracker.observe(0, 0, true, start)); - assert!(!tracker.observe(CONFIRM_FRAMES, 0, false, start + CONFIRM_AFTER)); - assert!(!tracker.observe(2 * CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER)); - assert!(tracker.observe(3 * CONFIRM_FRAMES, 0, true, start + 2 * CONFIRM_AFTER)); + assert_eq!(tracker.observe(0, 0, true, start, false), Step::Wait); + assert_eq!( + tracker.observe(CONFIRM_FRAMES, 0, false, start + CONFIRM_AFTER, false), + Step::Wait + ); + assert_eq!( + tracker.observe(2 * CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER, false), + Step::Wait + ); + assert_eq!( + tracker.observe( + 3 * CONFIRM_FRAMES, + 0, + true, + start + 2 * CONFIRM_AFTER, + false + ), + Step::Prove + ); + assert_eq!( + tracker.observe( + 3 * CONFIRM_FRAMES + 1, + 0, + true, + start + 2 * CONFIRM_AFTER, + true + ), + Step::Confirmed + ); } // ── Remote renderer rules ─────────────────────────────────────────────── From 8e43c5920bced39c047120bc7f5c251f556a0fbd Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 03:10:41 -0500 Subject: [PATCH 147/226] Use smaller packets or the PC's other address when no video arrives When the PC says it sent video and the iPad still has no picture, the iPad asks for smaller media datagrams (1,200, then 576 bytes, each with a fresh keyframe). If that doesn't help, it asks the PC to sign its other home network addresses with the pairing, confirms the same PC at one of them, and reconnects there. The address that worked is tried first next time. The size request is a field appended to KEYFRAME_REQUEST, and the frame count is appended to HEARTBEAT. Both are gated on FEATURE_MEDIA_SIZE and HOSTCAP_MEDIA_SIZE, so older PCs and iPads see the same bytes as before. A PC session that sent video and ended with the iPad's error goodbye without a counted frame now summarizes as no_video or no_frame instead of other. --- host/src/diagnostics.rs | 28 ++ host/src/transport/mod.rs | 8 +- host/src/transport/session.rs | 232 ++++++++++++++++- host/tests/e2e_synthetic.rs | 122 +++++++++ .../App/ConnectionManager.swift | 149 ++++++++++- ios/EternalMonitor/App/MovedPC.swift | 35 ++- ios/EternalMonitor/App/NoVideoFallback.swift | 97 +++++++ ios/EternalMonitor/App/SessionSummary.swift | 14 +- .../Network/ControlChannel.swift | 11 +- ios/EternalMonitor/Network/WireProtocol.swift | 28 +- ios/EternalMonitorTests/FakeUDPHost.swift | 72 +++++ .../NoVideoFallbackTests.swift | 246 ++++++++++++++++++ .../WireProtocolGoldenTests.swift | 39 ++- proto/src/v2/control.rs | 98 ++++++- proto/testdata/v2_vectors.txt | 3 + proto/tests/golden.rs | 70 ++++- 16 files changed, 1220 insertions(+), 32 deletions(-) create mode 100644 ios/EternalMonitor/App/NoVideoFallback.swift create mode 100644 ios/EternalMonitorTests/NoVideoFallbackTests.swift diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index ab3e4d47..87ce175f 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -418,6 +418,15 @@ pub fn session_summary( EndReason::LinkLost => "other", EndReason::Liveness | EndReason::Goodbye(ByeReason::Error) if !streamed => "no_video", EndReason::Liveness => "signal_lost", + // The iPad gave up on video that went out: none of it arrived, or + // no frame of it completed. + EndReason::Goodbye(ByeReason::Error) if !end.frames_received => { + if end.frags_received { + "no_frame" + } else { + "no_video" + } + } EndReason::Goodbye(ByeReason::Error) => "other", EndReason::Failed(StageFailure::Capture) => "capture_failed", EndReason::Failed(StageFailure::Encoder) => "encoder_failed", @@ -705,6 +714,7 @@ pub(crate) mod tests { reason, first_video: first_video_ms.map(Duration::from_millis), frames_received: first_video_ms.is_some(), + frags_received: first_video_ms.is_some(), duration: Duration::from_millis(duration_ms), } } @@ -777,6 +787,24 @@ pub(crate) mod tests { end(Goodbye(ByeReason::Error), Some(500), 30_000), ["steady", "streamed", "other", "goodbye"], ), + // The iPad gave up on video that went out: none of it arrived, + // or no frame of it completed. + ( + SessionEnd { + frames_received: false, + frags_received: false, + ..end(Goodbye(ByeReason::Error), Some(300), 10_000) + }, + ["video_sent", "streamed", "no_video", "goodbye"], + ), + ( + SessionEnd { + frames_received: false, + frags_received: true, + ..end(Goodbye(ByeReason::Error), Some(300), 10_000) + }, + ["video_sent", "streamed", "no_frame", "goodbye"], + ), ( end(Liveness, None, 3_000), ["accepted", "no_picture", "no_video", "liveness"], diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index c2f7b1da..0ea8dd42 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -582,7 +582,13 @@ pub async fn start_sender( } let payload = &nal.data; - let payload_size = media_payload_size(shared.max_dgram.load(Ordering::SeqCst) as usize).unwrap_or(MAX_MEDIA_PAYLOAD); + // The PC's packet size, or the smaller one this session's + // client asked for when none of the larger reached it. + let max_dgram = shared.session.lock().max_media_dgram(session_id).map_or( + shared.max_dgram.load(Ordering::SeqCst), + |asked| shared.max_dgram.load(Ordering::SeqCst).min(u32::from(asked)), + ); + let payload_size = media_payload_size(max_dgram as usize).unwrap_or(MAX_MEDIA_PAYLOAD); let frag_count_usize = payload.len().div_ceil(payload_size).max(1); // Held back until the client answers; the frames after it // wait for the keyframe its answer asks for. diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index db3ade05..9ad0a41b 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -14,8 +14,9 @@ use std::time::{Duration, Instant}; use eternal_wire::v2::control::{ ByeReason, ControlHeader, ControlMessage, FailedReason, HelloAck, HelloStatus, InputEvent, - KeyframeRequest, Nack, ReceiverReport, StreamConfig, FEATURE_STANDBY_STATUS, - FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_NACK, + KeyframeRequest, Nack, ReceiverReport, StreamConfig, FEATURE_MEDIA_SIZE, + FEATURE_STANDBY_STATUS, FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_MEDIA_SIZE, + HOSTCAP_NACK, MIN_MEDIA_DGRAM, }; use tracing::{info, warn}; @@ -216,6 +217,8 @@ pub struct SessionEnd { pub first_video: Option, /// The client's receiver reports counted complete frames. pub frames_received: bool, + /// The client's receiver reports counted any video fragment. + pub frags_received: bool, pub duration: Duration, } @@ -278,6 +281,13 @@ struct ActiveSession { first_video_at: Option, /// A receiver report counted complete frames. frames_received: bool, + /// A receiver report counted any video fragment. + frags_received: bool, + /// Video frames let through to this session ([`Session::may_send_media`]). + frames_sent: u32, + /// The largest media datagram the client asked for in this session + /// ([`KeyframeRequest::max_media_dgram`]). + max_media_dgram: Option, /// A Wi-Fi goodbye while the cable takes over. bye: Option, /// The msg_seq of input packets taken in this session. @@ -540,10 +550,14 @@ impl Session { let Some(session) = self .active .as_mut() - .filter(|session| session.session_id == session_id && !session.answered) + .filter(|session| session.session_id == session_id) else { return true; }; + if session.answered { + session.frames_sent = session.frames_sent.saturating_add(1); + return true; + } if session.withheld || session.unanswered_sent + bytes > UNANSWERED_MEDIA_BUDGET || host_spent + bytes > UNANSWERED_MEDIA_BUDGET @@ -552,10 +566,20 @@ impl Session { return false; } session.unanswered_sent += bytes; + session.frames_sent = session.frames_sent.saturating_add(1); self.unanswered_media = (host_spent + bytes, now); true } + /// The largest media datagram `session_id`'s client asked for, if it + /// asked for one. + pub fn max_media_dgram(&self, session_id: u32) -> Option { + self.active + .as_ref() + .filter(|session| session.session_id == session_id) + .and_then(|session| session.max_media_dgram) + } + /// The host-wide media to clients not yet heard that still counts at /// `now`. fn unanswered_media_at(&self, now: Instant) -> usize { @@ -1094,6 +1118,9 @@ impl Session { msg_seq_out: 0, first_video_at: None, frames_received: false, + frags_received: false, + frames_sent: 0, + max_media_dgram: None, bye: None, input_seqs: SeqWindow::default(), answered, @@ -1163,6 +1190,24 @@ impl Session { } actions.force_idr = session.heard(Heard::Keyframe, now); info!(reason = ?request.reason, "Keyframe request received"); + // Only a client that was told the host takes it asks; anything else + // is ignored, as before. Each smaller size restarts the picture at + // once: the frames sent so far gave the client no picture. + let asked = request.max_media_dgram; + if asked != 0 && session.info.feature_caps & FEATURE_MEDIA_SIZE != 0 { + let size = asked.clamp(MIN_MEDIA_DGRAM, eternal_wire::v2::MAX_DGRAM_SIZE as u16); + if session.max_media_dgram.is_none_or(|current| size < current) { + session.max_media_dgram = Some(size); + session.last_keyframe_grant = Some(now); + info!( + max_dgram = size, + frames_sent = session.frames_sent, + "Media packet size lowered for this session" + ); + actions.force_idr = true; + return actions; + } + } if session.grant_keyframe(now) { info!(reason = ?request.reason, "Keyframe requested by client — forcing IDR"); @@ -1178,6 +1223,7 @@ impl Session { actions.force_idr = session.heard(Heard::Report, now); session.last_report = Some(report); session.frames_received |= report.frames_complete > 0; + session.frags_received |= report.frags_received > 0; actions.report = Some(report); } } @@ -1356,11 +1402,14 @@ impl Session { if send_heartbeat { let peer = session.peer; let session_id = session.session_id; + let frames_sent = (session.info.feature_caps & FEATURE_MEDIA_SIZE != 0) + .then_some(session.frames_sent); let msg_seq = self.next_msg_seq(); let heartbeat = ControlMessage::Heartbeat(eternal_wire::v2::control::Heartbeat { host_time_us: crate::clock::host_now_us(), stream_config: self.negotiated_config(config), notice: config.notice(), + frames_sent, }); actions.replies.push(( peer, @@ -1563,6 +1612,7 @@ impl Session { reason: EndReason::Refused(status), first_video: None, frames_received: false, + frags_received: false, duration: Duration::ZERO, }), ..Actions::default() @@ -1638,6 +1688,13 @@ impl Session { auth_token: [u8; 16], reason: &str, ) -> HelloAck { + // Only the client that asks hears that the host takes smaller media + // datagrams; older iPads get the reply they always did. + let media_size = status == HelloStatus::Ok + && self.active.as_ref().is_some_and(|session| { + session.session_id == session_id + && session.info.feature_caps & FEATURE_MEDIA_SIZE != 0 + }); HelloAck { status, accepted_version: 2, @@ -1649,7 +1706,9 @@ impl Session { stream_config: self.negotiated_config(config), host_name: config.host_name(), auth_token, - host_caps: HOSTCAP_NACK | config.host_caps(), + host_caps: HOSTCAP_NACK + | config.host_caps() + | if media_size { HOSTCAP_MEDIA_SIZE } else { 0 }, reason: reason.to_string(), } } @@ -1805,6 +1864,7 @@ impl ActiveSession { .first_video_at .map(|at| at.saturating_duration_since(start)), frames_received: self.frames_received, + frags_received: self.frags_received, duration: now.saturating_duration_since(start), } } @@ -2418,6 +2478,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 10, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, }); let first = session.handle_control_authed(peer, request.clone(), &TestConfig, now); @@ -2458,6 +2519,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 10, reason, + max_media_dgram: 0, }) }; @@ -2758,6 +2820,164 @@ mod tests { assert!(matches!(message, ControlMessage::Heartbeat(_))); } + fn media_size_hello(nonce: u32, feature_caps: u16) -> ControlMessage { + let ControlMessage::Hello2(mut hello) = hello(nonce, 50000) else { + unreachable!() + }; + hello.feature_caps = feature_caps; + ControlMessage::Hello2(hello) + } + + fn sized_request(max_media_dgram: u16) -> ControlMessage { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 7, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram, + }) + } + + fn heartbeat_of(actions: &Actions) -> eternal_wire::v2::control::Heartbeat { + match eternal_wire::v2::control::parse_control(&actions.replies[0].1) + .unwrap() + .1 + { + ControlMessage::Heartbeat(heartbeat) => heartbeat, + other => panic!("{other:?}"), + } + } + + /// Only an iPad that says it may ask for smaller media datagrams hears + /// that the host takes them, and gets the frame count in heartbeats. + /// Any other iPad gets the reply and heartbeats it always did. + #[test] + fn media_size_is_offered_only_to_an_ipad_that_asks() { + let peer = addr([10, 0, 0, 5], 50000); + let now = Instant::now(); + for (feature_caps, offered) in [(FEATURE_MEDIA_SIZE, true), (0, false)] { + let mut session = Session::new(1234); + let accepted = session.handle_control( + peer, + 0, + media_size_hello(1, feature_caps), + &TestConfig, + now, + ); + let ack = parse_ack(&accepted.replies[0].1); + assert_eq!(ack.host_caps & HOSTCAP_MEDIA_SIZE != 0, offered); + assert_eq!(ack.host_caps & !HOSTCAP_MEDIA_SIZE, HOSTCAP_NACK); + let id = session.session_id().unwrap(); + for _ in 0..3 { + assert!(session.may_send_media(id, 1000, now)); + } + let ticked = session.tick(&TestConfig, true, now + Duration::from_millis(100)); + assert_eq!( + heartbeat_of(&ticked).frames_sent, + offered.then_some(3), + "{feature_caps}" + ); + // The heartbeat an older iPad gets is the plain one. + if !offered { + assert_eq!( + ticked.replies[0].1.len(), + eternal_wire::v2::control::CONTROL_HEADER_SIZE + + 8 + + eternal_wire::v2::control::STREAM_CONFIG_SIZE + ); + } + } + // A refusal never offers it. + let mut busy = Session::new(1234); + busy.handle_control( + peer, + 0, + media_size_hello(1, FEATURE_MEDIA_SIZE), + &TestConfig, + now, + ); + let refused = busy.handle_control( + addr([10, 0, 0, 6], 50000), + 0, + media_size_hello(2, FEATURE_MEDIA_SIZE), + &TestConfig, + now, + ); + let ack = parse_ack(&refused.replies[0].1); + assert_eq!(ack.status, HelloStatus::Busy); + assert_eq!(ack.host_caps & HOSTCAP_MEDIA_SIZE, 0); + } + + /// A smaller size restarts the picture at once, inside the keyframe + /// rate limit, and holds for the rest of the session: never larger, + /// never under the minimum. A new session starts at the PC's size. + #[test] + fn a_smaller_media_size_lowers_only_this_session_and_forces_an_idr() { + let peer = addr([10, 0, 0, 5], 50000); + let now = Instant::now(); + let mut session = Session::new(1234); + session.handle_control( + peer, + 0, + media_size_hello(1, FEATURE_MEDIA_SIZE), + &TestConfig, + now, + ); + let id = session.session_id().unwrap(); + assert_eq!(session.max_media_dgram(id), None); + + let plain = session.handle_control_authed(peer, sized_request(0), &TestConfig, now); + assert!(plain.force_idr); + assert_eq!(session.max_media_dgram(id), None); + + let at = |millis| now + Duration::from_millis(millis); + for (millis, asked, size, idr) in [ + (100, 1200, 1200, true), + (200, 1200, 1200, false), + (300, 1300, 1200, false), + (400, 576, 576, true), + (450, 100, 576, false), + ] { + let actions = + session.handle_control_authed(peer, sized_request(asked), &TestConfig, at(millis)); + assert_eq!(session.max_media_dgram(id), Some(size), "{asked}"); + assert_eq!(actions.force_idr, idr, "{asked}"); + assert!(actions.replies.is_empty() && actions.new_target.is_none()); + } + assert_eq!(session.max_media_dgram(id.wrapping_add(1)), None); + + session.handle_control( + peer, + 0, + media_size_hello(2, FEATURE_MEDIA_SIZE), + &TestConfig, + at(500), + ); + let next = session.session_id().unwrap(); + assert_ne!(next, id); + assert_eq!(session.max_media_dgram(next), None); + } + + /// An iPad that didn't say it may ask has its request read as the + /// keyframe request it always was. + #[test] + fn a_media_size_from_an_ipad_that_did_not_offer_is_ignored() { + let peer = addr([10, 0, 0, 5], 50000); + let now = Instant::now(); + let mut session = Session::new(1234); + session.handle_control(peer, 0, media_size_hello(1, 0), &TestConfig, now); + let id = session.session_id().unwrap(); + let first = session.handle_control_authed(peer, sized_request(576), &TestConfig, now); + assert!(first.force_idr); + let second = session.handle_control_authed( + peer, + sized_request(576), + &TestConfig, + now + Duration::from_millis(100), + ); + assert!(!second.force_idr, "rate limited as any keyframe request"); + assert_eq!(session.max_media_dgram(id), None); + } + struct PairConfig(parking_lot::Mutex); impl ConfigSource for PairConfig { fn stream_config(&self) -> StreamConfig { @@ -3418,6 +3638,7 @@ mod tests { reason: EndReason::Goodbye(ByeReason::UserDisconnect), first_video: Some(Duration::from_millis(40)), frames_received: true, + frags_received: false, duration: Duration::from_secs(2), } ); @@ -3668,6 +3889,7 @@ mod tests { reason: EndReason::Refused(HelloStatus::VersionUnsupported), first_video: None, frames_received: false, + frags_received: false, duration: Duration::ZERO, } ); @@ -3890,6 +4112,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 10, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, }); let ping = ControlMessage::Ping(eternal_wire::v2::control::Ping { t1_us: 1 }); let nack = ControlMessage::Nack(Nack { @@ -4368,6 +4591,7 @@ mod tests { stream_epoch: 7, last_complete_seq: 0, reason: KeyframeReason::Startup, + max_media_dgram: 0, }), ControlMessage::Nack(Nack { stream_epoch: 7, diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index d9bc5f80..48e3caf5 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -707,6 +707,7 @@ fn run_slow_usb(bytes_per_sec: f64, run_for: Duration) -> SlowUsbRun { stream_epoch: epoch, last_complete_seq: seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, }); write_datagram(&mut stream, &encode_control(session_id, msg_seq, &request)) .await @@ -913,6 +914,7 @@ fn lossy_wifi_recovery_measurement() { stream_epoch: c.stream_epoch, last_complete_seq: assembler.latest_completed_seq(), reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } } @@ -1584,6 +1586,123 @@ fn audio_stream_end_to_end() { } } +/// What one fake iPad saw for `window`: the largest media datagram, whether +/// a keyframe came, and the latest heartbeat frame count. +fn media_seen(receiver: &mut FakeReceiver, window: Duration) -> (usize, bool, Option) { + let started = Instant::now(); + let mut buffer = [0; 2048]; + let (mut largest, mut keyframe, mut frames_sent) = (0, false, None); + while started.elapsed() < window { + receiver.maybe_report(0, 0); + let Ok((len, _)) = receiver.socket.recv_from(&mut buffer) else { + continue; + }; + match classify(&buffer[..len]) { + Classified::Media { flags } => { + largest = largest.max(len); + keyframe |= flags & 1 != 0; + } + Classified::Control(_) => { + if let Ok((_, ControlMessage::Heartbeat(heartbeat))) = parse_control(&buffer[..len]) + { + frames_sent = heartbeat.frames_sent; + } + } + _ => {} + } + } + (largest, keyframe, frames_sent) +} + +/// An iPad that said it may ask for smaller media datagrams gets them for +/// the rest of its session, starting with a keyframe, and heartbeats that +/// count the frames sent. An iPad that didn't say so gets neither, and its +/// request is the keyframe request it always was. +#[test] +fn an_ipad_gets_the_smaller_media_datagrams_it_asks_for() { + use eternal_wire::v2::control::{FEATURE_MEDIA_SIZE, HOSTCAP_MEDIA_SIZE}; + let _env = test_env(); + init_test_tracing(); + ffmpeg_next::init().unwrap(); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + for name in ["ETERNAL_DROP", "ETERNAL_REORDER", "ETERNAL_JITTER_MS"] { + std::env::remove_var(name); + } + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".into()); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor_shared = shared.clone(); + let supervisor_sender = supervisor_tx.clone(); + let supervisor = std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + supervisor_shared, + GpuInfo::software_fallback(), + supervisor_sender, + supervisor_rx, + ) + }); + + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let sized = |max_media_dgram| { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 0, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram, + }) + }; + for offered in [true, false] { + let features = if offered { FEATURE_MEDIA_SIZE } else { 0 }; + let mut receiver = FakeReceiver::connect_full(listen_port, CAP_DECODE_H264, features); + assert_eq!(receiver.host_caps & HOSTCAP_MEDIA_SIZE != 0, offered); + let (largest, _, frames_sent) = media_seen(&mut receiver, Duration::from_secs(2)); + assert!( + largest > 1000, + "{offered}: the keyframe goes in full-size datagrams ({largest})" + ); + assert_eq!( + frames_sent.is_some_and(|frames| frames > 0), + offered, + "{frames_sent:?}" + ); + if !offered { + assert_eq!(frames_sent, None); + } + + receiver.send(&sized(576)); + // Frames already queued may still be large; what follows isn't. + let (_, queued_keyframe, _) = media_seen(&mut receiver, Duration::from_millis(500)); + let (largest, keyframe, _) = media_seen(&mut receiver, Duration::from_secs(2)); + if offered { + assert!( + queued_keyframe || keyframe, + "the smaller size starts with a keyframe" + ); + assert!(largest > 0 && largest <= 576, "{largest}"); + } else { + assert!(largest > 576, "{largest}"); + } + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + std::thread::sleep(Duration::from_millis(200)); + } + // The size was this session's: the next one starts at the PC's. + let mut receiver = + FakeReceiver::connect_full(listen_port, CAP_DECODE_H264, FEATURE_MEDIA_SIZE); + let (largest, _, _) = media_seen(&mut receiver, Duration::from_secs(2)); + assert!(largest > 1000, "{largest}"); + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + })); + let _ = supervisor_tx.send(SupervisorCommand::Shutdown); + supervisor.join().unwrap(); + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } +} + /// Native first-second audio measurement across repeated connects with the /// PC's real WASAPI loopback (inaudible: it only reads the mix). #[cfg(windows)] @@ -1789,6 +1908,7 @@ fn synthetic_stream_end_to_end_v2() { stream_epoch: header.stream_epoch, last_complete_seq: header.frame_seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } if let Some(at) = keyframe_requested_at { @@ -2210,6 +2330,7 @@ fn lossy_stream_recovers_and_adapts() { stream_epoch: epoch, last_complete_seq: seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } for frame in assembler.take_ready() { @@ -2379,6 +2500,7 @@ fn lossy_legacy_stream_uses_keyframe_recovery() { stream_epoch: current_epoch, last_complete_seq: highest_seq, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, })); } diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index a8381af5..e94613be 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -99,7 +99,8 @@ final class ConnectionManager: ObservableObject { /// understands why a PC isn't streaming. nonisolated static func featureCaps(wantsInput: Bool, wantsAudio: Bool, isUSB: Bool) -> UInt16 { (wantsInput ? Hello2.featureWantsInput : 0) | (wantsAudio ? Hello2.featureWantsAudio : 0) - | (isUSB ? 0 : Hello2.featureSupportsNack) | Hello2.featureChecksUpdates | Hello2.featureStandbyStatus + | (isUSB ? 0 : Hello2.featureSupportsNack | Hello2.featureMediaSize) | Hello2.featureChecksUpdates + | Hello2.featureStandbyStatus } private var awaitingPairing = false /// The PC answered that it isn't streaming; the attempt waits for it. @@ -239,6 +240,9 @@ final class ConnectionManager: ObservableObject { var reconnectGraceSeconds = 2.0 var reconnectBackoffSeconds: (Int) -> Double = { attempt in Double(min(1 << (attempt - 1), 15)) } var appleDevicesNoteSeconds = AppleDevicesSighting.lifetime + /// How long video the PC says it sent may go missing before the + /// next no-video step (see `NoVideoFallback`). + var noVideoStepSeconds = 1.5 } var timing: Timing /// The Wi-Fi link to a host's port. Tests pass a fake one, or replace it @@ -252,6 +256,9 @@ final class ConnectionManager: ObservableObject { var homeAddress: (String) -> Bool = MovedPC.isHomeAddress /// How this iPad reaches an address. Tests replace it. var localRoute: (String, UInt16) async -> MovedPC.Route? = { await MovedPC.route(to: $0, port: $1) } + /// Where the iPad sends to reach a PC address it was given. Tests on + /// loopback point the addresses a fake signs at other fakes. + var dialAddress: (String, UInt16) -> (host: String, port: UInt16) = { ($0, $1) } /// "host:port" of each address picked from Bonjour's list in this /// launch. Saved entries keep the mark once they show a picture. private var foundAddresses = Set() @@ -262,10 +269,21 @@ final class ConnectionManager: ObservableObject { let name: String /// Its pairing token, which the PC proved it holds. let token: Data + /// The PC's video didn't arrive at `from`; it still answers there. + var noVideo = false } /// The move the next attempt makes, and the one the current attempt made. private var pendingMove: Move? private var attemptMove: Move? + /// The PC's video didn't arrive at the address asked for before, and did + /// at another: the next attempt goes there first, and back to the address + /// asked for (kept here) if the PC doesn't answer there. + private var pendingRedirect: (host: String, port: UInt16)? + private var attemptRedirect: (host: String, port: UInt16)? + /// The current session's no-video steps, with a PC that takes them, and + /// the step whose result isn't known yet, as its log line names it. + private var noVideoFallback: NoVideoFallback? + private var noVideoStep: String? init(timing: Timing = Timing(), udpReceiver: @escaping (UInt16) -> any MediaLink = { UDPReceiver(port: $0) }, pairingStore: PairingStore? = nil) { @@ -360,10 +378,26 @@ final class ConnectionManager: ObservableObject { if let token { do { try pairingStore.save(token: token, hostName: "", address: PairingStore.address(host: host, port: port)) } catch { record(.warning, "pairing", "Could not save the scanned pairing on this iPad.") } + } else if let better = betterAddress(host: host, port: port) { + pendingRedirect = (host.trimmingCharacters(in: .whitespacesAndNewlines), port) + connect(target: .udp(host: better.host, port: better.port)) + pendingRedirect = nil + return } connect(target: .udp(host: host, port: port)) } + /// Where the same PC showed video after none arrived at `host:port`: an + /// address this iPad holds the same pairing for. + private func betterAddress(host: String, port: UInt16) -> NoVideoPaths.Address? { + let host = host.trimmingCharacters(in: .whitespacesAndNewlines) + guard let better = NoVideoPaths.better(than: host, port: port), + let token = (try? pairingStore.token(address: PairingStore.address(host: host, port: port))) ?? nil, + let there = (try? pairingStore.token(address: PairingStore.address(host: better.host, port: better.port))) ?? nil, + token == there else { return nil } + return better + } + func submitPairing() { guard awaitingPairing, let code = pairingPrompt?.submit() else { return } // The code goes in a new HELLO2. The refused one ends here, as on the @@ -380,6 +414,7 @@ final class ConnectionManager: ObservableObject { func forgetPairedHosts() { do { try pairingStore.forgetAll() } catch { connectionError = "Could not remove saved pairings from this iPad." } + NoVideoPaths.forgetAll() } /// A connection the user or a cable starts. It replaces any automatic @@ -433,6 +468,10 @@ final class ConnectionManager: ObservableObject { explicitPairingToken = nil attemptMove = isUSB ? nil : pendingMove pendingMove = nil + attemptRedirect = isUSB ? nil : pendingRedirect + pendingRedirect = nil + noVideoFallback = nil + noVideoStep = nil let connectionID = UUID() currentAttempt.withLock { $0 = connectionID } transportMode = isUSB ? "USB" : "WiFi" @@ -465,7 +504,12 @@ final class ConnectionManager: ObservableObject { audioStats = AudioStats() debugState = ConnectionDebugState(host: normalizedHost, port: port) record(.info, "connection", "Starting connection attempt=\(connectionID) transport=\(transportMode) to \(normalizedHost):\(port) drawing=\(drawing.enabled) input=\(wantsInput) audio=\(wantsAudio)") - if attemptMove != nil { record(.info, "connection", "Trying the saved PC at the new address Bonjour gave") } + if attemptMove != nil { + record(.info, "connection", attemptMove?.noVideo == true + ? "Trying the PC at another of its addresses: its video didn't arrive at the first" + : "Trying the saved PC at the new address Bonjour gave") + } + if attemptRedirect != nil { record(.info, "connection", "no_video_fallback step=remembered result=tried") } let decoder = VideoDecoder() let metrics = ReceiverMetrics() @@ -515,6 +559,8 @@ final class ConnectionManager: ObservableObject { if self.debugState.decodedFrames == 1 { self.firstFrameAt = ProcessInfo.processInfo.systemUptime self.record(.info, "decode", "First frame decoded successfully" + self.attemptTimings) + self.endNoVideoStep("video") + self.noVideoFallback = nil self.timeoutTask?.cancel() self.noPictureStreak = NoPictureStreak() self.state = .connected @@ -678,6 +724,8 @@ final class ConnectionManager: ObservableObject { self.lastTarget = (normalizedHost, port) } self.sessionAcceptedAt = acceptedAt + self.noVideoFallback = !isUSB && info.hostCaps & HelloAck.hostCapMediaSize != 0 + ? NoVideoFallback(wait: self.timing.noVideoStepSeconds) : nil self.acceptedAddresses.insert("\(normalizedHost):\(port)") self.noPictureStreak.answered(by: info.hostName, pairingToken: info.authToken, at: isUSB ? nil : "\(normalizedHost):\(port)") @@ -786,7 +834,14 @@ final class ConnectionManager: ObservableObject { self.disconnect(.legacyPC) } else { self.record(.warning, "ctrl", "HELLO2 handshake got no reply yet") - if !isUSB { self.lookForMovedPC(connectionID, host: normalizedHost, port: port) } + if let from = self.attemptRedirect { + // Not on this network now: back to the address asked for. + self.record(.info, "connection", "no_video_fallback step=remembered result=no_reply") + self.disconnect(.pcMoved) + self.startAttempt(.udp(host: from.host, port: from.port)) + } else if !isUSB { + self.lookForMovedPC(connectionID, host: normalizedHost, port: port) + } } } } @@ -830,6 +885,9 @@ final class ConnectionManager: ObservableObject { Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } self.hostStreamConfig = heartbeat.streamConfig + if let frames = heartbeat.framesSent { + self.noVideoFallback?.heard(framesSent: frames, at: ProcessInfo.processInfo.systemUptime) + } let locked = heartbeat.notice == Heartbeat.noticePCLocked if locked != self.pcLocked { self.pcLocked = locked @@ -992,6 +1050,7 @@ final class ConnectionManager: ObservableObject { } } + self.advanceNoVideoFallback() // Runs on EVERY tick, including silent ones: a dead host sends // neither media nor heartbeats, so skipping this when nothing // arrived is exactly when the liveness watchdog is needed. @@ -1200,13 +1259,91 @@ final class ConnectionManager: ObservableObject { return false } RecentConnectionStore.shared.moved(from: move.from, to: new, name: hostInfo?.hostName ?? move.name, - replacesOld: homeAddress(move.from.host)) - if homeAddress(move.from.host) { + replacesOld: move.noVideo || homeAddress(move.from.host)) + if move.noVideo { + // The PC still answers at the old address: its pairing stays, + // and connecting there goes here first. + NoVideoPaths.remember(from: .init(host: move.from.host, port: move.from.port), + to: .init(host: new.host, port: new.port)) + } else if homeAddress(move.from.host) { try? pairingStore.remove(address: PairingStore.address(host: move.from.host, port: move.from.port)) } return true } + /// Takes the next no-video step once it is due: the PC said it sent + /// video and no picture came of it (see `NoVideoFallback`). A path that + /// drops large datagrams often lets a frame's small last fragment by. + private func advanceNoVideoFallback() { + guard noVideoFallback != nil, state == .connecting, hostInfo != nil, !awaitingPairing, !waitingForStream, + let step = noVideoFallback?.next(at: ProcessInfo.processInfo.systemUptime) else { return } + endNoVideoStep("no_video") + switch step { + case .smallerPackets(let size): + // The PC is sending and the iPad is working on it: one fresh + // window, as when data starts to arrive. + if !didExtendTimeout { + didExtendTimeout = true + armConnectTimeout(seconds: timing.connectTimeoutSeconds) + } + noVideoStep = "step=smaller_packets size=\(size)" + controlChannel?.sendKeyframeRequest(streamEpoch: 0, lastCompleteSeq: 0, reason: .startup, maxMediaDgram: size) + case .otherAddress: + tryOtherAddress() + } + } + + /// Logs the pending no-video step's `result`. + private func endNoVideoStep(_ result: String) { + guard let step = noVideoStep else { return } + noVideoStep = nil + record(.info, "connection", "no_video_fallback \(step) result=\(result)") + } + + /// No picture came of the PC's video at this address, even in small + /// datagrams. Asked here, the PC signs its home network addresses with + /// this iPad's pairing; the first other one this iPad reaches over + /// Wi-Fi or Ethernet, where the PC shows again that it holds the pairing + /// and receives there, gets a new attempt. A forwarded or spoofed + /// address can't make either answer. + private func tryOtherAddress() { + let host = debugState.host, port = debugState.port + let done = { [weak self] (result: String) -> Void in + self?.record(.info, "connection", "no_video_fallback step=other_address result=\(result)") + } + guard attemptMove == nil else { return done("moved_before") } + guard let connectionID = currentAttempt.withLock({ $0 }), let name = hostInfo?.hostName, + let token = (try? pairingStore.token(address: PairingStore.address(host: host, port: port))) ?? nil + else { return done("unpaired") } + let current = { [weak self] in + guard let self else { return false } + return self.currentAttempt.withLock { $0 == connectionID } && self.state == .connecting + } + Task { @MainActor [weak self] in + guard let self else { return } + let identity = await MovedPC.identify(host: host, port: port, token: token, link: self.makeUDPLink(port)) + guard current() else { return } + guard let identity else { return done("no_answer") } + for other in MovedPC.otherAddresses(identity, asked: host, homeAddress: self.homeAddress) { + let target = self.dialAddress(other, identity.port) + let route = await self.localRoute(target.host, target.port) + guard current() else { return } + guard let route, MovedPC.isDirect(route) else { continue } + let same = await MovedPC.confirm(host: target.host, port: target.port, token: token, + link: self.makeUDPLink(target.port)) + guard current() else { return } + guard same else { continue } + done("moved") + self.disconnect(.noVideoPath) + self.pendingMove = Move(from: (host, port), name: name, token: token, noVideo: true) + self.explicitPairingToken = token + self.startAttempt(.udp(host: target.host, port: target.port)) + return + } + done("none") + } + } + /// Retires the automatic reconnect cycle, if one is running: its task /// stops at its next check and can no longer write state. private func endReconnectCycle() { @@ -1453,12 +1590,14 @@ final class ConnectionManager: ObservableObject { /// `.userDisconnect`. func disconnect(_ end: AttemptEnd = .userDisconnect) { if state != .disconnected { + endNoVideoStep("no_video") recordStreamSummary(reason: "disconnect=\(end.name)") record(.info, "connection", "Disconnect state=\(state) reason=\(end.name)") recordSummary(end) } awaitingPairing = false waitingForStream = false + noVideoFallback = nil pairingPrompt = nil currentAttempt.withLock { $0 = nil } timeoutTask?.cancel() diff --git a/ios/EternalMonitor/App/MovedPC.swift b/ios/EternalMonitor/App/MovedPC.swift index 6e53aa92..6c181fb1 100644 --- a/ios/EternalMonitor/App/MovedPC.swift +++ b/ios/EternalMonitor/App/MovedPC.swift @@ -85,12 +85,24 @@ enum MovedPC { guard text.split(separator: ".").count == 4, let bytes = IPv4Address(text)?.rawValue, bytes.count == 4 else { return nil } return bytes.reduce(0) { $0 << 8 | UInt32($1) } } - guard route.direct, !tunnels.contains(where: route.interface.hasPrefix), - let address = ipv4(route.address), let mask = ipv4(route.netmask), let candidate = ipv4(host), + guard isDirect(route), let address = ipv4(route.address), let mask = ipv4(route.netmask), let candidate = ipv4(host), mask != 0, mask != .max else { return false } return address & mask == candidate & mask } + /// Whether `route` is a direct Wi-Fi or Ethernet path, not a tunnel. + static func isDirect(_ route: Route) -> Bool { + route.direct && !tunnels.contains(where: route.interface.hasPrefix) + } + + /// The other home network addresses `identity` signed, in the PC's + /// order (those with a default gateway first), without `asked`. + static func otherAddresses(_ identity: WireIdentity, asked: String, + homeAddress: (String) -> Bool = isHomeAddress) -> [String] { + identity.addresses.map { $0.map(String.init).joined(separator: ".") } + .filter { $0 != asked && homeAddress($0) } + } + /// How this iPad reaches `host:port` now, from a UDP path to it that /// sends nothing; nil if no path is ready within 2 s. static func route(to host: String, port: UInt16) async -> Route? { @@ -161,6 +173,13 @@ enum MovedPC { /// wrong proof. The token itself is never sent. static func confirm(host: String, port: UInt16, token: Data, link: any MediaLink, attempts: Int = 8, interval: Double = 0.25) async -> Bool { + await identify(host: host, port: port, token: token, link: link, attempts: attempts, interval: interval) != nil + } + + /// As `confirm`, with the PC's signed answer: the port it receives the + /// stream on and its home network addresses, one of them `host`. + static func identify(host: String, port: UInt16, token: Data, link: any MediaLink, attempts: Int = 8, + interval: Double = 0.25) async -> WireIdentity? { var generator = SystemRandomNumberGenerator() let nonce = Data((0..<16).map { _ in UInt8.random(in: .min ... .max, using: &generator) }) let request = Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identify(WireIdentify(nonce: nonce))) @@ -169,7 +188,7 @@ enum MovedPC { let box = LinkBox(link) return await withCheckedContinuation { continuation in let done = OSAllocatedUnfairLock(initialState: false) - let finish: @Sendable (Bool) -> Void = { result in + let finish: @Sendable (WireIdentity?) -> Void = { result in guard done.withLock({ finished in defer { finished = true }; return !finished }) else { return } box.link.controlQueue.async { box.link.onControlDatagram = nil @@ -184,13 +203,13 @@ enum MovedPC { identity.version == WireIdentify.version, identity.nonce == nonce else { return } let signed = WireIdentity.message(nonce: nonce, port: identity.port, addresses: identity.addresses) guard HMAC.isValidAuthenticationCode(identity.proof, authenticating: signed, using: key) else { - return finish(false) + return finish(nil) } // The PC signs where it receives the stream. An answer for // another address isn't the saved PC at this one. - if identity.port == port, let asked, identity.addresses.contains(asked) { finish(true) } + if identity.port == port, let asked, identity.addresses.contains(asked) { finish(identity) } } - link.onError = { _ in finish(false) } + link.onError = { _ in finish(nil) } link.onListenerReady = { _ in for attempt in 0.. framesBeforeStep, sendingSince == nil { sendingSince = now } + } + + /// The step to take at `now`, while there is no picture; nil while the + /// PC sends nothing, or within `wait` of its sending. + mutating func next(at now: TimeInterval) -> Step? { + guard !finished, let since = sendingSince, now - since >= wait else { return nil } + sendingSince = nil + framesBeforeStep = framesSent + guard sizesAsked < Self.sizes.count else { + finished = true + return .otherAddress + } + sizesAsked += 1 + return .smallerPackets(Self.sizes[sizesAsked - 1]) + } +} + +/// Where a PC's video didn't arrive, and the address of the same PC where it +/// did, so the next connect to the first goes to the second. Kept in +/// UserDefaults as "host:port" pairs. +enum NoVideoPaths { + private static let key = "noVideoPaths" + private static let kept = 16 + + struct Address: Codable, Equatable { + var host: String + var port: UInt16 + } + + private struct Entry: Codable { + var from: Address + var to: Address + } + + /// The address where the PC at `host:port` showed video instead. + static func better(than host: String, port: UInt16) -> Address? { + load().last { $0.from == Address(host: host, port: port) }?.to + } + + static func remember(from: Address, to: Address) { + var entries = load().filter { $0.from != from && $0.from != to } + entries.append(Entry(from: from, to: to)) + save(Array(entries.suffix(kept))) + } + + static func forgetAll() { + UserDefaults.standard.removeObject(forKey: key) + } + + private static func load() -> [Entry] { + guard let data = UserDefaults.standard.data(forKey: key) else { return [] } + return (try? JSONDecoder().decode([Entry].self, from: data)) ?? [] + } + + private static func save(_ entries: [Entry]) { + guard let data = try? JSONEncoder().encode(entries) else { return } + UserDefaults.standard.set(data, forKey: key) + } +} diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index 4ae3449b..702d1e44 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -30,9 +30,14 @@ enum AttemptEnd: Equatable { /// iPadOS found no usable network for the path: Wi-Fi is off or /// joined to none (`ConnectionProblem.wifiOff`). case noNetwork - /// No answer at the saved address, and Bonjour found the same PC at - /// another: a new attempt goes there. + /// No answer at the address tried, and the same PC is known at another: + /// Bonjour found it there, or its video arrived there before. A new + /// attempt goes there. case pcMoved + /// The PC sent video that gave no picture at this address, even in + /// small datagrams, and showed it is the same PC at another: a new + /// attempt goes there. + case noVideoPath /// The PC found at a saved PC's new address turned out to be another PC. case otherPC case receiverFailed @@ -102,7 +107,7 @@ struct SessionSummary: Equatable { case .appBackground, .superseded, .pcGoodbye: return "app_ended" case .connectTimeout: return sessionId == nil ? "no_reply" : "no_picture" case .pcMoved: return "no_reply" - case .closedBeforePicture: return "no_picture" + case .closedBeforePicture, .noVideoPath: return "no_picture" default: return "error" } } @@ -111,6 +116,7 @@ struct SessionSummary: Equatable { switch end { case .connectTimeout where sessionId == nil, .pcMoved: return "no_reply" case .connectTimeout: return datagramsReceived == 0 ? "no_video" : "no_frame" + case .noVideoPath: return "no_video" case .pcStoppedResponding: return "signal_lost" case .cableUnplugged, .usbLinkFailed, .closedBeforePicture: return "usb_lost" case .noNetwork: return "no_network" @@ -123,7 +129,7 @@ struct SessionSummary: Equatable { switch end { case .userDisconnect, .usbTurnedOff: return "user" case .appBackground: return "background" - case .superseded, .retriedWithCode, .pcMoved: return "superseded" + case .superseded, .retriedWithCode, .pcMoved, .noVideoPath: return "superseded" case .pcGoodbye: return "goodbye" case .connectTimeout: return "timeout" case .pcStoppedResponding: return "liveness" diff --git a/ios/EternalMonitor/Network/ControlChannel.swift b/ios/EternalMonitor/Network/ControlChannel.swift index 5f540e64..18fbfd48 100644 --- a/ios/EternalMonitor/Network/ControlChannel.swift +++ b/ios/EternalMonitor/Network/ControlChannel.swift @@ -324,7 +324,10 @@ final class ControlChannel { timer.resume() } - func sendKeyframeRequest(streamEpoch: UInt32, lastCompleteSeq: UInt32, reason: KeyframeReason) { + /// `maxMediaDgram` asks the PC for media datagrams no larger, for the + /// rest of the session; only a PC with `HelloAck.hostCapMediaSize` gets it. + func sendKeyframeRequest(streamEpoch: UInt32, lastCompleteSeq: UInt32, reason: KeyframeReason, + maxMediaDgram: UInt16 = 0) { queue.async { [self] in guard sessionId != 0 else { return } let now = nowUs() @@ -332,12 +335,14 @@ final class ControlChannel { // dropped frames' own requests then keep the spacing busy, and // would always hide it from the PC's log. if reason != .startup, let last = lastKeyframeRequestUs, now &- last < Self.keyframeRequestSpacingUs { return } + let maxMediaDgram = hostCaps & HelloAck.hostCapMediaSize != 0 ? maxMediaDgram : 0 lastKeyframeRequestUs = now - onDiagnostic?("Requesting keyframe (\(reason))") + onDiagnostic?("Requesting keyframe (\(reason))" + (maxMediaDgram == 0 ? "" : " with media datagrams up to \(maxMediaDgram) bytes")) sendMessage(.keyframeRequest(KeyframeRequest( streamEpoch: streamEpoch, lastCompleteSeq: lastCompleteSeq, - reason: reason + reason: reason, + maxMediaDgram: maxMediaDgram ))) } } diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index 95342fcb..295fd1b3 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -264,6 +264,10 @@ struct Hello2: Equatable { static let standbyLength = 164 /// This app checks for its own updates, so the PC need not tell the user. 0.3.1 and older never set it. static let featureChecksUpdates: UInt16 = 1 << 4 + /// This iPad may ask for smaller media datagrams in a keyframe request, + /// and wants heartbeats to count the frames the PC sent. Bit 5 is taken + /// by another change in flight. + static let featureMediaSize: UInt16 = 1 << 6 static let maxNameLength = 64 var protoMin: UInt8 = 2 @@ -313,6 +317,10 @@ struct HelloAck: Equatable { /// Every 0.3.4 and later PC sets it in replies that accept a session; /// see `OldPC`. static let hostCapContiguousFrameSeq: UInt16 = 1 << 5 + /// The PC takes `KeyframeRequest.maxMediaDgram` for the rest of the + /// session, and its heartbeats carry `framesSent`. Only an iPad that + /// sets `Hello2.featureMediaSize` hears it. + static let hostCapMediaSize: UInt16 = 1 << 7 var status: HelloStatus var acceptedVersion: UInt8 var clientNonce: UInt32 @@ -340,6 +348,10 @@ struct Heartbeat: Equatable { static let noticeNone: UInt8 = 0 /// The PC is locked or shows a permission prompt, so it can't capture. static let noticePCLocked: UInt8 = 1 + /// The video frames the PC sent this session, from a PC that offered + /// `HelloAck.hostCapMediaSize`; nil from others. Appended after + /// `notice`, which is then always written. + var framesSent: UInt32? } enum ByeReason: UInt8 { @@ -361,6 +373,11 @@ struct KeyframeRequest: Equatable { var streamEpoch: UInt32 var lastCompleteSeq: UInt32 var reason: KeyframeReason + /// The largest media datagram to send from now on in this session; 0 + /// asks nothing. Appended: only for a PC with `HelloAck.hostCapMediaSize`. + var maxMediaDgram: UInt16 = 0 + /// The smallest a request may ask for, as the PC's own setting. + static let minMediaDgram: UInt16 = 576 } struct Nack: Equatable { @@ -549,13 +566,15 @@ extension Wire { case .heartbeat(let hb): body.appendLE(hb.hostTimeUs) hb.streamConfig.encode(into: &body) - if hb.notice != Heartbeat.noticeNone { body.append(hb.notice) } + if hb.notice != Heartbeat.noticeNone || hb.framesSent != nil { body.append(hb.notice) } + if let frames = hb.framesSent { body.appendLE(frames) } case .bye(let reason): body.append(reason.rawValue) case .keyframeRequest(let k): body.appendLE(k.streamEpoch) body.appendLE(k.lastCompleteSeq) body.append(k.reason.rawValue) + if k.maxMediaDgram != 0 { body.appendLE(k.maxMediaDgram) } case .nack(let n): precondition(n.isValid) body.appendLE(n.streamEpoch) @@ -671,7 +690,8 @@ extension Wire { guard let hostTimeUs = r.readU64(), let cfg = StreamConfig.decode(&r) else { return nil } message = .heartbeat(Heartbeat(hostTimeUs: hostTimeUs, streamConfig: cfg, - notice: r.readU8() ?? Heartbeat.noticeNone)) + notice: r.readU8() ?? Heartbeat.noticeNone, + framesSent: r.remaining >= 4 ? r.readU32() : nil)) case .bye: guard let raw = r.readU8(), let reason = ByeReason(rawValue: raw) else { return nil } message = .bye(reason) @@ -680,8 +700,8 @@ extension Wire { let last: UInt32 = r.readU32(), let raw = r.readU8(), let reason = KeyframeReason(rawValue: raw) else { return nil } - message = .keyframeRequest( - KeyframeRequest(streamEpoch: epoch, lastCompleteSeq: last, reason: reason)) + message = .keyframeRequest(KeyframeRequest(streamEpoch: epoch, lastCompleteSeq: last, reason: reason, + maxMediaDgram: r.remaining >= 2 ? r.readU16() ?? 0 : 0)) case .receiverReport: message = parseReceiverReport(&r) case .nack: diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index fdb71a7a..8bc148e9 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -285,6 +285,78 @@ final class FakeUDPHost: @unchecked Sendable { return requests } + /// What `servePath` saw the iPad do. + struct PathRun { + /// The media datagram sizes the iPad asked for, in turn. + var sizesAsked: [UInt16] = [] + /// Keyframe requests that asked for no size. + var plainKeyframeRequests = 0 + var identifyAnswers = 0 + } + + /// Plays a PC whose media reaches the iPad only in datagrams of at most + /// `limit` bytes (0: none at all), as over a path that drops larger ones. + /// Accepts the next HELLO2 with `hostCaps`, then until `stop` says so, + /// about every 100 ms, sends a heartbeat (counting the frames sent when + /// `countFrames`) and `frame` as the next keyframe, cut at the size the + /// iPad asked for. Answers identify requests as the PC holding `token` + /// receiving on `addresses`, to whichever socket asks. + func servePath(limit: Int, frame: Data, hostCaps: UInt16, countFrames: Bool = true, token: Data = Data(17...32), + addresses: [Data] = [Data([127, 0, 0, 1])], until stop: () -> Bool) -> PathRun { + var run = PathRun() + var size = Wire.maxDatagramSize + var framesSent: UInt32 = 0 + var lastSent = Date.distantPast + let deadline = Date().addingTimeInterval(60) + while !stop(), Date() < deadline { + if let (data, from, length) = receive() { + switch Wire.parseControl(data)?.message { + case .hello2(let hello)? where client == nil: + client = (from, length) + lastNonce = hello.clientNonce + send(Self.ack(nonce: hello.clientNonce, authToken: token, hostCaps: hostCaps)) + case .keyframeRequest(let request)?: + if request.maxMediaDgram == 0 { + run.plainKeyframeRequests += 1 + } else { + run.sizesAsked.append(request.maxMediaDgram) + size = min(size, Int(request.maxMediaDgram)) + } + case .identify(let request)?: + run.identifyAnswers += 1 + let answer = Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .identity(WireIdentity( + nonce: request.nonce, port: port, addresses: addresses, + proof: MovedPC.proof(token: token, nonce: request.nonce, port: port, addresses: addresses)))) + var to = from + _ = answer.withUnsafeBytes { bytes in + withUnsafePointer(to: &to) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { + sendto(fd, bytes.baseAddress, bytes.count, 0, $0, length) + } + } + } + default: + break + } + } + guard client != nil, Date().timeIntervalSince(lastSent) >= 0.1 else { continue } + lastSent = Date() + framesSent += 1 + let payload = size - MediaHeader.size + let count = (frame.count + payload - 1) / payload + for index in 0.. [PairingRecord] { values.map { PairingRecord(account: $0.key, token: $0.value) } } + func save(account: String, token: Data) throws { values[account] = token } + func remove(account: String?) throws { + if let account { values.removeValue(forKey: account) } else { values.removeAll() } + } +} + +/// A PC that accepts the session and sends video, over a path that drops +/// large datagrams or all of them: the iPad asks for smaller ones, then +/// tries the PC's other address. +@MainActor +final class NoVideoFallbackTests: XCTestCase { + nonisolated private static let token = Data(17...32) + /// The other address the first fake signs; the manager dials it at the + /// second fake. + nonisolated private static let otherAddress = Data([192, 0, 2, 7]) + nonisolated private static let bothAddresses = [Data([127, 0, 0, 1]), otherAddress] + + override func setUp() async throws { + let store = RecentConnectionStore.shared + let saved = store.connections + let defaults = ["recentConnections", "lastHost", "lastPort", "noVideoPaths"] + .map { ($0, UserDefaults.standard.object(forKey: $0)) } + addTeardownBlock { @MainActor in + store.connections = saved + for (key, value) in defaults { UserDefaults.standard.set(value, forKey: key) } + } + UserDefaults.standard.removeObject(forKey: "noVideoPaths") + store.connections = [] + } + + private func until(_ description: String, seconds: Double = 60, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + /// A manager whose steps come 0.3 s apart, paired with "Test PC" at + /// `pc`, reaching every address directly, and dialling the other + /// address the PC signs at `other`. + private func manager(pairedAt pc: FakeUDPHost, other: FakeUDPHost? = nil, + connectTimeout: Double = 60) throws -> (ConnectionManager, PairingStore) { + let pairings = PairingStore(storage: MemoryPairings()) + try pairings.save(token: Self.token, hostName: "Test PC", address: PairingStore.address(host: "127.0.0.1", port: pc.port)) + var timing = ConnectionManager.Timing(connectTimeoutSeconds: connectTimeout, reconnectGraceSeconds: 0.1, + reconnectBackoffSeconds: { _ in 0.05 }) + timing.noVideoStepSeconds = 0.3 + let manager = ConnectionManager(timing: timing, pairingStore: pairings) + manager.homeAddress = { _ in true } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "127.0.0.1", netmask: "255.0.0.0") } + if let other { + manager.dialAddress = { host, port in host == "192.0.2.7" ? ("127.0.0.1", other.port) : (host, port) } + } + return (manager, pairings) + } + + /// Every line the manager logs, across attempts (each clears the list). + private func lines(_ manager: ConnectionManager) -> (() -> [String], AnyCancellable) { + var lines: [String] = [] + let sink = manager.$diagnostics.sink { if let line = $0.last?.message { lines.append(line) } } + return ({ lines }, sink) + } + + // MARK: - The steps + + /// Each step waits for frames the PC says it sent after the last one, + /// and none comes while the PC sends nothing. + func testStepsWaitForVideoThePCSaysItSent() { + var steps = NoVideoFallback(wait: 1.5) + XCTAssertNil(steps.next(at: 100)) + steps.heard(framesSent: 0, at: 0) + XCTAssertNil(steps.next(at: 100), "a PC that sent nothing yet") + steps.heard(framesSent: 3, at: 100) + XCTAssertNil(steps.next(at: 101)) + XCTAssertEqual(steps.next(at: 101.5), .smallerPackets(1200)) + XCTAssertNil(steps.next(at: 110)) + steps.heard(framesSent: 3, at: 110) + XCTAssertNil(steps.next(at: 120), "nothing sent since the step") + steps.heard(framesSent: 4, at: 120) + steps.heard(framesSent: 9, at: 121) + XCTAssertNil(steps.next(at: 121.4)) + XCTAssertEqual(steps.next(at: 121.5), .smallerPackets(576)) + steps.heard(framesSent: 12, at: 122) + XCTAssertEqual(steps.next(at: 123.5), .otherAddress) + steps.heard(framesSent: 20, at: 124) + XCTAssertNil(steps.next(at: 200), "then no more") + } + + // MARK: - Connecting + + /// Datagrams over 1,000 bytes never arrive; only each frame's short last + /// fragment does. The iPad asks for 1,200 bytes, then 576, and streams. + func testAPathThatDropsLargeDatagramsStreamsWithSmallerOnes() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let streaming = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: 1000, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("streaming") { manager.state == .connected } + streaming.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, [1200, 576]) + XCTAssertEqual(run.identifyAnswers, 0, "smaller datagrams were enough") + let steps = logged().filter { $0.hasPrefix("no_video_fallback ") } + XCTAssertEqual(steps, ["no_video_fallback step=smaller_packets size=1200 result=no_video", + "no_video_fallback step=smaller_packets size=576 result=video"]) + } + + /// The first address carries no video at all. Asked there, the PC signs + /// another of its addresses; at that one it shows again that it holds + /// the pairing before the token goes there, and the iPad streams from it + /// with no tap. Connecting to the first address later goes there first. + func testAnAddressThatDropsAllVideoMovesToTheVerifiedOtherAddress() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, pairings) = try manager(pairedAt: first, other: other) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let moved = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: first.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + first.servePath(limit: 0, frame: frame, hostCaps: HelloAck.hostCapMediaSize, + addresses: Self.bothAddresses) { moved.withLock { $0 } } + } + let accepted = await Task.detached { + other.answerIdentify(token: Self.token) && other.acceptSession(authToken: Self.token) + }.value + XCTAssertTrue(accepted, "the iPad never checked and tried the other address") + moved.withLock { $0 = true } + let run = await served.value + try await until("handshake there") { manager.hostInfo != nil && manager.debugState.port == other.port } + other.sendKeyframe(frame) + try await until("streaming") { manager.state == .connected } + + XCTAssertEqual(run.sizesAsked, [1200, 576]) + XCTAssertGreaterThan(run.identifyAnswers, 0) + XCTAssertFalse(other.helloBeforeIdentify, "nothing with the token before the PC showed it is the same there") + XCTAssertEqual(other.lastAuthToken, Self.token) + let all = logged() + let steps = all.filter { $0.hasPrefix("no_video_fallback ") } + XCTAssertEqual(steps, ["no_video_fallback step=smaller_packets size=1200 result=no_video", + "no_video_fallback step=smaller_packets size=576 result=no_video", + "no_video_fallback step=other_address result=moved"]) + XCTAssertTrue(all.contains("Disconnect state=connecting reason=noVideoPath"), all.joined(separator: "\n")) + let summary = all.first { $0.hasPrefix("session_summary ") } ?? "" + XCTAssertTrue(summary.contains(" stage=accepted outcome=no_picture failure=no_video fallback=none end=superseded "), summary) + + // The working address is the saved one now; the first keeps its + // pairing, as the PC still answers there. + XCTAssertEqual(RecentConnectionStore.shared.connections.first?.port, other.port) + XCTAssertNil(RecentConnectionStore.shared.entry(host: "127.0.0.1", port: first.port)) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: other.port)), Self.token) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) + XCTAssertEqual(UserDefaults.standard.integer(forKey: "lastPort"), Int(other.port)) + + // Connecting to the first address later goes to the other first. + manager.cancel() + manager.connect(host: "127.0.0.1", port: first.port) + XCTAssertTrue(manager.diagnostics.contains { $0.message == "no_video_fallback step=remembered result=tried" }) + async let atFirst = Task.detached { first.received(within: 1) }.value + let answered = await Task.detached { other.acceptSession(authToken: Self.token) }.value + XCTAssertTrue(answered, "the address that showed video goes first") + let firstTypes = await atFirst + XCTAssertFalse(firstTypes.contains(.hello2), "\(firstTypes)") + + // Where the other address doesn't answer, the attempt goes back to + // the first. + manager.cancel() + manager.connect(host: "127.0.0.1", port: first.port) + let backAtFirst = await Task.detached { first.helloAttempts(within: 8) }.value + XCTAssertGreaterThan(backAtFirst, 0) + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=no_reply"), + logged().joined(separator: "\n")) + } + + /// The other address the PC signs answers without the pairing's proof, + /// as a forwarder or another device would: no HELLO2 and no token go + /// there, and the attempt ends as before. + func testAnOtherAddressThatDoesNotShowItIsTheSamePCIsLeftAlone() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, _) = try manager(pairedAt: first, other: other, connectTimeout: 8) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let ended = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: first.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + first.servePath(limit: 0, frame: frame, hostCaps: HelloAck.hostCapMediaSize, + addresses: Self.bothAddresses) { ended.withLock { $0 } } + } + let types = await Task.detached { () -> [Wire.PacketType] in + guard other.answerIdentify(token: Data(2...17)) else { return [] } + return other.received(within: 3) + }.value + XCTAssertFalse(types.contains(.hello2), "\(types)") + XCTAssertFalse(other.helloBeforeIdentify) + try await until("timed out", seconds: 20) { manager.state == .disconnected } + ended.withLock { $0 = true } + _ = await served.value + XCTAssertEqual(manager.problem, .noPicture) + XCTAssertTrue(logged().contains("no_video_fallback step=other_address result=none"), logged().joined(separator: "\n")) + XCTAssertNil(UserDefaults.standard.data(forKey: "noVideoPaths")) + } + + /// A PC that doesn't offer smaller datagrams, as every PC before this + /// version: the iPad asks for nothing new and asks no identity, and the + /// attempt times out as it always did. + func testAnOlderPCIsLeftAsBefore() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc, connectTimeout: 3) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let ended = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: 0, frame: frame, hostCaps: 0, countFrames: false) { ended.withLock { $0 } } + } + try await until("timed out", seconds: 20) { manager.state == .disconnected } + ended.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, []) + XCTAssertEqual(run.identifyAnswers, 0) + XCTAssertEqual(manager.problem, .noPicture) + let all = logged() + XCTAssertFalse(all.contains { $0.hasPrefix("no_video_fallback ") }, all.joined(separator: "\n")) + XCTAssertTrue(all.contains { $0.hasPrefix("Session accepted by 127.0.0.1:") && $0.contains("no video datagrams arrived") }) + } +} diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index b7d2273e..b33977cf 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,7 +35,7 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 37, + Self.vectors.count, 40, "golden vector count drifted — update both test suites together" ) } @@ -408,6 +408,43 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(old.notice, Heartbeat.noticeNone) } + /// A PC that takes smaller media datagrams says so, to an iPad that + /// asked; the request and the frame count are appended to the bodies + /// earlier versions read, so those read them as before. + func testMediaSizeVectors() throws { + XCTAssertEqual(HelloAck.hostCapMediaSize, 0x0080) + XCTAssertEqual(Hello2.featureMediaSize, 0x0040) + XCTAssertEqual(KeyframeRequest.minMediaDgram, 576) + + let ackData = try vector("hello_ack_media_size") + let (ackHeader, ackMessage) = try XCTUnwrap(Wire.parseControl(ackData)) + guard case .helloAck(let ack) = ackMessage else { return XCTFail("wrong type") } + XCTAssertEqual(ack.hostCaps, HelloAck.hostCapNack | HelloAck.hostCapContiguousFrameSeq | HelloAck.hostCapMediaSize) + XCTAssertEqual(Wire.encodeControl(sessionId: ackHeader.sessionId, msgSeq: ackHeader.msgSeq, message: ackMessage), ackData) + + let request = try vector("keyframe_request_media_size") + let (requestHeader, requestMessage) = try XCTUnwrap(Wire.parseControl(request)) + XCTAssertEqual(requestHeader.msgSeq, 51) + XCTAssertEqual(requestMessage, .keyframeRequest(KeyframeRequest(streamEpoch: 3, lastCompleteSeq: 0, reason: .startup, + maxMediaDgram: 1200))) + XCTAssertEqual(Wire.encodeControl(sessionId: 0x1234_5678, msgSeq: 51, message: requestMessage), request) + XCTAssertEqual(request.count, try vector("keyframe_request").count + 2) + + let counted = try vector("heartbeat_frames_sent") + let (heartbeatHeader, heartbeatMessage) = try XCTUnwrap(Wire.parseControl(counted)) + guard case .heartbeat(let hb) = heartbeatMessage else { return XCTFail("wrong type") } + XCTAssertEqual(hb.framesSent, 1234) + XCTAssertEqual(hb.notice, Heartbeat.noticeNone) + XCTAssertEqual(hb.streamConfig, Self.sampleConfig) + XCTAssertEqual(Wire.encodeControl(sessionId: 0x1234_5678, msgSeq: heartbeatHeader.msgSeq, message: heartbeatMessage), + counted) + let plain = try vector("heartbeat") + XCTAssertEqual(counted.count, plain.count + 5) + XCTAssertEqual(counted.dropFirst(16).prefix(plain.count - 16), plain.dropFirst(16)) + guard case .heartbeat(let old) = try XCTUnwrap(Wire.parseControl(plain)).message else { return XCTFail("wrong type") } + XCTAssertNil(old.framesSent) + } + func testByeVector() throws { let data = try vector("bye_background") let (_, message) = try XCTUnwrap(Wire.parseControl(data)) diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index e3df226f..e383fa28 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -44,6 +44,11 @@ pub const FEATURE_STANDBY_STATUS: u16 = 1 << 3; /// The iPad app checks for its own updates (0.3.2 and later), so a HELLO2 /// without it comes from 0.3.1 or older. Older hosts ignore it. pub const FEATURE_CHECKS_UPDATES: u16 = 1 << 4; +/// The iPad may ask for smaller media datagrams in a +/// [`KeyframeRequest::max_media_dgram`], and wants [`Heartbeat::frames_sent`] +/// to tell a PC that sends nothing from video lost on the way (0.3.5 and +/// later). Bit 5 is taken by another change in flight. +pub const FEATURE_MEDIA_SIZE: u16 = 1 << 6; pub const HOSTCAP_NACK: u16 = 1 << 0; pub const HOSTCAP_AUDIO: u16 = 1 << 1; pub const HOSTCAP_USB: u16 = 1 << 2; @@ -57,6 +62,14 @@ pub const HOSTCAP_PEN: u16 = 1 << 4; /// session, and iPads read an accepted reply without it as a PC to update, /// so it stays set there. pub const HOSTCAP_CONTIGUOUS_FRAME_SEQ: u16 = 1 << 5; +/// The host honours [`KeyframeRequest::max_media_dgram`] for the rest of the +/// session and puts [`Heartbeat::frames_sent`] in its heartbeats. Set only +/// for an iPad whose HELLO2 has [`FEATURE_MEDIA_SIZE`], so an older iPad +/// gets the same bytes as before. Bit 6 is taken by another change in flight. +pub const HOSTCAP_MEDIA_SIZE: u16 = 1 << 7; +/// The smallest media datagram an iPad may ask for: the IPv4 minimum +/// reassembly size, also the smallest packet size the PC's own setting takes. +pub const MIN_MEDIA_DGRAM: u16 = 576; /// [`StreamConfig::flags`] bits. pub const STREAM_FLAG_SOFTWARE_ENCODER: u8 = 1 << 0; @@ -271,6 +284,10 @@ pub struct Heartbeat { pub stream_config: StreamConfig, /// What the iPad should say over the picture, for as long as it lasts. pub notice: u8, + /// Video frames sent to this session so far, for an iPad that set + /// [`FEATURE_MEDIA_SIZE`]; `None` for others. Appended after `notice`, + /// which is then always written. + pub frames_sent: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -323,6 +340,11 @@ pub struct KeyframeRequest { pub stream_epoch: u32, pub last_complete_seq: u32, pub reason: KeyframeReason, + /// The largest media datagram the client asks for from now on in this + /// session, at least [`MIN_MEDIA_DGRAM`]; 0 asks nothing. Appended + /// (0.3.5): sent only to a host with [`HOSTCAP_MEDIA_SIZE`], and older + /// hosts ignore it as any trailing body bytes. + pub max_media_dgram: u16, } /// Requests missing fragments from one frame. Indices are unique and sorted. @@ -566,15 +588,21 @@ pub fn encode_control(session_id: u32, msg_seq: u32, message: &ControlMessage) - let mut cfg = [0u8; STREAM_CONFIG_SIZE]; hb.stream_config.encode_into(&mut cfg); body.extend_from_slice(&cfg); - if hb.notice != NOTICE_NONE { + if hb.notice != NOTICE_NONE || hb.frames_sent.is_some() { body.push(hb.notice); } + if let Some(frames) = hb.frames_sent { + body.extend_from_slice(&frames.to_le_bytes()); + } } ControlMessage::Bye(reason) => body.push(*reason as u8), ControlMessage::KeyframeRequest(k) => { body.extend_from_slice(&k.stream_epoch.to_le_bytes()); body.extend_from_slice(&k.last_complete_seq.to_le_bytes()); body.push(k.reason as u8); + if k.max_media_dgram != 0 { + body.extend_from_slice(&k.max_media_dgram.to_le_bytes()); + } } ControlMessage::Nack(n) => { n.validate().expect("invalid outgoing NACK"); @@ -785,6 +813,11 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), } else { NOTICE_NONE }, + frames_sent: if r.remaining() >= 4 { + Some(r.u32()?) + } else { + None + }, }), PacketType::Bye => ControlMessage::Bye( ByeReason::from_u8(r.u8()?).ok_or(WireError::InvalidField("bye_reason"))?, @@ -794,6 +827,7 @@ pub fn parse_control(datagram: &[u8]) -> Result<(ControlHeader, ControlMessage), last_complete_seq: r.u32()?, reason: KeyframeReason::from_u8(r.u8()?) .ok_or(WireError::InvalidField("keyframe_reason"))?, + max_media_dgram: if r.remaining() >= 2 { r.u16()? } else { 0 }, }), PacketType::Nack => { let stream_epoch = r.u32()?; @@ -1134,12 +1168,26 @@ mod tests { host_time_us: 987_654_321, stream_config: sample_config(), notice: NOTICE_NONE, + frames_sent: None, + }), + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: sample_config(), + notice: NOTICE_NONE, + frames_sent: Some(0), }), ControlMessage::Bye(ByeReason::AppBackground), ControlMessage::KeyframeRequest(KeyframeRequest { stream_epoch: 3, last_complete_seq: 4111, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, + }), + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 3, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram: 1200, }), ControlMessage::ReceiverReport(ReceiverReport { stream_epoch: 3, @@ -1232,6 +1280,7 @@ mod tests { host_time_us: 987_654_321, stream_config: sample_config(), notice, + frames_sent: None, }) }; // No notice: the bytes every earlier iPad already reads. @@ -1252,6 +1301,53 @@ mod tests { assert_eq!(parse_control(&plain).unwrap().1, heartbeat(NOTICE_NONE)); } + /// The media size fields are appended: without them the bytes are the + /// ones every earlier PC and iPad reads, and a body cut back to the old + /// length, as an older parser reads it, is the request without them. + #[test] + fn the_media_size_fields_are_appended_only_when_set() { + let request = |max_media_dgram| { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 3, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram, + }) + }; + let plain = encode_control(7, 1, &request(0)); + assert_eq!(plain.len(), CONTROL_HEADER_SIZE + 9); + let sized = encode_control(7, 1, &request(576)); + assert_eq!(sized.len(), plain.len() + 2); + assert_eq!(sized[8..plain.len()], plain[8..]); + assert_eq!(&sized[plain.len()..], &576u16.to_le_bytes()); + assert_eq!(parse_control(&sized).unwrap().1, request(576)); + assert_eq!(parse_control(&plain).unwrap().1, request(0)); + + let heartbeat = |notice, frames_sent| { + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: sample_config(), + notice, + frames_sent, + }) + }; + let plain = encode_control(7, 1, &heartbeat(NOTICE_NONE, None)); + // The count needs the notice byte before it, even with no notice. + let counted = encode_control(7, 1, &heartbeat(NOTICE_NONE, Some(0x0102_0304))); + assert_eq!(counted.len(), plain.len() + 5); + assert_eq!(counted[8..plain.len()], plain[8..]); + assert_eq!(&counted[plain.len()..], &[NOTICE_NONE, 4, 3, 2, 1]); + assert_eq!( + parse_control(&counted).unwrap().1, + heartbeat(NOTICE_NONE, Some(0x0102_0304)) + ); + let locked = encode_control(7, 1, &heartbeat(NOTICE_PC_LOCKED, Some(9))); + assert_eq!( + parse_control(&locked).unwrap().1, + heartbeat(NOTICE_PC_LOCKED, Some(9)) + ); + } + #[test] fn trailing_bytes_are_ignored_for_forward_compat() { let mut datagram = encode_control(1, 1, &ControlMessage::Ping(Ping { t1_us: 5 })); diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 82419b11..98386fcd 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -23,6 +23,8 @@ heartbeat 454d021200001800785634122a000000b168de3a0000000003000000000aa0053c0000 heartbeat_pc_locked 454d021200001900785634122a000000b168de3a0000000003000000000aa0053c000000c0e1e40001 bye_background 454d021300000100785634122b00000001 keyframe_request 454d021400000900785634122c000000030000000f10000000 +keyframe_request_media_size 454d021400000b007856341233000000030000000000000002b004 +heartbeat_frames_sent 454d021200001d007856341234000000b168de3a0000000003000000000aa0053c000000c0e1e40000d2040000 receiver_report 454d021500002400785634122d00000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00 ping 454d021600000800785634122e0000000807060504030201 pong 454d021700001800785634122e000000080706050403020110070605040302012007060504030201 @@ -32,6 +34,7 @@ nack 454d021900001100785634123100000003000000891300002c0103000011002b01 receiver_report_v030 454d021500003200785634123200000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00d204000037020000590000003c00 hello_ack_paired 454d02110000390000000000030000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f100100 hello_ack_contiguous_seq 454d02110000390000000000050000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f102100 +hello_ack_media_size 454d02110000390000000000060000000002efbeadde78563412e803f401b80b03000000000aa0053c000000c0e1e40006414c492d50430102030405060708090a0b0c0d0e0f10a100 hello2_v030 454d02100000410000000000040000000202efbeadde94260300010074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e20100 hello2_standby_padded 454d02100000940000000000040000000202efbeadde94260300090074098406ba044203780e416c69277320695061642050726fefcdab8967452301780102030405060708090a0b0c0d0e0f1040e201000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 identify 454d021a00004400000000000100000001a0a1a2a3a4a5a6a7a8a9aaabacadaeaf000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index 968c6cc1..2a383437 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -249,6 +249,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { host_time_us: 987_654_321, stream_config: stream_config(), notice: NOTICE_NONE, + frames_sent: None, }), ), ( @@ -259,6 +260,7 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { host_time_us: 987_654_321, stream_config: stream_config(), notice: NOTICE_PC_LOCKED, + frames_sent: None, }), ), ( @@ -275,6 +277,29 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { stream_epoch: 3, last_complete_seq: 4111, reason: KeyframeReason::GapLoss, + max_media_dgram: 0, + }), + ), + ( + "keyframe_request_media_size", + 0x1234_5678, + 51, + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: 3, + last_complete_seq: 0, + reason: KeyframeReason::Startup, + max_media_dgram: 1200, + }), + ), + ( + "heartbeat_frames_sent", + 0x1234_5678, + 52, + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: stream_config(), + notice: NOTICE_NONE, + frames_sent: Some(1234), }), ), ( @@ -398,7 +423,12 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { ack.host_caps = HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ; vectors.push(( "hello_ack_contiguous_seq", - encode_control(0, 5, &ControlMessage::HelloAck(ack)), + encode_control(0, 5, &ControlMessage::HelloAck(ack.clone())), + )); + ack.host_caps = HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ | HOSTCAP_MEDIA_SIZE; + vectors.push(( + "hello_ack_media_size", + encode_control(0, 6, &ControlMessage::HelloAck(ack)), )); let (_, ControlMessage::Hello2(mut hello)) = parse_control( &vectors @@ -704,3 +734,41 @@ fn contiguous_frame_seq_capability_round_trips() { assert_eq!(caps("hello_ack_paired") & HOSTCAP_CONTIGUOUS_FRAME_SEQ, 0); assert_eq!(HOSTCAP_CONTIGUOUS_FRAME_SEQ, 0x0020); } + +/// A host that takes smaller media datagrams says so; the request and the +/// heartbeat count are appended to the bodies earlier versions read. +#[test] +fn media_size_vectors_round_trip_and_extend_the_old_bodies() { + let vectors: BTreeMap<_, _> = canonical_vectors().into_iter().collect(); + let message = |name: &str| parse_control(&vectors[name]).unwrap().1; + let ControlMessage::HelloAck(ack) = message("hello_ack_media_size") else { + panic!("hello_ack_media_size") + }; + assert_eq!( + ack.host_caps, + HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ | HOSTCAP_MEDIA_SIZE + ); + assert_eq!(HOSTCAP_MEDIA_SIZE, 0x0080); + assert_eq!(FEATURE_MEDIA_SIZE, 0x0040); + assert_eq!(MIN_MEDIA_DGRAM, 576); + + let ControlMessage::KeyframeRequest(request) = message("keyframe_request_media_size") else { + panic!("keyframe_request_media_size") + }; + assert_eq!(request.max_media_dgram, 1200); + assert_eq!(request.reason, KeyframeReason::Startup); + let sized = &vectors["keyframe_request_media_size"]; + let plain = &vectors["keyframe_request"]; + assert_eq!(sized.len(), plain.len() + 2); + + let ControlMessage::Heartbeat(heartbeat) = message("heartbeat_frames_sent") else { + panic!("heartbeat_frames_sent") + }; + assert_eq!(heartbeat.frames_sent, Some(1234)); + assert_eq!(heartbeat.notice, NOTICE_NONE); + let counted = &vectors["heartbeat_frames_sent"]; + let plain = &vectors["heartbeat"]; + assert_eq!(counted.len(), plain.len() + 5); + // Past the msg_seq, the old heartbeat is a prefix of the new one. + assert_eq!(counted[16..plain.len()], plain[16..]); +} From 5f8270174ff3377762346f6de05e6aa9005b3d24 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 03:10:45 -0500 Subject: [PATCH 148/226] Count an unread root device as an unreadable driver state The scan of root devices used to drop a device it couldn't locate, or whose hardware IDs it couldn't read. A failed read could then leave no device of the driver, which counted as off. The next successful read would record a start for a driver that was already running. - The scan now keeps every present root device as one of: - not the driver's: its hardware IDs were read and don't name it, or it has none (CR_NO_SUCH_VALUE); - the driver's, with its status; - unread. - Any unread device, or a driver device whose status can't be read, makes the state unreadable. - A pure run_from classifies the scan, so a test can inject a failed read: the state is unreadable and no record is kept. --- host/src/vdd_device.rs | 239 ++++++++++++++++++++++++++++++++--------- 1 file changed, 187 insertions(+), 52 deletions(-) diff --git a/host/src/vdd_device.rs b/host/src/vdd_device.rs index 1f403c15..cd76ac66 100644 --- a/host/src/vdd_device.rs +++ b/host/src/vdd_device.rs @@ -144,17 +144,20 @@ pub fn query() -> DeviceState { CM_Get_Child, CM_DRP_DRIVER, CR_SUCCESS, }; - let Some(devnodes) = driver_devnodes() else { + let Some(devnodes) = root_devnodes() else { return DeviceState::Missing; }; - let states = devnodes.into_iter().filter_map(|devinst| { - let (flags, problem) = devnode_status(devinst)?; + let states = devnodes.into_iter().filter_map(|devnode| { + let Devnode::Driver { devinst, status } = devnode else { + return None; + }; + let (flags, problem) = status?; let mut child = 0u32; let monitor = (unsafe { CM_Get_Child(&mut child, devinst, 0) } == CR_SUCCESS).then(|| { devnode_status(child) .is_some_and(|(flags, _)| flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0) }); - let has_driver = registry_property(devinst, CM_DRP_DRIVER).is_some(); + let has_driver = matches!(registry_property(devinst, CM_DRP_DRIVER), Ok(Some(_))); Some(DeviceState::from_devnode( flags, problem, has_driver, monitor, )) @@ -162,41 +165,78 @@ pub fn query() -> DeviceState { DeviceState::best(states) } -/// The driver as Configuration Manager reads it now -/// (`vdd_settings::DriverRun`): its one running start (this boot and its -/// device's last arrival), off when every device of the driver reads as -/// not started (or there is none), and unreadable when a read fails or +/// One present root device, as Configuration Manager read it. +#[cfg(any(windows, test))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Devnode { + /// Its hardware IDs were read (or it has none) and don't name the driver. + Other, + /// The driver's device, with its status flags and problem code, `None` + /// when they couldn't be read. + Driver { + devinst: u32, + status: Option<(u32, u32)>, + }, + /// It couldn't be located, or its hardware IDs couldn't be read: it may + /// be the driver's. + Unread, +} + +/// The driver as Configuration Manager reads it (`vdd_settings::DriverRun`), +/// from the present root devices (`None` when they couldn't be listed): +/// off when every device of the driver reads as not started, or there is +/// none; running with the start `start` reads for its one running device; +/// unreadable when any read failed, a device started with a problem, or /// more than one runs. +#[cfg(any(windows, test))] +pub(crate) fn run_from( + devnodes: Option<&[Devnode]>, + start: impl FnOnce(u32) -> Option, +) -> crate::vdd_settings::DriverRun { + use crate::vdd_settings::DriverRun; + let Some(devnodes) = devnodes else { + return DriverRun::Unreadable; + }; + let mut running = None; + for devnode in devnodes { + match *devnode { + Devnode::Other => {} + Devnode::Unread | Devnode::Driver { status: None, .. } => return DriverRun::Unreadable, + Devnode::Driver { + devinst, + status: Some((flags, _)), + } => { + if flags & DN_STARTED == 0 { + continue; + } + if flags & DN_HAS_PROBLEM != 0 || running.replace(devinst).is_some() { + return DriverRun::Unreadable; + } + } + } + } + match running { + None => DriverRun::Off, + Some(devinst) => start(devinst).map_or(DriverRun::Unreadable, DriverRun::Running), + } +} + +/// The driver as Configuration Manager reads it now (`run_from`). #[cfg(windows)] pub fn run_state() -> crate::vdd_settings::DriverRun { - use crate::vdd_settings::{DriverInstance, DriverRun}; + run_from(root_devnodes().as_deref(), driver_start) +} + +/// The start of the driver running on `devinst`: this boot and the +/// device's last arrival. +#[cfg(windows)] +fn driver_start(devinst: u32) -> Option { use windows::Win32::Devices::DeviceAndDriverInstallation::{ CM_Get_DevNode_PropertyW, CR_SUCCESS, }; use windows::Win32::Devices::Properties::{ DEVPKEY_Device_LastArrivalDate, DEVPROPTYPE, DEVPROP_TYPE_FILETIME, }; - - let Some(devnodes) = driver_devnodes() else { - return DriverRun::Unreadable; - }; - let mut running = Vec::new(); - for devinst in devnodes { - let Some((flags, _)) = devnode_status(devinst) else { - return DriverRun::Unreadable; - }; - if flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0 { - running.push(devinst); - } else if flags & DN_STARTED != 0 { - // Started with a problem: neither cleanly off nor running. - return DriverRun::Unreadable; - } - } - let devinst = match running[..] { - [] => return DriverRun::Off, - [devinst] => devinst, - _ => return DriverRun::Unreadable, - }; let (mut arrival, mut kind, mut size) = (0u64, DEVPROPTYPE(0), 8u32); let result = unsafe { CM_Get_DevNode_PropertyW( @@ -209,12 +249,12 @@ pub fn run_state() -> crate::vdd_settings::DriverRun { ) }; if result != CR_SUCCESS || kind != DEVPROP_TYPE_FILETIME || size != 8 || arrival == 0 { - return DriverRun::Unreadable; - } - match boot_id() { - Some(boot_id) => DriverRun::Running(DriverInstance { boot_id, arrival }), - None => DriverRun::Unreadable, + return None; } + Some(crate::vdd_settings::DriverInstance { + boot_id: boot_id()?, + arrival, + }) } /// Windows' count of boots, which the kernel raises at each one. @@ -238,10 +278,11 @@ fn boot_id() -> Option { .then_some(value) } -/// The present root devices with the driver's hardware ID, or `None` when -/// Windows can't list them. +/// The present root devices (`Devnode`), or `None` when Windows can't list +/// them. A device that can't be located or whose hardware IDs can't be read +/// is kept as `Unread`, never dropped. #[cfg(windows)] -fn driver_devnodes() -> Option> { +fn root_devnodes() -> Option> { use windows::core::{w, PCWSTR}; use windows::Win32::Devices::DeviceAndDriverInstallation::{ CM_Get_Device_ID_ListW, CM_Get_Device_ID_List_SizeW, CM_Locate_DevNodeW, CM_DRP_HARDWAREID, @@ -261,20 +302,29 @@ fn driver_devnodes() -> Option> { Some( ids.split(|&unit| unit == 0) .filter(|id| !id.is_empty()) - .filter_map(|id| { + .map(|id| { let id: Vec = id.iter().copied().chain(Some(0)).collect(); let mut devinst = 0u32; if unsafe { CM_Locate_DevNodeW(&mut devinst, PCWSTR(id.as_ptr()), CM_LOCATE_DEVNODE_NORMAL) } != CR_SUCCESS { - return None; + return Devnode::Unread; + } + match registry_property(devinst, CM_DRP_HARDWAREID) { + Err(()) => Devnode::Unread, + Ok(Some(hardware_ids)) + if hardware_ids.split(|&unit| unit == 0).any(|hwid| { + String::from_utf16_lossy(hwid).eq_ignore_ascii_case(HARDWARE_ID) + }) => + { + Devnode::Driver { + devinst, + status: devnode_status(devinst), + } + } + Ok(_) => Devnode::Other, } - let hardware_ids = registry_property(devinst, CM_DRP_HARDWAREID)?; - hardware_ids - .split(|&unit| unit == 0) - .any(|hwid| String::from_utf16_lossy(hwid).eq_ignore_ascii_case(HARDWARE_ID)) - .then_some(devinst) }) .collect(), ) @@ -290,16 +340,18 @@ fn devnode_status(devinst: u32) -> Option<(u32, u32)> { .then_some((status.0, problem.0)) } +/// A device's registry property: `Ok(None)` when it has none +/// (`CR_NO_SUCH_VALUE`), `Err` when the read failed. #[cfg(windows)] -fn registry_property(devinst: u32, which: u32) -> Option> { +fn registry_property(devinst: u32, which: u32) -> Result>, ()> { use windows::Win32::Devices::DeviceAndDriverInstallation::{ - CM_Get_DevNode_Registry_PropertyW, CR_BUFFER_SMALL, CR_SUCCESS, + CM_Get_DevNode_Registry_PropertyW, CR_BUFFER_SMALL, CR_NO_SUCH_VALUE, CR_SUCCESS, }; let mut bytes = 0u32; - let result = - unsafe { CM_Get_DevNode_Registry_PropertyW(devinst, which, None, None, &mut bytes, 0) }; - if result != CR_BUFFER_SMALL || bytes == 0 { - return None; + match unsafe { CM_Get_DevNode_Registry_PropertyW(devinst, which, None, None, &mut bytes, 0) } { + CR_NO_SUCH_VALUE => return Ok(None), + CR_BUFFER_SMALL if bytes > 0 => {} + _ => return Err(()), } let mut buffer = vec![0u16; bytes.div_ceil(2) as usize]; (unsafe { @@ -312,7 +364,8 @@ fn registry_property(devinst: u32, which: u32) -> Option> { 0, ) } == CR_SUCCESS) - .then_some(buffer) + .then_some(Some(buffer)) + .ok_or(()) } /// Set the mode of the virtual display `device` (a GDI name like @@ -394,6 +447,88 @@ pub fn query() -> DeviceState { mod tests { use super::*; + /// Review r6: a root device whose location or hardware IDs couldn't be + /// read may be the driver's, so the driver's state is unreadable, not + /// off, and an Enable after it keeps no record. One whose hardware IDs + /// were read and don't name the driver is not the driver's. + #[test] + fn a_device_that_could_not_be_read_makes_the_driver_unreadable() { + use crate::vdd_settings::{DriverCursor, DriverInstance, DriverRun}; + let start = DriverInstance { + boot_id: 205, + arrival: 30, + }; + let starts = |devinst| (devinst == 7).then_some(start); + let off = Devnode::Driver { + devinst: 7, + status: Some((0, 22)), + }; + let on = Devnode::Driver { + devinst: 7, + status: Some((DN_STARTED, 0)), + }; + let unread = [Devnode::Other, off, Devnode::Unread]; + let before = run_from(Some(&unread), starts); + assert_eq!(before, DriverRun::Unreadable); + assert_eq!( + crate::vdd_settings::record_after_enable( + before, + run_from(Some(&[Devnode::Other, on]), starts), + Some(""), + DriverCursor::Software, + ), + None, + "no record after an unreadable before" + ); + + for (devnodes, run) in [ + (None, DriverRun::Unreadable), + (Some(&[][..]), DriverRun::Off), + (Some(&[Devnode::Other][..]), DriverRun::Off), + (Some(&[Devnode::Other, off][..]), DriverRun::Off), + (Some(&[Devnode::Unread][..]), DriverRun::Unreadable), + (Some(&[Devnode::Unread, on][..]), DriverRun::Unreadable), + (Some(&[Devnode::Other, on][..]), DriverRun::Running(start)), + ( + Some( + &[Devnode::Driver { + devinst: 7, + status: None, + }][..], + ), + DriverRun::Unreadable, + ), + ( + Some( + &[Devnode::Driver { + devinst: 7, + status: Some((DN_STARTED | DN_HAS_PROBLEM, 43)), + }][..], + ), + DriverRun::Unreadable, + ), + ( + Some( + &[ + on, + Devnode::Driver { + devinst: 8, + status: Some((DN_STARTED, 0)), + }, + ][..], + ), + DriverRun::Unreadable, + ), + ] { + assert_eq!(run_from(devnodes, starts), run, "{devnodes:?}"); + } + assert_eq!( + run_from(Some(&[on]), |_| None), + DriverRun::Unreadable, + "its start couldn't be read" + ); + } + #[test] fn the_enable_task_result_names_the_device_state() { assert_eq!(DeviceState::from_task_result(0), None); From f3c6ef5ded670f863ee1258c524364c0dd445c5d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 02:52:01 -0500 Subject: [PATCH 149/226] Read vdd_settings.xml saved in a Windows code page A settings file saved in the system's ANSI code page, as Notepad does, was not UTF-8 or UTF-16, so the host kept the driver's old modes and skipped the graphics card pin. The driver reads the file with XmlLite, which takes UTF-8 and UTF-16 but no double-byte code page: declared, it reads nothing; undeclared or declared UTF-8, it stops at the first non-ASCII byte. The host now decodes such a file in the code page its declaration names, or in the system's ANSI code page, and writes it back as UTF-8 with a matching declaration, which the driver reads to the end. The original is kept as vdd_settings.xml.bak- first. Bytes that are no character in that code page leave the file alone, as before. --- host/src/vdd_settings.rs | 417 ++++++++++++++++-- host/tests/fixtures/README.md | 6 + .../fixtures/vdd_settings-25.5.2-cp936.xml | 98 ++++ .../fixtures/vdd_settings-25.5.2-cp949.xml | 98 ++++ 4 files changed, 570 insertions(+), 49 deletions(-) create mode 100644 host/tests/fixtures/vdd_settings-25.5.2-cp936.xml create mode 100644 host/tests/fixtures/vdd_settings-25.5.2-cp949.xml diff --git a/host/src/vdd_settings.rs b/host/src/vdd_settings.rs index 1a6f275c..cc93ff5d 100644 --- a/host/src/vdd_settings.rs +++ b/host/src/vdd_settings.rs @@ -389,13 +389,13 @@ pub fn with_gpu(xml: &str, gpu: &str) -> Option { /// would leave it none. A file that does not parse as expected is left /// alone, and the error says why. pub fn pin_gpu_in(path: &Path, gpu: &str) -> io::Result { - let current = read_settings(path)?; + let (current, legacy) = read_settings(path)?; let updated = gpu_edit(¤t, gpu) .map_err(|reason| io::Error::new(io::ErrorKind::InvalidData, reason))?; if updated == current { return Ok(false); } - replace(path, &updated)?; + replace_settings(path, legacy, &updated)?; Ok(true) } @@ -551,7 +551,7 @@ pub fn apply_pin( wanted: Option<&str>, ) -> io::Result { let invalid = |reason| io::Error::new(io::ErrorKind::InvalidData, reason); - let current = read_settings(path)?; + let (current, legacy) = read_settings(path)?; let named = named_gpu(¤t).map_err(invalid)?; let ours = recorded_pins(record); let ours: Vec<&str> = ours.iter().map(String::as_str).collect(); @@ -563,7 +563,9 @@ pub fn apply_pin( }; let updated = gpu_edit(¤t, &name).map_err(invalid)?; let new_record = (outcome == PinOutcome::Named).then_some(name.as_str()); - record_around(record, new_record, &ours, || replace(path, &updated))?; + record_around(record, new_record, &ours, || { + replace_settings(path, legacy, &updated) + })?; Ok(outcome) } @@ -622,13 +624,13 @@ pub fn write_with_pin( modes: &[VddMode], wanted: Option<&str>, ) -> io::Result { - let named = match read_settings(path) { - Ok(current) => match named_gpu(¤t) { - Ok(named) => named, - Err(_) if owned && wanted.is_some() => None, + let (named, legacy) = match read_settings(path) { + Ok((current, legacy)) => match named_gpu(¤t) { + Ok(named) => (named, legacy), + Err(_) if owned && wanted.is_some() => (None, legacy), Err(reason) => return Err(io::Error::new(io::ErrorKind::InvalidData, reason)), }, - Err(error) if error.kind() == io::ErrorKind::NotFound => None, + Err(error) if error.kind() == io::ErrorKind::NotFound => (None, false), Err(error) => return Err(error), }; let ours = recorded_pins(record); @@ -639,7 +641,12 @@ pub fn write_with_pin( PinChange::Write(card) => (Some(card), PinOutcome::Named), PinChange::Reset => (None, PinOutcome::Reset), }; - let write = || write_to(path, modes, name.as_deref()); + let write = || { + if legacy { + back_up(path)?; + } + write_to(path, modes, name.as_deref()) + }; match outcome { PinOutcome::Named => record_around(record, name.as_deref(), &ours, write)?, PinOutcome::Reset => record_around(record, None, &ours, write)?, @@ -662,31 +669,42 @@ pub fn pin_result( } } -/// The settings file as text. Another tool may have saved it as UTF-16 -/// (Windows PowerShell's `Out-File` does), with a byte order mark or, as XML -/// allows, without one; it is read and, when this host writes it, written -/// back as UTF-8, the encoding the driver's own file has and every file this -/// host writes. A UTF-16 document's `encoding="utf-16"` declaration becomes -/// `utf-8` with it, so the declaration always names what is on disk. UTF-8 -/// keeps any byte order mark. Any other encoding, UTF-32 included, is left -/// alone, and the error says so. -fn read_settings(path: &Path) -> io::Result { - decode_settings(&std::fs::read(path)?).ok_or_else(|| { +/// The settings file as text, and whether it was in a legacy code page. +/// Another tool may have saved it as UTF-16 (Windows PowerShell's +/// `Out-File` does), with a byte order mark or, as XML allows, without one; +/// or in a Windows code page, as Notepad's ANSI does, keeping the driver's +/// `encoding='utf-8'` declaration or naming the code page. It is read and, +/// when this host writes it, written back as UTF-8, the encoding the +/// driver's own file has and every file this host writes. The driver's XML +/// reader takes UTF-8 and UTF-16 but not double-byte code pages: declared, +/// it reads nothing; otherwise it stops at the first byte it cannot read, +/// keeping only the settings before it. A UTF-16 or code page +/// document's declaration names UTF-8 with it, so the declaration always +/// names what is on disk. UTF-8 keeps any byte order mark. Any other +/// encoding, UTF-32 included, is left alone, and the error says so. +fn read_settings(path: &Path) -> io::Result<(String, bool)> { + decode_settings(&std::fs::read(path)?, ANSI).ok_or_else(|| { io::Error::new( io::ErrorKind::InvalidData, - "the settings file is neither UTF-8 nor UTF-16", + "the settings file is in no encoding this host reads", ) }) } -fn decode_settings(bytes: &[u8]) -> Option { - let text = decode_text(bytes)?; +/// `CP_ACP`: the system's ANSI code page, which Notepad and other older +/// tools save text in. +const ANSI: u32 = 0; + +/// `bytes` as text, and whether they were in a code page; text that is not +/// UTF and declares no other encoding is in code page `ansi`. +fn decode_settings(bytes: &[u8], ansi: u32) -> Option<(String, bool)> { + let (text, legacy) = decode_text(bytes, ansi)?; // XML never holds NUL: text with one was read in the wrong encoding, // such as UTF-32 without a byte order mark, which reads as UTF-8. - (!text.contains('\0')).then_some(text) + (!text.contains('\0')).then_some((text, legacy)) } -fn decode_text(bytes: &[u8]) -> Option { +fn decode_text(bytes: &[u8], ansi: u32) -> Option<(String, bool)> { // UTF-32's byte order marks; the little-endian one starts like UTF-16's. if bytes.starts_with(&[0xFF, 0xFE, 0, 0]) || bytes.starts_with(&[0, 0, 0xFE, 0xFF]) { return None; @@ -697,7 +715,7 @@ fn decode_text(bytes: &[u8]) -> Option { // " (bytes, false), [0, b'<', 0, b'?', ..] => (bytes, true), - _ => return String::from_utf8(bytes.to_vec()).ok(), + _ => return decode_bytes(bytes, ansi), }; let (pairs, odd) = units.as_chunks::<2>(); if !odd.is_empty() { @@ -713,26 +731,135 @@ fn decode_text(bytes: &[u8]) -> Option { let text = char::decode_utf16(units) .collect::>() .ok()?; - Some(declared_utf8(text)) + let label = encoding_label(text.as_bytes()).map(|label| text[label].to_ascii_lowercase()); + match label.as_deref() { + Some("utf-16" | "utf-16le" | "utf-16be") => Some((declared_utf8(text), false)), + _ => Some((text, false)), + } } -/// `xml` with its declaration's UTF-16 encoding named as UTF-8, quote and -/// all else unchanged. -fn declared_utf8(xml: String) -> String { - let Some(end) = xml.starts_with("")).flatten() else { - return xml; +/// Bytes in an 8-bit encoding: UTF-8, or the code page the declaration +/// names. Without one, bytes that are not UTF-8 are in code page `ansi`; +/// so are bytes declared UTF-8 that are not, as Notepad leaves the driver's +/// file when it saves it as ANSI. +fn decode_bytes(bytes: &[u8], ansi: u32) -> Option<(String, bool)> { + let label = encoding_label(bytes).map(|label| bytes[label].to_ascii_lowercase()); + let code_page = match label.as_deref() { + None | Some(b"utf-8") => match String::from_utf8(bytes.to_vec()) { + Ok(text) => return Some((text, false)), + Err(_) => ansi, + }, + Some(label) => match std::str::from_utf8(label).ok().and_then(code_page) { + Some(code_page) => code_page, + // The declaration is not one this host edits. + None => { + return String::from_utf8(bytes.to_vec()) + .ok() + .map(|text| (text, false)) + } + }, }; + let text = decode_code_page(bytes, code_page)?; + Some((declared_utf8(text), true)) +} + +/// The Windows multi-byte code page an XML encoding name stands for. The +/// driver's reader takes none of these, declared or not; it does take the +/// single-byte ones, so a file declaring one stays as it is. +fn code_page(label: &str) -> Option { + let numbered = ["windows-", "x-windows-", "cp", "x-cp"] + .iter() + .find_map(|prefix| label.strip_prefix(prefix)?.parse().ok()); + let code_page = numbered.or(match label { + "shift_jis" | "shift-jis" | "sjis" | "ms_kanji" | "windows-31j" => Some(932), + "gb2312" | "gbk" | "x-gbk" => Some(936), + "euc-kr" | "ks_c_5601-1987" | "ks_c_5601-1989" | "ksc5601" | "uhc" => Some(949), + "big5" => Some(950), + "gb18030" => Some(54936), + _ => None, + })?; + [932, 936, 949, 950, 54936] + .contains(&code_page) + .then_some(code_page) +} + +/// `bytes` in Windows code page `code_page`; `None` when any byte is not +/// a character there, or off Windows. +#[cfg(windows)] +fn decode_code_page(bytes: &[u8], code_page: u32) -> Option { + use windows::Win32::Globalization::{MultiByteToWideChar, MB_ERR_INVALID_CHARS}; + if bytes.is_empty() { + return Some(String::new()); + } + let length = unsafe { MultiByteToWideChar(code_page, MB_ERR_INVALID_CHARS, bytes, None) }; + let mut wide = vec![0u16; usize::try_from(length).ok().filter(|&n| n > 0)?]; + let length = + unsafe { MultiByteToWideChar(code_page, MB_ERR_INVALID_CHARS, bytes, Some(&mut wide)) }; + String::from_utf16(wide.get(..usize::try_from(length).ok().filter(|&n| n > 0)?)?).ok() +} + +#[cfg(not(windows))] +fn decode_code_page(_bytes: &[u8], _code_page: u32) -> Option { + None +} + +/// Where the XML declaration at the start of `xml` names its encoding. +fn encoding_label(xml: &[u8]) -> Option> { + let end = xml + .starts_with(b""))??; let declaration = &xml[..end]; - for quote in ['"', '\''] { - for label in ["utf-16", "utf-16le", "utf-16be"] { - let pattern = format!("encoding={quote}{label}{quote}"); - if let Some(at) = declaration.to_ascii_lowercase().find(&pattern) { - let utf8 = format!("encoding={quote}utf-8{quote}"); - return format!("{}{utf8}{}", &xml[..at], &xml[at + pattern.len()..]); - } - } + let at = declaration.windows(9).position(|w| w == b"encoding=")? + 9; + let quote = *declaration.get(at).filter(|&&c| c == b'"' || c == b'\'')?; + let length = declaration[at + 1..].iter().position(|&c| c == quote)?; + Some(at + 1..at + 1 + length) +} + +/// `xml` with its declaration's encoding named as UTF-8, quote and all else +/// unchanged. +fn declared_utf8(xml: String) -> String { + match encoding_label(xml.as_bytes()) { + Some(label) => format!("{}utf-8{}", &xml[..label.start], &xml[label.end..]), + None => xml, } - xml +} + +/// Keep a file read from a code page as it was, beside it, before its +/// first replacement on a day: `vdd_settings.xml.bak-`. +fn back_up(path: &Path) -> io::Result<()> { + let mut backup = path.as_os_str().to_owned(); + backup.push(format!(".bak-{}", utc_date(std::time::SystemTime::now()))); + let backup = std::path::PathBuf::from(backup); + if backup.exists() { + return Ok(()); + } + std::fs::copy(path, backup).map(drop) +} + +/// `time`'s UTC date as `YYYYMMDD` (Howard Hinnant's civil_from_days). +fn utc_date(time: std::time::SystemTime) -> String { + let seconds = time + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |since| since.as_secs()); + let days = (seconds / 86_400) as i64 + 719_468; + let era = days / 146_097; + let doe = days - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = yoe + era * 400 + i64::from(month <= 2); + format!("{year:04}{month:02}{day:02}") +} + +/// Replace the settings file with `contents`, backing it up first when it +/// was read from a code page (`legacy`). +fn replace_settings(path: &Path, legacy: bool, contents: &str) -> io::Result<()> { + if legacy { + back_up(path)?; + } + replace(path, contents) } /// Write atomically (tmp + rename) so the driver never reads a torn file. @@ -1763,18 +1890,25 @@ mod tests { dir.join("vdd_settings.xml"), dir.join("app").join("vdd-pin.txt"), ); - let ansi = VENDOR + // Not UTF-8, in an encoding this host does not know. + let unknown = VENDOR + .replace("encoding='utf-8'", "encoding='x-unknown'") .replace(">default<", ">Radeon\u{AE}<") .replace('\u{AE}', ""); - let mut ansi = ansi.into_bytes(); - let at = ansi.windows(7).position(|w| w == b"Radeon<").unwrap() + 6; - ansi.insert(at, 0xAE); + let mut unknown = unknown.into_bytes(); + let at = unknown.windows(7).position(|w| w == b"Radeon<").unwrap() + 6; + unknown.insert(at, 0xAE); + // A double-byte character cut short. + let mut broken = declared(CP949, "euc-kr"); + let at = broken.iter().position(|&b| b >= 0x80).unwrap(); + broken[at + 1] = b' '; let mut odd = utf16(VENDOR, false, true); odd.push(b' '); let mut lone = utf16(VENDOR, false, true); lone.extend_from_slice(&0xD800u16.to_le_bytes()); for (name, bytes) in [ - ("ansi", ansi), + ("unknown encoding", unknown), + ("broken code page", broken), ("odd utf-16", odd), ("lone surrogate", lone), ] { @@ -1784,12 +1918,197 @@ mod tests { assert_eq!(error.kind(), io::ErrorKind::InvalidData, "{name}"); assert_eq!( error.to_string(), - "the settings file is neither UTF-8 nor UTF-16" + "the settings file is in no encoding this host reads" ); - let written = write_with_pin(&path, &record, false, &default_modes(), Some("x")); - assert!(written.is_err(), "{name}"); + for owned in [false, true] { + let written = write_with_pin(&path, &record, owned, &default_modes(), Some("x")); + assert!(written.is_err(), "{name}"); + } assert_eq!(std::fs::read(&path).unwrap(), bytes, "{name}"); + assert_eq!(backups(&dir), Vec::>::new(), "{name}"); + } + std::fs::remove_dir_all(&dir).unwrap(); + } + + /// The driver's file saved in code page 949 as Notepad's ANSI does: its + /// `encoding='utf-8'` declaration stays. A comment holds a folder path in + /// non-ASCII text, and so does the card's name. + const CP949: &[u8] = include_bytes!("../tests/fixtures/vdd_settings-25.5.2-cp949.xml"); + const VENDOR_DECLARATION: &str = "\r\n"; + + /// The code page 949 fixture and its code page 936 twin, each with its + /// code page, the text it holds, and the card it names. + #[cfg(windows)] + fn code_page_fixtures() -> [(&'static [u8], u32, String, &'static str); 2] { + let text = |comment: &str, card: &str| { + VENDOR_FILE + .replacen( + "\r\n", + &format!("\r\n \r\n"), + 1, + ) + .replace(">default<", &format!(">{card}<")) + }; + const CP949_CARD: &str = "Microsoft 기본 렌더링 드라이버"; + const CP936_CARD: &str = "Microsoft 基本呈现驱动程序"; + [ + ( + CP949, + 949, + text("가상 디스플레이 설정: D:\\도구\\디스플레이", CP949_CARD), + CP949_CARD, + ), + ( + include_bytes!("../tests/fixtures/vdd_settings-25.5.2-cp936.xml"), + 936, + text("虚拟显示器设置: D:\\工具\\显示器", CP936_CARD), + CP936_CARD, + ), + ] + } + + /// A code page fixture declaring `label` as its encoding instead. + fn declared(fixture: &[u8], label: &str) -> Vec { + let body = fixture.strip_prefix(VENDOR_DECLARATION.as_bytes()).unwrap(); + let declaration = format!("\r\n"); + [declaration.as_bytes(), body].concat() + } + + /// The backups beside the settings file in `dir`. + fn backups(dir: &Path) -> Vec> { + let mut found: Vec<_> = std::fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| { + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + name.starts_with("vdd_settings.xml.bak-") + }) + .collect(); + found.sort(); + found + .iter() + .map(|path| std::fs::read(path).unwrap()) + .collect() + } + + /// Field triage round 3: a settings file saved in a Windows code page + /// was "not valid UTF-8", so the second screen kept the driver's old + /// modes and the card was never named. Declared UTF-8 as Notepad leaves + /// it, or not declared, it is in the system's code page, given here. + #[cfg(windows)] + #[test] + fn a_code_page_settings_file_is_read_in_the_systems_code_page() { + const INTEL: &str = "Intel(R) HD Graphics 630"; + for (bytes, code_page, text, card) in code_page_fixtures() { + let undeclared = bytes.strip_prefix(VENDOR_DECLARATION.as_bytes()).unwrap(); + let body = text.strip_prefix(VENDOR_DECLARATION).unwrap(); + for (bytes, text) in [(bytes, text.as_str()), (undeclared, body)] { + assert_eq!( + decode_settings(bytes, code_page), + Some((text.to_string(), true)), + "{code_page}" + ); + assert_eq!(named_gpu(text), Ok(Some(card.to_string()))); + assert_eq!(with_gpu(text, INTEL), Some(text.replace(card, INTEL))); + } + // UTF-8 stays UTF-8 whatever the system's code page. + assert_eq!( + decode_settings(text.as_bytes(), code_page), + Some((text.clone(), false)) + ); + } + } + + /// A declaration may name the code page instead. Either way the file is + /// written back as UTF-8, declared so, which the driver's reader takes to + /// the end, and the file as it was is kept beside it. + #[cfg(windows)] + #[test] + fn a_code_page_settings_file_is_written_back_as_utf8_with_a_backup() { + let dir = std::env::temp_dir().join(format!("vdd-encoding-cp-{}", std::process::id())); + let (path, record) = ( + dir.join("vdd_settings.xml"), + dir.join("app").join("vdd-pin.txt"), + ); + const INTEL: &str = "Intel(R) HD Graphics 630"; + const AMD: &str = "AMD Radeon(TM) Graphics"; + let modes = modes_for_client(1640, 2360, 60); + let labels = [ + ( + 949, + &["euc-kr", "ks_c_5601-1987", "windows-949", "CP949"][..], + ), + (936, &["gb2312", "GBK", "windows-936", "cp936"][..]), + ]; + for (bytes, code_page, text, card) in code_page_fixtures() { + let utf8 = text.replacen( + VENDOR_DECLARATION, + "\r\n", + 1, + ); + let (_, labels) = labels.iter().find(|(page, _)| *page == code_page).unwrap(); + for label in *labels { + let declared = declared(bytes, label); + let fresh = || { + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(&path, &declared).unwrap(); + }; + + // Only the card changes, as the driver will read it. + fresh(); + let pinned = apply_pin(&path, &record, true, Some(INTEL)); + assert_eq!(pinned.ok(), Some(PinOutcome::Named), "{label}"); + let written = std::fs::read_to_string(&path).expect(label); + assert_eq!(written, utf8.replace(card, INTEL), "{label}"); + assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + // The file is UTF-8 now: the next write keeps the backup. + let pinned = apply_pin(&path, &record, true, Some(AMD)); + assert_eq!(pinned.ok(), Some(PinOutcome::Named), "{label}"); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + utf8.replace(card, AMD) + ); + assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + + // Match the iPad's resolution writes the whole file, keeping + // the card another app named. + fresh(); + let written = write_with_pin(&path, &record, false, &modes, Some(INTEL)); + assert_eq!( + written.ok(), + Some(PinOutcome::Kept(NAMED_ELSEWHERE)), + "{label}" + ); + let text = std::fs::read_to_string(&path).expect(label); + assert_eq!(text, render(&modes, Some(card)), "{label}"); + assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + } } std::fs::remove_dir_all(&dir).unwrap(); } + + #[test] + fn encoding_names_are_windows_code_pages() { + for (label, expected) in [ + ("euc-kr", Some(949)), + ("ks_c_5601-1987", Some(949)), + ("windows-949", Some(949)), + ("cp949", Some(949)), + ("gb2312", Some(936)), + ("gbk", Some(936)), + ("x-windows-936", Some(936)), + ("windows-1252", None), + ("iso-8859-1", None), + ("utf-16", None), + ("x-unknown", None), + ] { + assert_eq!(code_page(label), expected, "{label}"); + } + let at = |seconds| std::time::UNIX_EPOCH + std::time::Duration::from_secs(seconds); + assert_eq!(utc_date(at(0)), "19700101"); + assert_eq!(utc_date(at(951_868_800)), "20000301"); + assert_eq!(utc_date(at(1_709_251_199)), "20240229"); + assert_eq!(utc_date(at(1_794_096_000)), "20261108"); + } } diff --git a/host/tests/fixtures/README.md b/host/tests/fixtures/README.md index 767c2c01..028b7d6b 100644 --- a/host/tests/fixtures/README.md +++ b/host/tests/fixtures/README.md @@ -5,3 +5,9 @@ 25.5.2 (`Virtual Display Driver (HDR)/vdd_settings.xml`), byte for byte. It is MIT licensed by its authors. `host/src/vdd_settings.rs` tests that the host edits this file in place. + +`vdd_settings-25.5.2-cp949.xml` and `vdd_settings-25.5.2-cp936.xml` are that +file with a comment holding a folder path and a card name in non-ASCII text, +saved in Windows code page 949 and 936 as Notepad's ANSI does: the +`encoding='utf-8'` declaration stays. They were made with Python's `cp949` and +`gbk` codecs, so they do not depend on the Windows conversion under test. diff --git a/host/tests/fixtures/vdd_settings-25.5.2-cp936.xml b/host/tests/fixtures/vdd_settings-25.5.2-cp936.xml new file mode 100644 index 00000000..c42a794c --- /dev/null +++ b/host/tests/fixtures/vdd_settings-25.5.2-cp936.xml @@ -0,0 +1,98 @@ + + + + + 1 + + + Microsoft + + + + 60 + 90 + 120 + 144 + 165 + 244 + + + + 800 + 600 + 30 + + + 1366 + 768 + 30 + + + 1920 + 1080 + 30 + + + 2560 + 1440 + 30 + + + 3840 + 2160 + 30 + + + + + + true + false + + false + + + + + + + false + false + RGB + + + + true + + 128 + + 128 + + true + + 2 + + + + + false + + false + + false + + + \ No newline at end of file diff --git a/host/tests/fixtures/vdd_settings-25.5.2-cp949.xml b/host/tests/fixtures/vdd_settings-25.5.2-cp949.xml new file mode 100644 index 00000000..a31024c7 --- /dev/null +++ b/host/tests/fixtures/vdd_settings-25.5.2-cp949.xml @@ -0,0 +1,98 @@ + + + + + 1 + + + Microsoft ⺻ ̹ + + + + 60 + 90 + 120 + 144 + 165 + 244 + + + + 800 + 600 + 30 + + + 1366 + 768 + 30 + + + 1920 + 1080 + 30 + + + 2560 + 1440 + 30 + + + 3840 + 2160 + 30 + + + + + + true + false + + false + + + + + + + false + false + RGB + + + + true + + 128 + + 128 + + true + + 2 + + + + + false + + false + + false + + + \ No newline at end of file From 60f0b8b129a4619565f4db8409ddd2b849cc4e66 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 03:21:55 -0500 Subject: [PATCH 150/226] Translate the no-display notice --- .../Resources/Localizable.xcstrings | 60 +++++++++++++++++++ .../LocalizationTests.swift | 4 +- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/ios/EternalMonitor/Resources/Localizable.xcstrings b/ios/EternalMonitor/Resources/Localizable.xcstrings index c8c00b61..7c85e091 100644 --- a/ios/EternalMonitor/Resources/Localizable.xcstrings +++ b/ios/EternalMonitor/Resources/Localizable.xcstrings @@ -9191,11 +9191,71 @@ "comment" : "Over the held picture while no display is on at the PC: a closed laptop lid, or screens that are off or unplugged", "extractionState" : "manual", "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "لا توجد شاشة نشطة على PC. شغّل شاشة للمتابعة." + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "Dein PC hat keinen aktiven Bildschirm. Schalte einen Bildschirm ein, um fortzufahren." + } + }, "en" : { "stringUnit" : { "state" : "translated", "value" : "Your PC has no active display. Turn on a screen to continue." } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "Tu PC no tiene una pantalla activa. Enciende una pantalla para continuar." + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "Votre PC n’a pas d’écran actif. Allumez un écran pour continuer." + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "आपके PC पर कोई स्क्रीन चालू नहीं है। जारी रखने के लिए एक स्क्रीन चालू करें।" + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC に有効な画面がありません。続行するには画面をオンにしてください。" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC에 활성 화면이 없습니다. 계속하려면 화면을 켜세요." + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "Seu PC não tem uma tela ativa. Ligue uma tela para continuar." + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "На PC нет активного экрана. Включите экран, чтобы продолжить." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "PC 没有活动的显示器。打开显示器后即可继续。" + } } } }, diff --git a/ios/EternalMonitorTests/LocalizationTests.swift b/ios/EternalMonitorTests/LocalizationTests.swift index 4fbb0ad6..0db3ce8d 100644 --- a/ios/EternalMonitorTests/LocalizationTests.swift +++ b/ios/EternalMonitorTests/LocalizationTests.swift @@ -22,9 +22,7 @@ final class LocalizationTests: XCTestCase { /// English strings whose translations come in a separate pass. Until /// then every language shows them in English. - private let awaitingTranslation: Set = [ - "Your PC has no active display. Turn on a screen to continue.", - ] + private let awaitingTranslation: Set = [] /// Every language has exactly English's strings, none empty, each with /// the same placeholders. From 233001df06e2e69c27b5cf7467bcc9a4ad03b776 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 03:23:24 -0500 Subject: [PATCH 151/226] Keep the macOS release build free of an unused second screen method remember_panel_lost is only called from attach, which builds on Windows, in tests and in debug builds, so a macOS release build found it unused and release clippy failed. It now has the same cfg as attach. CI's macOS job also lints the workspace as released (libraries and binaries; the tests use debug-only fakes), which fails on this base and catches the next one. --- .github/workflows/ci.yml | 7 +++++++ host/src/capture/second_screen.rs | 1 + 2 files changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 691a0493..6239f5cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,6 +111,13 @@ jobs: env: PKG_CONFIG_PATH: /opt/homebrew/opt/ffmpeg@7/lib/pkgconfig run: cargo clippy --workspace --all-targets --locked -- -D warnings + - name: Lint workspace as released + # Code gated on debug_assertions or Windows can leave items unused in + # a macOS release build. Libraries and binaries only: the tests use + # debug-only fakes and don't build in release. + env: + PKG_CONFIG_PATH: /opt/homebrew/opt/ffmpeg@7/lib/pkgconfig + run: cargo clippy --workspace --release --locked -- -D warnings - name: Test workspace (includes synthetic end-to-end stream) env: PKG_CONFIG_PATH: /opt/homebrew/opt/ffmpeg@7/lib/pkgconfig diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index e406fbed..2591f54a 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -434,6 +434,7 @@ impl State { /// The display took the built-in panel away: save that until Second /// screen is chosen again. Neither a blocked mirror nor a withheld /// desktop excuses it, since trying again takes the panel again. + #[cfg(any(windows, test, debug_assertions))] fn remember_panel_lost(&mut self, now: Instant) { self.clear_blocked_retry(); if let Some(hold) = self.memory.note_failure(PANEL_LOST, now) { From 900097192486c2be3ea369d201dafda072590114 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 03:32:46 -0500 Subject: [PATCH 152/226] Check the remembered address's proof and cap the media-size keyframes - A remembered address gets HELLO2 and the token only after the PC there proves it holds the pairing, as a move with the same-PC checks. Without the proof, the entry is forgotten and the attempt goes to the address tapped. Entries are bound to a digest of the pairing token. - The other-address step only tries addresses on the iPad's own subnet, as a PC that Bonjour moved does. - Only the first two smaller sizes in a session skip the keyframe rate limit; later ones are ordinary keyframe requests. - The iPad's summary for a move after no picture says no_frame when some video arrived, as the PC's does. --- host/src/transport/session.rs | 53 ++++++++++-- .../App/ConnectionManager.swift | 80 ++++++++++++++---- ios/EternalMonitor/App/MovedPC.swift | 8 +- ios/EternalMonitor/App/NoVideoFallback.swift | 28 +++++-- ios/EternalMonitor/App/SessionSummary.swift | 2 +- .../NoVideoFallbackTests.swift | 83 +++++++++++++++++-- .../SessionSummaryTests.swift | 6 ++ 7 files changed, 217 insertions(+), 43 deletions(-) diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 9ad0a41b..9355dc00 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -109,6 +109,10 @@ struct RefusalMinute { } /// Honor at most one keyframe request per this window (PLI storm guard). pub const KEYFRAME_REQUEST_MIN_INTERVAL: Duration = Duration::from_millis(500); +/// Smaller media sizes per session whose keyframe skips that window: the +/// iPad asks for two (1,200 then 576 bytes). Any later one is an ordinary, +/// rate-limited keyframe request. +const MEDIA_SIZE_IDRS: u8 = 2; /// Media bytes, headers included, a Wi-Fi client's address gets before /// anything of its session comes back from it: room for the first picture, @@ -286,8 +290,10 @@ struct ActiveSession { /// Video frames let through to this session ([`Session::may_send_media`]). frames_sent: u32, /// The largest media datagram the client asked for in this session - /// ([`KeyframeRequest::max_media_dgram`]). + /// ([`KeyframeRequest::max_media_dgram`]), and how many times it went + /// down. max_media_dgram: Option, + media_size_lowered: u8, /// A Wi-Fi goodbye while the cable takes over. bye: Option, /// The msg_seq of input packets taken in this session. @@ -1121,6 +1127,7 @@ impl Session { frags_received: false, frames_sent: 0, max_media_dgram: None, + media_size_lowered: 0, bye: None, input_seqs: SeqWindow::default(), answered, @@ -1191,21 +1198,24 @@ impl Session { actions.force_idr = session.heard(Heard::Keyframe, now); info!(reason = ?request.reason, "Keyframe request received"); // Only a client that was told the host takes it asks; anything else - // is ignored, as before. Each smaller size restarts the picture at - // once: the frames sent so far gave the client no picture. + // is ignored, as before. The first smaller sizes restart the picture + // at once: the frames sent so far gave the client no picture. let asked = request.max_media_dgram; if asked != 0 && session.info.feature_caps & FEATURE_MEDIA_SIZE != 0 { let size = asked.clamp(MIN_MEDIA_DGRAM, eternal_wire::v2::MAX_DGRAM_SIZE as u16); if session.max_media_dgram.is_none_or(|current| size < current) { session.max_media_dgram = Some(size); - session.last_keyframe_grant = Some(now); + session.media_size_lowered = session.media_size_lowered.saturating_add(1); info!( max_dgram = size, frames_sent = session.frames_sent, "Media packet size lowered for this session" ); - actions.force_idr = true; - return actions; + if session.media_size_lowered <= MEDIA_SIZE_IDRS { + session.last_keyframe_grant = Some(now); + actions.force_idr = true; + return actions; + } } } @@ -2945,6 +2955,37 @@ mod tests { } assert_eq!(session.max_media_dgram(id.wrapping_add(1)), None); + // A client that keeps asking for less, a byte at a time, gets two + // keyframes past the rate limit, then no more than any other client. + session.handle_control( + peer, + 0, + media_size_hello(3, FEATURE_MEDIA_SIZE), + &TestConfig, + at(1_000), + ); + let id = session.session_id().unwrap(); + let mut forced = Vec::new(); + for step in 0..825u64 { + let actions = session.handle_control_authed( + peer, + sized_request(1400 - step as u16), + &TestConfig, + at(1_000 + step * 10), + ); + if actions.force_idr { + forced.push(step * 10); + } + } + assert_eq!(session.max_media_dgram(id), Some(MIN_MEDIA_DGRAM)); + // Two at once, then one per rate-limit window from the second. + assert_eq!(forced[..2], [0, 10]); + assert!(forced[2..] + .windows(2) + .all(|pair| pair[1] - pair[0] >= KEYFRAME_REQUEST_MIN_INTERVAL.as_millis() as u64)); + assert_eq!(forced[2], 510); + assert!(forced.len() <= 2 + 8240 / 500, "{forced:?}"); + session.handle_control( peer, 0, diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index e94613be..9b87e04e 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -280,6 +280,9 @@ final class ConnectionManager: ObservableObject { /// asked for (kept here) if the PC doesn't answer there. private var pendingRedirect: (host: String, port: UInt16)? private var attemptRedirect: (host: String, port: UInt16)? + /// The proof the PC at a remembered address is checked for, before any + /// attempt starts (see `connectViaRemembered`). + private var redirectCheck: UUID? /// The current session's no-video steps, with a PC that takes them, and /// the step whose result isn't known yet, as its log line names it. private var noVideoFallback: NoVideoFallback? @@ -378,24 +381,59 @@ final class ConnectionManager: ObservableObject { if let token { do { try pairingStore.save(token: token, hostName: "", address: PairingStore.address(host: host, port: port)) } catch { record(.warning, "pairing", "Could not save the scanned pairing on this iPad.") } - } else if let better = betterAddress(host: host, port: port) { - pendingRedirect = (host.trimmingCharacters(in: .whitespacesAndNewlines), port) - connect(target: .udp(host: better.host, port: better.port)) - pendingRedirect = nil + } else if let remembered = rememberedAddress(host: host, port: port) { + connectViaRemembered(from: (host.trimmingCharacters(in: .whitespacesAndNewlines), port), remembered) return } connect(target: .udp(host: host, port: port)) } - /// Where the same PC showed video after none arrived at `host:port`: an - /// address this iPad holds the same pairing for. - private func betterAddress(host: String, port: UInt16) -> NoVideoPaths.Address? { + /// Where the PC paired at `host:port` showed video after none arrived + /// there, kept for this pairing, with the pairing and the PC's name. + private func rememberedAddress(host: String, port: UInt16) -> (to: NoVideoPaths.Address, token: Data, name: String)? { let host = host.trimmingCharacters(in: .whitespacesAndNewlines) - guard let better = NoVideoPaths.better(than: host, port: port), - let token = (try? pairingStore.token(address: PairingStore.address(host: host, port: port))) ?? nil, - let there = (try? pairingStore.token(address: PairingStore.address(host: better.host, port: better.port))) ?? nil, - token == there else { return nil } - return better + let address = PairingStore.address(host: host, port: port) + guard let token = (try? pairingStore.token(address: address)) ?? nil, + let to = NoVideoPaths.better(than: host, port: port, pairing: token), + let name = pairingStore.pcName(address: address) ?? RecentConnectionStore.shared.entry(host: host, port: port)?.name + else { return nil } + return (to, token, name) + } + + /// Connecting to `from`, where the PC's video didn't arrive before, goes + /// to the address where it did, once the PC there shows it holds this + /// pairing and receives there: the address may belong to another device + /// now. Only then does a HELLO2 with the token go there, as a move, with + /// the same-PC checks of one. Without the proof the entry is forgotten + /// and the attempt goes to `from`. + private func connectViaRemembered(from: (host: String, port: UInt16), + _ remembered: (to: NoVideoPaths.Address, token: Data, name: String)) { + endReconnectCycle() + let check = UUID() + redirectCheck = check + connectionError = nil + problem = nil + state = .connecting + Task { @MainActor [weak self] in + guard let self else { return } + let to = remembered.to + let same = await MovedPC.confirm(host: to.host, port: to.port, token: remembered.token, + link: self.makeUDPLink(to.port)) + guard self.redirectCheck == check else { return } + self.redirectCheck = nil + self.state = .disconnected + guard same else { + NoVideoPaths.forget(from: .init(host: from.host, port: from.port)) + self.startAttempt(.udp(host: from.host, port: from.port)) + self.record(.info, "connection", "no_video_fallback step=remembered result=not_confirmed") + return + } + self.pendingMove = Move(from: from, name: remembered.name, token: remembered.token, noVideo: true) + self.pendingRedirect = from + self.explicitPairingToken = remembered.token + self.startAttempt(.udp(host: to.host, port: to.port)) + self.pendingRedirect = nil + } } func submitPairing() { @@ -1264,7 +1302,7 @@ final class ConnectionManager: ObservableObject { // The PC still answers at the old address: its pairing stays, // and connecting there goes here first. NoVideoPaths.remember(from: .init(host: move.from.host, port: move.from.port), - to: .init(host: new.host, port: new.port)) + to: .init(host: new.host, port: new.port), pairing: move.token) } else if homeAddress(move.from.host) { try? pairingStore.remove(address: PairingStore.address(host: move.from.host, port: move.from.port)) } @@ -1302,9 +1340,9 @@ final class ConnectionManager: ObservableObject { /// No picture came of the PC's video at this address, even in small /// datagrams. Asked here, the PC signs its home network addresses with - /// this iPad's pairing; the first other one this iPad reaches over - /// Wi-Fi or Ethernet, where the PC shows again that it holds the pairing - /// and receives there, gets a new attempt. A forwarded or spoofed + /// this iPad's pairing; the first other one on this iPad's own network, + /// where the PC shows again that it holds the pairing and receives + /// there, gets a new attempt. A forwarded or spoofed /// address can't make either answer. private func tryOtherAddress() { let host = debugState.host, port = debugState.port @@ -1328,7 +1366,9 @@ final class ConnectionManager: ObservableObject { let target = self.dialAddress(other, identity.port) let route = await self.localRoute(target.host, target.port) guard current() else { return } - guard let route, MovedPC.isDirect(route) else { continue } + // As for a PC Bonjour moved: only an address on this iPad's + // own network, so nothing relays the stream elsewhere. + guard let route, MovedPC.isLocal(target.host, over: route) else { continue } let same = await MovedPC.confirm(host: target.host, port: target.port, token: token, link: self.makeUDPLink(target.port)) guard current() else { return } @@ -1589,6 +1629,12 @@ final class ConnectionManager: ObservableObject { /// Ends the attempt. `end` says why; only the user's own action is /// `.userDisconnect`. func disconnect(_ end: AttemptEnd = .userDisconnect) { + // Still checking the remembered address: nothing went out yet. + if redirectCheck != nil { + redirectCheck = nil + state = .disconnected + return + } if state != .disconnected { endNoVideoStep("no_video") recordStreamSummary(reason: "disconnect=\(end.name)") diff --git a/ios/EternalMonitor/App/MovedPC.swift b/ios/EternalMonitor/App/MovedPC.swift index 6c181fb1..5e0e0723 100644 --- a/ios/EternalMonitor/App/MovedPC.swift +++ b/ios/EternalMonitor/App/MovedPC.swift @@ -85,16 +85,12 @@ enum MovedPC { guard text.split(separator: ".").count == 4, let bytes = IPv4Address(text)?.rawValue, bytes.count == 4 else { return nil } return bytes.reduce(0) { $0 << 8 | UInt32($1) } } - guard isDirect(route), let address = ipv4(route.address), let mask = ipv4(route.netmask), let candidate = ipv4(host), + guard route.direct, !tunnels.contains(where: route.interface.hasPrefix), + let address = ipv4(route.address), let mask = ipv4(route.netmask), let candidate = ipv4(host), mask != 0, mask != .max else { return false } return address & mask == candidate & mask } - /// Whether `route` is a direct Wi-Fi or Ethernet path, not a tunnel. - static func isDirect(_ route: Route) -> Bool { - route.direct && !tunnels.contains(where: route.interface.hasPrefix) - } - /// The other home network addresses `identity` signed, in the PC's /// order (those with a default gateway first), without `asked`. static func otherAddresses(_ identity: WireIdentity, asked: String, diff --git a/ios/EternalMonitor/App/NoVideoFallback.swift b/ios/EternalMonitor/App/NoVideoFallback.swift index b2270e5f..b7828db9 100644 --- a/ios/EternalMonitor/App/NoVideoFallback.swift +++ b/ios/EternalMonitor/App/NoVideoFallback.swift @@ -1,3 +1,4 @@ +import CryptoKit import Foundation /// A PC accepted the session and sends video, and no picture comes of it: @@ -54,8 +55,9 @@ struct NoVideoFallback: Equatable { } /// Where a PC's video didn't arrive, and the address of the same PC where it -/// did, so the next connect to the first goes to the second. Kept in -/// UserDefaults as "host:port" pairs. +/// did, so the next connect to the first goes to the second once the PC +/// there proves it holds the pairing. Each entry is bound to that pairing by +/// a digest of its token. Kept in UserDefaults. enum NoVideoPaths { private static let key = "noVideoPaths" private static let kept = 16 @@ -68,19 +70,31 @@ enum NoVideoPaths { private struct Entry: Codable { var from: Address var to: Address + /// SHA-256 of the pairing token, in hex. + var pairing: String } - /// The address where the PC at `host:port` showed video instead. - static func better(than host: String, port: UInt16) -> Address? { - load().last { $0.from == Address(host: host, port: port) }?.to + /// The address where the PC paired with `pairing` at `host:port` showed + /// video instead. + static func better(than host: String, port: UInt16, pairing: Data) -> Address? { + let digest = digest(pairing) + return load().last { $0.from == Address(host: host, port: port) && $0.pairing == digest }?.to } - static func remember(from: Address, to: Address) { + static func remember(from: Address, to: Address, pairing: Data) { var entries = load().filter { $0.from != from && $0.from != to } - entries.append(Entry(from: from, to: to)) + entries.append(Entry(from: from, to: to, pairing: digest(pairing))) save(Array(entries.suffix(kept))) } + static func forget(from: Address) { + save(load().filter { $0.from != from }) + } + + private static func digest(_ token: Data) -> String { + SHA256.hash(data: token).map { String(format: "%02x", $0) }.joined() + } + static func forgetAll() { UserDefaults.standard.removeObject(forKey: key) } diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index 702d1e44..e2af7146 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -116,7 +116,7 @@ struct SessionSummary: Equatable { switch end { case .connectTimeout where sessionId == nil, .pcMoved: return "no_reply" case .connectTimeout: return datagramsReceived == 0 ? "no_video" : "no_frame" - case .noVideoPath: return "no_video" + case .noVideoPath: return datagramsReceived == 0 ? "no_video" : "no_frame" case .pcStoppedResponding: return "signal_lost" case .cableUnplugged, .usbLinkFailed, .closedBeforePicture: return "usb_lost" case .noNetwork: return "no_network" diff --git a/ios/EternalMonitorTests/NoVideoFallbackTests.swift b/ios/EternalMonitorTests/NoVideoFallbackTests.swift index 8070989b..9b0d4d25 100644 --- a/ios/EternalMonitorTests/NoVideoFallbackTests.swift +++ b/ios/EternalMonitorTests/NoVideoFallbackTests.swift @@ -168,24 +168,95 @@ final class NoVideoFallbackTests: XCTestCase { XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) XCTAssertEqual(UserDefaults.standard.integer(forKey: "lastPort"), Int(other.port)) - // Connecting to the first address later goes to the other first. + // Connecting to the first address later goes to the other first, + // once the PC there shows again that it holds the pairing. manager.cancel() manager.connect(host: "127.0.0.1", port: first.port) - XCTAssertTrue(manager.diagnostics.contains { $0.message == "no_video_fallback step=remembered result=tried" }) async let atFirst = Task.detached { first.received(within: 1) }.value - let answered = await Task.detached { other.acceptSession(authToken: Self.token) }.value + let answered = await Task.detached { + other.answerIdentify(token: Self.token) && other.acceptSession(authToken: Self.token) + }.value XCTAssertTrue(answered, "the address that showed video goes first") + XCTAssertFalse(other.helloBeforeIdentify, "nothing with the token before the proof") let firstTypes = await atFirst XCTAssertFalse(firstTypes.contains(.hello2), "\(firstTypes)") + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=tried"), logged().joined(separator: "\n")) - // Where the other address doesn't answer, the attempt goes back to - // the first. + // Where nothing at the other address shows the proof, the attempt + // goes to the first, and the other is forgotten. manager.cancel() manager.connect(host: "127.0.0.1", port: first.port) let backAtFirst = await Task.detached { first.helloAttempts(within: 8) }.value XCTAssertGreaterThan(backAtFirst, 0) - XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=no_reply"), + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=not_confirmed"), logged().joined(separator: "\n")) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token)) + } + + /// The remembered address belongs to another PC now: one with pairing + /// on, which can't make this pairing's proof, or one with pairing off, + /// which would take any HELLO2 and doesn't answer the identity check. + /// The token never goes there, the pairing stays, and the attempt goes + /// to the address tapped. + func testARememberedAddressThatIsNowAnotherPCNeverGetsTheToken() async throws { + for pairingOn in [true, false] { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, pairings) = try manager(pairedAt: first) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: first.port), + to: .init(host: "127.0.0.1", port: other.port), pairing: Self.token) + manager.connect(host: "127.0.0.1", port: first.port) + async let atOther = Task.detached { () -> [Wire.PacketType] in + if pairingOn, !other.answerIdentify(token: Data(2...17)) { return [] } + return other.received(within: 4) + }.value + let accepted = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(accepted, "\(pairingOn): the tapped address got no HELLO2") + XCTAssertEqual(first.lastAuthToken, Self.token, "\(pairingOn)") + let types = await atOther + XCTAssertFalse(types.contains(.hello2), "\(pairingOn): \(types)") + XCTAssertFalse(other.helloBeforeIdentify, "\(pairingOn)") + if !pairingOn { XCTAssertEqual(Set(types), [.identify]) } + try await until("handshake at the tapped address") { manager.hostInfo != nil } + XCTAssertEqual(manager.debugState.port, first.port) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token)) + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=not_confirmed"), "\(pairingOn)") + } + // An entry kept for another pairing is never used for this one. + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: 1), to: .init(host: "127.0.0.1", port: 2), + pairing: Data(2...17)) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: 1, pairing: Self.token)) + XCTAssertEqual(NoVideoPaths.better(than: "127.0.0.1", port: 1, pairing: Data(2...17))?.port, 2) + } + + /// The other address the PC signs is outside this iPad's own network: + /// it is never asked, as for a PC Bonjour moved there. + func testAnOtherAddressOffThisNetworkIsNeverAsked() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, _) = try manager(pairedAt: first, other: other, connectTimeout: 8) + defer { manager.cancel() } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "10.0.0.5", netmask: "255.255.255.0") } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let ended = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: first.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + first.servePath(limit: 0, frame: frame, hostCaps: HelloAck.hostCapMediaSize, + addresses: Self.bothAddresses) { ended.withLock { $0 } } + } + try await until("asked the PC for its addresses", seconds: 20) { + logged().contains("no_video_fallback step=other_address result=none") + } + let types = await Task.detached { other.received(within: 1) }.value + XCTAssertEqual(types, []) + try await until("timed out", seconds: 20) { manager.state == .disconnected } + ended.withLock { $0 = true } + let run = await served.value + XCTAssertGreaterThan(run.identifyAnswers, 0) } /// The other address the PC signs answers without the pairing's proof, diff --git a/ios/EternalMonitorTests/SessionSummaryTests.swift b/ios/EternalMonitorTests/SessionSummaryTests.swift index 83f2fc5a..0ae21b58 100644 --- a/ios/EternalMonitorTests/SessionSummaryTests.swift +++ b/ios/EternalMonitorTests/SessionSummaryTests.swift @@ -63,6 +63,8 @@ final class SessionSummaryTests: XCTestCase { func testEachWayAnAttemptEndsHasItsSummary() { var noFrame = summary(.connectTimeout, duration: 10) noFrame.datagramsReceived = 12 + var movedNoFrame = summary(.noVideoPath, duration: 7) + movedNoFrame.datagramsReceived = 3 var pairing = summary(.userDisconnect, session: nil, duration: 30) pairing.awaitingPairing = true var retried = summary(.retriedWithCode, session: nil, duration: 4) @@ -82,6 +84,10 @@ final class SessionSummaryTests: XCTestCase { (summary(.otherPC), ["accepted", "error", "other", "error"]), (summary(.connectTimeout, duration: 10), ["accepted", "no_picture", "no_video", "timeout"]), (noFrame, ["accepted", "no_picture", "no_frame", "timeout"]), + // Moved to the PC's other address: as the PC says, nothing came, + // or no frame of what came completed. + (summary(.noVideoPath, duration: 7), ["accepted", "no_picture", "no_video", "superseded"]), + (movedNoFrame, ["accepted", "no_picture", "no_frame", "superseded"]), (summary(.pcStoppedResponding, firstFrame: 0.3, duration: 90), ["steady", "streamed", "signal_lost", "liveness"]), (summary(.pictureStalled, firstFrame: 0.3, duration: 90), ["steady", "streamed", "other", "error"]), (summary(.cableUnplugged, usb: true, firstFrame: 0.3), ["first_frame", "streamed", "usb_lost", "error"]), From bb028550ce5bff3ff99e75e41ffcfcf4af541cdf Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 03:46:41 -0500 Subject: [PATCH 153/226] Hand a lost duplication during the pointer rebuild to the handler When the pointer's answer changes, the cached frame is rebuilt from the staging texture. If that copy hit ACCESS_LOST, the error ended the capture loop. The stage then opened capture afresh after at least 500 ms, which dropped the cached picture and skipped the quick re-creation of the duplication. - ACCESS_LOST from the copy now goes to the loop's existing handler, as one from ReleaseFrame or the readback does. The cache is kept. - While a loss is pending, the rebuild waits until the handler has run. - Any other error, such as DEVICE_REMOVED, still ends the loop, so the pipeline re-creates the device. - The decision is redraw_cached, generic over the error, so a test can inject the ACCESS_LOST. --- host/src/capture/dxgi.rs | 78 +++++++++++++++++++++++----------------- host/src/capture/mod.rs | 69 +++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 32 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 59357927..f72ea660 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -19,9 +19,9 @@ use super::recovery::{ use super::{ acquired, client_connected, composite_pointer, frame_budget_for, frame_buffer, heartbeat, idle_heartbeat, keepalive_allowed, keepalive_due, pointer_apart, pointer_place, - read_back_image, reconcile_virtual_display, recovery_poll, resolve_target, Acquired, FrameSlot, - FrameUse, IdleGate, OutputInfo, Picture, PointerPlace, RawFrame, ACQUIRE_TIMEOUT_MS, - FPS_WINDOW, FRAME_PADDING, PICTURE_HOLD, + read_back_image, reconcile_virtual_display, recovery_poll, redraw_cached, resolve_target, + Acquired, FrameSlot, FrameUse, IdleGate, OutputInfo, Picture, PointerPlace, RawFrame, Redraw, + ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, PICTURE_HOLD, }; use crate::control::{CaptureTarget, SharedControl}; use crate::gpu_preference::Outcome; @@ -391,36 +391,50 @@ pub fn run_capture_loop( let draws_pointer = pointer_apart(shared.client_draws_pointer(), place); if picture.seen_image && last_frame_has_pointer == draws_pointer { let mut pixel_buf = frame_buffer(frame_bytes); - copy_staging( - &device_context, - &staging_texture, - tex_height, - row_bytes, - &mut pixel_buf, + // A lost duplication goes to the handler below, which keeps the + // cache; the rebuild is tried again after it. + let copied = redraw_cached( + lost_after_acquire.is_some(), + || { + copy_staging( + &device_context, + &staging_texture, + tex_height, + row_bytes, + &mut pixel_buf, + ) + }, + |error| error.code() == DXGI_ERROR_ACCESS_LOST, )?; - if composite_pointer(draws_pointer, place, cursor_visible, cursor.has_shape) { - draw_cursor( - &mut pixel_buf, - tex_width, - tex_height, - row_bytes, - &cursor, - cursor_x, - cursor_y, - ); - } - spare = None; - last_frame = Arc::new(pixel_buf); - last_frame_has_pointer = !draws_pointer; - if client_connected(&shared) { - resend_last_frame( - slot, - &shared, - frame_number, - &mut last_send, - &last_frame, - (tex_width, tex_height), - ); + match copied { + Redraw::Later => {} + Redraw::Lost(error) => lost_after_acquire = Some(error), + Redraw::Copied => { + if composite_pointer(draws_pointer, place, cursor_visible, cursor.has_shape) { + draw_cursor( + &mut pixel_buf, + tex_width, + tex_height, + row_bytes, + &cursor, + cursor_x, + cursor_y, + ); + } + spare = None; + last_frame = Arc::new(pixel_buf); + last_frame_has_pointer = !draws_pointer; + if client_connected(&shared) { + resend_last_frame( + slot, + &shared, + frame_number, + &mut last_send, + &last_frame, + (tex_width, tex_height), + ); + } + } } } diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 5f3ba7c2..5ee270de 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -205,6 +205,41 @@ pub(crate) fn acquired(last_present_time: i64, seen_image: bool, draws_pointer: } } +/// What the cached-frame rebuild for a changed pointer answer does with its +/// copy of the staging texture. +#[cfg(any(windows, test))] +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Redraw { + /// Copied: draw the pointer as the answer wants and keep the frame. + Copied, + /// A lost duplication is waiting for the loop's handler: rebuild after + /// it, with the cache kept as it is. + Later, + /// The copy hit ACCESS_LOST: hand it to the loop's handler, which + /// re-creates the duplication and keeps the cached picture. + Lost(E), +} + +/// Copies the staging texture for the rebuild (`copy`), unless a loss is +/// pending. ACCESS_LOST (`access_lost`) goes to the loop's handler as one +/// after an acquire does; any other error, such as DEVICE_REMOVED, ends +/// the capture loop so the pipeline re-creates the device. +#[cfg(any(windows, test))] +pub(crate) fn redraw_cached( + pending_loss: bool, + copy: impl FnOnce() -> Result<(), E>, + access_lost: impl FnOnce(&E) -> bool, +) -> Result, E> { + if pending_loss { + return Ok(Redraw::Later); + } + match copy() { + Ok(()) => Ok(Redraw::Copied), + Err(error) if access_lost(&error) => Ok(Redraw::Lost(error)), + Err(error) => Err(error), + } +} + pub(crate) fn frame_budget_for(target_fps: u32) -> Duration { let fps = target_fps.max(1) as u64; Duration::from_micros(1_000_000 / fps) @@ -1463,6 +1498,40 @@ mod tests { } } + /// Review r7: an ACCESS_LOST from the rebuild's copy goes to the loop's + /// handler, which keeps the cached picture, instead of ending the + /// capture loop; a pending loss is handled before the rebuild is tried + /// again; any other error still ends the loop. + #[test] + fn a_lost_duplication_during_the_pointer_rebuild_goes_to_the_handler() { + const ACCESS_LOST: u32 = 0x887A_0026; + const DEVICE_REMOVED: u32 = 0x887A_0005; + let lost = |code: &u32| *code == ACCESS_LOST; + assert_eq!(redraw_cached(false, || Ok(()), lost), Ok(Redraw::Copied)); + assert_eq!( + redraw_cached(false, || Err(ACCESS_LOST), lost), + Ok(Redraw::Lost(ACCESS_LOST)) + ); + assert_eq!( + redraw_cached(false, || Err(DEVICE_REMOVED), lost), + Err(DEVICE_REMOVED), + "the device itself goes: the pipeline re-creates it" + ); + let mut copied = false; + assert_eq!( + redraw_cached( + true, + || { + copied = true; + Ok::<(), u32>(()) + }, + lost + ), + Ok(Redraw::Later) + ); + assert!(!copied, "not tried while a loss waits for the handler"); + } + /// One pointer, never none and never two. Review r3: with the pointer /// in the picture Desktop Duplication can still report it visible, /// with a shape, at a stale spot, and drawing that over Windows' own From e909a94095f6f8b877d32ffa88c110b7526fda4b Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 03:50:43 -0500 Subject: [PATCH 154/226] Retire a remembered address that fails, and check it is on this network - Before the identity check, a remembered address must be on this iPad's own subnet, as the other-address step requires; otherwise the entry is forgotten and the attempt goes to the address tapped. - When the attempt at a remembered address fails after its proof (no HELLO2 answer, no picture before the timeout, or another PC by name or pairing), the entry is forgotten and a new attempt goes to the address tapped. - Try again and a resume go to the address tapped while the remembered one is checked. - A golden vector covers a heartbeat that carries both the no-display notice and the frame count. --- .../App/ConnectionManager.swift | 54 ++++++++++----- .../NoVideoFallbackTests.swift | 65 +++++++++++++++++++ .../WireProtocolGoldenTests.swift | 11 +++- proto/testdata/v2_vectors.txt | 1 + proto/tests/golden.rs | 20 ++++++ 5 files changed, 134 insertions(+), 17 deletions(-) diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index b1e0357c..307eafb8 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -404,14 +404,17 @@ final class ConnectionManager: ObservableObject { } /// Connecting to `from`, where the PC's video didn't arrive before, goes - /// to the address where it did, once the PC there shows it holds this - /// pairing and receives there: the address may belong to another device - /// now. Only then does a HELLO2 with the token go there, as a move, with - /// the same-PC checks of one. Without the proof the entry is forgotten - /// and the attempt goes to `from`. + /// to the address where it did, once that address is on this iPad's own + /// network and the PC there shows it holds this pairing and receives + /// there: the address may belong to another device now. Only then does a + /// HELLO2 with the token go there, as a move, with the same-PC checks of + /// one. Otherwise the entry is forgotten and the attempt goes to `from`, + /// as it does if that move fails (see `leaveRememberedAddress`). private func connectViaRemembered(from: (host: String, port: UInt16), _ remembered: (to: NoVideoPaths.Address, token: Data, name: String)) { endReconnectCycle() + // Try again and a resume go to the address tapped. + lastTarget = from let check = UUID() redirectCheck = check connectionError = nil @@ -420,17 +423,23 @@ final class ConnectionManager: ObservableObject { Task { @MainActor [weak self] in guard let self else { return } let to = remembered.to + let toFrom = { [weak self] (result: String) in + guard let self else { return } + self.redirectCheck = nil + self.state = .disconnected + NoVideoPaths.forget(from: .init(host: from.host, port: from.port)) + self.startAttempt(.udp(host: from.host, port: from.port)) + self.record(.info, "connection", "no_video_fallback step=remembered result=\(result)") + } + let route = await self.localRoute(to.host, to.port) + guard self.redirectCheck == check else { return } + guard let route, MovedPC.isLocal(to.host, over: route) else { return toFrom("not_local") } let same = await MovedPC.confirm(host: to.host, port: to.port, token: remembered.token, link: self.makeUDPLink(to.port)) guard self.redirectCheck == check else { return } + guard same else { return toFrom("not_confirmed") } self.redirectCheck = nil self.state = .disconnected - guard same else { - NoVideoPaths.forget(from: .init(host: from.host, port: from.port)) - self.startAttempt(.udp(host: from.host, port: from.port)) - self.record(.info, "connection", "no_video_fallback step=remembered result=not_confirmed") - return - } self.pendingMove = Move(from: from, name: remembered.name, token: remembered.token, noVideo: true) self.pendingRedirect = from self.explicitPairingToken = remembered.token @@ -439,6 +448,19 @@ final class ConnectionManager: ObservableObject { } } + /// The attempt at a remembered address (see `connectViaRemembered`) + /// passed its proof and failed after it: it ends with `end`, the entry is + /// forgotten, and a new attempt goes to the address tapped. False when + /// this attempt didn't go to a remembered address. + private func leaveRememberedAddress(_ end: AttemptEnd, result: String) -> Bool { + guard let from = attemptRedirect else { return false } + NoVideoPaths.forget(from: .init(host: from.host, port: from.port)) + record(.info, "connection", "no_video_fallback step=remembered result=\(result)") + disconnect(end) + startAttempt(.udp(host: from.host, port: from.port)) + return true + } + func submitPairing() { guard awaitingPairing, let code = pairingPrompt?.submit() else { return } // The code goes in a new HELLO2. The refused one ends here, as on the @@ -759,6 +781,7 @@ final class ConnectionManager: ObservableObject { if let move = self.attemptMove { guard Self.isSamePC(move, info) else { self.record(.warning, "connection", "The PC at the new address isn't the saved one") + if self.leaveRememberedAddress(.otherPC, result: "other_pc") { return } self.disconnect(.otherPC) self.problem = .noReply(usb: false) return @@ -811,6 +834,7 @@ final class ConnectionManager: ObservableObject { // it is another PC, and the saved pairing stays. if self.attemptMove != nil, status == .unauthorized || status == .rateLimited { self.record(.warning, "connection", "The PC at the new address isn't the saved one: it asked for pairing") + if self.leaveRememberedAddress(.otherPC, result: "other_pc") { return } self.disconnect(.otherPC) self.problem = .noReply(usb: false) return @@ -876,11 +900,8 @@ final class ConnectionManager: ObservableObject { self.disconnect(.legacyPC) } else { self.record(.warning, "ctrl", "HELLO2 handshake got no reply yet") - if let from = self.attemptRedirect { - // Not on this network now: back to the address asked for. - self.record(.info, "connection", "no_video_fallback step=remembered result=no_reply") - self.disconnect(.pcMoved) - self.startAttempt(.udp(host: from.host, port: from.port)) + if self.leaveRememberedAddress(.pcMoved, result: "no_reply") { + // Back to the address tapped. } else if !isUSB { self.lookForMovedPC(connectionID, host: normalizedHost, port: port) } @@ -1427,6 +1448,7 @@ final class ConnectionManager: ObservableObject { let level = debug.timeoutLevel(handshakeDone: handshakeDone, answeredHereBefore: self.acceptedAddresses.contains("\(debug.host):\(debug.port)")) self.record(level, "timeout", debug.timeoutSummary(handshakeDone: handshakeDone) + self.attemptTimings) + if self.leaveRememberedAddress(.connectTimeout, result: "no_picture") { return } // Only an attempt the PC answered counts. if handshakeDone { self.noPictureStreak.endedWithoutPicture() } // A PC that said it isn't streaming and then went quiet (it diff --git a/ios/EternalMonitorTests/NoVideoFallbackTests.swift b/ios/EternalMonitorTests/NoVideoFallbackTests.swift index 9b0d4d25..3a676c61 100644 --- a/ios/EternalMonitorTests/NoVideoFallbackTests.swift +++ b/ios/EternalMonitorTests/NoVideoFallbackTests.swift @@ -232,6 +232,71 @@ final class NoVideoFallbackTests: XCTestCase { XCTAssertEqual(NoVideoPaths.better(than: "127.0.0.1", port: 1, pairing: Data(2...17))?.port, 2) } + /// A remembered address that is outside this iPad's own network now + /// gets nothing: no identity check, no HELLO2. The attempt goes to the + /// address tapped, and Try again goes there too. + func testARememberedAddressOffThisNetworkIsNeverAsked() async throws { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, pairings) = try manager(pairedAt: first) + defer { manager.cancel() } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "10.0.0.5", netmask: "255.255.255.0") } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: first.port), + to: .init(host: "127.0.0.1", port: other.port), pairing: Self.token) + manager.connect(host: "127.0.0.1", port: first.port) + let accepted = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(accepted) + XCTAssertEqual(first.lastAuthToken, Self.token) + let types = await Task.detached { other.received(within: 1) }.value + XCTAssertEqual(types, []) + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=not_local"), logged().joined(separator: "\n")) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token)) + XCTAssertEqual(try pairings.token(address: PairingStore.address(host: "127.0.0.1", port: first.port)), Self.token) + } + + /// The PC at the remembered address passed its proof, then the attempt + /// there failed: HELLO2 went unanswered, the session showed no frame, + /// or the PC that answered had another name. The entry is forgotten, + /// the attempt goes to the address tapped at once, and so does the + /// next tap, with no identity check at the remembered address. + func testARememberedAddressThatFailsAfterItsProofIsRetired() async throws { + for failure in ["no_reply", "no_picture", "other_pc"] { + let first = FakeUDPHost(), other = FakeUDPHost() + let (manager, _) = try manager(pairedAt: first, connectTimeout: 3) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoPaths.remember(from: .init(host: "127.0.0.1", port: first.port), + to: .init(host: "127.0.0.1", port: other.port), pairing: Self.token) + manager.connect(host: "127.0.0.1", port: first.port) + let atOther = await Task.detached { () -> Bool in + guard other.answerIdentify(token: Self.token) else { return false } + switch failure { + case "no_picture": return other.acceptSession(authToken: Self.token) + case "other_pc": return other.acceptSession(hostName: "Other PC", authToken: Data(repeating: 0, count: 16)) + default: return other.helloAttempts(within: 1) > 0 + } + }.value + XCTAssertTrue(atOther, "\(failure): the proof passed and HELLO2 went there") + let atFirst = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(atFirst, "\(failure): then to the address tapped") + try await until("\(failure): handshake at the address tapped", seconds: 20) { + manager.hostInfo != nil && manager.debugState.port == first.port + } + XCTAssertTrue(logged().contains("no_video_fallback step=remembered result=\(failure)"), logged().joined(separator: "\n")) + XCTAssertNil(NoVideoPaths.better(than: "127.0.0.1", port: first.port, pairing: Self.token), failure) + _ = await Task.detached { other.received(within: 0.5) }.value + + manager.cancel() + manager.connect(host: "127.0.0.1", port: first.port) + let again = await Task.detached { first.acceptSession() }.value + XCTAssertTrue(again, "\(failure): the next tap goes to the address tapped") + let types = await Task.detached { other.received(within: 0.5) }.value + XCTAssertFalse(types.contains(.identify) || types.contains(.hello2), "\(failure): \(types)") + } + } + /// The other address the PC signs is outside this iPad's own network: /// it is never asked, as for a PC Bonjour moved there. func testAnOtherAddressOffThisNetworkIsNeverAsked() async throws { diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index b33977cf..02f33b20 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,7 +35,7 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 40, + Self.vectors.count, 41, "golden vector count drifted — update both test suites together" ) } @@ -443,6 +443,15 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(counted.dropFirst(16).prefix(plain.count - 16), plain.dropFirst(16)) guard case .heartbeat(let old) = try XCTUnwrap(Wire.parseControl(plain)).message else { return XCTFail("wrong type") } XCTAssertNil(old.framesSent) + + // A notice and the count together: the notice byte, then the count. + let both = try vector("heartbeat_no_display_frames_sent") + let (bothHeader, bothMessage) = try XCTUnwrap(Wire.parseControl(both)) + guard case .heartbeat(let noDisplay) = bothMessage else { return XCTFail("wrong type") } + XCTAssertEqual(noDisplay.notice, Heartbeat.noticeNoDisplay) + XCTAssertEqual(noDisplay.framesSent, 77) + XCTAssertEqual(Array(both.dropFirst(plain.count)), [Heartbeat.noticeNoDisplay, 77, 0, 0, 0]) + XCTAssertEqual(Wire.encodeControl(sessionId: 0x1234_5678, msgSeq: bothHeader.msgSeq, message: bothMessage), both) } func testByeVector() throws { diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 98386fcd..622c0037 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -25,6 +25,7 @@ bye_background 454d021300000100785634122b00000001 keyframe_request 454d021400000900785634122c000000030000000f10000000 keyframe_request_media_size 454d021400000b007856341233000000030000000000000002b004 heartbeat_frames_sent 454d021200001d007856341234000000b168de3a0000000003000000000aa0053c000000c0e1e40000d2040000 +heartbeat_no_display_frames_sent 454d021200001d007856341235000000b168de3a0000000003000000000aa0053c000000c0e1e400024d000000 receiver_report 454d021500002400785634122d00000003000000881300007e1300000a000000c0d40100fa000000340800005702010041011c00 ping 454d021600000800785634122e0000000807060504030201 pong 454d021700001800785634122e000000080706050403020110070605040302012007060504030201 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index 2a383437..3294c423 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -302,6 +302,17 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { frames_sent: Some(1234), }), ), + ( + "heartbeat_no_display_frames_sent", + 0x1234_5678, + 53, + ControlMessage::Heartbeat(Heartbeat { + host_time_us: 987_654_321, + stream_config: stream_config(), + notice: NOTICE_NO_DISPLAY, + frames_sent: Some(77), + }), + ), ( "receiver_report", 0x1234_5678, @@ -771,4 +782,13 @@ fn media_size_vectors_round_trip_and_extend_the_old_bodies() { assert_eq!(counted.len(), plain.len() + 5); // Past the msg_seq, the old heartbeat is a prefix of the new one. assert_eq!(counted[16..plain.len()], plain[16..]); + + // A notice and the count together: the notice byte, then the count. + let ControlMessage::Heartbeat(both) = message("heartbeat_no_display_frames_sent") else { + panic!("heartbeat_no_display_frames_sent") + }; + assert_eq!(both.notice, NOTICE_NO_DISPLAY); + assert_eq!(both.frames_sent, Some(77)); + let bytes = &vectors["heartbeat_no_display_frames_sent"]; + assert_eq!(&bytes[plain.len()..], &[NOTICE_NO_DISPLAY, 77, 0, 0, 0]); } From 8ef2b6126d1d21a39950ea0e9f251e8d8033b19a Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 04:03:48 -0500 Subject: [PATCH 155/226] Give an unanswered HELLO2 time to give up before the connect timeout in the remembered-address test --- ios/EternalMonitorTests/NoVideoFallbackTests.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ios/EternalMonitorTests/NoVideoFallbackTests.swift b/ios/EternalMonitorTests/NoVideoFallbackTests.swift index 3a676c61..17785f0a 100644 --- a/ios/EternalMonitorTests/NoVideoFallbackTests.swift +++ b/ios/EternalMonitorTests/NoVideoFallbackTests.swift @@ -263,7 +263,9 @@ final class NoVideoFallbackTests: XCTestCase { func testARememberedAddressThatFailsAfterItsProofIsRetired() async throws { for failure in ["no_reply", "no_picture", "other_pc"] { let first = FakeUDPHost(), other = FakeUDPHost() - let (manager, _) = try manager(pairedAt: first, connectTimeout: 3) + // Well past HELLO2's 2 s of retries, so an unanswered HELLO2 always + // gives up before the connect timeout, even on a slow runner. + let (manager, _) = try manager(pairedAt: first, connectTimeout: 6) defer { manager.cancel() } let (logged, sink) = lines(manager) defer { sink.cancel() } From fcaf6af6b0888ef3785ba0d4d2d83b9670143ef3 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 04:05:15 -0500 Subject: [PATCH 156/226] Convert code page settings files the card does not change A settings file read from a code page is now written back as UTF-8 even when its graphics card stays, so the driver reads all of it. UTF-8 with stray bytes is no longer read in the system's code page when it starts with a byte order mark, or, in a single-byte code page, when it still holds UTF-8 beyond ASCII; its UTF-8 text would be garbled. A second conversion the same day keeps its original as vdd_settings.xml.bak--. Each conversion is logged once. Tests cover the system code page through apply_pin and write_with_pin. --- host/src/vdd_settings.rs | 277 +++++++++++++++++++++++++++++++++------ 1 file changed, 237 insertions(+), 40 deletions(-) diff --git a/host/src/vdd_settings.rs b/host/src/vdd_settings.rs index cc93ff5d..c45d9fb7 100644 --- a/host/src/vdd_settings.rs +++ b/host/src/vdd_settings.rs @@ -384,18 +384,19 @@ pub fn with_gpu(xml: &str, gpu: &str) -> Option { } /// Point the settings file the host does not otherwise write at `gpu`. -/// True when it changed. A missing file stays missing: without one the -/// driver offers its built-in modes, and a document holding only a GPU -/// would leave it none. A file that does not parse as expected is left -/// alone, and the error says why. +/// True when it changed: a file read from a code page is written back as +/// UTF-8 even when it already names `gpu`. A missing file stays missing: +/// without one the driver offers its built-in modes, and a document holding +/// only a GPU would leave it none. A file that does not parse as expected +/// is left alone, and the error says why. pub fn pin_gpu_in(path: &Path, gpu: &str) -> io::Result { let (current, legacy) = read_settings(path)?; let updated = gpu_edit(¤t, gpu) .map_err(|reason| io::Error::new(io::ErrorKind::InvalidData, reason))?; - if updated == current { + if updated == current && !legacy { return Ok(false); } - replace_settings(path, legacy, &updated)?; + write_settings(path, legacy, || replace(path, &updated))?; Ok(true) } @@ -543,7 +544,9 @@ pub fn remember_pins(record: &Path, names: &[&str]) -> io::Result<()> { /// Make the settings file at `path` name `wanted`, changing only the card, /// under the rules of `pin_change`. `record` holds the names this host /// recorded writing. A missing file stays missing, and one that does not -/// parse as expected is left alone; the error says why. +/// parse as expected is left alone; the error says why. A file read from a +/// code page is written back as UTF-8 even when its card stays, so the +/// driver reads all of it. pub fn apply_pin( path: &Path, record: &Path, @@ -555,16 +558,22 @@ pub fn apply_pin( let named = named_gpu(¤t).map_err(invalid)?; let ours = recorded_pins(record); let ours: Vec<&str> = ours.iter().map(String::as_str).collect(); + let unchanged = |outcome| { + if legacy { + write_settings(path, legacy, || replace(path, ¤t))?; + } + Ok(outcome) + }; let (name, outcome) = match pin_change(named.as_deref(), wanted, owned, &ours) { - PinChange::Leave => return Ok(PinOutcome::Unchanged), - PinChange::Refuse(reason) => return Ok(PinOutcome::Kept(reason)), + PinChange::Leave => return unchanged(PinOutcome::Unchanged), + PinChange::Refuse(reason) => return unchanged(PinOutcome::Kept(reason)), PinChange::Write(card) => (card, PinOutcome::Named), PinChange::Reset => ("default".to_string(), PinOutcome::Reset), }; let updated = gpu_edit(¤t, &name).map_err(invalid)?; let new_record = (outcome == PinOutcome::Named).then_some(name.as_str()); record_around(record, new_record, &ours, || { - replace_settings(path, legacy, &updated) + write_settings(path, legacy, || replace(path, &updated)) })?; Ok(outcome) } @@ -641,12 +650,7 @@ pub fn write_with_pin( PinChange::Write(card) => (Some(card), PinOutcome::Named), PinChange::Reset => (None, PinOutcome::Reset), }; - let write = || { - if legacy { - back_up(path)?; - } - write_to(path, modes, name.as_deref()) - }; + let write = || write_settings(path, legacy, || write_to(path, modes, name.as_deref())); match outcome { PinOutcome::Named => record_around(record, name.as_deref(), &ours, write)?, PinOutcome::Reset => record_around(record, None, &ours, write)?, @@ -741,12 +745,17 @@ fn decode_text(bytes: &[u8], ansi: u32) -> Option<(String, bool)> { /// Bytes in an 8-bit encoding: UTF-8, or the code page the declaration /// names. Without one, bytes that are not UTF-8 are in code page `ansi`; /// so are bytes declared UTF-8 that are not, as Notepad leaves the driver's -/// file when it saves it as ANSI. +/// file when it saves it as ANSI. A UTF-8 file with stray bytes is neither: +/// one with a byte order mark, or, in a single-byte code page, one that +/// still holds UTF-8 beyond ASCII, which that code page's text all but +/// never does. Read in the code page, its UTF-8 text would be garbled. fn decode_bytes(bytes: &[u8], ansi: u32) -> Option<(String, bool)> { let label = encoding_label(bytes).map(|label| bytes[label].to_ascii_lowercase()); let code_page = match label.as_deref() { None | Some(b"utf-8") => match String::from_utf8(bytes.to_vec()) { Ok(text) => return Some((text, false)), + Err(_) if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) => return None, + Err(_) if single_byte(ansi) && holds_utf8(bytes) => return None, Err(_) => ansi, }, Some(label) => match std::str::from_utf8(label).ok().and_then(code_page) { @@ -803,6 +812,24 @@ fn decode_code_page(_bytes: &[u8], _code_page: u32) -> Option { None } +/// Whether Windows code page `code_page` has one byte per character. +#[cfg(windows)] +fn single_byte(code_page: u32) -> bool { + use windows::Win32::Globalization::{GetCPInfo, CPINFO}; + let mut info = CPINFO::default(); + unsafe { GetCPInfo(code_page, &mut info) }.is_ok() && info.MaxCharSize == 1 +} + +#[cfg(not(windows))] +fn single_byte(_code_page: u32) -> bool { + false +} + +/// Whether `bytes` hold a UTF-8 character beyond ASCII anywhere. +fn holds_utf8(bytes: &[u8]) -> bool { + bytes.utf8_chunks().any(|chunk| !chunk.valid().is_ascii()) +} + /// Where the XML declaration at the start of `xml` names its encoding. fn encoding_label(xml: &[u8]) -> Option> { let end = xml @@ -824,16 +851,34 @@ fn declared_utf8(xml: String) -> String { } } -/// Keep a file read from a code page as it was, beside it, before its -/// first replacement on a day: `vdd_settings.xml.bak-`. -fn back_up(path: &Path) -> io::Result<()> { - let mut backup = path.as_os_str().to_owned(); - backup.push(format!(".bak-{}", utc_date(std::time::SystemTime::now()))); - let backup = std::path::PathBuf::from(backup); - if backup.exists() { - return Ok(()); +/// Keep the file at `path` as it is, beside it, as +/// `vdd_settings.xml.bak-`, or `-2`, `-3`… when today's backups +/// hold something else; one that holds the same bytes is kept as it is. +/// The backup's file name. +fn back_up(path: &Path) -> io::Result { + use std::io::Write; + let original = std::fs::read(path)?; + let name = path.file_name().unwrap_or_default().to_string_lossy(); + let date = utc_date(std::time::SystemTime::now()); + let mut n = 1; + loop { + let backup = match n { + 1 => format!("{name}.bak-{date}"), + n => format!("{name}.bak-{date}-{n}"), + }; + let at = path.with_file_name(&backup); + match std::fs::read(&at) { + Ok(held) if held == original => return Ok(backup), + Ok(_) => n += 1, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + let mut file = std::fs::File::create_new(&at)?; + file.write_all(&original)?; + file.sync_all()?; + return Ok(backup); + } + Err(error) => return Err(error), + } } - std::fs::copy(path, backup).map(drop) } /// `time`'s UTC date as `YYYYMMDD` (Howard Hinnant's civil_from_days). @@ -853,13 +898,22 @@ fn utc_date(time: std::time::SystemTime) -> String { format!("{year:04}{month:02}{day:02}") } -/// Replace the settings file with `contents`, backing it up first when it -/// was read from a code page (`legacy`). -fn replace_settings(path: &Path, legacy: bool, contents: &str) -> io::Result<()> { - if legacy { - back_up(path)?; +/// Replace the settings file with `write`. One read from a code page +/// (`legacy`) is backed up first, and its conversion logged once done. +fn write_settings( + path: &Path, + legacy: bool, + write: impl FnOnce() -> io::Result<()>, +) -> io::Result<()> { + let backup = legacy.then(|| back_up(path)).transpose()?; + write()?; + if let Some(backup) = backup { + tracing::info!( + backup, + "Wrote vdd_settings.xml as UTF-8; the original is kept beside it" + ); } - replace(path, contents) + Ok(()) } /// Write atomically (tmp + rename) so the driver never reads a torn file. @@ -1891,13 +1945,13 @@ mod tests { dir.join("app").join("vdd-pin.txt"), ); // Not UTF-8, in an encoding this host does not know. - let unknown = VENDOR - .replace("encoding='utf-8'", "encoding='x-unknown'") - .replace(">default<", ">Radeon\u{AE}<") - .replace('\u{AE}', ""); - let mut unknown = unknown.into_bytes(); - let at = unknown.windows(7).position(|w| w == b"Radeon<").unwrap() + 6; - unknown.insert(at, 0xAE); + let unknown = stray_byte(&VENDOR.replace("encoding='utf-8'", "encoding='x-unknown'")); + // UTF-8 with a stray byte: after a byte order mark, or with UTF-8 + // beyond ASCII after it (the vendor file's comments) or before it. + let bom = [&[0xEF, 0xBB, 0xBF][..], &stray_byte(VENDOR)].concat(); + let mixed_after = stray_byte(VENDOR_FILE); + let mixed_before = + stray_byte(&VENDOR.replace("", "")); // A double-byte character cut short. let mut broken = declared(CP949, "euc-kr"); let at = broken.iter().position(|&b| b >= 0x80).unwrap(); @@ -1906,12 +1960,21 @@ mod tests { odd.push(b' '); let mut lone = utf16(VENDOR, false, true); lone.extend_from_slice(&0xD800u16.to_le_bytes()); + // Off Windows no code page is read; on it, the system's code page + // reads this (a_file_in_the_systems_code_page_is_written_back_as_utf8). + let ansi = cfg!(not(windows)).then(|| ("ansi", stray_byte(VENDOR))); for (name, bytes) in [ ("unknown encoding", unknown), + ("utf-8 bom, stray byte", bom), + ("utf-8 after a stray byte", mixed_after), + ("utf-8 before a stray byte", mixed_before), ("broken code page", broken), ("odd utf-16", odd), ("lone surrogate", lone), - ] { + ] + .into_iter() + .chain(ansi) + { std::fs::write(&path, &bytes).unwrap(); let pinned = apply_pin(&path, &record, false, Some("Intel(R) HD Graphics 630")); let error = pinned.expect_err(name); @@ -1930,6 +1993,15 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + /// `text` naming the card `Radeon` followed by the byte 0xAE, which is + /// `®` in code page 1252 and not UTF-8. + fn stray_byte(text: &str) -> Vec { + let mut bytes = text.replace(">default<", ">Radeon<").into_bytes(); + let at = bytes.windows(7).position(|w| w == b"Radeon<").unwrap() + 6; + bytes.insert(at, 0xAE); + bytes + } + /// The driver's file saved in code page 949 as Notepad's ANSI does: its /// `encoding='utf-8'` declaration stays. A comment holds a folder path in /// non-ASCII text, and so does the card's name. @@ -2071,6 +2143,25 @@ mod tests { ); assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + // When the card stays, another app's or the driver's choice, + // the file is still written as UTF-8, so the driver reads it + // to the end. + for (wanted, outcome) in [ + (Some(INTEL), PinOutcome::Kept(NAMED_ELSEWHERE)), + (Some(card), PinOutcome::Unchanged), + (None, PinOutcome::Unchanged), + ] { + fresh(); + let pinned = apply_pin(&path, &record, false, wanted); + assert_eq!(pinned.ok(), Some(outcome), "{label} {wanted:?}"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), utf8, "{label}"); + assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + } + fresh(); + assert_eq!(pin_gpu_in(&path, card).ok(), Some(true), "{label}"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), utf8, "{label}"); + assert_eq!(backups(&dir), vec![declared.clone()], "{label}"); + // Match the iPad's resolution writes the whole file, keeping // the card another app named. fresh(); @@ -2088,6 +2179,112 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + /// Review r1: the system's code page, which reads a file declared UTF-8 + /// that is not or declaring nothing, had no test of its own. Here it is + /// the driver's file saved as ANSI on a system whose code page is 1252, + /// as on CI and the test PC, naming a card outside ASCII. Another code + /// page reads these bytes as other characters, so the test needs 1252. + #[cfg(windows)] + #[test] + fn a_file_in_the_systems_code_page_is_written_back_as_utf8() { + let ansi_code_page = unsafe { windows::Win32::Globalization::GetACP() }; + if ansi_code_page != 1252 { + eprintln!("skipped: the system's code page is {ansi_code_page}, not 1252"); + return; + } + let dir = std::env::temp_dir().join(format!("vdd-encoding-acp-{}", std::process::id())); + let (path, record) = ( + dir.join("vdd_settings.xml"), + dir.join("app").join("vdd-pin.txt"), + ); + const INTEL: &str = "Intel(R) HD Graphics 630"; + const CARD: &str = "Radeon\u{AE}"; + let text = VENDOR_FILE.replace(">default<", &format!(">{CARD}<")); + // As Notepad saves it: `®` is 0xAE and the comments' `’` 0x92. + let ansi: Vec = text + .chars() + .map(|c| match c { + '\u{AE}' => 0xAE, + '\u{2019}' => 0x92, + c => u8::try_from(c).ok().filter(u8::is_ascii).unwrap(), + }) + .collect(); + let fresh = || { + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(&path, &ansi).unwrap(); + }; + assert_eq!(decode_settings(&ansi, ANSI), Some((text.clone(), true))); + + // The card another app named stays, or this one, or the driver's + // choice; the file is written as UTF-8 all the same. + for (owned, wanted, outcome, card) in [ + (false, Some(INTEL), PinOutcome::Kept(NAMED_ELSEWHERE), CARD), + (false, Some(CARD), PinOutcome::Unchanged, CARD), + (true, Some(INTEL), PinOutcome::Named, INTEL), + ] { + fresh(); + let pinned = apply_pin(&path, &record, owned, wanted); + assert_eq!(pinned.ok(), Some(outcome), "{wanted:?}"); + let written = std::fs::read_to_string(&path).unwrap(); + assert_eq!(written, text.replace(CARD, card), "{wanted:?}"); + assert_eq!(backups(&dir), vec![ansi.clone()], "{wanted:?}"); + } + + // Match the iPad's resolution writes the whole file. + fresh(); + let modes = modes_for_client(1640, 2360, 60); + let written = write_with_pin(&path, &record, false, &modes, Some(INTEL)); + assert_eq!(written.ok(), Some(PinOutcome::Kept(NAMED_ELSEWHERE))); + let written = std::fs::read_to_string(&path).unwrap(); + assert_eq!(written, render(&modes, Some(CARD))); + assert_eq!(backups(&dir), vec![ansi.clone()]); + std::fs::remove_dir_all(&dir).unwrap(); + } + + /// Review r1: UTF-8 with a stray byte read in a single-byte code page + /// garbles its UTF-8 text. A double-byte code page's text may hold + /// what reads as UTF-8, so only single-byte code pages refuse it. + #[cfg(windows)] + #[test] + fn utf8_with_a_stray_byte_is_not_read_in_a_single_byte_code_page() { + let text = VENDOR.replace(">default<", ">Radeon\u{AE}<"); + assert_eq!( + decode_settings(&stray_byte(VENDOR), 1252), + Some((text, true)) + ); + let bom = [&[0xEF, 0xBB, 0xBF][..], &stray_byte(VENDOR)].concat(); + assert_eq!(decode_settings(&bom, 1252), None); + assert_eq!(decode_settings(&stray_byte(VENDOR_FILE), 1252), None); + assert!(single_byte(1252) && !single_byte(949) && !single_byte(936)); + } + + /// Review r1: a second conversion the same day kept no copy of the file + /// it replaced. + #[test] + fn each_original_of_a_day_is_backed_up_once() { + let dir = std::env::temp_dir().join(format!("vdd-backup-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("vdd_settings.xml"); + let date = utc_date(std::time::SystemTime::now()); + let mut names = Vec::new(); + for original in ["one", "two", "one", "three", "two"] { + std::fs::write(&path, original).unwrap(); + names.push(back_up(&path).unwrap()); + } + let name = |suffix: &str| format!("vdd_settings.xml.bak-{date}{suffix}"); + assert_eq!( + names, + [name(""), name("-2"), name(""), name("-3"), name("-2")] + ); + assert_eq!( + backups(&dir), + [b"one".to_vec(), b"two".to_vec(), b"three".to_vec()] + ); + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn encoding_names_are_windows_code_pages() { for (label, expected) in [ From 8f660c284e64fa45276601c6eddfdbdb8497c574 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 04:13:22 -0500 Subject: [PATCH 157/226] Bound the wait for a presented frame, and accept only this span's answer Fourth review of #206 found three more gaps; this closes them. - A window whose paints return without a surface keeps running UI passes, so the watcher's 4 s without a pass never came, and its screenshots were asked for again and again with no notice. The watcher now also raises the notice when the window has run UI passes for 12 s without being confirmed drawing (a working one is at about 3.5 s). Both bounds count from the latest of the start and the last time the window was held, so a minimized or dragged window suspends them. - Any screenshot marked as the proof was accepted, also a late one from before a surface error or a new device. Each ask now carries its span (a new one after each surface error, skipped frame or device) and a request id, and only an answer to the current span's asks confirms. - The watcher's decisions are a step function the tests call with each state in turn, instead of a thread they hoped had looked. --- host/src/gui/driver_wait.rs | 272 +++++++++++++++++++----------------- host/src/gui/mod.rs | 1 + host/src/gui/renderer.rs | 257 +++++++++++++++++++++++----------- 3 files changed, 319 insertions(+), 211 deletions(-) diff --git a/host/src/gui/driver_wait.rs b/host/src/gui/driver_wait.rs index d71fa772..9ae0900b 100644 --- a/host/src/gui/driver_wait.rs +++ b/host/src/gui/driver_wait.rs @@ -38,6 +38,12 @@ pub(super) const POLL: Duration = Duration::from_millis(250); /// How long OK hides the notice while the wait goes on. pub(super) const REMIND_AFTER: Duration = Duration::from_secs(15); +/// How long the window may run UI passes without being confirmed drawing. +/// A working one is confirmed about 3.5 s after its first pass (3 s and 10 +/// frames of drawing, then the screenshot that proves a present); one whose +/// paints return without a surface keeps passing and never is. +pub(super) const PROOF_WITHIN: Duration = Duration::from_secs(12); + /// How the user closed the notice. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Answer { @@ -287,27 +293,72 @@ where .is_some() } +/// What the first-paint watcher has seen, and when it says the window is +/// stuck. Every look is made from the watcher's thread, in order. +pub(super) struct Watch { + started: Instant, + limit: Duration, + proof_within: Duration, + last_held: Option, + first_pass: Option, +} + +impl Watch { + pub(super) fn new(started: Instant, limit: Duration, proof_within: Duration) -> Self { + Self { + started, + limit, + proof_within, + last_held: None, + first_pass: None, + } + } + + /// Whether the notice should be up at `at`, given the window thread's + /// last UI pass and whether the window is held now (minimized, or its + /// title bar or system menu held, when a working window runs no UI pass + /// either). Counted from the latest of the start and the last time it + /// was held: `limit` without a UI pass, or `proof_within` of UI passes + /// without the window confirmed drawing. + pub(super) fn look(&mut self, at: Instant, last_pass: Option, held: bool) -> bool { + if held { + self.last_held = Some(at); + return false; + } + if let Some(pass) = last_pass { + self.first_pass.get_or_insert(pass); + } + let since = self + .last_held + .map_or(self.started, |held| held.max(self.started)); + let quiet = last_pass.map_or(since, |pass| pass.max(since)); + let silent = at.saturating_duration_since(quiet) >= self.limit; + let unproven = self.first_pass.is_some_and(|first| { + at.saturating_duration_since(first.max(since)) >= self.proof_within + }); + silent || unproven + } +} + /// Watches the window from another thread until the wait ends /// ([`FirstPaintWatch::finisher`], called when the window is confirmed -/// drawing, or [`FirstPaintWatch::stop`]). The notice goes up once `limit` -/// has passed since the latest of: the watch's start, the window thread's -/// last UI pass (`last_pass`), and the last time the watcher saw the window -/// held (`held`: minimized, or its title bar held, when a working window -/// runs no UI pass either). Cancel runs `quit`, which ends the process: the +/// drawing, or [`FirstPaintWatch::stop`]), and raises the notice when +/// [`Watch::look`] says so. Cancel runs `quit`, which ends the process: the /// window's thread may never return. +#[allow(clippy::too_many_arguments)] pub(super) fn watch_first_paint( prompt: Prompt, limit: Duration, + proof_within: Duration, poll: Duration, now: impl Fn() -> Instant + Send + 'static, last_pass: impl Fn() -> Option + Send + 'static, held: impl Fn() -> bool + Send + 'static, quit: impl FnOnce() + Send + 'static, ) -> FirstPaintWatch { - let started = now(); + let mut watch = Watch::new(now(), limit, proof_within); let watched = prompt.clone(); - let mut last_held = None; - let watch = move || loop { + let run = move || loop { match watched.phase(now()) { Phase::Quit => { info!("Quit while the window waited for the graphics driver"); @@ -318,22 +369,14 @@ pub(super) fn watch_first_paint( Phase::Waiting => {} } let at = now(); - if held() { - last_held = Some(at); - } - let quiet_since = [Some(started), last_pass(), last_held] - .into_iter() - .flatten() - .max() - .unwrap_or(started); - if at.saturating_duration_since(quiet_since) >= limit { + if watch.look(at, last_pass(), held()) { watched.raise(at); } std::thread::sleep(poll); }; let thread = std::thread::Builder::new() .name("first-paint-watch".into()) - .spawn(watch) + .spawn(run) .map_err(|error| warn!(%error, "Could not watch the window's first frames")) .ok(); FirstPaintWatch { prompt, thread } @@ -461,7 +504,7 @@ impl Notice for MessageNotice { #[cfg(test)] mod tests { use super::*; - use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering::SeqCst}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst}; use std::sync::mpsc; /// For a call that hangs until the test releases it. @@ -829,137 +872,105 @@ mod tests { assert_eq!(fake.events.all(), ["shown", "closed"]); } - /// A clock the test moves on by hand, and what the renderer and the live - /// window check report, in its time. - struct World { - start: Instant, - ms: Arc, - last_pass: Arc>>, - held: Arc, - } + const LIMIT: Duration = Duration::from_secs(4); + const PROOF: Duration = Duration::from_secs(12); - impl World { - fn new() -> Self { - Self { - start: Instant::now(), - ms: Arc::default(), - last_pass: Arc::default(), - held: Arc::default(), - } - } - fn at(&self, ms: u64) { - self.ms.store(ms, SeqCst); - } - fn pass_at(&self, ms: u64) { - *self.last_pass.lock() = Some(self.start + Duration::from_millis(ms)); - self.at(ms); - } - fn watch(&self, fake: &Fake, quit: impl FnOnce() + Send + 'static) -> FirstPaintWatch { - let (start, ms) = (self.start, Arc::clone(&self.ms)); - let last_pass = Arc::clone(&self.last_pass); - let held = Arc::clone(&self.held); - watch_first_paint( - Prompt::new(fake.clone()), - SHORT.limit, - SLICE, - move || start + Duration::from_millis(ms.load(SeqCst)), - move || *last_pass.lock(), - move || held.load(SeqCst), - quit, - ) - } - /// Lets the watcher look a few times. - fn settle(&self) { - std::thread::sleep(Duration::from_millis(20)); - } + /// Looks at the window as the watcher does, each at a time in ms from + /// the start: (ms, last UI pass in ms, held). The times the notice + /// would be raised at. + fn looks(script: &[(u64, Option, bool)]) -> Vec { + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let mut watch = Watch::new(start, LIMIT, PROOF); + script + .iter() + .filter(|(ms, pass, held)| watch.look(at(*ms), pass.map(at), *held)) + .map(|(ms, _, _)| *ms) + .collect() } - fn never_quit() -> impl FnOnce() + Send + 'static { - || panic!("quit") + /// UI passes every 200 ms from `from` to `to` ms, looked at each 250 ms. + fn passing(from: u64, to: u64) -> Vec<(u64, Option, bool)> { + (from..=to) + .step_by(250) + .map(|ms| (ms, Some(ms / 200 * 200), false)) + .collect() } - const LIMIT_MS: u64 = SHORT.limit.as_millis() as u64; + #[test] + fn a_window_that_passes_and_draws_in_time_shows_no_notice() { + // Confirmed after about 3.5 s, well before either bound. + assert!(looks(&passing(0, 3_500)).is_empty()); + } #[test] - fn a_window_that_draws_in_time_shows_no_notice() { - let fake = Fake::default(); - let world = World::new(); - let watch = world.watch(&fake, never_quit()); - // UI passes keep coming; then the window is confirmed drawing. - for ms in [100, 300, 500, 700] { - world.pass_at(ms); - world.settle(); - } - (watch.finisher())(); - watch.stop().unwrap(); - assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); + fn a_failed_first_acquire_then_a_stuck_recreation_raises_the_notice_4_s_after_the_last_pass() { + // One pass at 200 ms, whose frame's surface acquire failed; the + // recreation at the start of the next paint never returns. + let script: Vec<_> = (250..=4_500) + .step_by(50) + .map(|ms| (ms, Some(200), false)) + .collect(); + assert_eq!(looks(&script)[0], 4_200); } #[test] - fn a_failed_first_acquire_then_a_stuck_recreation_gets_the_notice_until_the_window_draws() { - let fake = Fake::default(); - let world = World::new(); - let watch = world.watch(&fake, never_quit()); - // The first pass ran and its frame's surface acquire failed, so the - // renderer did not confirm the window; recreating the surface at the - // start of the next paint never returns: no more passes. - world.pass_at(200); - world.at(200 + LIMIT_MS - 1); - world.settle(); - assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); - world.at(200 + LIMIT_MS); - fake.events.wait_for("shown", 1); - // OK, then the reminder while it is still stuck. - *fake.answer.lock() = Some(Answer::Wait); - fake.events.wait_for("ok", 1); - world.at(200 + LIMIT_MS + REMIND_AFTER.as_millis() as u64 + 1); - fake.events.wait_for("shown", 2); - // It comes back and is confirmed drawing: the notice goes. - (watch.finisher())(); - watch.stop().unwrap(); - assert_eq!(fake.events.all(), ["shown", "ok", "shown", "closed"]); + fn passes_without_a_proof_of_a_presented_frame_raise_the_notice_at_the_bound() { + // The surface is gone and every paint returns before drawing: UI + // passes go on, the screenshots are never answered. + let raised = looks(&passing(0, 12_500)); + assert_eq!(raised.first(), Some(&12_000), "{raised:?}"); } #[test] - fn a_minimized_window_gets_no_notice_and_is_counted_from_when_it_is_restored() { - let fake = Fake::default(); - let world = World::new(); - let watch = world.watch(&fake, never_quit()); + fn held_windows_suspend_both_bounds_and_count_again_from_when_they_are_let_go() { // One pass, then minimized: eframe runs no pass for a minimized - // window, so nothing reaches the renderer; only the live check sees - // it. - world.pass_at(100); - world.held.store(true, SeqCst); - for ms in (200..=100 * LIMIT_MS).step_by(LIMIT_MS as usize) { - world.at(ms); - world.settle(); - } - assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); - // Restored at `back`, and its passes never come. - let back = 100 * LIMIT_MS + 1; - world.at(back); - world.settle(); - world.held.store(false, SeqCst); - world.at(back + LIMIT_MS - 1); - world.settle(); - assert!(fake.events.all().is_empty(), "{:?}", fake.events.all()); - world.at(back + LIMIT_MS + 1); - fake.events.wait_for("shown", 1); - watch.stop().unwrap(); - assert_eq!(fake.events.all(), ["shown", "closed"]); + // window, so only the live check sees it. Held for a minute: nothing. + let mut script = vec![(100, Some(100), false)]; + script.extend((250..=60_000).step_by(250).map(|ms| (ms, Some(100), true))); + assert!(looks(&script).is_empty()); + // Restored at 60 s, and no pass comes: the notice 4 s later. + let mut stuck = script.clone(); + stuck.extend( + (60_250..=64_500) + .step_by(250) + .map(|ms| (ms, Some(100), false)), + ); + assert_eq!(looks(&stuck).first(), Some(&64_000)); + // Restored and passing, but never confirmed: the proof bound counts + // from the release, not from the first pass. + let mut unproven = script.clone(); + unproven.extend(passing(60_250, 72_500)); + assert_eq!(looks(&unproven).first(), Some(&72_000)); + // The title bar held while passes stop counts the same way. + let mut dragged = passing(0, 2_000); + dragged.extend( + (2_250..=20_000) + .step_by(250) + .map(|ms| (ms, Some(2_000), true)), + ); + assert!(looks(&dragged).is_empty()); } #[test] fn cancel_while_the_window_is_stuck_quits_from_the_watcher() { let fake = Fake::default(); - let world = World::new(); let quits = Arc::new(AtomicUsize::new(0)); let count = Arc::clone(&quits); *fake.answer.lock() = Some(Answer::Quit); - let watch = world.watch(&fake, move || { - count.fetch_add(1, SeqCst); - }); - world.at(LIMIT_MS); + // Nothing passes: the notice goes up at once with a zero limit. + let watch = watch_first_paint( + Prompt::new(fake.clone()), + Duration::ZERO, + PROOF, + SLICE, + Instant::now, + || None, + || false, + move || { + count.fetch_add(1, SeqCst); + }, + ); watch.thread.unwrap().join().unwrap(); assert_eq!(quits.load(SeqCst), 1); assert_eq!(fake.events.all(), ["shown", "cancel"]); @@ -979,6 +990,7 @@ mod tests { let watch = watch_first_paint( prompt, REMIND_AFTER, + PROOF, SLICE, Instant::now, || None, @@ -1011,14 +1023,16 @@ mod tests { } start + SHORT.limit * 2 }; + let never_quit = || panic!("quit"); let watch = watch_first_paint( Prompt::new(fake.clone()), SHORT.limit, + PROOF, SLICE, now, || None, || false, - never_quit(), + never_quit, ); clock_read.recv().unwrap(); (watch.finisher())(); diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 27a6b560..79c9e882 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -1835,6 +1835,7 @@ pub fn run_gui( let watch = driver_wait::watch_first_paint( prompt, probe_limits.limit, + driver_wait::PROOF_WITHIN, driver_wait::POLL, std::time::Instant::now, renderer::last_pass, diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 01f620a5..64a3e3d5 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -646,8 +646,15 @@ fn save_marker(directory: &Path, owner: ProcessId, marker: &Marker) -> bool { .is_ok_and(|json| fs::write(marker_path(directory, owner), json).is_ok()) } -/// Marks the screenshot that proves a frame was presented. -struct PresentProof; +/// What a screenshot that proves a frame was presented carries: the span it +/// was asked in (a new one after each surface error, skipped frame or new +/// device) and which ask it answers. Only an answer to the current span's +/// asks counts. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct PresentProof { + span: u64, + request: u64, +} /// Called at the start of each UI pass; clears this launch's marker once the /// window has kept drawing without a surface error. @@ -662,11 +669,19 @@ pub(super) fn frame_started(ctx: &egui::Context) { if let Ok(mut last) = LAST_PASS.lock() { *last = Some(Instant::now()); } - let proof = ctx.input(|input| { - input.events.iter().any(|event| { - matches!(event, egui::Event::Screenshot { user_data, .. } - if user_data.data.as_ref().is_some_and(|data| data.is::())) - }) + let proofs: Vec = ctx.input(|input| { + input + .events + .iter() + .filter_map(|event| match event { + egui::Event::Screenshot { user_data, .. } => user_data + .data + .as_ref()? + .downcast_ref::() + .copied(), + _ => None, + }) + .collect() }); let Ok(mut tracker) = PRESENT.lock() else { return; @@ -676,15 +691,15 @@ pub(super) fn frame_started(ctx: &egui::Context) { failures(), visible, Instant::now(), - proof, + &proofs, ); let drawing = tracker.confirmed; drop(tracker); - if step == Step::Prove { + if let Step::Prove(proof) = step { // egui-wgpu answers only after it acquired the surface, drew the // frame into it and submitted it, right before the present. ctx.send_viewport_cmd(egui::ViewportCommand::Screenshot(egui::UserData::new( - PresentProof, + proof, ))); } if step == Step::Confirmed { @@ -910,24 +925,29 @@ fn finish_launch(directory: &Path, marker: &Marker, owner: ProcessId, drew: bool struct PresentTracker { span: Option, confirmed: bool, + /// The id the next span gets, and the next ask. + next_span: u64, + next_request: u64, } /// Visible frames since the last surface error. -#[derive(Clone, Copy)] +#[derive(Clone)] struct Span { + id: u64, since: Instant, first_frame: u64, failures: u64, - /// The frame that asked for the proof in this span. - asked: Option, + /// The frame that last asked for the proof in this span, and the asks + /// made in it. + asked: Option<(u64, std::ops::RangeInclusive)>, } /// What a UI pass does for the confirmation. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Step { Wait, - /// Ask for the proof: a screenshot of this frame. - Prove, + /// Ask for the proof: a screenshot of this frame, carrying this. + Prove(PresentProof), /// Confirmed, once. Confirmed, } @@ -937,50 +957,63 @@ impl PresentTracker { Self { span: None, confirmed: false, + next_span: 0, + next_request: 0, } } /// After [`CONFIRM_AFTER`] and [`CONFIRM_FRAMES`] of visible frames with /// no new surface error, asks for the proof (again after - /// [`PROOF_AGAIN_AFTER`] frames without it), and confirms once it came - /// (`proof`) in the same span. A surface error or a minimized window - /// starts the span again. + /// [`PROOF_AGAIN_AFTER`] frames without it), and confirms once an + /// answer to one of this span's asks came (`proofs`). A surface error, + /// a skipped frame, a new device or a minimized window starts the span + /// again, and answers to an earlier span's asks are ignored. fn observe( &mut self, frame: u64, failures: u64, visible: bool, now: Instant, - proof: bool, + proofs: &[PresentProof], ) -> Step { if self.confirmed { return Step::Wait; } + let request = self.next_request; match self.span.as_mut() { Some(span) if visible && span.failures == failures => { - if let Some(asked) = span.asked { - if proof { + let ask = if let Some((asked, requests)) = &span.asked { + if proofs + .iter() + .any(|proof| proof.span == span.id && requests.contains(&proof.request)) + { self.confirmed = true; return Step::Confirmed; } - if frame >= asked + PROOF_AGAIN_AFTER { - span.asked = Some(frame); - return Step::Prove; - } - } else if frame >= span.first_frame + CONFIRM_FRAMES - && now.duration_since(span.since) >= CONFIRM_AFTER - { - span.asked = Some(frame); - return Step::Prove; + (frame >= asked + PROOF_AGAIN_AFTER).then_some(*requests.start()..=request) + } else { + (frame >= span.first_frame + CONFIRM_FRAMES + && now.duration_since(span.since) >= CONFIRM_AFTER) + .then_some(request..=request) + }; + if let Some(requests) = ask { + span.asked = Some((frame, requests)); + self.next_request += 1; + return Step::Prove(PresentProof { + span: span.id, + request, + }); } } _ => { self.span = visible.then_some(Span { + id: self.next_span, since: now, first_frame: frame, failures, asked: None, }); + self.next_span += 1; } } Step::Wait @@ -2511,6 +2544,16 @@ mod tests { } /// Visible frames every 200 ms, as the GUI repaints, counted from `start`. + const NO_PROOF: &[PresentProof] = &[]; + + /// The proof a step asks for; panics on any other step. + fn ask(step: Step) -> PresentProof { + match step { + Step::Prove(proof) => proof, + other => panic!("{other:?}"), + } + } + /// Frames `frames` a 200 ms apart, the proof answering the frame after /// each ask: whether the window was confirmed. fn run( @@ -2519,16 +2562,19 @@ mod tests { frames: std::ops::Range, failures: u64, ) -> bool { - let mut asked = false; + let mut answer = None; frames.into_iter().any(|frame| { + let proofs: Vec = answer.take().into_iter().collect(); let step = tracker.observe( frame, failures, true, start + Duration::from_millis(200 * frame), - std::mem::take(&mut asked), + &proofs, ); - asked = step == Step::Prove; + if let Step::Prove(proof) = step { + answer = Some(proof); + } step == Step::Confirmed }) } @@ -2539,34 +2585,28 @@ mod tests { let mut tracker = PresentTracker::new(); // Frames alone are not enough: egui may lay out one frame many times. for frame in 0..100 { - assert_eq!(tracker.observe(frame, 0, true, start, false), Step::Wait); + assert_eq!(tracker.observe(frame, 0, true, start, NO_PROOF), Step::Wait); } + let proof = ask(tracker.observe(100, 0, true, start + CONFIRM_AFTER, NO_PROOF)); assert_eq!( - tracker.observe(100, 0, true, start + CONFIRM_AFTER, false), - Step::Prove - ); - assert_eq!( - tracker.observe(101, 0, true, start + CONFIRM_AFTER, true), + tracker.observe(101, 0, true, start + CONFIRM_AFTER, &[proof]), Step::Confirmed ); let mut tracker = PresentTracker::new(); // Nor is time alone, with one frame laid out repeatedly. - assert_eq!(tracker.observe(0, 0, true, start, false), Step::Wait); + assert_eq!(tracker.observe(0, 0, true, start, NO_PROOF), Step::Wait); assert_eq!( - tracker.observe(0, 0, true, start + 10 * CONFIRM_AFTER, false), + tracker.observe(0, 0, true, start + 10 * CONFIRM_AFTER, NO_PROOF), Step::Wait ); + let proof = ask(tracker.observe(CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER, NO_PROOF)); assert_eq!( - tracker.observe(CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER, false), - Step::Prove - ); - assert_eq!( - tracker.observe(CONFIRM_FRAMES + 1, 0, true, start + CONFIRM_AFTER, true), + tracker.observe(CONFIRM_FRAMES + 1, 0, true, start + CONFIRM_AFTER, &[proof]), Step::Confirmed ); // Once. assert_eq!( - tracker.observe(CONFIRM_FRAMES + 2, 0, true, start + CONFIRM_AFTER, true), + tracker.observe(CONFIRM_FRAMES + 2, 0, true, start + CONFIRM_AFTER, &[proof]), Step::Wait ); } @@ -2576,35 +2616,91 @@ mod tests { let start = Instant::now(); let later = start + CONFIRM_AFTER; let mut tracker = PresentTracker::new(); - assert_eq!(tracker.observe(0, 0, true, start, false), Step::Wait); - assert_eq!( - tracker.observe(CONFIRM_FRAMES, 0, true, later, false), - Step::Prove - ); + assert_eq!(tracker.observe(0, 0, true, start, NO_PROOF), Step::Wait); + let first = ask(tracker.observe(CONFIRM_FRAMES, 0, true, later, NO_PROOF)); // The surface was removed and its recreation failed: egui-wgpu's // paints return before drawing, with no surface error, and the UI // passes go on. No screenshot answers, so nothing is confirmed; the // ask is repeated now and then. for frame in CONFIRM_FRAMES + 1..CONFIRM_FRAMES + PROOF_AGAIN_AFTER { - assert_eq!(tracker.observe(frame, 0, true, later, false), Step::Wait); + assert_eq!(tracker.observe(frame, 0, true, later, NO_PROOF), Step::Wait); } + let again = + ask(tracker.observe(CONFIRM_FRAMES + PROOF_AGAIN_AFTER, 0, true, later, NO_PROOF)); + assert_eq!(again.span, first.span); + assert_ne!(again.request, first.request); + // A late answer to the first ask of this span still proves a frame. assert_eq!( - tracker.observe(CONFIRM_FRAMES + PROOF_AGAIN_AFTER, 0, true, later, false), - Step::Prove + tracker.observe( + CONFIRM_FRAMES + PROOF_AGAIN_AFTER + 1, + 0, + true, + later, + &[first] + ), + Step::Confirmed ); - // A proof after a surface error is not this span's. + } + + #[test] + fn a_late_answer_from_before_a_surface_error_or_a_new_device_proves_nothing() { + let start = Instant::now(); + let later = start + CONFIRM_AFTER; + // `failures` counts surface errors, frames the painter skipped and + // devices: a lost device skips frames and its replacement is one more. + for (what, after) in [ + ("a surface error", 1), + ("skipped frames and a new device", 4), + ] { + let mut tracker = PresentTracker::new(); + tracker.observe(0, 0, true, start, NO_PROOF); + let old = ask(tracker.observe(CONFIRM_FRAMES, 0, true, later, NO_PROOF)); + // The old frame's answer arrives only after the change. + let frame = CONFIRM_FRAMES + 1; + assert_eq!( + tracker.observe(frame, after, true, later, &[old]), + Step::Wait, + "{what}" + ); + let new_start = later; + let new_later = new_start + CONFIRM_AFTER; + assert_eq!( + tracker.observe(frame + 1, after, true, new_start, &[old]), + Step::Wait, + "{what}" + ); + let new = + ask(tracker.observe(frame + 1 + CONFIRM_FRAMES, after, true, new_later, &[old])); + assert_ne!(new.span, old.span, "{what}"); + // Still only the old answer: nothing. + assert_eq!( + tracker.observe(frame + 2 + CONFIRM_FRAMES, after, true, new_later, &[old]), + Step::Wait, + "{what}" + ); + // The new span's own answer confirms. + assert_eq!( + tracker.observe( + frame + 3 + CONFIRM_FRAMES, + after, + true, + new_later, + &[old, new] + ), + Step::Confirmed, + "{what}" + ); + } + // An answer naming this span but an ask it never made proves nothing. let mut tracker = PresentTracker::new(); - tracker.observe(0, 0, true, start, false); - assert_eq!( - tracker.observe(CONFIRM_FRAMES, 0, true, later, false), - Step::Prove - ); - assert_eq!( - tracker.observe(CONFIRM_FRAMES + 1, 1, true, later, true), - Step::Wait - ); + tracker.observe(0, 0, true, start, NO_PROOF); + let asked = ask(tracker.observe(CONFIRM_FRAMES, 0, true, later, NO_PROOF)); + let forged = PresentProof { + request: asked.request + 7, + ..asked + }; assert_eq!( - tracker.observe(CONFIRM_FRAMES + 2, 1, true, later, true), + tracker.observe(CONFIRM_FRAMES + 1, 0, true, later, &[forged]), Step::Wait ); } @@ -2613,12 +2709,12 @@ mod tests { fn a_lost_surface_that_is_recreated_late_is_not_a_working_window() { let start = Instant::now(); let mut tracker = PresentTracker::new(); - assert_eq!(tracker.observe(0, 0, true, start, false), Step::Wait); + assert_eq!(tracker.observe(0, 0, true, start, NO_PROOF), Step::Wait); // Lost: egui reports it and recreates the surface on the next frame, // which can fail without a report and without a present. for (frame, ms) in [(1, 16), (2, 32), (3, 48)] { assert_eq!( - tracker.observe(frame, 1, true, start + Duration::from_millis(ms), false), + tracker.observe(frame, 1, true, start + Duration::from_millis(ms), NO_PROOF), Step::Wait ); } @@ -2706,32 +2802,29 @@ mod tests { fn a_minimized_window_is_not_confirmed() { let start = Instant::now(); let mut tracker = PresentTracker::new(); - assert_eq!(tracker.observe(0, 0, true, start, false), Step::Wait); + assert_eq!(tracker.observe(0, 0, true, start, NO_PROOF), Step::Wait); assert_eq!( - tracker.observe(CONFIRM_FRAMES, 0, false, start + CONFIRM_AFTER, false), + tracker.observe(CONFIRM_FRAMES, 0, false, start + CONFIRM_AFTER, NO_PROOF), Step::Wait ); assert_eq!( - tracker.observe(2 * CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER, false), + tracker.observe(2 * CONFIRM_FRAMES, 0, true, start + CONFIRM_AFTER, NO_PROOF), Step::Wait ); - assert_eq!( - tracker.observe( - 3 * CONFIRM_FRAMES, - 0, - true, - start + 2 * CONFIRM_AFTER, - false - ), - Step::Prove - ); + let proof = ask(tracker.observe( + 3 * CONFIRM_FRAMES, + 0, + true, + start + 2 * CONFIRM_AFTER, + NO_PROOF, + )); assert_eq!( tracker.observe( 3 * CONFIRM_FRAMES + 1, 0, true, start + 2 * CONFIRM_AFTER, - true + &[proof] ), Step::Confirmed ); From 034caf60cbd0e7c0ce0f100c51b07bb59a0eff93 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 04:20:17 -0500 Subject: [PATCH 158/226] Keep the HUD up in the pairing UI test The pairing test waited for the stream's Disconnect button after reading the pairing field back. Once the right code pairs, the sheet closes and the field is gone, so that read retries for up to several seconds on a busy runner. The stream screen opens on the first decoded frame, and its HUD hides five seconds later. In three CI failures the wait for the button began 5.8 to 6.6 s after the last digit, after the HUD had already hidden, so the button was never found although the stream was up. The test now launches with -assumeVoiceOver YES, which only stops the HUD from hiding by itself. Disconnect then stays from the first frame until the test acts, and the same 10 s wait proves that the right code paired and the stream started, and later that the token reconnect streams. HUDHideTests still covers the HUD hiding by itself. --- ios/EternalMonitorUITests/PairingFlowTests.swift | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ios/EternalMonitorUITests/PairingFlowTests.swift b/ios/EternalMonitorUITests/PairingFlowTests.swift index 9d02ce09..aad0a8aa 100644 --- a/ios/EternalMonitorUITests/PairingFlowTests.swift +++ b/ios/EternalMonitorUITests/PairingFlowTests.swift @@ -8,7 +8,13 @@ final class PairingFlowTests: UITestCase { let host = try XCTUnwrap(environment["EM_PAIRING_HOST"]) XCTAssertEqual(code.count, 6) let app = XCUIApplication() - app.launchArguments = ["-didSeeOnboarding", "YES", "-allowUSB", "NO", "-controlPC", "NO", "-playPCaudio", "NO", "-pictureFit", "fit"] + // VoiceOver keeps the HUD up, so Disconnect is there from the first + // frame until the test acts. Otherwise the HUD hides five seconds + // after the stream opens, and reading the closed sheet's field back + // can take that long on a starved runner (6.6 s twice on CI). + // HUDHideTests covers the HUD hiding by itself. + app.launchArguments = ["-didSeeOnboarding", "YES", "-allowUSB", "NO", "-controlPC", "NO", "-playPCaudio", "NO", + "-pictureFit", "fit", "-assumeVoiceOver", "YES"] app.launchEnvironment["EM_AUTOCONNECT"] = host app.launchEnvironment["EM_E2E_LOG"] = "1" launchCleanly(app) From 57cd87d4c1c213584b48f986175c7d851f82a417 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 04:30:41 -0500 Subject: [PATCH 159/226] Make the encoding tests independent of the system code page and date The mixed UTF-8 cases are refused only in a single-byte system code page, so the file-level cases now run only there and both byte orders are tested in code page 1252 directly. The backup test passes its date instead of reading the clock. --- host/src/vdd_settings.rs | 42 +++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/host/src/vdd_settings.rs b/host/src/vdd_settings.rs index c45d9fb7..84a38899 100644 --- a/host/src/vdd_settings.rs +++ b/host/src/vdd_settings.rs @@ -856,10 +856,14 @@ fn declared_utf8(xml: String) -> String { /// hold something else; one that holds the same bytes is kept as it is. /// The backup's file name. fn back_up(path: &Path) -> io::Result { + back_up_dated(path, &utc_date(std::time::SystemTime::now())) +} + +/// `back_up` on `date`. +fn back_up_dated(path: &Path, date: &str) -> io::Result { use std::io::Write; let original = std::fs::read(path)?; let name = path.file_name().unwrap_or_default().to_string_lossy(); - let date = utc_date(std::time::SystemTime::now()); let mut n = 1; loop { let backup = match n { @@ -1949,9 +1953,19 @@ mod tests { // UTF-8 with a stray byte: after a byte order mark, or with UTF-8 // beyond ASCII after it (the vendor file's comments) or before it. let bom = [&[0xEF, 0xBB, 0xBF][..], &stray_byte(VENDOR)].concat(); - let mixed_after = stray_byte(VENDOR_FILE); - let mixed_before = - stray_byte(&VENDOR.replace("", "")); + // A double-byte system code page may read these as its own text, so + // they are refused only in a single-byte one (and off Windows, where + // no code page is read); utf8_with_a_stray_byte_is_not_read_in_a_ + // single_byte_code_page tests both in 1252 on any system. + let mixed = if cfg!(not(windows)) || single_byte(ANSI) { + vec![ + ("utf-8 after a stray byte", mixed_after()), + ("utf-8 before a stray byte", mixed_before()), + ] + } else { + eprintln!("skipped the mixed UTF-8 cases: the system's code page is not single-byte"); + Vec::new() + }; // A double-byte character cut short. let mut broken = declared(CP949, "euc-kr"); let at = broken.iter().position(|&b| b >= 0x80).unwrap(); @@ -1966,13 +1980,12 @@ mod tests { for (name, bytes) in [ ("unknown encoding", unknown), ("utf-8 bom, stray byte", bom), - ("utf-8 after a stray byte", mixed_after), - ("utf-8 before a stray byte", mixed_before), ("broken code page", broken), ("odd utf-16", odd), ("lone surrogate", lone), ] .into_iter() + .chain(mixed) .chain(ansi) { std::fs::write(&path, &bytes).unwrap(); @@ -2002,6 +2015,16 @@ mod tests { bytes } + /// UTF-8 beyond ASCII after a stray byte: the vendor file's comments. + fn mixed_after() -> Vec { + stray_byte(VENDOR_FILE) + } + + /// UTF-8 beyond ASCII before a stray byte. + fn mixed_before() -> Vec { + stray_byte(&VENDOR.replace("", "")) + } + /// The driver's file saved in code page 949 as Notepad's ANSI does: its /// `encoding='utf-8'` declaration stays. A comment holds a folder path in /// non-ASCII text, and so does the card's name. @@ -2255,7 +2278,8 @@ mod tests { ); let bom = [&[0xEF, 0xBB, 0xBF][..], &stray_byte(VENDOR)].concat(); assert_eq!(decode_settings(&bom, 1252), None); - assert_eq!(decode_settings(&stray_byte(VENDOR_FILE), 1252), None); + assert_eq!(decode_settings(&mixed_after(), 1252), None); + assert_eq!(decode_settings(&mixed_before(), 1252), None); assert!(single_byte(1252) && !single_byte(949) && !single_byte(936)); } @@ -2267,11 +2291,11 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("vdd_settings.xml"); - let date = utc_date(std::time::SystemTime::now()); + let date = "20261108"; let mut names = Vec::new(); for original in ["one", "two", "one", "three", "two"] { std::fs::write(&path, original).unwrap(); - names.push(back_up(&path).unwrap()); + names.push(back_up_dated(&path, date).unwrap()); } let name = |suffix: &str| format!("vdd_settings.xml.bak-{date}{suffix}"); assert_eq!( From 3aaa902f6588131ecd19d9ed49a6af57352ca144 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 05:04:02 -0500 Subject: [PATCH 160/226] Accept the second screen step as where a session-locked loop is stuck a_stuck_transport_loop_is_named_with_its_step holds the session lock and expected the loop to be named stuck at its tick. The loop takes that lock at the tick and again, after the tick's replies, at the second screen's step (client_heard in second_screen::next_step). A hold that lands between the two leaves the loop stuck in SecondScreen, and the watch rightly says so. It happened once in candidate CI. The test now takes either step that locks the session, and still requires one stall of at least 2 s, then a move from the same step. Holding the lock as the loop marks SecondScreen, the old test fails as in CI: Stalled { phase: SecondScreen, silent_ms: 2059 }. The new one passes. --- host/tests/transport_stall.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/host/tests/transport_stall.rs b/host/tests/transport_stall.rs index 38446046..20d75212 100644 --- a/host/tests/transport_stall.rs +++ b/host/tests/transport_stall.rs @@ -1,7 +1,7 @@ //! A transport loop stuck in one call is named by the supervisor's watch, with //! the step it is stuck in, and again when it moves. Drives the real loop and //! blocks it the way a stuck call would: the session lock it needs at its -//! next tick is held elsewhere. +//! next tick, or at the second screen's step after it, is held elsewhere. use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; @@ -97,13 +97,21 @@ fn a_stuck_transport_loop_is_named_with_its_step() { Instant::now() + Duration::from_secs(4), ); drop(held); + // The loop takes the session lock at its tick and again, after each + // tick's replies, for the second screen's next step. Whichever it + // reaches first while the lock is held is where it is stuck. + let [Verdict::Stalled { + phase: stuck_in, + silent_ms, + }] = stuck.as_slice() + else { + panic!("{stuck:?}"); + }; assert!( - matches!( - stuck.as_slice(), - [Verdict::Stalled { phase: Phase::Tick, silent_ms }] if *silent_ms >= 2_000 - ), + matches!(stuck_in, Phase::Tick | Phase::SecondScreen), "{stuck:?}" ); + assert!(*silent_ms >= 2_000, "{stuck:?}"); let moved = watch_until( &mut watch, &shared, @@ -113,7 +121,7 @@ fn a_stuck_transport_loop_is_named_with_its_step() { assert!( matches!( moved.as_slice(), - [Verdict::Moving { phase: Phase::Tick, stalled_ms }] if *stalled_ms >= 2_000 + [Verdict::Moving { phase, stalled_ms }] if phase == stuck_in && *stalled_ms >= 2_000 ), "{moved:?}" ); From 32d5b37e5910b2cf798f371b144eb3b3d9946320 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 05:21:14 -0500 Subject: [PATCH 161/226] Leave release clippy out of CI The macOS runner is CI's bottleneck, and a release-profile clippy checks every dependency again in a second profile. It stays a manual check before a release. --- .github/workflows/ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6239f5cf..691a0493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,13 +111,6 @@ jobs: env: PKG_CONFIG_PATH: /opt/homebrew/opt/ffmpeg@7/lib/pkgconfig run: cargo clippy --workspace --all-targets --locked -- -D warnings - - name: Lint workspace as released - # Code gated on debug_assertions or Windows can leave items unused in - # a macOS release build. Libraries and binaries only: the tests use - # debug-only fakes and don't build in release. - env: - PKG_CONFIG_PATH: /opt/homebrew/opt/ffmpeg@7/lib/pkgconfig - run: cargo clippy --workspace --release --locked -- -D warnings - name: Test workspace (includes synthetic end-to-end stream) env: PKG_CONFIG_PATH: /opt/homebrew/opt/ffmpeg@7/lib/pkgconfig From 229904ac7f07727d8727e42f615b45b8adcc82dc Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 05:24:46 -0500 Subject: [PATCH 162/226] Start the audio stage tests' fake tone at its first read The fake source timed its tone from when it was made. The Opus open comes between that and the session's first read, and on a loaded PC it can take over 250 ms. The whole tone then predated the session's start, the stage dropped every frame as stale, and the test saw no packets. The tone now starts at the first read, as a real capture's audio does, and one test holds the open for 300 ms to keep it that way. --- host/src/audio/stage.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/host/src/audio/stage.rs b/host/src/audio/stage.rs index 9367cb24..a0f7f157 100644 --- a/host/src/audio/stage.rs +++ b/host/src/audio/stage.rs @@ -912,14 +912,20 @@ mod tests { /// Ten 20 ms blocks of tone, then it notes the audio status in `seen` /// and stops the stream. + /// + /// The tone starts at the first read, as a real capture's audio does. + /// Timed from when the source was made, it predated the session's start + /// whenever the Opus open between the two took over 250 ms, as it can on + /// a loaded PC, and the stage rightly dropped every frame as stale. fn tone_then_stop( shared: &SharedControl, seen: &std::rc::Rc>>, ) -> Source { let (shared, seen) = (shared.clone(), seen.clone()); - let start = crate::clock::host_now_us() + 50_000; + let mut start = None; let mut index = 0; Source::Fake(Box::new(move || { + let start = *start.get_or_insert_with(crate::clock::host_now_us); if index == 10 { *seen.borrow_mut() = Some(audio_status()); shared.running.store(false, Ordering::SeqCst); @@ -1103,7 +1109,11 @@ mod tests { if opens.borrow().len() <= 4 { busy() } else { - Ok(tone_then_stop(&shared, &seen)) + let source = tone_then_stop(&shared, &seen); + // Real time before the first read, as a loaded PC's Opus + // open takes: the tone must still be sent. + std::thread::sleep(Duration::from_millis(300)); + Ok(source) } }; let (lines, packets) = run_stepped(&shared, open, &clock); From f7ceae324007d1b1d40969b0a53a88e83ed6b6c4 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 05:41:55 -0500 Subject: [PATCH 163/226] Ride out GitHub's 5xx spells on release downloads GitHub's release downloads answered 504 for minutes at a time: the nightly Beta's download check (verify(), 6 tries 10 s apart) failed on them, and the installer builds downloaded the bundled driver once, with no retry. verify() now gives HTTP 429 and 5xx, from the HEAD or the hashing download, and dropped connections a window of their own: 12 tries, each wait 5 s longer up to 30 s. A missing file and a size or hash mismatch keep the short window they had, for an asset that is new or was just renamed. The size and SHA-256 must still match exactly. The driver and license downloads in beta-installer.yml and release.yml retry 429, 5xx and dropped connections up to 5 times over 2 minutes; any other 4xx fails at once. The pinned driver SHA-256 check that follows is unchanged. --- .github/workflows/beta-installer.yml | 24 ++++++- .github/workflows/release.yml | 24 ++++++- scripts/nightly.py | 33 ++++++++-- scripts/test_nightly.py | 93 ++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+), 10 deletions(-) diff --git a/.github/workflows/beta-installer.yml b/.github/workflows/beta-installer.yml index 654477f6..8aa26e2c 100644 --- a/.github/workflows/beta-installer.yml +++ b/.github/workflows/beta-installer.yml @@ -100,12 +100,30 @@ jobs: - name: Download the bundled virtual display driver shell: pwsh run: | + # GitHub's download servers sometimes answer 5xx for a minute or two. + # Up to 5 tries over 2 minutes; another 4xx fails at once. The next + # step checks the driver's pinned SHA-256 either way. + function Get-WithRetry($uri, $out) { + $waits = 10, 20, 30, 60 + for ($try = 1; ; $try++) { + try { + Invoke-WebRequest -Uri $uri -OutFile $out + return + } catch { + $status = [int]$_.Exception.Response.StatusCode + $transient = $status -eq 0 -or $status -eq 429 -or $status -ge 500 + if (-not $transient -or $try -gt $waits.Count) { throw } + Write-Warning "Download of $uri failed (try $try): $($_.Exception.Message). Retrying in $($waits[$try - 1]) s." + Start-Sleep -Seconds $waits[$try - 1] + } + } + } $dest = "installer\vendor\vdd" $url = "https://github.com/VirtualDrivers/Virtual-Display-Driver/releases/download/$env:VDD_RELEASE_TAG/$env:VDD_ASSET" - Invoke-WebRequest -Uri $url -OutFile (Join-Path $dest $env:VDD_ASSET) + Get-WithRetry $url (Join-Path $dest $env:VDD_ASSET) # Its MIT license text, redistributed alongside the binary. - Invoke-WebRequest -Uri "https://raw.githubusercontent.com/VirtualDrivers/Virtual-Display-Driver/$env:VDD_RELEASE_TAG/LICENSE" ` - -OutFile (Join-Path $dest "LICENSE-VirtualDisplayDriver.txt") + Get-WithRetry "https://raw.githubusercontent.com/VirtualDrivers/Virtual-Display-Driver/$env:VDD_RELEASE_TAG/LICENSE" ` + (Join-Path $dest "LICENSE-VirtualDisplayDriver.txt") - name: Verify the driver against the pinned SHA-256 shell: pwsh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a66da814..83476248 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -101,12 +101,30 @@ jobs: - name: Download the bundled virtual display driver shell: pwsh run: | + # GitHub's download servers sometimes answer 5xx for a minute or two. + # Up to 5 tries over 2 minutes; another 4xx fails at once. The next + # step checks the driver's pinned SHA-256 either way. + function Get-WithRetry($uri, $out) { + $waits = 10, 20, 30, 60 + for ($try = 1; ; $try++) { + try { + Invoke-WebRequest -Uri $uri -OutFile $out + return + } catch { + $status = [int]$_.Exception.Response.StatusCode + $transient = $status -eq 0 -or $status -eq 429 -or $status -ge 500 + if (-not $transient -or $try -gt $waits.Count) { throw } + Write-Warning "Download of $uri failed (try $try): $($_.Exception.Message). Retrying in $($waits[$try - 1]) s." + Start-Sleep -Seconds $waits[$try - 1] + } + } + } $dest = "installer\vendor\vdd" $url = "https://github.com/VirtualDrivers/Virtual-Display-Driver/releases/download/$env:VDD_RELEASE_TAG/$env:VDD_ASSET" - Invoke-WebRequest -Uri $url -OutFile (Join-Path $dest $env:VDD_ASSET) + Get-WithRetry $url (Join-Path $dest $env:VDD_ASSET) # Its MIT license text, redistributed alongside the binary. - Invoke-WebRequest -Uri "https://raw.githubusercontent.com/VirtualDrivers/Virtual-Display-Driver/$env:VDD_RELEASE_TAG/LICENSE" ` - -OutFile (Join-Path $dest "LICENSE-VirtualDisplayDriver.txt") + Get-WithRetry "https://raw.githubusercontent.com/VirtualDrivers/Virtual-Display-Driver/$env:VDD_RELEASE_TAG/LICENSE" ` + (Join-Path $dest "LICENSE-VirtualDisplayDriver.txt") - name: Verify the driver against the pinned SHA-256 shell: pwsh diff --git a/scripts/nightly.py b/scripts/nightly.py index 0e45d504..3b9cf294 100644 --- a/scripts/nightly.py +++ b/scripts/nightly.py @@ -241,13 +241,26 @@ def assets(release, name): return [a for a in (release or {}).get("assets", []) if a["name"] == name] -def verify(github, url, size, sha256=None, tries=6, wait=10, sleep=time.sleep): +def transient(status): + """An HTTP status GitHub's download servers give while briefly unwell, which says + nothing about the file.""" + return status == 429 or 500 <= status <= 599 + + +def verify(github, url, size, sha256=None, tries=6, wait=10, transient_tries=12, sleep=time.sleep): """url serves size bytes (with sha256, those exact bytes) to an anonymous visitor. - A new or renamed asset can take a few seconds to show, so it retries.""" - for attempt in range(tries): + + A new or renamed asset can take a few seconds to show, so a missing or different + file is checked again: tries times, wait seconds apart. A server error (HTTP 429 or + 5xx) or a dropped connection says nothing about the file, so those have a longer + window of their own: transient_tries, each wait 5 s longer up to 30 s (about four + minutes). Either way the size and SHA-256 must match exactly.""" + misses = errors = 0 + while True: try: status, length = github.head(url) seen = f"HTTP {status}, {length} bytes" + flaky = transient(status) if status == 200 and length == size: if sha256 is None: return @@ -255,9 +268,21 @@ def verify(github, url, size, sha256=None, tries=6, wait=10, sleep=time.sleep): if got == sha256: return seen = f"SHA-256 {got}" + except urllib.error.HTTPError as error: + seen = f"HTTP {error.code}" + flaky = transient(error.code) except OSError as error: seen = str(error) - if attempt + 1 < tries: + flaky = True + if flaky: + errors += 1 + if errors >= transient_tries: + break + sleep(min(5 * errors, 30)) + else: + misses += 1 + if misses >= tries: + break sleep(wait) raise PublishError(f"{url}: {seen}; expected {size} bytes" + (f", SHA-256 {sha256}" if sha256 else "")) diff --git a/scripts/test_nightly.py b/scripts/test_nightly.py index 6d9e2d85..c8b6de30 100644 --- a/scripts/test_nightly.py +++ b/scripts/test_nightly.py @@ -9,6 +9,7 @@ import sys import tempfile import unittest +import urllib.error sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) import nightly # noqa: E402 @@ -297,6 +298,98 @@ def sha256(self, url): return hashlib.sha256(data + (b"x" if self.corrupt else b"")).hexdigest() +class ScriptedDownloads: + """An asset URL that answers HEAD and the SHA-256 download from scripts, one + entry per call, then keeps answering with the last entry. An entry that is an + exception is raised.""" + + def __init__(self, heads, hashes=()): + self.heads, self.hashes = list(heads), list(hashes) + self.calls = [] + + @staticmethod + def answer(script): + entry = script.pop(0) if len(script) > 1 else script[0] + if isinstance(entry, BaseException): + raise entry + return entry + + def head(self, url): + self.calls.append("head") + return self.answer(self.heads) + + def sha256(self, url): + self.calls.append("sha256") + return self.answer(self.hashes) + + +def http_error(code): + return urllib.error.HTTPError(URL, code, f"HTTP {code}", {}, None) + + +URL = "https://github.com/o/r/releases/download/beta/EternalMonitor-Setup.exe" +SIZE = 1234 +OK = (200, SIZE) + + +class VerifyTest(unittest.TestCase): + """verify() rides out GitHub's 5xx spells but never accepts a different file.""" + + def verify(self, github, sha256=None): + waits = [] + try: + nightly.verify(github, URL, SIZE, sha256, sleep=waits.append) + except nightly.PublishError as error: + return waits, str(error) + return waits, None + + def test_a_5xx_spell_is_waited_out_with_growing_waits(self): + github = ScriptedDownloads([(504, -1)] * 11 + [OK], [HASH]) + waits, failed = self.verify(github, HASH) + self.assertIsNone(failed) + self.assertEqual(waits, [5, 10, 15, 20, 25, 30, 30, 30, 30, 30, 30]) + + def test_a_5xx_spell_that_goes_on_fails_after_twelve_tries(self): + github = ScriptedDownloads([(502, -1)]) + waits, failed = self.verify(github) + self.assertEqual(github.calls.count("head"), 12) + self.assertIn("HTTP 502", failed) + + def test_rate_limits_and_dropped_connections_are_waited_out_too(self): + github = ScriptedDownloads([(429, -1), urllib.error.URLError("connection reset"), + TimeoutError("timed out"), OK]) + self.assertEqual(self.verify(github), ([5, 10, 15], None)) + + def test_a_5xx_while_hashing_is_waited_out(self): + github = ScriptedDownloads([OK], [http_error(504), http_error(503), HASH]) + self.assertEqual(self.verify(github, HASH), ([5, 10], None)) + + def test_a_missing_file_has_the_short_window_it_had(self): + # A new or renamed asset can take a few seconds to show. + github = ScriptedDownloads([(404, -1)] * 3 + [OK]) + self.assertEqual(self.verify(github), ([10, 10, 10], None)) + github = ScriptedDownloads([(404, -1)]) + waits, failed = self.verify(github) + self.assertEqual((len(waits), github.calls.count("head")), (5, 6)) + self.assertIn("HTTP 404", failed) + + def test_a_different_file_is_never_taken_for_a_server_error(self): + wrong_size = ScriptedDownloads([(200, SIZE + 1)]) + waits, failed = self.verify(wrong_size, HASH) + self.assertEqual(wrong_size.calls.count("head"), 6) + self.assertNotIn("sha256", wrong_size.calls) + self.assertIn(f"{SIZE + 1} bytes", failed) + wrong_bytes = ScriptedDownloads([OK], ["cd" * 32]) + waits, failed = self.verify(wrong_bytes, HASH) + self.assertEqual(wrong_bytes.calls.count("sha256"), 6) + self.assertIn("SHA-256 " + "cd" * 32, failed) + self.assertEqual(waits, [10] * 5) + + def test_server_errors_do_not_use_up_the_window_for_a_late_file(self): + github = ScriptedDownloads([(504, -1)] * 4 + [(404, -1)] * 5 + [OK]) + self.assertEqual(self.verify(github), ([5, 10, 15, 20] + [10] * 5, None)) + + OLD_SHA, NEW_SHA = "1" * 40, "2" * 40 WRITES = ("create", "edit", "upload", "rename", "delete", "move_tag") From 6e7397f8dbc2067a6a3dc4cb00bb9755c00d76eb Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 05:46:09 -0500 Subject: [PATCH 164/226] Retry a release download cut off partway A chunked download that drops partway raises http.client.IncompleteRead, which is no OSError, so verify() failed at once, and reconcile() saw no PublishError. verify() now counts http.client.HTTPException with dropped connections, in the 12-try window. --- scripts/nightly.py | 14 +++++++++----- scripts/test_nightly.py | 13 +++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/scripts/nightly.py b/scripts/nightly.py index 3b9cf294..51547eac 100644 --- a/scripts/nightly.py +++ b/scripts/nightly.py @@ -28,6 +28,7 @@ import argparse import datetime import hashlib +import http.client import json import os import re @@ -252,9 +253,10 @@ def verify(github, url, size, sha256=None, tries=6, wait=10, transient_tries=12, A new or renamed asset can take a few seconds to show, so a missing or different file is checked again: tries times, wait seconds apart. A server error (HTTP 429 or - 5xx) or a dropped connection says nothing about the file, so those have a longer - window of their own: transient_tries, each wait 5 s longer up to 30 s (about four - minutes). Either way the size and SHA-256 must match exactly.""" + 5xx) or a dropped connection, also one cut off partway through a download, says + nothing about the file, so those have a longer window of their own: + transient_tries, each wait 5 s longer up to 30 s (about four minutes). Either way + the size and SHA-256 must match exactly.""" misses = errors = 0 while True: try: @@ -271,8 +273,10 @@ def verify(github, url, size, sha256=None, tries=6, wait=10, transient_tries=12, except urllib.error.HTTPError as error: seen = f"HTTP {error.code}" flaky = transient(error.code) - except OSError as error: - seen = str(error) + except (OSError, http.client.HTTPException) as error: + # A dropped connection, also one cut off partway through the download + # (http.client.IncompleteRead, which is no OSError). + seen = f"{type(error).__name__}: {error}" flaky = True if flaky: errors += 1 diff --git a/scripts/test_nightly.py b/scripts/test_nightly.py index c8b6de30..a68865aa 100644 --- a/scripts/test_nightly.py +++ b/scripts/test_nightly.py @@ -1,6 +1,7 @@ """Beta channel builds must sort below the release they lead to and never touch other releases.""" import copy import hashlib +import http.client import itertools import os import pathlib @@ -360,6 +361,18 @@ def test_rate_limits_and_dropped_connections_are_waited_out_too(self): TimeoutError("timed out"), OK]) self.assertEqual(self.verify(github), ([5, 10, 15], None)) + def test_a_download_cut_off_partway_is_waited_out(self): + github = ScriptedDownloads([OK], [http.client.IncompleteRead(b"x" * 10, SIZE - 10), + http.client.RemoteDisconnected("closed"), HASH]) + self.assertEqual(self.verify(github, HASH), ([5, 10], None)) + + def test_downloads_cut_off_every_time_fail_after_twelve_tries(self): + github = ScriptedDownloads([OK], [http.client.IncompleteRead(b"", SIZE)]) + waits, failed = self.verify(github, HASH) + self.assertEqual(github.calls.count("sha256"), 12) + self.assertEqual(waits, [5, 10, 15, 20, 25, 30, 30, 30, 30, 30, 30]) + self.assertIn("IncompleteRead", failed) + def test_a_5xx_while_hashing_is_waited_out(self): github = ScriptedDownloads([OK], [http_error(504), http_error(503), HASH]) self.assertEqual(self.verify(github, HASH), ([5, 10], None)) From 09dd5f0bf01e9f72a18e7e3fe3eafae017637750 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 05:14:10 -0500 Subject: [PATCH 165/226] Give AMF its last picture again only when no picture is due #198 gave an AMF session its last picture again when capture's next one took more than 20 ms, to bring out a held keyframe. While the desktop moves at 50 fps or below, the next picture is due about that long after the last one: a 50 Hz display streamed at 60 fps, or 30 fps content, had a picture encoded twice whenever a frame came a little late. The keyframe-request test, fed a frame every 20 ms, saw one picture twice in 1 run of 5 on AMF. While the desktop moves, the wait is now a frame and a half, at least 20 ms. The frame is the target rate's, or capture's longest recent gap when it is longer. A gap of 100 ms or more (a resent still picture) means a still desktop, where no picture is due and the wait stays 20 ms, so a new viewer's first keyframe comes out as soon as before. The stage counts pictures given again (keyframe_resubmits). The AMF tests now check that a stream fed a frame every 20 ms gives none again and encodes no picture twice, unless the feeder itself stalled for a frame and a half; on a still desktop only the resent picture may repeat, as often as it was given again, and AMF must have been given it again. A fake-clock test replays jittered captures at 60, 50 and 30 fps through the rule, and another checks the wait while moving and when still. --- host/src/encoder/mod.rs | 240 ++++++++++++++++++++++++++++++++++---- host/src/stats.rs | 4 + host/tests/amf_encoder.rs | 130 ++++++++++++++++++--- 3 files changed, 336 insertions(+), 38 deletions(-) diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index 3a6b19a8..bb94dabe 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -180,14 +180,19 @@ fn run_encode_loop( }; let mut owed_keyframe: Option = None; + let mut cadence = CaptureCadence::default(); loop { - let raw_frame = match owed_keyframe.as_mut() { + let (raw_frame, given_again) = match owed_keyframe.as_mut() { None => match frames.blocking_take() { - Some(frame) => frame, + Some(frame) => (frame, false), None => break, }, Some(owed) => match owed.next(&frames) { - Next::Frame(frame) => frame, + Next::Frame(frame) => (frame, false), + Next::Again(frame) => { + PIPELINE_STATS.lock().keyframe_resubmits += 1; + (frame, true) + } Next::Closed => break, Next::GaveUp => { info!( @@ -199,6 +204,9 @@ fn run_encode_loop( } }, }; + if !given_again { + cadence.captured(raw_frame.timestamp); + } if !shared.running.load(Ordering::SeqCst) || !crate::supervisor::generation_is_current(generation) { @@ -844,8 +852,13 @@ fn run_encode_loop( continue; } encoder.bitrate_reset_unconfirmed = false; - owed_keyframe = - OwedKeyframe::after_encode(owed_keyframe.take(), picture, keyframe_out, encode_start); + owed_keyframe = OwedKeyframe::after_encode( + owed_keyframe.take(), + picture, + keyframe_out, + encode_start, + keyframe_flush_wait(target_fps, cadence.moving_interval()), + ); } info!("Capture channel closed, encoder shutting down"); @@ -2398,11 +2411,67 @@ pub fn holds_last_frame(encoder_name: &str) -> bool { matches!(encoder_name, "h264_amf" | "hevc_amf") } -/// How long the encode loop waits for capture's next picture before it gives -/// an encoder that [holds its last frame](holds_last_frame) the last picture -/// again, to bring out a session's first keyframe. It is about one frame at -/// 60 fps, and longer than AMF takes to encode one. +/// The shortest wait for capture's next picture before an encoder that +/// [holds its last frame](holds_last_frame) is given the last picture again, +/// to bring out a session's first keyframe: longer than AMF takes to encode +/// one. See [`keyframe_flush_wait`]. const KEYFRAME_FLUSH_WAIT: Duration = Duration::from_millis(20); + +/// A gap between capture's pictures at least this long is a still desktop: +/// capture resends a still picture for a keyframe request only after 100 ms +/// (its `KEYFRAME_RESEND`), and otherwise at its 750 ms keepalive. +const MOVING_GAP: Duration = Duration::from_millis(100); + +/// How long the encode loop waits for capture's next picture, after encoding +/// one, before it gives the last picture again. While the desktop moves +/// (`moving_interval` is capture's longest recent gap), a frame and a half: +/// of the target rate, or of that gap when capture delivers less often, as a +/// 50 Hz display streamed at 60 fps or 30 fps content does. A flat 20 ms gave +/// those pictures to AMF twice. On a still desktop no picture is due, so the +/// shortest wait brings the keyframe out soonest. +fn keyframe_flush_wait(target_fps: u32, moving_interval: Option) -> Duration { + match moving_interval { + Some(gap) => { + (crate::capture::frame_budget_for(target_fps).max(gap) * 3 / 2).max(KEYFRAME_FLUSH_WAIT) + } + None => KEYFRAME_FLUSH_WAIT, + } +} + +/// The pace of capture's own pictures, not counting those given again. +#[derive(Default)] +struct CaptureCadence { + last: Option, + gaps: std::collections::VecDeque, +} + +impl CaptureCadence { + const KEPT: usize = 8; + + fn captured(&mut self, at: Instant) { + if let Some(last) = self.last.replace(at) { + if self.gaps.len() == Self::KEPT { + self.gaps.pop_front(); + } + self.gaps.push_back(at.saturating_duration_since(last)); + } + } + + /// The longest recent gap between pictures while the desktop moves, or + /// `None` when the last picture came after a still spell (or is the + /// first). + fn moving_interval(&self) -> Option { + let latest = *self.gaps.back()?; + (latest < MOVING_GAP).then(|| { + self.gaps + .iter() + .copied() + .filter(|gap| *gap < MOVING_GAP) + .max() + .unwrap_or(latest) + }) + } +} /// How long after the keyframe went in the last picture is given again, /// before the keyframe is left to capture's next picture: an AMD driver can /// take a few hundred ms to encode a fresh session's first frame, and @@ -2416,12 +2485,16 @@ struct OwedKeyframe { picture: Option, /// When the picture stops being given again. until: Instant, + /// How long to wait for capture's next picture first. + wait: Duration, given: u32, } /// What the encode loop encodes next while a keyframe is owed. enum Next { Frame(RawFrame), + /// The last picture, given again. + Again(RawFrame), Closed, /// The deadline passed: capture's next picture brings the keyframe out. GaveUp, @@ -2432,29 +2505,35 @@ impl OwedKeyframe { Self { picture: None, until, + wait: KEYFRAME_FLUSH_WAIT, given: 0, } } /// What is owed after a frame was submitted at `submitted`: nothing once /// the keyframe came out or the encoder holds no frame, else `picture` - /// to give again, until [`KEYFRAME_FLUSH_FOR`] after the first try. + /// to give again after `wait`, until [`KEYFRAME_FLUSH_FOR`] after the + /// first try. fn after_encode( owed: Option, picture: Option, keyframe_out: bool, submitted: Instant, + wait: Duration, ) -> Option { let picture = picture.filter(|_| !keyframe_out)?; let mut owed = owed.unwrap_or_else(|| Self::until(submitted + KEYFRAME_FLUSH_FOR)); owed.picture = Some(picture); + owed.wait = wait; Some(owed) } - /// Capture's next picture if one arrives within [`KEYFRAME_FLUSH_WAIT`], - /// else the last picture again, until the deadline. + /// Capture's next picture if one arrives within the wait, else the last + /// picture again, until the deadline. fn next(&mut self, frames: &FrameSlot) -> Next { - let wait = KEYFRAME_FLUSH_WAIT.min(self.until.saturating_duration_since(Instant::now())); + let wait = self + .wait + .min(self.until.saturating_duration_since(Instant::now())); match frames.take_within(wait, &mut self.picture) { Taken::Frame(frame) => Next::Frame(frame), Taken::Closed => Next::Closed, @@ -2466,7 +2545,7 @@ impl OwedKeyframe { given = self.given, "Giving the encoder its last picture again to bring out the keyframe" ); - Next::Frame(picture) + Next::Again(picture) } _ => Next::GaveUp, }, @@ -6264,20 +6343,135 @@ mod tests { .checked_sub(Duration::from_millis(200)) .unwrap(); // The keyframe went in 200 ms ago; sending and receiving took that long. - let owed = OwedKeyframe::after_encode(None, Some(raw(1, &kept)), false, submitted) - .expect("the keyframe is owed"); + let owed = OwedKeyframe::after_encode( + None, + Some(raw(1, &kept)), + false, + submitted, + KEYFRAME_FLUSH_WAIT, + ) + .expect("the keyframe is owed"); assert_eq!(owed.until, submitted + KEYFRAME_FLUSH_FOR); // A retry submitted later keeps the first deadline. - let owed = - OwedKeyframe::after_encode(Some(owed), Some(raw(1, &kept)), false, Instant::now()) - .expect("still owed"); + let owed = OwedKeyframe::after_encode( + Some(owed), + Some(raw(1, &kept)), + false, + Instant::now(), + KEYFRAME_FLUSH_WAIT, + ) + .expect("still owed"); assert_eq!(owed.until, submitted + KEYFRAME_FLUSH_FOR); assert!( - OwedKeyframe::after_encode(Some(owed), Some(raw(1, &kept)), true, Instant::now()) - .is_none(), + OwedKeyframe::after_encode( + Some(owed), + Some(raw(1, &kept)), + true, + Instant::now(), + KEYFRAME_FLUSH_WAIT + ) + .is_none(), "the keyframe came out" ); - assert!(OwedKeyframe::after_encode(None, None, false, submitted).is_none()); + assert!( + OwedKeyframe::after_encode(None, None, false, submitted, KEYFRAME_FLUSH_WAIT).is_none() + ); + } + + /// Replays a capture schedule on a fake clock through the rule the encode + /// loop uses, as if a keyframe were owed after every picture: encoding + /// ends `ENCODE` after a picture is captured, and the picture is given + /// again if capture's next one has not come by then plus the wait. + /// Returns how many pictures would have been given again. + fn resubmits_on_a_moving_desktop(target_fps: u32, frame: Duration) -> usize { + const ENCODE: Duration = Duration::from_millis(2); + let start = Instant::now(); + // Up to a quarter frame early or late, so gaps run from half a frame + // to a frame and a half. + let quarter = frame / 4; + let jitter = [ + Duration::ZERO, + quarter, + Duration::ZERO, + quarter, + quarter / 2, + ]; + let early = [false, false, false, true, true]; + let captured: Vec = (0..240u32) + .map(|n| { + let at = start + Duration::from_secs(1) + frame * n; + let k = n as usize % jitter.len(); + if early[k] { + at - jitter[k] + } else { + at + jitter[k] + } + }) + .collect(); + let mut cadence = CaptureCadence::default(); + let mut resubmits = 0; + for (n, pair) in captured.windows(2).enumerate() { + cadence.captured(pair[0]); + let wait = keyframe_flush_wait(target_fps, cadence.moving_interval()); + // The pipeline's first picture has no pace to go by. It is + // encoded before any viewer arrives (see capture's IdleGate). + if n > 0 && pair[1] > pair[0] + ENCODE + wait { + resubmits += 1; + } + } + resubmits + } + + #[test] + fn a_moving_desktop_never_has_its_picture_given_again_at_60_50_or_30_fps() { + let ms = |n: u64| Duration::from_micros(n); + for (target_fps, frame, what) in [ + (60, ms(16_667), "60 fps"), + (60, ms(20_000), "a 50 Hz display streamed at 60 fps"), + (50, ms(20_000), "50 fps"), + (30, ms(33_333), "30 fps"), + (60, ms(33_333), "30 fps content streamed at 60 fps"), + ] { + assert_eq!( + resubmits_on_a_moving_desktop(target_fps, frame), + 0, + "{what}: a picture was given again before capture's next one" + ); + } + } + + #[test] + fn a_still_desktop_has_its_picture_given_again_after_the_shortest_wait() { + let start = Instant::now(); + for target_fps in [60, 50, 30] { + let mut cadence = CaptureCadence::default(); + // Moving at the target rate, then still until the resent picture. + let frame = crate::capture::frame_budget_for(target_fps); + for n in 0..10 { + cadence.captured(start + frame * n); + } + assert_eq!( + keyframe_flush_wait(target_fps, cadence.moving_interval()), + frame * 3 / 2, + "{target_fps} fps while the desktop moves" + ); + cadence.captured(start + frame * 9 + Duration::from_millis(750)); + assert_eq!(cadence.moving_interval(), None, "{target_fps} fps: still"); + assert_eq!( + keyframe_flush_wait(target_fps, cadence.moving_interval()), + KEYFRAME_FLUSH_WAIT, + "{target_fps} fps on a still desktop" + ); + } + // Past 75 fps a frame and a half is shorter than AMF's encode. + let mut cadence = CaptureCadence::default(); + for n in 0..10 { + cadence.captured(start + crate::capture::frame_budget_for(120) * n); + } + assert_eq!( + keyframe_flush_wait(120, cadence.moving_interval()), + KEYFRAME_FLUSH_WAIT + ); } #[test] @@ -6287,7 +6481,7 @@ mod tests { let mut owed = OwedKeyframe::until(Instant::now() + Duration::from_secs(60)); owed.picture = Some(raw(1, &kept)); let started = Instant::now(); - let Next::Frame(frame) = owed.next(&consumer) else { + let Next::Again(frame) = owed.next(&consumer) else { panic!("the picture was not given again"); }; assert_eq!(frame.frame_number, 1); diff --git a/host/src/stats.rs b/host/src/stats.rs index b84b3155..cd1e3d91 100644 --- a/host/src/stats.rs +++ b/host/src/stats.rs @@ -42,6 +42,9 @@ pub struct PipelineStats { /// (CPU) encoding. Surfaced as a warning banner in the GUI so a tester is never silently /// stuck on slow software encoding. pub using_software_fallback: bool, + /// Pictures given to an encoder again to bring out a keyframe it held + /// (see `encoder::holds_last_frame`). + pub keyframe_resubmits: u64, /// NVENC refused to open because the NVIDIA driver is older than the /// linked FFmpeg's NVENC needs. The fallback notice then says to update /// it. @@ -114,6 +117,7 @@ impl PipelineStats { bitrate_bps: 0, codec_name: String::new(), using_software_fallback: false, + keyframe_resubmits: 0, nvidia_driver_outdated: false, driver_update_vendors: Vec::new(), encode_timestamps: VecDeque::with_capacity(128), diff --git a/host/tests/amf_encoder.rs b/host/tests/amf_encoder.rs index cdd2cf9e..786af5c9 100644 --- a/host/tests/amf_encoder.rs +++ b/host/tests/amf_encoder.rs @@ -109,6 +109,10 @@ struct StageRun { packets: Vec, software_fallback: bool, codec_name: String, + /// Pictures the stage gave the encoder again (`keyframe_resubmits`). + resubmits: u64, + /// The longest time between two published frames. + longest_gap: Duration, } /// Feeds `FRAME_COUNT` frames through a fresh encode stage at @@ -130,6 +134,7 @@ fn run_stage( ..GpuInfo::software_fallback() }; let generation = eternal_host::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let resubmits_before = PIPELINE_STATS.lock().keyframe_resubmits; let stage_shared = shared.clone(); let stage = std::thread::spawn(move || { run_encode_stage(consumer, tx, stage_shared, gpu, generation).map_err(|e| e.to_string()) @@ -143,6 +148,8 @@ fn run_stage( packets }); + let mut last_publish: Option = None; + let mut longest_gap = Duration::ZERO; for number in 0..FRAME_COUNT { if keyframe_at == Some(number) { shared.force_next_idr.store(true, Ordering::SeqCst); @@ -150,6 +157,9 @@ fn run_stage( let (width, height) = size_of(number); let mut data = frame_buffer((width * height * 4) as usize); render_synthetic_frame(&mut data, width, height, number); + if let Some(last) = last_publish.replace(Instant::now()) { + longest_gap = longest_gap.max(last.elapsed()); + } producer.publish(RawFrame { frame_number: number, timestamp: Instant::now(), @@ -173,6 +183,40 @@ fn run_stage( packets, software_fallback: stats.using_software_fallback, codec_name: stats.codec_name.clone(), + resubmits: stats.keyframe_resubmits - resubmits_before, + longest_gap, + } +} + +/// A stream whose frames come every `FRAME_INTERVAL` is never still, so no +/// picture may be given to the encoder again and none encoded twice. Only if +/// the producer itself stalled for a frame and a half (the encode loop's wait, +/// see `keyframe_flush_wait`) may a picture repeat, and then only one the +/// stage reports giving again. +fn assert_no_duplicate_encodes(run: &StageRun, pictures: &[(u32, u32, u64)]) { + let counters: Vec = pictures.iter().map(|picture| picture.2).collect(); + assert!( + counters.windows(2).all(|pair| pair[0] <= pair[1]), + "frame counters go back: {counters:?}" + ); + let repeats = counters + .windows(2) + .filter(|pair| pair[0] == pair[1]) + .count() as u64; + if run.longest_gap < FRAME_INTERVAL * 3 / 2 { + assert_eq!( + (repeats, run.resubmits), + (0, 0), + "pictures encoded twice, and given again, while frames came every \ + {FRAME_INTERVAL:?} at most {:?} apart: {counters:?}", + run.longest_gap + ); + } else { + assert!( + repeats <= run.resubmits, + "{repeats} pictures encoded twice but {} given again: {counters:?}", + run.resubmits + ); } } @@ -242,7 +286,8 @@ fn assert_hevc_keyframe(packet: &NALUnit, what: &str) { /// picture is at most `KEYFRAME_WINDOW` frames past the request, and a decoder /// that joins there must decode everything after it. fn assert_keyframe_on_request(run: &StageRun, codec: Id) { - decode(codec, &run.packets.iter().collect::>()); + let all = decode(codec, &run.packets.iter().collect::>()); + assert_no_duplicate_encodes(run, &all); let keyframes: Vec = run .packets .iter() @@ -274,10 +319,7 @@ fn assert_keyframe_on_request(run: &StageRun, codec: Id) { waited for the encoder's own IDR; keyframes at {keyframes:?}", counters[0] ); - assert!( - counters.windows(2).all(|pair| pair[0] < pair[1]), - "frame counters do not advance: {counters:?}" - ); + // Advancing counters across the whole stream: `assert_no_duplicate_encodes`. } /// Size schedule for the mode-change runs: small, then large, then small. @@ -292,7 +334,8 @@ fn mode_changes(number: u64) -> (u32, u32) { /// After each mode change the stream must restart on a keyframe at the new /// size that a fresh decoder can join. fn assert_follows_mode_changes(run: &StageRun, codec: Id) { - decode(codec, &run.packets.iter().collect::>()); + let all = decode(codec, &run.packets.iter().collect::>()); + assert_no_duplicate_encodes(run, &all); for (from, size) in [(30u64, LARGE), (60, SMALL)] { let start = run .packets @@ -506,9 +549,8 @@ const WARM_UP_QUIET: Duration = Duration::from_millis(300); /// Streams a few frames and waits for the encoder to encode them, goes still, /// then requests a keyframe with one resent picture and nothing after it, as -/// for a reconnecting viewer. Returns how long that picture's keyframe took -/// to leave the stage, or `None` if it did not within [`STILL_WINDOW`]. -fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { +/// for a reconnecting viewer. +fn keyframe_on_a_still_desktop(encoder: &str) -> StillRun { ffmpeg_next::init().unwrap(); let (producer, consumer) = frame_slot(); let (tx, mut rx) = tokio::sync::mpsc::channel(NAL_CHANNEL_CAPACITY); @@ -520,6 +562,7 @@ fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { ..GpuInfo::software_fallback() }; let generation = eternal_host::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + let resubmits_before = PIPELINE_STATS.lock().keyframe_resubmits; let stage_shared = shared.clone(); let stage = std::thread::spawn(move || { run_encode_stage(consumer, tx, stage_shared, gpu, generation).map_err(|e| e.to_string()) @@ -571,7 +614,6 @@ fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { } // A viewer connects to the still desktop. - const RESENT: u64 = 100; let requested = Instant::now(); shared.force_next_idr.store(true, Ordering::SeqCst); publish(RESENT); @@ -596,7 +638,13 @@ fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { Err(_) => panic!("{encoder} encode stage panicked"), } collector.join().unwrap(); - let software_fallback = PIPELINE_STATS.lock().using_software_fallback; + let (software_fallback, resubmits) = { + let stats = PIPELINE_STATS.lock(); + ( + stats.using_software_fallback, + stats.keyframe_resubmits - resubmits_before, + ) + }; eprintln!( "{encoder}: still-desktop keyframe after {keyframe:?}; packets {:?}", seen.iter() @@ -608,19 +656,71 @@ fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { )) .collect::>() ); - (keyframe, software_fallback) + StillRun { + keyframe, + software_fallback, + resubmits, + sequences: seen.iter().map(|(_, nal)| nal.sequence).collect(), + } +} + +/// The picture resent for the viewer, after the warm-up's frames 0 to 9. +const RESENT: u64 = 100; + +struct StillRun { + /// How long the resent picture's keyframe took to leave the stage, or + /// `None` if it did not within [`STILL_WINDOW`]. + keyframe: Option, + software_fallback: bool, + resubmits: u64, + /// The capture number of each packet, in order. + sequences: Vec, } fn assert_still_keyframe(encoder: &str) { - let (keyframe, software_fallback) = keyframe_on_a_still_desktop(encoder); + let run = keyframe_on_a_still_desktop(encoder); assert!( - encoder == "libx264" || !software_fallback, + encoder == "libx264" || !run.software_fallback, "{encoder} fell back to libx264" ); assert!( - keyframe.is_some(), + run.keyframe.is_some(), "{encoder} held the requested keyframe until the next captured frame" ); + // Only the still picture may be encoded twice, and only by being given + // again: the warm-up's frames came every 20 ms. + let warm_up: Vec = run + .sequences + .iter() + .copied() + .filter(|&sequence| sequence != RESENT) + .collect(); + assert!( + warm_up.windows(2).all(|pair| pair[0] < pair[1]), + "{encoder} encoded a moving picture twice: {:?}", + run.sequences + ); + let still = run + .sequences + .iter() + .filter(|&&sequence| sequence == RESENT) + .count() as u64; + assert!( + still.saturating_sub(1) <= run.resubmits, + "{encoder}: the still picture came out {still} times, given again {}", + run.resubmits + ); + if eternal_host::encoder::holds_last_frame(encoder) { + assert!( + run.resubmits >= 1, + "{encoder} brought out the keyframe without being given the picture again" + ); + } else { + assert_eq!( + run.resubmits, 0, + "{encoder} returns each frame, so nothing is given again" + ); + } } #[test] From b88fc92f1c01fc3e859eaa83573a8ba159eaaf9c Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 06:21:26 -0500 Subject: [PATCH 166/226] Let the first-frame probe answer as the app does Since #200 the host sends a viewer that has not answered only its first picture. The app pings the moment its HELLO_ACK lands; the probe sent only a receiver report every 300 ms. On reconnects that answer came after the first keyframe, the next keyframe waited for AMF's reopen limit and capture's keepalive, and the probe reported about 1.2 s that an iPad does not see. It now pings when its HELLO_ACK arrives. --- host/examples/first_frame_probe.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/host/examples/first_frame_probe.rs b/host/examples/first_frame_probe.rs index f11ed850..c828e80f 100644 --- a/host/examples/first_frame_probe.rs +++ b/host/examples/first_frame_probe.rs @@ -11,7 +11,7 @@ use std::time::{Duration, Instant}; use eternal_wire::reassembly::{AddOutcome, Reassembler}; use eternal_wire::v2::control::{ - encode_control, parse_control, ControlMessage, Hello2, HelloStatus, ReceiverReport, + encode_control, parse_control, ControlMessage, Hello2, HelloStatus, Ping, ReceiverReport, CAP_DECODE_H264, }; use eternal_wire::v2::media::MediaHeader; @@ -102,7 +102,19 @@ fn main() { if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(bytes) { assert_eq!(ack.status, HelloStatus::Ok, "pairing must be off"); session_id = ack.session_id; - acked_at.get_or_insert_with(Instant::now); + if acked_at.is_none() { + acked_at = Some(Instant::now()); + // As the app does when its HELLO_ACK lands: the host + // sends a viewer that has not answered only its + // first picture. + msg_seq += 1; + let ping = ControlMessage::Ping(Ping { + t1_us: started.elapsed().as_micros() as u64, + }); + socket + .send_to(&encode_control(session_id, msg_seq, &ping), &host) + .unwrap(); + } } } Classified::Media { .. } if session_id != 0 => { From 2e58ecbc01b1f2b21e0be752cbc6967df332221c Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 06:36:36 -0500 Subject: [PATCH 167/226] Remember the media size a path needed When smaller media datagrams gave the first picture, the iPad keeps that size for the PC's address. The next session there asks for it as soon as the PC accepts, through the keyframe request's size field, instead of waiting through the steps again. The size is bound to the pairing (a digest of its token) and to the network the iPad reached the PC over. It lasts a week from when it was last needed, and is forgotten once a session at the address gives a picture at the PC's own size. PCs that don't take smaller sizes are never asked, as before. --- .../App/ConnectionManager.swift | 50 ++++++ ios/EternalMonitor/App/NoVideoFallback.swift | 90 ++++++++++- ios/EternalMonitorTests/FakeUDPHost.swift | 6 + .../NoVideoFallbackTests.swift | 152 +++++++++++++++++- 4 files changed, 296 insertions(+), 2 deletions(-) diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 4e197d61..74ef73b7 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -293,6 +293,12 @@ final class ConnectionManager: ObservableObject { /// the step whose result isn't known yet, as its log line names it. private var noVideoFallback: NoVideoFallback? private var noVideoStep: String? + /// The media size the current session runs at, when smaller than the + /// PC's own; whether a step this session asked for it, rather than the + /// size kept for this path; and where it is kept (see `NoVideoSizes`). + private var sessionMediaSize: UInt16? + private var mediaSizeFromStep = false + private var mediaSizeKey: NoVideoSizes.Key? init(timing: Timing = Timing(), udpReceiver: @escaping (UInt16) -> any MediaLink = { UDPReceiver(port: $0) }, pairingStore: PairingStore? = nil) { @@ -481,6 +487,7 @@ final class ConnectionManager: ObservableObject { do { try pairingStore.forgetAll() } catch { connectionError = "Could not remove saved pairings from this iPad." } NoVideoPaths.forgetAll() + NoVideoSizes.forgetAll() } /// A connection the user or a cable starts. It replaces any automatic @@ -538,6 +545,9 @@ final class ConnectionManager: ObservableObject { pendingRedirect = nil noVideoFallback = nil noVideoStep = nil + sessionMediaSize = nil + mediaSizeFromStep = false + mediaSizeKey = nil let connectionID = UUID() currentAttempt.withLock { $0 = connectionID } transportMode = isUSB ? "USB" : "WiFi" @@ -628,6 +638,7 @@ final class ConnectionManager: ObservableObject { self.record(.info, "decode", "First frame decoded successfully" + self.attemptTimings) self.endNoVideoStep("video") self.noVideoFallback = nil + self.settleMediaSize() self.timeoutTask?.cancel() self.noPictureStreak = NoPictureStreak() self.state = .connected @@ -797,6 +808,11 @@ final class ConnectionManager: ObservableObject { self.sessionAcceptedAt = acceptedAt self.noVideoFallback = !isUSB && info.hostCaps & HelloAck.hostCapMediaSize != 0 ? NoVideoFallback(wait: self.timing.noVideoStepSeconds) : nil + let sessionToken = PairingStore.valid(info.authToken) ? info.authToken : pairingToken + if self.noVideoFallback != nil, PairingStore.valid(sessionToken) { + self.startWithRememberedSize(connectionID, address: .init(host: normalizedHost, port: port), + pairing: sessionToken) + } self.acceptedAddresses.insert("\(normalizedHost):\(port)") self.noPictureStreak.answered(by: info.hostName, pairingToken: info.authToken, at: isUSB ? nil : "\(normalizedHost):\(port)") @@ -1364,12 +1380,46 @@ final class ConnectionManager: ObservableObject { armConnectTimeout(seconds: timing.connectTimeoutSeconds) } noVideoStep = "step=smaller_packets size=\(size)" + sessionMediaSize = size + mediaSizeFromStep = true controlChannel?.sendKeyframeRequest(streamEpoch: 0, lastCompleteSeq: 0, reason: .startup, maxMediaDgram: size) case .otherAddress: tryOtherAddress() } } + /// A session at an address where the PC's video needed a smaller media + /// size before, over the same network, asks for it at once instead of + /// waiting through the steps again (see `NoVideoSizes`). + private func startWithRememberedSize(_ connectionID: UUID, address: NoVideoPaths.Address, pairing: Data) { + Task { @MainActor [weak self] in + guard let self else { return } + let route = await self.localRoute(address.host, address.port) + guard self.currentAttempt.withLock({ $0 == connectionID }) else { return } + let key = NoVideoSizes.Key(address: address, pairing: pairing, network: NoVideoSizes.network(route)) + self.mediaSizeKey = key + // A picture came first, at the PC's size or after a step. + guard self.state == .connecting else { return self.settleMediaSize() } + guard self.sessionMediaSize == nil, let size = NoVideoSizes.size(for: key, now: self.now()) else { return } + self.sessionMediaSize = size + self.noVideoFallback?.skipSizes(downTo: size) + self.controlChannel?.sendKeyframeRequest(streamEpoch: 0, lastCompleteSeq: 0, reason: .startup, maxMediaDgram: size) + self.record(.info, "connection", "no_video_fallback step=remembered_size size=\(size) result=asked") + } + } + + /// The session shows a picture: keep the smaller size a step found, or + /// forget the size kept for this address once the PC's own size works, + /// as after a change of network. + private func settleMediaSize() { + guard let key = mediaSizeKey else { return } + if let size = sessionMediaSize { + if mediaSizeFromStep { NoVideoSizes.remember(size, for: key, now: now()) } + } else if NoVideoSizes.forget(key) { + record(.info, "connection", "no_video_fallback step=remembered_size result=forgotten") + } + } + /// Logs the pending no-video step's `result`. private func endNoVideoStep(_ result: String) { guard let step = noVideoStep else { return } diff --git a/ios/EternalMonitor/App/NoVideoFallback.swift b/ios/EternalMonitor/App/NoVideoFallback.swift index b7828db9..467142ee 100644 --- a/ios/EternalMonitor/App/NoVideoFallback.swift +++ b/ios/EternalMonitor/App/NoVideoFallback.swift @@ -39,6 +39,12 @@ struct NoVideoFallback: Equatable { if framesSent > framesBeforeStep, sendingSince == nil { sendingSince = now } } + /// The session already runs at `size`: the steps go on with the sizes + /// below it. + mutating func skipSizes(downTo size: UInt16) { + sizesAsked = max(sizesAsked, Self.sizes.filter { $0 >= size }.count) + } + /// The step to take at `now`, while there is no picture; nil while the /// PC sends nothing, or within `wait` of its sending. mutating func next(at now: TimeInterval) -> Step? { @@ -91,7 +97,8 @@ enum NoVideoPaths { save(load().filter { $0.from != from }) } - private static func digest(_ token: Data) -> String { + /// SHA-256 of a pairing token, in hex. + static func digest(_ token: Data) -> String { SHA256.hash(data: token).map { String(format: "%02x", $0) }.joined() } @@ -109,3 +116,84 @@ enum NoVideoPaths { UserDefaults.standard.set(data, forKey: key) } } + +/// The media datagram size a PC's video needed at an address, so the next +/// session there starts with it instead of waiting through the steps again. +/// Each entry is bound to the pairing by a digest of its token, and to the +/// network this iPad reached the PC over. It lasts a week from when the +/// size was last needed, and goes as soon as a session at the address shows +/// a picture at the PC's own size. Kept in UserDefaults. +enum NoVideoSizes { + static let lifetime: TimeInterval = 7 * 24 * 60 * 60 + private static let key = "noVideoSizes" + private static let kept = 16 + + /// A session's address, pairing and network. + struct Key: Equatable { + var address: NoVideoPaths.Address + var pairing: Data + /// `network(_:)` of this iPad's route to the address. + var network: String + } + + private struct Entry: Codable { + var address: NoVideoPaths.Address + /// SHA-256 of the pairing token, in hex. + var pairing: String + var network: String + var size: UInt16 + var at: Date + } + + /// This iPad's side of a route: its interface, address and netmask. + static func network(_ route: MovedPC.Route?) -> String { + route.map { "\($0.interface) \($0.address)/\($0.netmask)" } ?? "-" + } + + /// The size to start a session at `key` with: one kept for its address + /// and pairing, on the same network, less than `lifetime` old. + static func size(for key: Key, now: Date) -> UInt16? { + guard let entry = entry(key), entry.network == key.network, key.network != "-", + now >= entry.at, now.timeIntervalSince(entry.at) < lifetime else { return nil } + return entry.size + } + + static func remember(_ size: UInt16, for key: Key, now: Date) { + var entries = load().filter { !matches($0, key) } + entries.append(Entry(address: key.address, pairing: NoVideoPaths.digest(key.pairing), network: key.network, size: size, at: now)) + save(Array(entries.suffix(kept))) + } + + /// Forgets the size kept for `key`'s address and pairing, on any + /// network; true if there was one. + @discardableResult + static func forget(_ key: Key) -> Bool { + let entries = load() + let kept = entries.filter { !matches($0, key) } + guard kept.count != entries.count else { return false } + save(kept) + return true + } + + static func forgetAll() { + UserDefaults.standard.removeObject(forKey: key) + } + + private static func entry(_ key: Key) -> Entry? { + load().last { matches($0, key) } + } + + private static func matches(_ entry: Entry, _ key: Key) -> Bool { + entry.address == key.address && entry.pairing == NoVideoPaths.digest(key.pairing) + } + + private static func load() -> [Entry] { + guard let data = UserDefaults.standard.data(forKey: key) else { return [] } + return (try? JSONDecoder().decode([Entry].self, from: data)) ?? [] + } + + private static func save(_ entries: [Entry]) { + guard let data = try? JSONEncoder().encode(entries) else { return } + UserDefaults.standard.set(data, forKey: key) + } +} diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index 8bc148e9..c8b14a26 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -301,6 +301,12 @@ final class FakeUDPHost: @unchecked Sendable { /// `countFrames`) and `frame` as the next keyframe, cut at the size the /// iPad asked for. Answers identify requests as the PC holding `token` /// receiving on `addresses`, to whichever socket asks. + /// Takes the next HELLO2 as a new client's, as a PC does for the iPad's + /// next session. + func forgetClient() { + client = nil + } + func servePath(limit: Int, frame: Data, hostCaps: UInt16, countFrames: Bool = true, token: Data = Data(17...32), addresses: [Data] = [Data([127, 0, 0, 1])], until stop: () -> Bool) -> PathRun { var run = PathRun() diff --git a/ios/EternalMonitorTests/NoVideoFallbackTests.swift b/ios/EternalMonitorTests/NoVideoFallbackTests.swift index 17785f0a..f11017c2 100644 --- a/ios/EternalMonitorTests/NoVideoFallbackTests.swift +++ b/ios/EternalMonitorTests/NoVideoFallbackTests.swift @@ -27,13 +27,14 @@ final class NoVideoFallbackTests: XCTestCase { override func setUp() async throws { let store = RecentConnectionStore.shared let saved = store.connections - let defaults = ["recentConnections", "lastHost", "lastPort", "noVideoPaths"] + let defaults = ["recentConnections", "lastHost", "lastPort", "noVideoPaths", "noVideoSizes"] .map { ($0, UserDefaults.standard.object(forKey: $0)) } addTeardownBlock { @MainActor in store.connections = saved for (key, value) in defaults { UserDefaults.standard.set(value, forKey: key) } } UserDefaults.standard.removeObject(forKey: "noVideoPaths") + UserDefaults.standard.removeObject(forKey: "noVideoSizes") store.connections = [] } @@ -94,6 +95,155 @@ final class NoVideoFallbackTests: XCTestCase { XCTAssertNil(steps.next(at: 200), "then no more") } + // MARK: - The size kept for a path + + /// The route `manager(pairedAt:)` gives, and the one of another network. + nonisolated private static let homeRoute = MovedPC.Route(interface: "en0", direct: true, address: "127.0.0.1", + netmask: "255.0.0.0") + nonisolated private static let otherRoute = MovedPC.Route(interface: "en0", direct: true, address: "10.0.0.5", + netmask: "255.255.255.0") + + private func sizeKey(_ pc: FakeUDPHost, pairing: Data = NoVideoFallbackTests.token, + route: MovedPC.Route = NoVideoFallbackTests.homeRoute) -> NoVideoSizes.Key { + NoVideoSizes.Key(address: .init(host: "127.0.0.1", port: pc.port), pairing: pairing, + network: NoVideoSizes.network(route)) + } + + /// A kept size answers only for its address, pairing and network, for a + /// week from when it was needed, and goes with `forget` on any network. + func testAKeptSizeIsBoundToItsAddressPairingNetworkAndWeek() { + let pc = FakeUDPHost(), elsewhere = FakeUDPHost() + let at = Date(timeIntervalSince1970: 1_800_000_000) + let key = sizeKey(pc) + NoVideoSizes.remember(576, for: key, now: at) + XCTAssertEqual(NoVideoSizes.size(for: key, now: at), 576) + XCTAssertEqual(NoVideoSizes.size(for: key, now: at.addingTimeInterval(NoVideoSizes.lifetime - 1)), 576) + XCTAssertNil(NoVideoSizes.size(for: key, now: at.addingTimeInterval(NoVideoSizes.lifetime)), "a week on") + XCTAssertNil(NoVideoSizes.size(for: key, now: at.addingTimeInterval(-1)), "a clock set back") + XCTAssertNil(NoVideoSizes.size(for: sizeKey(pc, pairing: Data(2...17)), now: at), "another pairing") + XCTAssertNil(NoVideoSizes.size(for: sizeKey(elsewhere), now: at), "another address") + XCTAssertNil(NoVideoSizes.size(for: sizeKey(pc, route: Self.otherRoute), now: at), "another network") + XCTAssertNil(NoVideoSizes.size(for: NoVideoSizes.Key(address: key.address, pairing: key.pairing, + network: NoVideoSizes.network(nil)), now: at), "no route") + // Needed again later: the week starts over, at the newer size. + NoVideoSizes.remember(1200, for: key, now: at.addingTimeInterval(86_400)) + XCTAssertEqual(NoVideoSizes.size(for: key, now: at.addingTimeInterval(NoVideoSizes.lifetime)), 1200) + XCTAssertFalse(NoVideoSizes.forget(sizeKey(pc, pairing: Data(2...17)))) + XCTAssertTrue(NoVideoSizes.forget(sizeKey(pc, route: Self.otherRoute))) + XCTAssertNil(NoVideoSizes.size(for: key, now: at.addingTimeInterval(86_400))) + } + + /// The size a path needed is kept: the next session at that address + /// asks for it as soon as the PC accepts, with no steps, and streams. + func testTheNextSessionStartsWithTheSizeThePathNeeded() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let at = Date(timeIntervalSince1970: 1_800_000_000) + manager.now = { at } + // The app's own wait: a slow runner's picture at the kept size comes + // well before any further step. + manager.timing.noVideoStepSeconds = 1.5 + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let frame = try FakeUDPHost.keyframe() + var runs: [FakeUDPHost.PathRun] = [] + for session in 1...2 { + let streaming = OSAllocatedUnfairLock(initialState: false) + pc.forgetClient() + manager.connect(host: "127.0.0.1", port: pc.port) + let served = Task.detached { + pc.servePath(limit: 1000, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("session \(session) streaming") { manager.state == .connected } + streaming.withLock { $0 = true } + runs.append(await served.value) + manager.cancel() + } + XCTAssertEqual(runs[0].sizesAsked, [1200, 576]) + XCTAssertEqual(NoVideoSizes.size(for: sizeKey(pc), now: at), 576) + XCTAssertEqual(runs[1].sizesAsked, [576], "asked at once, once") + let steps = logged().filter { $0.hasPrefix("no_video_fallback ") } + XCTAssertEqual(steps, ["no_video_fallback step=smaller_packets size=1200 result=no_video", + "no_video_fallback step=smaller_packets size=576 result=video", + "no_video_fallback step=remembered_size size=576 result=asked"]) + } + + /// A size kept for this address over another network isn't used, and + /// once the PC's own size gives a picture, it is forgotten. + func testAKeptSizeIsForgottenOnceThePCsOwnSizeWorks() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + NoVideoSizes.remember(576, for: sizeKey(pc, route: Self.otherRoute), now: Date()) + let streaming = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: .max, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("streaming") { manager.state == .connected } + try await until("forgotten", seconds: 10) { + logged().contains("no_video_fallback step=remembered_size result=forgotten") + } + streaming.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, []) + XCTAssertNil(NoVideoSizes.size(for: sizeKey(pc, route: Self.otherRoute), now: Date())) + XCTAssertFalse(NoVideoSizes.forget(sizeKey(pc))) + } + + /// A week after the size was needed, the next session starts at the + /// PC's own size again; where the path still needs less, the steps find + /// it and the week starts over. + func testAnExpiredSizeIsNotUsed() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let needed = Date(timeIntervalSince1970: 1_800_000_000) + let later = needed.addingTimeInterval(NoVideoSizes.lifetime + 60) + manager.now = { later } + NoVideoSizes.remember(576, for: sizeKey(pc), now: needed) + let streaming = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: 1000, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("streaming") { manager.state == .connected } + streaming.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, [1200, 576]) + XCTAssertEqual(NoVideoSizes.size(for: sizeKey(pc), now: later.addingTimeInterval(NoVideoSizes.lifetime - 120)), 576) + } + + /// A PC that doesn't take smaller sizes is asked for none, a kept size + /// or not, and the kept size stays as it was. + func testAnOlderPCIsNeverAskedForAKeptSize() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let (logged, sink) = lines(manager) + defer { sink.cancel() } + let at = Date() + NoVideoSizes.remember(576, for: sizeKey(pc), now: at) + let streaming = OSAllocatedUnfairLock(initialState: false) + manager.connect(host: "127.0.0.1", port: pc.port) + let frame = try FakeUDPHost.keyframe() + let served = Task.detached { + pc.servePath(limit: .max, frame: frame, hostCaps: 0, countFrames: false) { streaming.withLock { $0 } } + } + try await until("streaming") { manager.state == .connected } + try await Task.sleep(nanoseconds: 500_000_000) + streaming.withLock { $0 = true } + let run = await served.value + XCTAssertEqual(run.sizesAsked, []) + XCTAssertFalse(logged().contains { $0.hasPrefix("no_video_fallback ") }, logged().joined(separator: "\n")) + XCTAssertEqual(NoVideoSizes.size(for: sizeKey(pc), now: at), 576) + } + // MARK: - Connecting /// Datagrams over 1,000 bytes never arrive; only each frame's short last From 70eb57cd09f421a9e5062d346291e39933a0bfef Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 06:48:04 -0500 Subject: [PATCH 168/226] Tell the encoder which pictures capture resent from its cache The re-submit wait guessed a still picture from the gap before it. A pointer redraw resends the cached picture without the 100 ms resend limit, so a still picture 50 ms after the last real one waited 75 ms instead of 20; and motion resuming after a 750 ms idle spell waited only 20 ms, so 30 fps content had its first picture given to AMF again. RawFrame now carries `cached`, set by capture's resends (the keepalive, a keyframe request's resend, the pointer redraw, recovery) and by the synthetic source's still resends. After a cached picture the wait is 20 ms. After a real one it is a frame and a half of the target rate, or of the pace measured between real pictures only; that pace is kept across idle spells and forgotten after 10 s. The AMF tests tie each repeated picture to a feeder stall right after that frame, at most one repeat per 20 ms of it, and require no picture given again when the feeder never stalled; a deterministic test checks the checker. keyframe_resubmits is cleared for each new viewer. --- host/src/capture/dxgi.rs | 2 + host/src/capture/mod.rs | 6 + host/src/capture/synthetic.rs | 5 + host/src/encoder/mod.rs | 258 ++++++++++++++++++++-------------- host/src/stats.rs | 13 +- host/tests/amf_encoder.rs | 121 ++++++++++++---- 6 files changed, 271 insertions(+), 134 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index f72ea660..dfeca2a2 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -675,6 +675,7 @@ pub fn run_capture_loop( data: Arc::clone(&data), width: tex_width, height: tex_height, + cached: false, }); last_send = now; } else { @@ -1012,6 +1013,7 @@ fn resend_last_frame( data: Arc::clone(last_frame), width, height, + cached: true, }); *last_send = now; } diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 5ee270de..097dedc8 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -258,6 +258,10 @@ pub struct RawFrame { pub data: Arc>, pub width: u32, pub height: u32, + /// A picture capture had already, sent again for a still desktop: the + /// keepalive, a keyframe request's resend, or the cached picture + /// redrawn with or without the pointer. The desktop gave no new one. + pub cached: bool, } /// Readable bytes after a frame's last row: FFmpeg's @@ -1343,6 +1347,7 @@ mod tests { data: Arc::new(Vec::new()), width: 2, height: 2, + cached: false, }; let short = Duration::from_millis(5); assert!(matches!( @@ -1383,6 +1388,7 @@ mod tests { data: Arc::clone(&pool[n]), width: 2, height: 2, + cached: false, }; let mut kept = Some(frame(0)); producer.publish(frame(1)); diff --git a/host/src/capture/synthetic.rs b/host/src/capture/synthetic.rs index cf23baa5..9de5bc68 100644 --- a/host/src/capture/synthetic.rs +++ b/host/src/capture/synthetic.rs @@ -113,7 +113,10 @@ pub fn run_capture_loop( heartbeat(&shared.hb_capture_loop_ms); + // A still picture sent again, as DXGI capture marks its resends. + let mut cached = false; let data = if let Some(scene) = pointer.as_mut() { + cached = shared.client_draws_pointer(); scene.tick(&shared, &mut buffers, frame_start, frame_number) } else { let data = match still.as_ref() { @@ -125,6 +128,7 @@ pub fn run_capture_loop( true, ) => { + cached = true; Arc::clone(picture) } Some(_) => { @@ -151,6 +155,7 @@ pub fn run_capture_loop( data, width, height, + cached, }); #[cfg(debug_assertions)] if std::mem::take(&mut proving) { diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index bb94dabe..99cfde2f 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -205,7 +205,7 @@ fn run_encode_loop( }, }; if !given_again { - cadence.captured(raw_frame.timestamp); + cadence.captured(raw_frame.timestamp, raw_frame.cached); } if !shared.running.load(Ordering::SeqCst) || !crate::supervisor::generation_is_current(generation) @@ -857,7 +857,7 @@ fn run_encode_loop( picture, keyframe_out, encode_start, - keyframe_flush_wait(target_fps, cadence.moving_interval()), + keyframe_flush_wait(target_fps, raw_frame.cached, cadence.pace()), ); } @@ -2417,28 +2417,31 @@ pub fn holds_last_frame(encoder_name: &str) -> bool { /// one. See [`keyframe_flush_wait`]. const KEYFRAME_FLUSH_WAIT: Duration = Duration::from_millis(20); -/// A gap between capture's pictures at least this long is a still desktop: -/// capture resends a still picture for a keyframe request only after 100 ms -/// (its `KEYFRAME_RESEND`), and otherwise at its 750 ms keepalive. +/// A gap between capture's real pictures shorter than this is the desktop's +/// pace while it moves; a longer one is an idle spell, which says nothing +/// about the pace (capture resends a still picture after 100 ms for a +/// keyframe request, else at its 750 ms keepalive). const MOVING_GAP: Duration = Duration::from_millis(100); +/// After an idle spell this long, the pace seen before it is forgotten. +const PACE_FORGOTTEN: Duration = Duration::from_secs(10); /// How long the encode loop waits for capture's next picture, after encoding -/// one, before it gives the last picture again. While the desktop moves -/// (`moving_interval` is capture's longest recent gap), a frame and a half: -/// of the target rate, or of that gap when capture delivers less often, as a -/// 50 Hz display streamed at 60 fps or 30 fps content does. A flat 20 ms gave -/// those pictures to AMF twice. On a still desktop no picture is due, so the -/// shortest wait brings the keyframe out soonest. -fn keyframe_flush_wait(target_fps: u32, moving_interval: Option) -> Duration { - match moving_interval { - Some(gap) => { - (crate::capture::frame_budget_for(target_fps).max(gap) * 3 / 2).max(KEYFRAME_FLUSH_WAIT) - } - None => KEYFRAME_FLUSH_WAIT, - } +/// `cached` (see [`RawFrame::cached`]), before it gives that picture again. +/// After a cached picture of a still desktop no picture is due, so the +/// shortest wait brings the keyframe out soonest. After a real one, a frame +/// and a half: of the target rate, or of capture's pace while the desktop +/// moves (`pace`) when that is slower, as for a 50 Hz display streamed at +/// 60 fps or 30 fps content. A flat 20 ms gave those pictures to AMF twice. +fn keyframe_flush_wait(target_fps: u32, cached: bool, pace: Option) -> Duration { + if cached { + return KEYFRAME_FLUSH_WAIT; + } + let frame = crate::capture::frame_budget_for(target_fps).max(pace.unwrap_or_default()); + (frame * 3 / 2).max(KEYFRAME_FLUSH_WAIT) } -/// The pace of capture's own pictures, not counting those given again. +/// The pace of capture's real pictures, measured only between real pictures +/// and kept across idle spells: motion that resumes keeps its pace. #[derive(Default)] struct CaptureCadence { last: Option, @@ -2448,30 +2451,31 @@ struct CaptureCadence { impl CaptureCadence { const KEPT: usize = 8; - fn captured(&mut self, at: Instant) { + /// A picture from capture (not one given again). + fn captured(&mut self, at: Instant, cached: bool) { + if cached { + return; + } if let Some(last) = self.last.replace(at) { - if self.gaps.len() == Self::KEPT { - self.gaps.pop_front(); + let gap = at.saturating_duration_since(last); + if gap < MOVING_GAP { + if self.gaps.len() == Self::KEPT { + self.gaps.pop_front(); + } + self.gaps.push_back(gap); + } else if gap >= PACE_FORGOTTEN { + self.gaps.clear(); } - self.gaps.push_back(at.saturating_duration_since(last)); } } - /// The longest recent gap between pictures while the desktop moves, or - /// `None` when the last picture came after a still spell (or is the - /// first). - fn moving_interval(&self) -> Option { - let latest = *self.gaps.back()?; - (latest < MOVING_GAP).then(|| { - self.gaps - .iter() - .copied() - .filter(|gap| *gap < MOVING_GAP) - .max() - .unwrap_or(latest) - }) + /// The longest recent gap between real pictures while the desktop moved, + /// if any were seen. + fn pace(&self) -> Option { + self.gaps.iter().copied().max() } } + /// How long after the keyframe went in the last picture is given again, /// before the keyframe is left to capture's next picture: an AMD driver can /// take a few hundred ms to encode a fresh session's first frame, and @@ -6282,6 +6286,7 @@ mod tests { data: std::sync::Arc::clone(data), width: 2, height: 2, + cached: false, } } @@ -6378,100 +6383,142 @@ mod tests { ); } - /// Replays a capture schedule on a fake clock through the rule the encode - /// loop uses, as if a keyframe were owed after every picture: encoding - /// ends `ENCODE` after a picture is captured, and the picture is given + const ENCODE: Duration = Duration::from_millis(2); + + /// Replays capture on a fake clock through the rule the encode loop + /// uses, as if a keyframe were owed after every picture: encoding ends + /// [`ENCODE`] after a picture is captured, and the picture is given /// again if capture's next one has not come by then plus the wait. - /// Returns how many pictures would have been given again. - fn resubmits_on_a_moving_desktop(target_fps: u32, frame: Duration) -> usize { - const ENCODE: Duration = Duration::from_millis(2); - let start = Instant::now(); - // Up to a quarter frame early or late, so gaps run from half a frame - // to a frame and a half. + /// `pictures` are capture times, each marked cached or real. Returns the + /// indices of the pictures that would have been given again. + fn given_again(target_fps: u32, pictures: &[(Instant, bool)]) -> Vec { + let mut cadence = CaptureCadence::default(); + let mut again = Vec::new(); + for (n, pair) in pictures.windows(2).enumerate() { + let (at, cached) = pair[0]; + cadence.captured(at, cached); + let wait = keyframe_flush_wait(target_fps, cached, cadence.pace()); + if pair[1].0 > at + ENCODE + wait { + again.push(n); + } + } + again + } + + /// Real pictures every `frame`, each up to a quarter frame early or late, + /// so gaps run from half a frame to a frame and a half. + fn moving(start: Instant, frame: Duration, count: u32) -> Vec<(Instant, bool)> { let quarter = frame / 4; - let jitter = [ - Duration::ZERO, - quarter, - Duration::ZERO, - quarter, - quarter / 2, - ]; - let early = [false, false, false, true, true]; - let captured: Vec = (0..240u32) + let offsets = [0i32, 1, 0, -1, -2]; + (0..count) .map(|n| { - let at = start + Duration::from_secs(1) + frame * n; - let k = n as usize % jitter.len(); - if early[k] { - at - jitter[k] - } else { - at + jitter[k] - } + let at = start + frame * n; + let at = match offsets[n as usize % offsets.len()] { + k if k >= 0 => at + quarter * k as u32, + k => at - quarter * (-k) as u32 / 2, + }; + (at, false) }) - .collect(); - let mut cadence = CaptureCadence::default(); - let mut resubmits = 0; - for (n, pair) in captured.windows(2).enumerate() { - cadence.captured(pair[0]); - let wait = keyframe_flush_wait(target_fps, cadence.moving_interval()); + .collect() + } + + #[test] + fn a_moving_desktop_never_has_its_picture_given_again_at_60_50_or_30_fps() { + let start = Instant::now() + Duration::from_secs(1); + let us = Duration::from_micros; + for (target_fps, frame, what) in [ + (60, us(16_667), "60 fps"), + (60, us(20_000), "a 50 Hz display streamed at 60 fps"), + (50, us(20_000), "50 fps"), + (30, us(33_333), "30 fps"), + (60, us(33_333), "30 fps content streamed at 60 fps"), + ] { + let pictures = moving(start, frame, 240); // The pipeline's first picture has no pace to go by. It is // encoded before any viewer arrives (see capture's IdleGate). - if n > 0 && pair[1] > pair[0] + ENCODE + wait { - resubmits += 1; - } + let again: Vec = given_again(target_fps, &pictures) + .into_iter() + .filter(|&n| n > 0) + .collect(); + assert!(again.is_empty(), "{what}: pictures {again:?} given again"); } - resubmits } #[test] - fn a_moving_desktop_never_has_its_picture_given_again_at_60_50_or_30_fps() { - let ms = |n: u64| Duration::from_micros(n); - for (target_fps, frame, what) in [ - (60, ms(16_667), "60 fps"), - (60, ms(20_000), "a 50 Hz display streamed at 60 fps"), - (50, ms(20_000), "50 fps"), - (30, ms(33_333), "30 fps"), - (60, ms(33_333), "30 fps content streamed at 60 fps"), + fn motion_that_resumes_after_an_idle_spell_keeps_its_pace() { + let start = Instant::now() + Duration::from_secs(1); + for (target_fps, frame) in [ + (60, Duration::from_micros(16_667)), + (60, Duration::from_micros(20_000)), + (60, Duration::from_micros(33_333)), + (30, Duration::from_micros(33_333)), ] { - assert_eq!( - resubmits_on_a_moving_desktop(target_fps, frame), - 0, - "{what}: a picture was given again before capture's next one" + let mut pictures = moving(start, frame, 20); + let last = pictures.last().unwrap().0; + // Still for 750 ms, with the keepalive's cached resend, then + // moving again at the same pace. + pictures.push((last + Duration::from_millis(750), true)); + let resumed = last + Duration::from_millis(1500); + pictures.extend(moving(resumed, frame, 20)); + // Judged from the first picture after the resend: the last one + // before the idle spell is given again, as the desktop went + // still after it. + let resent = 20; + let again: Vec = given_again(target_fps, &pictures) + .into_iter() + .filter(|&n| n > resent) + .collect(); + assert!( + again.is_empty(), + "{target_fps} fps, {frame:?} frames: real pictures {again:?} given again" ); } + // Past a long idle spell the pace is forgotten: the target's again. + let mut cadence = CaptureCadence::default(); + for (at, cached) in moving(start, Duration::from_micros(33_333), 10) { + cadence.captured(at, cached); + } + cadence.captured(start + PACE_FORGOTTEN * 2, false); + assert_eq!(cadence.pace(), None); + assert_eq!( + keyframe_flush_wait(60, false, cadence.pace()), + crate::capture::frame_budget_for(60) * 3 / 2 + ); } #[test] - fn a_still_desktop_has_its_picture_given_again_after_the_shortest_wait() { - let start = Instant::now(); + fn a_cached_picture_is_given_again_after_the_shortest_wait() { + let start = Instant::now() + Duration::from_secs(1); for target_fps in [60, 50, 30] { - let mut cadence = CaptureCadence::default(); - // Moving at the target rate, then still until the resent picture. let frame = crate::capture::frame_budget_for(target_fps); - for n in 0..10 { - cadence.captured(start + frame * n); + let mut pictures = moving(start, frame, 10); + let last = pictures.last().unwrap().0; + // A pointer redraw resends the cached picture after only 50 ms, + // then the desktop stays still. + pictures.push((last + Duration::from_millis(50), true)); + pictures.push((last + Duration::from_millis(800), true)); + let mut cadence = CaptureCadence::default(); + for &(at, cached) in &pictures[..pictures.len() - 1] { + cadence.captured(at, cached); } assert_eq!( - keyframe_flush_wait(target_fps, cadence.moving_interval()), - frame * 3 / 2, - "{target_fps} fps while the desktop moves" + keyframe_flush_wait(target_fps, true, cadence.pace()), + KEYFRAME_FLUSH_WAIT, + "{target_fps} fps: a cached picture" + ); + assert!( + given_again(target_fps, &pictures).contains(&10), + "{target_fps} fps: the redrawn picture was not given again" ); - cadence.captured(start + frame * 9 + Duration::from_millis(750)); - assert_eq!(cadence.moving_interval(), None, "{target_fps} fps: still"); + // A real picture waits a frame and a half of the pace. assert_eq!( - keyframe_flush_wait(target_fps, cadence.moving_interval()), - KEYFRAME_FLUSH_WAIT, - "{target_fps} fps on a still desktop" + keyframe_flush_wait(target_fps, false, cadence.pace()), + frame.max(cadence.pace().unwrap()) * 3 / 2, + "{target_fps} fps: a real picture" ); } // Past 75 fps a frame and a half is shorter than AMF's encode. - let mut cadence = CaptureCadence::default(); - for n in 0..10 { - cadence.captured(start + crate::capture::frame_budget_for(120) * n); - } - assert_eq!( - keyframe_flush_wait(120, cadence.moving_interval()), - KEYFRAME_FLUSH_WAIT - ); + assert_eq!(keyframe_flush_wait(120, false, None), KEYFRAME_FLUSH_WAIT); } #[test] @@ -6635,6 +6682,7 @@ mod tests { data: std::sync::Arc::new(data), width: 1280, height: 720, + cached: false, }); self.number += 1; std::thread::sleep(std::time::Duration::from_millis(33)); diff --git a/host/src/stats.rs b/host/src/stats.rs index cd1e3d91..cb032cb9 100644 --- a/host/src/stats.rs +++ b/host/src/stats.rs @@ -43,7 +43,7 @@ pub struct PipelineStats { /// stuck on slow software encoding. pub using_software_fallback: bool, /// Pictures given to an encoder again to bring out a keyframe it held - /// (see `encoder::holds_last_frame`). + /// (see `encoder::holds_last_frame`), for the connected viewer. pub keyframe_resubmits: u64, /// NVENC refused to open because the NVIDIA driver is older than the /// linked FFmpeg's NVENC needs. The fallback notice then says to update @@ -179,6 +179,7 @@ impl PipelineStats { self.transport_retransmits = 0; self.pointer_states_sent = 0; self.pointer_shapes_sent = 0; + self.keyframe_resubmits = 0; self.transport_timestamps.clear(); self.bandwidth_samples.clear(); self.bandwidth_bps = 0.0; @@ -313,6 +314,16 @@ impl PipelineStats { mod tests { use super::PipelineStats; + #[test] + fn a_new_viewer_starts_counting_pictures_given_again_from_zero() { + let mut stats = PipelineStats { + keyframe_resubmits: 3, + ..PipelineStats::default() + }; + stats.reset_connection_stats(); + assert_eq!(stats.keyframe_resubmits, 0); + } + #[test] fn reset_for_restart_preserves_runtime_configuration() { let mut stats = PipelineStats::new(); diff --git a/host/tests/amf_encoder.rs b/host/tests/amf_encoder.rs index 786af5c9..78203758 100644 --- a/host/tests/amf_encoder.rs +++ b/host/tests/amf_encoder.rs @@ -111,8 +111,8 @@ struct StageRun { codec_name: String, /// Pictures the stage gave the encoder again (`keyframe_resubmits`). resubmits: u64, - /// The longest time between two published frames. - longest_gap: Duration, + /// When each frame was published, by frame number. + published: Vec, } /// Feeds `FRAME_COUNT` frames through a fresh encode stage at @@ -148,8 +148,7 @@ fn run_stage( packets }); - let mut last_publish: Option = None; - let mut longest_gap = Duration::ZERO; + let mut published = Vec::new(); for number in 0..FRAME_COUNT { if keyframe_at == Some(number) { shared.force_next_idr.store(true, Ordering::SeqCst); @@ -157,15 +156,14 @@ fn run_stage( let (width, height) = size_of(number); let mut data = frame_buffer((width * height * 4) as usize); render_synthetic_frame(&mut data, width, height, number); - if let Some(last) = last_publish.replace(Instant::now()) { - longest_gap = longest_gap.max(last.elapsed()); - } + published.push(Instant::now()); producer.publish(RawFrame { frame_number: number, timestamp: Instant::now(), data: Arc::new(data), width, height, + cached: false, }); std::thread::sleep(FRAME_INTERVAL); } @@ -184,40 +182,57 @@ fn run_stage( software_fallback: stats.using_software_fallback, codec_name: stats.codec_name.clone(), resubmits: stats.keyframe_resubmits - resubmits_before, - longest_gap, + published, } } /// A stream whose frames come every `FRAME_INTERVAL` is never still, so no -/// picture may be given to the encoder again and none encoded twice. Only if -/// the producer itself stalled for a frame and a half (the encode loop's wait, -/// see `keyframe_flush_wait`) may a picture repeat, and then only one the -/// stage reports giving again. +/// picture may be given to the encoder again and none encoded twice. A +/// picture may repeat only where the feeder itself stalled right after it, +/// for a frame and a half (the encode loop's wait, see `keyframe_flush_wait`), +/// at most once per 20 ms of that stall, and no more often in all than the +/// stage reports giving pictures again. Everywhere else counters advance. fn assert_no_duplicate_encodes(run: &StageRun, pictures: &[(u32, u32, u64)]) { let counters: Vec = pictures.iter().map(|picture| picture.2).collect(); assert!( counters.windows(2).all(|pair| pair[0] <= pair[1]), "frame counters go back: {counters:?}" ); - let repeats = counters - .windows(2) - .filter(|pair| pair[0] == pair[1]) - .count() as u64; - if run.longest_gap < FRAME_INTERVAL * 3 / 2 { - assert_eq!( - (repeats, run.resubmits), - (0, 0), - "pictures encoded twice, and given again, while frames came every \ - {FRAME_INTERVAL:?} at most {:?} apart: {counters:?}", - run.longest_gap - ); - } else { + let mut repeats = std::collections::BTreeMap::::new(); + for pair in counters.windows(2).filter(|pair| pair[0] == pair[1]) { + *repeats.entry(pair[0]).or_default() += 1; + } + for (&counter, ×) in &repeats { + let stall = run + .published + .get(counter as usize + 1) + .zip(run.published.get(counter as usize)) + .map(|(next, this)| next.saturating_duration_since(*this)); + let allowed = stall + .filter(|stall| *stall >= FRAME_INTERVAL * 3 / 2) + .map_or(0, |stall| (stall.as_millis() / 20) as u32); assert!( - repeats <= run.resubmits, - "{repeats} pictures encoded twice but {} given again: {counters:?}", - run.resubmits + times <= allowed, + "frame {counter} encoded {} times, though the next frame came {stall:?} \ + after it: {counters:?}", + times + 1 ); } + let stalled = run + .published + .windows(2) + .any(|pair| pair[1].saturating_duration_since(pair[0]) >= FRAME_INTERVAL * 3 / 2); + assert!( + stalled || run.resubmits == 0, + "{} pictures given again though no frame came later than {FRAME_INTERVAL:?}", + run.resubmits + ); + let total: u64 = repeats.values().map(|×| u64::from(times)).sum(); + assert!( + total <= run.resubmits, + "{total} pictures encoded again but {} given again: {counters:?}", + run.resubmits + ); } /// Decodes `packets` with a fresh, strict software decoder and returns each @@ -363,6 +378,54 @@ fn assert_follows_mode_changes(run: &StageRun, codec: Id) { } } +#[test] +fn a_repeat_is_excused_only_by_the_stall_right_after_it() { + // Frames every 20 ms, except a 60 ms stall after frame 10. + let start = Instant::now(); + let published: Vec = (0..60u32) + .map(|n| { + start + + FRAME_INTERVAL * n + + if n > 10 { + FRAME_INTERVAL * 2 + } else { + Duration::ZERO + } + }) + .collect(); + let run = |resubmits| StageRun { + packets: Vec::new(), + software_fallback: false, + codec_name: String::new(), + resubmits, + published: published.clone(), + }; + let pictures = |repeated: u64| -> Vec<(u32, u32, u64)> { + (0..60u64) + .flat_map(|n| std::iter::repeat_n((0, 0, n), if n == repeated { 2 } else { 1 })) + .collect() + }; + assert_no_duplicate_encodes(&run(1), &pictures(10)); + for (resubmits, repeated, what) in [ + (1, 50, "a repeat far from the stall"), + (0, 10, "a repeat the stage did not give again"), + (1, 9, "a repeat just before the stall"), + ] { + let failed = std::panic::catch_unwind(|| { + assert_no_duplicate_encodes(&run(resubmits), &pictures(repeated)) + }) + .is_err(); + assert!(failed, "{what} was accepted"); + } + // Pictures given again with no stall at all. + let steady = StageRun { + published: (0..60u32).map(|n| start + FRAME_INTERVAL * n).collect(), + ..run(1) + }; + let counters: Vec<(u32, u32, u64)> = (0..60u64).map(|n| (0, 0, n)).collect(); + assert!(std::panic::catch_unwind(|| assert_no_duplicate_encodes(&steady, &counters)).is_err()); +} + #[test] fn software_encoder_follows_capture_mode_changes() { let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); @@ -585,6 +648,8 @@ fn keyframe_on_a_still_desktop(encoder: &str) -> StillRun { data: Arc::new(data), width, height, + // The warm-up's frames are new; the resent one is the cache. + cached: number == RESENT, }); }; From d0683572bbbd6db258a40558a7c10fbe77b86c52 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 06:49:32 -0500 Subject: [PATCH 169/226] Keep a remembered media size per network Remembering a size on one network no longer replaces the size kept for the same PC on another: lookup and replacement now include the network. A picture at the PC's own size still forgets the sizes kept for the address on every network. --- ios/EternalMonitor/App/NoVideoFallback.swift | 24 +++++------ .../NoVideoFallbackTests.swift | 41 ++++++++++++++++++- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/ios/EternalMonitor/App/NoVideoFallback.swift b/ios/EternalMonitor/App/NoVideoFallback.swift index 467142ee..b3c8f442 100644 --- a/ios/EternalMonitor/App/NoVideoFallback.swift +++ b/ios/EternalMonitor/App/NoVideoFallback.swift @@ -120,9 +120,10 @@ enum NoVideoPaths { /// The media datagram size a PC's video needed at an address, so the next /// session there starts with it instead of waiting through the steps again. /// Each entry is bound to the pairing by a digest of its token, and to the -/// network this iPad reached the PC over. It lasts a week from when the -/// size was last needed, and goes as soon as a session at the address shows -/// a picture at the PC's own size. Kept in UserDefaults. +/// network this iPad reached the PC over: one address may keep a size per +/// network. It lasts a week from when the size was last needed, and every +/// size kept for the address goes as soon as a session there shows a +/// picture at the PC's own size. Kept in UserDefaults. enum NoVideoSizes { static let lifetime: TimeInterval = 7 * 24 * 60 * 60 private static let key = "noVideoSizes" @@ -153,23 +154,23 @@ enum NoVideoSizes { /// The size to start a session at `key` with: one kept for its address /// and pairing, on the same network, less than `lifetime` old. static func size(for key: Key, now: Date) -> UInt16? { - guard let entry = entry(key), entry.network == key.network, key.network != "-", + guard key.network != "-", let entry = load().last(where: { matches($0, key, network: true) }), now >= entry.at, now.timeIntervalSince(entry.at) < lifetime else { return nil } return entry.size } static func remember(_ size: UInt16, for key: Key, now: Date) { - var entries = load().filter { !matches($0, key) } + var entries = load().filter { !matches($0, key, network: true) } entries.append(Entry(address: key.address, pairing: NoVideoPaths.digest(key.pairing), network: key.network, size: size, at: now)) save(Array(entries.suffix(kept))) } - /// Forgets the size kept for `key`'s address and pairing, on any + /// Forgets the sizes kept for `key`'s address and pairing, on every /// network; true if there was one. @discardableResult static func forget(_ key: Key) -> Bool { let entries = load() - let kept = entries.filter { !matches($0, key) } + let kept = entries.filter { !matches($0, key, network: false) } guard kept.count != entries.count else { return false } save(kept) return true @@ -179,12 +180,11 @@ enum NoVideoSizes { UserDefaults.standard.removeObject(forKey: key) } - private static func entry(_ key: Key) -> Entry? { - load().last { matches($0, key) } - } - - private static func matches(_ entry: Entry, _ key: Key) -> Bool { + /// Whether `entry` is kept for `key`'s address and pairing, and for its + /// network when `network` says so. + private static func matches(_ entry: Entry, _ key: Key, network: Bool) -> Bool { entry.address == key.address && entry.pairing == NoVideoPaths.digest(key.pairing) + && (!network || entry.network == key.network) } private static func load() -> [Entry] { diff --git a/ios/EternalMonitorTests/NoVideoFallbackTests.swift b/ios/EternalMonitorTests/NoVideoFallbackTests.swift index f11017c2..78f362c5 100644 --- a/ios/EternalMonitorTests/NoVideoFallbackTests.swift +++ b/ios/EternalMonitorTests/NoVideoFallbackTests.swift @@ -128,9 +128,15 @@ final class NoVideoFallbackTests: XCTestCase { // Needed again later: the week starts over, at the newer size. NoVideoSizes.remember(1200, for: key, now: at.addingTimeInterval(86_400)) XCTAssertEqual(NoVideoSizes.size(for: key, now: at.addingTimeInterval(NoVideoSizes.lifetime)), 1200) + // Each network keeps its own size; `forget` drops them all. + let other = sizeKey(pc, route: Self.otherRoute) + NoVideoSizes.remember(576, for: other, now: at.addingTimeInterval(86_400)) + XCTAssertEqual(NoVideoSizes.size(for: key, now: at.addingTimeInterval(86_400)), 1200) + XCTAssertEqual(NoVideoSizes.size(for: other, now: at.addingTimeInterval(86_400)), 576) XCTAssertFalse(NoVideoSizes.forget(sizeKey(pc, pairing: Data(2...17)))) - XCTAssertTrue(NoVideoSizes.forget(sizeKey(pc, route: Self.otherRoute))) + XCTAssertTrue(NoVideoSizes.forget(other)) XCTAssertNil(NoVideoSizes.size(for: key, now: at.addingTimeInterval(86_400))) + XCTAssertNil(NoVideoSizes.size(for: other, now: at.addingTimeInterval(86_400))) } /// The size a path needed is kept: the next session at that address @@ -169,6 +175,39 @@ final class NoVideoFallbackTests: XCTestCase { "no_video_fallback step=remembered_size size=576 result=asked"]) } + /// Each network the iPad reaches the PC over keeps its own size: one + /// path needed 576 bytes, another 1,200. Back on the first, the session + /// asks for 576 at once. + func testEachNetworkKeepsItsOwnSize() async throws { + let pc = FakeUDPHost() + let (manager, _) = try manager(pairedAt: pc) + defer { manager.cancel() } + let at = Date(timeIntervalSince1970: 1_800_000_000) + manager.now = { at } + manager.timing.noVideoStepSeconds = 1.5 + let frame = try FakeUDPHost.keyframe() + var runs: [FakeUDPHost.PathRun] = [] + // Network A drops datagrams over 1,000 bytes, network B over 1,250. + for (route, limit) in [(Self.homeRoute, 1000), (Self.otherRoute, 1250), (Self.homeRoute, 1000)] { + manager.localRoute = { _, _ in route } + let streaming = OSAllocatedUnfairLock(initialState: false) + pc.forgetClient() + manager.connect(host: "127.0.0.1", port: pc.port) + let served = Task.detached { + pc.servePath(limit: limit, frame: frame, hostCaps: HelloAck.hostCapMediaSize) { streaming.withLock { $0 } } + } + try await until("streaming over \(route.address)") { manager.state == .connected } + streaming.withLock { $0 = true } + runs.append(await served.value) + manager.cancel() + } + XCTAssertEqual(runs[0].sizesAsked, [1200, 576]) + XCTAssertEqual(runs[1].sizesAsked, [1200]) + XCTAssertEqual(runs[2].sizesAsked, [576], "network A's size, at once") + XCTAssertEqual(NoVideoSizes.size(for: sizeKey(pc), now: at), 576) + XCTAssertEqual(NoVideoSizes.size(for: sizeKey(pc, route: Self.otherRoute), now: at), 1200) + } + /// A size kept for this address over another network isn't used, and /// once the PC's own size gives a picture, it is forgotten. func testAKeptSizeIsForgottenOnceThePCsOwnSizeWorks() async throws { From f879fb4b267ab364e8b040aad67de3abab754765 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 06:50:48 -0500 Subject: [PATCH 170/226] Keep the session-summary tests from sharing the fallback slot a_session_ended_with_a_fallback_in_one_pass_names_it again ended with fallback=none in parallel runs. a_pen_refresh_windows_holds_never_stops_the_loop, added since, runs the real transport loop and starts a session through act, which replaces the process-wide fallback slot, without holding FALLBACK_SLOT. It now holds it, and note_session_start asserts in tests that the slot is held, so the next test that starts a session without it fails when run alone. Three log captures added since also skipped logging::prepare_log_capture. a_lone_display_or_lost_panel_reaches_the_uploaded_summary captured no summary at all when another test thread logged a summary line first; the capture and second-screen attach tests now call it too. --- host/src/capture/mod.rs | 1 + host/src/capture/second_screen.rs | 1 + host/src/diagnostics.rs | 8 ++++++++ host/src/transport/mod.rs | 2 ++ 4 files changed, 12 insertions(+) diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 5ee270de..b4e430a9 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -2082,6 +2082,7 @@ mod tests { let primary = out(r"\\.\DISPLAY1", 0, 0); let chosen = CaptureTarget::Output(r"\\.\DISPLAY2".to_string()); let unplugged = vec![primary.clone()]; + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { for _ in 0..20 { assert!(!target_moved(&unplugged, &chosen, &primary)); diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index a396fb17..5c5a5600 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -3662,6 +3662,7 @@ mod tests { .with_ansi(false) .with_writer(move || Sink(Arc::clone(&writer))) .finish(); + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || attach(&shared, &driver, ATTACH_TIMEOUT)); let lines = String::from_utf8(out.lock().clone()).unwrap(); assert!( diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 85dd7547..6dd76446 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -248,6 +248,13 @@ static SESSION_FALLBACK: parking_lot::Mutex = /// `session_id` started: fallbacks logged from now on are its own. pub fn note_session_start(session_id: u32, shared: &SharedControl) { + // The slot is the process's, so a test that starts a session would + // replace another test's. + #[cfg(test)] + assert!( + tests::FALLBACK_SLOT.is_locked(), + "a test that starts a session holds diagnostics::tests::FALLBACK_SLOT" + ); *SESSION_FALLBACK.lock() = SessionFallback { session_id: Some(session_id), mode: Some(Arc::clone(&shared.capture_target)), @@ -1091,6 +1098,7 @@ pub(crate) mod tests { .with(SummaryCapture(summaries.clone())); let shared = SharedControl::new(9876, 15_000_000); *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + crate::logging::prepare_log_capture(); tracing::subscriber::with_default(subscriber, || { for (id, reason) in [(1, "vdd_only"), (2, "panel_lost")] { note_session_start(id, &shared); diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 42667586..d53977c0 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -2155,6 +2155,8 @@ mod tests { fn a_pen_refresh_windows_holds_never_stops_the_loop() { use crate::input::{CaptureGeometry, InputThread, Sink, KIND_KEY, PHASE_BEGAN}; use eternal_wire::v2::control::{HelloStatus, InputEvent, FEATURE_WANTS_INPUT}; + // The loop's session starts through `act`, which uses the fallback slot. + let _slot = crate::diagnostics::tests::FALLBACK_SLOT.lock(); /// Holds the first refresh until told to go on. struct StuckRefresh { From 12cc34eb8a95b5dda4d3be8a9252897cf5f2c8fe Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 06:52:49 -0500 Subject: [PATCH 171/226] End the second screen's attach wait early when its device runs with no monitor In the field the Enable task succeeds, the driver's device runs, and Windows never adds a monitor for it: every such attempt waited out the whole 30 s attach timeout before falling back to the mirror, and every retry did the same. Turning the device off and on again, as each retry does, never brought the monitor back. Where the second screen works, the monitor is there by the time the Enable task returns. While the attach waits, the worker now reads the device's state once a second. When it has run with no monitor for 10 s, the attempt ends as an attach timeout, with its own log line (vdd_device=no_monitor and the time waited) and the same snapshot, notice, turn off and remembered failure as before. A monitor that is there but not yet extended, a device that can't be read, or one that has not started keeps the whole wait. Tests: a fake driver whose device runs with no monitor ends the wait after 10 s; a ready, unreadable or unstarted device keeps the full wait; one with no monitor that attaches before 10 s still becomes the second screen. --- host/src/capture/mod.rs | 9 ++ host/src/capture/second_screen.rs | 140 +++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 2 deletions(-) diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 5ee270de..5dd3fd87 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -490,6 +490,8 @@ const VDD_POLL_INTERVAL: Duration = Duration::from_millis(120); enum AttachResult { Attached(OutputInfo), TimedOut, + /// The driver runs, but Windows added no monitor for it, and won't. + NoMonitor, Cancelled, } @@ -502,6 +504,7 @@ fn wait_for_virtual_output( mut poll: impl FnMut() -> Option>, mut now: impl FnMut() -> Instant, mut wait: impl FnMut(Duration), + mut no_monitor: impl FnMut() -> bool, ) -> AttachResult { let deadline = now() + timeout; loop { @@ -514,6 +517,9 @@ fn wait_for_virtual_output( if now() >= deadline { return AttachResult::TimedOut; } + if no_monitor() { + return AttachResult::NoMonitor; + } wait(VDD_POLL_INTERVAL); } } @@ -1948,6 +1954,7 @@ mod tests { "driver startup must not trip the 3s/5s watchdogs" ); }, + || false, ); let AttachResult::Attached(output) = result else { panic!("virtual display did not attach") @@ -1980,6 +1987,7 @@ mod tests { || Some(Vec::new()), || start + elapsed.get(), |duration| elapsed.set(elapsed.get() + duration), + || false, ); assert!(matches!(result, AttachResult::TimedOut)); assert!(elapsed.get() >= timeout && elapsed.get() < timeout + VDD_POLL_INTERVAL); @@ -2000,6 +2008,7 @@ mod tests { elapsed.set(elapsed.get() + duration); cancelled_at.set(true); }, + || false, ); assert!(matches!(result, AttachResult::Cancelled)); } diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index a396fb17..9c82b5f2 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -103,6 +103,17 @@ pub(crate) const EXTEND_TIMEOUT: Duration = Duration::from_secs(30); #[cfg(any(windows, test, debug_assertions))] pub(crate) const BLOCKED_EXTEND_RETRY: Duration = Duration::from_secs(120); +/// How long the device may run with no monitor, while the attach waits, +/// before the attempt ends. Where the second screen works, the monitor is +/// there by the time the Enable task returns; where it never came, waiting +/// out the whole attach timeout only delayed the fallback. +#[cfg(any(windows, test, debug_assertions))] +pub(crate) const NO_MONITOR_WAIT: Duration = Duration::from_secs(10); + +/// How often the attach wait reads the device's state. +#[cfg(any(windows, test, debug_assertions))] +const DEVICE_READ_EVERY: Duration = Duration::from_secs(1); + /// How long quitting waits for the worker to exit after the last Enable. const QUIT_JOIN: Duration = Duration::from_secs(5); @@ -1492,6 +1503,11 @@ pub(crate) trait Driver { /// Say why the display did not appear or showed no picture, while it is /// still on. fn report(&self, _failure: Failure) {} + /// The device's state while the attach waits, `None` when it can't be + /// read. + fn device(&self) -> Option { + None + } fn now(&self) -> Instant; fn sleep(&self, duration: Duration); } @@ -1525,6 +1541,33 @@ pub(crate) enum Failure { Unproven(&'static str), } +/// Whether the device has run with no monitor for [`NO_MONITOR_WAIT`], +/// read every [`DEVICE_READ_EVERY`] while the attach waits. Any other state, +/// or one that can't be read, starts the count again. +#[cfg(any(windows, test, debug_assertions))] +#[derive(Default)] +struct NoMonitorWatch { + since: Option, + next_read: Option, +} + +#[cfg(any(windows, test, debug_assertions))] +impl NoMonitorWatch { + fn gave_up(&mut self, now: Instant, read: impl FnOnce() -> Option) -> bool { + if self.next_read.is_some_and(|at| now < at) { + return false; + } + self.next_read = Some(now + DEVICE_READ_EVERY); + if read() == Some(DeviceState::NoMonitor) { + let since = *self.since.get_or_insert(now); + now.duration_since(since) >= NO_MONITOR_WAIT + } else { + self.since = None; + false + } + } +} + /// However [`attach`] ends, its claim ends with it: a start that did not /// finish leaves the status for the next tick to claim again. #[cfg(any(windows, test, debug_assertions))] @@ -1619,12 +1662,14 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura let result = if turned_on != TurnedOn::Beside { AttachResult::TimedOut } else { + let mut watch = NoMonitorWatch::default(); wait_for_virtual_output( &before, timeout, || selected().then(|| listed(shared, driver)), || driver.now(), |duration| driver.sleep(duration), + || watch.gave_up(driver.now(), || driver.device()), ) }; #[cfg(windows)] @@ -1682,8 +1727,16 @@ pub(crate) fn attach(shared: &SharedControl, driver: &impl Driver, timeout: Dura state.panel_lost_this_run = true; state.remember_panel_lost(now); } - AttachResult::TimedOut => { - if turned_on == TurnedOn::Alone { + ended @ (AttachResult::TimedOut | AttachResult::NoMonitor) => { + if matches!(ended, AttachResult::NoMonitor) { + warn!( + second_screen_fallback = "attach_timeout", + vdd_device = "no_monitor", + waited_ms = driver.now().duration_since(waiting_since).as_millis() as u64, + "Windows added no monitor for the running virtual display — capturing the \ + primary display instead" + ); + } else if turned_on == TurnedOn::Alone { warn!( second_screen_fallback = "vdd_only", "Windows put the desktop on the virtual display alone — capturing the \ @@ -1881,6 +1934,10 @@ impl Driver for WindowsDriver<'_> { chosen } + fn device(&self) -> Option { + Some(crate::vdd_device::query()) + } + fn turned_on(&self) -> TurnedOn { if crate::display_guard::last_turn_on_lost_panel() { TurnedOn::PanelLost @@ -2029,6 +2086,9 @@ mod tests { on_sleep: &'a dyn Fn(Duration), enable_fails: Option, turned_on: TurnedOn, + /// What the device reads while it is on and no display has attached + /// yet; `None` when it can't be read. Attached, it reads ready. + device_until_attached: Option, calls: RefCell>, } @@ -2049,6 +2109,7 @@ mod tests { on_sleep, enable_fails: None, turned_on: TurnedOn::Beside, + device_until_attached: None, calls: RefCell::new(Vec::new()), } } @@ -2059,6 +2120,16 @@ mod tests { } impl Driver for FakeDriver<'_> { + fn device(&self) -> Option { + if self.on_since.get().is_none() { + return Some(DeviceState::Problem(22)); + } + if self.outputs().iter().any(|output| !output.is_primary) { + return Some(DeviceState::Ready); + } + self.device_until_attached + } + fn outputs(&self) -> Vec { let mut outputs = vec![output("primary", true)]; let attached = self @@ -2133,6 +2204,71 @@ mod tests { } } + /// Field data (Windows 10 and 11): the Enable task succeeds, the device + /// runs, and Windows never adds its monitor. The attempt waited out the + /// whole attach timeout every time. It now ends once the device has run + /// with no monitor for NO_MONITOR_WAIT, as an attach timeout, with one + /// Enable and one Disable, and is remembered as before. + #[test] + fn a_device_that_runs_with_no_monitor_ends_the_wait_early() { + let shared = second_screen_selected(); + connect_viewer(&shared); + let start = Instant::now(); + assert_eq!(next_step(&shared, start), Step::Attach); + let on_sleep = |_: Duration| {}; + let mut driver = FakeDriver::new(secs(2.0), None, &on_sleep); + driver.device_until_attached = Some(DeviceState::NoMonitor); + attach(&shared, &driver, ATTACH_TIMEOUT); + + let toggles = driver.toggles.borrow().clone(); + assert_eq!(toggles.len(), 2, "{toggles:?}"); + let waited = toggles[1].0 - toggles[0].0; + assert!(!toggles[1].1, "{toggles:?}"); + assert!( + waited >= NO_MONITOR_WAIT && waited < NO_MONITOR_WAIT + DEVICE_READ_EVERY + secs(0.5), + "waited {waited:?}" + ); + assert!(driver + .calls() + .contains(&Call::Report(Failure::AttachTimeout))); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + assert_eq!(remembered_failure(&shared), Some("attach_timeout")); + } + + /// The early end needs a device that runs with no monitor all along. A + /// monitor that is there (Windows kept the desktop off it, and the user + /// may still choose Extend), a device that can't be read, or one that + /// attaches before NO_MONITOR_WAIT keeps the whole wait, or attaches. + #[test] + fn only_a_device_with_no_monitor_all_along_ends_the_wait_early() { + for device in [ + Some(DeviceState::Ready), + Some(DeviceState::NotStarted), + None, + ] { + let shared = second_screen_selected(); + connect_viewer(&shared); + assert_eq!(next_step(&shared, Instant::now()), Step::Attach); + let on_sleep = |_: Duration| {}; + let mut driver = FakeDriver::new(secs(2.0), None, &on_sleep); + driver.device_until_attached = device; + attach(&shared, &driver, ATTACH_TIMEOUT); + let toggles = driver.toggles.borrow().clone(); + let waited = toggles[1].0 - toggles[0].0; + assert!(waited >= ATTACH_TIMEOUT, "{device:?}: waited {waited:?}"); + } + // No monitor for a while, then the display attaches in time. + let shared = second_screen_selected(); + connect_viewer(&shared); + assert_eq!(next_step(&shared, Instant::now()), Step::Attach); + let on_sleep = |_: Duration| {}; + let mut driver = FakeDriver::new(secs(2.0), Some(NO_MONITOR_WAIT - secs(1.0)), &on_sleep); + driver.device_until_attached = Some(DeviceState::NoMonitor); + attach(&shared, &driver, ATTACH_TIMEOUT); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Active); + assert_eq!(remembered_failure(&shared), None); + } + /// A 0.3.2 log and the 0.3.1 field data: the display never attaches, /// the iPad sends its goodbye at 10 s, and USB dials again 1 s later. /// Here no transport step restarts the pipeline while the display From 3254f68179fa32d754c4b2dcedbcc1f41f9e41f9 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:04:39 -0500 Subject: [PATCH 172/226] Keep the iPad's session alive while the stream restarts A stream restart that waited longer than the iPad's three-second liveness dropped it: while the PC waited to restart, the standby responder only answered HELLO2s, and nothing sent the kept session heartbeats. - While the PC is restarting, the standby responder now sends the kept session a heartbeat each interval, with the stream config it last had, as a running stream does. Only an iPad that has answered its session gets them, and only while restarting: not after Stop or the storm brake. - It hears the session's reports, pings, keyframe requests and goodbye through the session, so an iPad that goes silent still expires. - After 20 seconds kept alive without a frame, the session ends with the goodbye that tells the iPad to connect again; its HELLO2 then hears that the PC is restarting. A frame sent resets the limit, and the storm brake still ends it sooner. - The picture comes back on the same session, starting with a keyframe. --- host/src/transport/session.rs | 242 +++++++++++++++++++++++++++++++++- host/src/transport/standby.rs | 96 ++++++++++++-- host/tests/hello_standby.rs | 173 ++++++++++++++++++++++++ 3 files changed, 499 insertions(+), 12 deletions(-) diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index c2399eac..7860b096 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -129,6 +129,27 @@ pub const UNANSWERED_MEDIA_BUDGET: usize = 512 * 1024; const UNANSWERED_KEPT: usize = 64; const UNANSWERED_FOR: Duration = Duration::from_secs(60); +/// How long a session is kept alive through restarts with no picture. Past +/// this, its iPad is told to connect again: it shows why while the PC +/// keeps restarting, instead of a frozen picture. Restarts that fail +/// quickly end sooner, at the storm brake. +pub const RESTART_HOLD_LIMIT: Duration = Duration::from_secs(20); + +/// Lets integration tests reach [`RESTART_HOLD_LIMIT`] in milliseconds. +#[cfg(feature = "test-hooks")] +pub static TEST_RESTART_HOLD_MS: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(RESTART_HOLD_LIMIT.as_millis() as u64); + +#[cfg(feature = "test-hooks")] +fn restart_hold_limit() -> Duration { + Duration::from_millis(TEST_RESTART_HOLD_MS.load(std::sync::atomic::Ordering::SeqCst)) +} + +#[cfg(not(feature = "test-hooks"))] +fn restart_hold_limit() -> Duration { + RESTART_HOLD_LIMIT +} + /// What the transport must do after feeding the session an event. #[derive(Debug, Default)] pub struct Actions { @@ -305,6 +326,12 @@ struct ActiveSession { unanswered_sent: usize, /// A frame was held back for the budget: the answer asks for a keyframe. withheld: bool, + /// The stream config its last heartbeat carried, which heartbeats sent + /// while the PC restarts carry again. + last_stream_config: Option, + /// Since when it has been kept alive through restarts without a frame + /// ([`RESTART_HOLD_LIMIT`]). + restart_held_since: Option, } /// What a client sent last, for the liveness log lines. @@ -560,6 +587,7 @@ impl Session { else { return true; }; + session.restart_held_since = None; if session.answered { session.frames_sent = session.frames_sent.saturating_add(1); return true; @@ -1164,6 +1192,8 @@ impl Session { answered, unanswered_sent, withheld: false, + last_stream_config: None, + restart_held_since: None, }); // Only a new pairing carries the token. iPads keep the token they @@ -1445,10 +1475,20 @@ impl Session { let session_id = session.session_id; let frames_sent = (session.info.feature_caps & FEATURE_MEDIA_SIZE != 0) .then_some(session.frames_sent); + let stream_config = match self.not_streaming { + // No stream: the config of the one the iPad last had. + Some(_) => session + .last_stream_config + .unwrap_or_else(|| self.negotiated_config(config)), + None => self.negotiated_config(config), + }; + if let Some(session) = self.active.as_mut() { + session.last_stream_config = Some(stream_config); + } let msg_seq = self.next_msg_seq(); let heartbeat = ControlMessage::Heartbeat(eternal_wire::v2::control::Heartbeat { host_time_us: crate::clock::host_now_us(), - stream_config: self.negotiated_config(config), + stream_config, notice: config.notice(), frames_sent, }); @@ -1460,6 +1500,44 @@ impl Session { actions } + /// One heartbeat interval while the PC restarts its stream and the + /// standby responder holds the port. A session kept across the restart + /// stays alive: its iPad gets heartbeats as from a running stream, so it + /// holds its last picture instead of losing the signal, and its silence + /// still expires it. Only an iPad that has answered gets them. After + /// [`RESTART_HOLD_LIMIT`] without a frame, the session ends with the + /// goodbye that tells its iPad to connect again. Does nothing unless the + /// PC is restarting. + pub fn restart_tick(&mut self, config: &impl ConfigSource, now: Instant) -> Actions { + if self.not_streaming != Some(NotStreaming::Restarting) { + return Actions::default(); + } + let Some(session) = self.active.as_mut() else { + return self.tick(config, false, now); + }; + let since = *session.restart_held_since.get_or_insert(now); + if now.saturating_duration_since(since) >= restart_hold_limit() { + info!( + session_id = session.session_id, + held_ms = now.saturating_duration_since(since).as_millis() as u64, + "The stream is taking too long to restart: telling the iPad to connect again" + ); + let ended = self.end_saying( + ByeReason::Error, + EndReason::Failed(StageFailure::Other), + now, + ); + return Actions { + goodbye: self.take_goodbye(), + ended, + client_lost: true, + ..Actions::default() + }; + } + let answered = session.answered; + self.tick(config, answered, now) + } + /// Ends the active session for liveness, with the goodbye that tells its /// client to connect again: `error`, which every 0.3.x iPad answers with /// a reconnect. Late packets of the session get it again @@ -5486,4 +5564,166 @@ mod tests { (HelloStatus::Error, "TEST-HOST") ); } + + /// The standby responder's config: no stream, epoch 0. + struct RestartConfig; + impl ConfigSource for RestartConfig { + fn stream_config(&self) -> StreamConfig { + StreamConfig::default() + } + fn host_name(&self) -> String { + "TEST-HOST".to_string() + } + } + + fn heartbeats(actions: &Actions, peer: PeerId, session_id: u32) -> Vec { + actions + .replies + .iter() + .filter(|(to, _)| *to == peer) + .filter_map( + |(_, bytes)| match eternal_wire::v2::control::parse_control(bytes) { + Ok((header, ControlMessage::Heartbeat(hb))) + if header.session_id == session_id => + { + Some(hb.stream_config) + } + _ => None, + }, + ) + .collect() + } + + /// A streaming session whose iPad has answered, then the PC restarts. + fn restarting_session(peer: PeerId, now: Instant) -> (Session, u32) { + let mut session = Session::new(1234); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + session.handle_control_authed(peer, report(10), &TestConfig, now); + let id = session.session_id().unwrap(); + assert_eq!( + heartbeats(&session.tick(&TestConfig, true, now), peer, id).len(), + 1 + ); + session.set_not_streaming(Some(NotStreaming::Restarting)); + (session, id) + } + + /// While the PC restarts, the kept session gets a heartbeat each + /// interval, with the stream config it last had, and stays alive for as + /// long as its iPad keeps reporting: longer than the iPad's liveness. + #[test] + fn a_restart_keeps_the_ipads_session_alive_with_heartbeats() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let (mut session, id) = restarting_session(peer, now); + for second in 1..=12 { + let at = now + Duration::from_secs(second); + session.handle_control_authed(peer, report(10), &RestartConfig, at); + let actions = session.restart_tick(&RestartConfig, at); + assert!( + !actions.client_lost && actions.goodbye.is_none(), + "second {second}" + ); + let configs = heartbeats(&actions, peer, id); + assert_eq!(configs.len(), 1, "second {second}"); + assert_eq!( + configs[0].stream_epoch, 7, + "the last stream's config, not epoch 0" + ); + } + assert_eq!(session.session_id(), Some(id)); + } + + /// Only an iPad that answered its session gets them, and only while the + /// PC restarts: not after Stop or the storm brake, nor while streaming. + #[test] + fn restart_heartbeats_go_only_to_an_answered_session_while_restarting() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + let id = session.session_id().unwrap(); + session.set_not_streaming(Some(NotStreaming::Restarting)); + let actions = session.restart_tick(&RestartConfig, now + Duration::from_millis(500)); + assert!(heartbeats(&actions, peer, id).is_empty(), "never answered"); + + let (mut session, id) = restarting_session(peer, now); + for state in [ + None, + Some(NotStreaming::Stopped), + Some(NotStreaming::Failed(FailedReason::Encoder)), + ] { + session.set_not_streaming(state); + let actions = session.restart_tick(&RestartConfig, now + Duration::from_secs(1)); + assert!(actions.replies.is_empty(), "{state:?}"); + } + assert!(session.session_id() == Some(id)); + let mut empty = Session::new(1); + empty.set_not_streaming(Some(NotStreaming::Restarting)); + assert!(empty.restart_tick(&RestartConfig, now).replies.is_empty()); + } + + /// An iPad that goes silent during the restart is still expired, with + /// the goodbye that tells it to connect again. + #[test] + fn a_silent_ipad_still_expires_during_a_restart() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let (mut session, _) = restarting_session(peer, now); + let mut lost = false; + for second in 1..=12 { + let actions = session.restart_tick(&RestartConfig, now + Duration::from_secs(second)); + lost |= actions.client_lost; + } + assert!(lost && !session.is_active()); + } + + /// Past the hold limit without a frame, the session ends with the + /// goodbye that tells its iPad to connect again, and its HELLO2 then + /// hears that the PC is restarting. A frame resets the limit. + #[test] + fn a_restart_that_outlasts_the_hold_limit_ends_the_session() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let (mut session, id) = restarting_session(peer, now); + session.restart_tick(&RestartConfig, now); + // A frame went out in between (a generation streamed briefly). + let half = now + restart_hold_limit() / 2; + session.set_not_streaming(None); + assert!(session.may_send_media(id, 1000, half)); + session.set_not_streaming(Some(NotStreaming::Restarting)); + session.handle_control_authed(peer, report(10), &RestartConfig, half); + session.restart_tick(&RestartConfig, half); + let before = half + restart_hold_limit() - Duration::from_millis(1); + session.handle_control_authed(peer, report(10), &RestartConfig, before); + assert!( + !session.restart_tick(&RestartConfig, before).client_lost, + "the frame reset it" + ); + + let past = half + restart_hold_limit(); + let actions = session.restart_tick(&RestartConfig, past); + assert!(actions.client_lost && !session.is_active()); + assert_eq!( + actions.ended.unwrap().reason, + EndReason::Failed(StageFailure::Other) + ); + let (to, bytes) = actions.goodbye.expect("a goodbye"); + assert_eq!(to, peer); + assert!(matches!( + eternal_wire::v2::control::parse_control(&bytes), + Ok((header, ControlMessage::Bye(ByeReason::Error))) if header.session_id == id + )); + let mut again = match hello(2, 50000) { + ControlMessage::Hello2(hello) => hello, + _ => unreachable!(), + }; + again.feature_caps |= FEATURE_STANDBY_STATUS; + let answer = + session.handle_control(peer, 0, ControlMessage::Hello2(again), &RestartConfig, past); + assert_eq!( + parse_ack(&answer.replies[0].1).status, + HelloStatus::Restarting + ); + } } diff --git a/host/src/transport/standby.rs b/host/src/transport/standby.rs index ee49214f..c4247f32 100644 --- a/host/src/transport/standby.rs +++ b/host/src/transport/standby.rs @@ -12,6 +12,14 @@ //! starts a session or a stream, and lets go of the port before the //! supervisor checks it for the next generation. //! +//! While the PC restarts its stream, a session kept across the restart +//! stays alive here: its iPad gets heartbeats as from a running stream, and +//! its reports and pings are heard as there (see +//! [`Session::restart_tick`]). It used to lose the signal after three +//! seconds, while a restart can wait up to half a minute. +//! +//! [`Session::restart_tick`]: super::session::Session::restart_tick +//! //! [`Session::set_not_streaming`]: super::session::Session::set_not_streaming use std::net::SocketAddr; @@ -108,6 +116,8 @@ async fn serve( stopped: &mut oneshot::Receiver<()>, ) -> bool { let mut goodbyes = tokio::time::interval(GOODBYE_CHECK); + let mut heartbeat = tokio::time::interval(super::session::HEARTBEAT_INTERVAL); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); let mut network_check = tokio::time::interval(NETWORK_CHECK); network_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { @@ -122,6 +132,11 @@ async fn serve( return true; } } + _ = heartbeat.tick() => { + let config = config(shared, bound); + let actions = shared.session.lock().restart_tick(&config, super::now()); + carry_out(links, shared, actions).await; + } // A Stop during a restart pause ends a session no transport // can tell: say it here. _ = goodbyes.tick() => { @@ -185,8 +200,40 @@ async fn open(listen_port: u16, shared: &SharedControl) -> (TransportLinks, Sock } } -/// Answers a HELLO2 or an IDENTIFY. Everything else waits for the next -/// generation: a session kept across a restart picks up there. +fn config(shared: &SharedControl, bound: SocketAddr) -> super::SharedConfigSource<'_> { + super::SharedConfigSource { + shared, + stream_epoch: 0, + pen_available: false, + listen: bound, + } +} + +/// Sends what the session decided while no stream runs: heartbeats and +/// pongs to the kept session, its goodbye, and the summary of a session +/// that ended. +async fn carry_out( + links: &TransportLinks, + shared: &SharedControl, + actions: super::session::Actions, +) { + if let Some(end) = &actions.ended { + crate::diagnostics::log_session_end(end, shared); + } + for (destination, datagram) in &actions.replies { + if let Err(error) = links.send_to(datagram, *destination).await { + debug!(peer = %destination, %error, "Could not reach the iPad while the stream restarts"); + } + } + if let Some(goodbye) = actions.goodbye { + super::say_goodbye(links, goodbye).await; + } +} + +/// Answers a HELLO2 or an IDENTIFY, and hears the kept session's reports, +/// pings, keyframe requests and goodbye, so it stays alive while the PC +/// restarts (see [`super::session::Session::restart_tick`]). Input and +/// repair requests wait for the next generation. async fn answer( links: &TransportLinks, shared: &SharedControl, @@ -202,17 +249,44 @@ async fn answer( } _ => return, } - let Ok((header, message @ (ControlMessage::Hello2(_) | ControlMessage::Identify(_)))) = - parse_control(datagram) - else { + let Ok((header, message)) = parse_control(datagram) else { return; }; - let config = super::SharedConfigSource { - shared, - stream_epoch: 0, - pen_available: false, - listen: bound, - }; + let config = config(shared, bound); + if matches!( + message, + ControlMessage::ReceiverReport(_) + | ControlMessage::Ping(_) + | ControlMessage::KeyframeRequest(_) + | ControlMessage::Bye(_) + ) { + let actions = { + let mut session = shared.session.lock(); + if session.not_streaming() != Some(super::session::NotStreaming::Restarting) { + return; + } + // Only the kept session's own messages get anywhere: the + // session checks their id and source, as the stream does. + session.handle_datagram( + peer, + &header, + message, + datagram.len(), + &config, + super::now(), + ) + }; + // A keyframe request asks for an IDR, which the next generation + // starts with anyway. + carry_out(links, shared, actions).await; + return; + } + if !matches!( + message, + ControlMessage::Hello2(_) | ControlMessage::Identify(_) + ) { + return; + } let actions = { let mut session = shared.session.lock(); // The supervisor clears it only after stopping this responder; diff --git a/host/tests/hello_standby.rs b/host/tests/hello_standby.rs index e76f2458..43a24709 100644 --- a/host/tests/hello_standby.rs +++ b/host/tests/hello_standby.rs @@ -113,6 +113,10 @@ impl Host { impl Drop for Host { fn drop(&mut self) { eternal_host::supervisor::TEST_BACKOFF_BASE_MS.store(500, Ordering::SeqCst); + eternal_host::transport::session::TEST_RESTART_HOLD_MS.store( + eternal_host::transport::session::RESTART_HOLD_LIMIT.as_millis() as u64, + Ordering::SeqCst, + ); #[cfg(debug_assertions)] eternal_host::network::set_test_adapters(None); self.shared.stop(); @@ -935,3 +939,172 @@ mod identify { ); } } + +/// What a streaming iPad heard over a stretch of time. +#[derive(Default, Debug)] +struct Heard { + /// When each heartbeat for the session came. + heartbeats: Vec, + /// Each media datagram for the session: when, its epoch, and whether + /// it was part of a keyframe. + media: Vec<(Instant, u32, bool)>, + /// Goodbyes for the session. + goodbyes: Vec, +} + +impl FakeIpad { + /// Streams as an iPad does for `within`, or until `done` says so: + /// reports every 250 ms for `session_id`, and notes what comes back. + fn stream(&self, session_id: u32, within: Duration, done: impl Fn(&Heard) -> bool) -> Heard { + use eternal_wire::v2::control::ReceiverReport; + use eternal_wire::v2::media::MediaHeader; + let mut heard = Heard::default(); + let deadline = Instant::now() + within; + let mut next_report = Instant::now(); + let mut seq = 100; + let mut buffer = [0; 2048]; + while Instant::now() < deadline && !done(&heard) { + if Instant::now() >= next_report { + seq += 1; + let report = ControlMessage::ReceiverReport(ReceiverReport { + frames_complete: 1, + ..ReceiverReport::default() + }); + self.send(&encode_control(session_id, seq, &report)); + next_report += Duration::from_millis(250); + } + let Ok(len) = self.socket.recv(&mut buffer) else { + continue; + }; + let bytes = &buffer[..len]; + let at = Instant::now(); + match eternal_wire::v2::classify(bytes) { + eternal_wire::v2::Classified::Control(_) => match parse_control(bytes) { + Ok((header, ControlMessage::Heartbeat(_))) + if header.session_id == session_id => + { + heard.heartbeats.push(at) + } + Ok((header, ControlMessage::Bye(reason))) + if header.session_id == session_id => + { + heard.goodbyes.push(reason) + } + _ => {} + }, + eternal_wire::v2::Classified::Media { .. } => { + if let Ok((header, _)) = MediaHeader::decode(bytes) { + if header.session_id == session_id { + heard + .media + .push((at, header.stream_epoch, header.is_keyframe)); + } + } + } + _ => {} + } + } + heard + } +} + +/// The encoder fails once, after this many frames: a capture or encoder +/// error that the supervisor restarts the stream for. +fn fail_the_encoder_once(after: u32) -> impl FnOnce(&SharedControl) { + move |shared| { + shared.pairing.lock().required = false; + std::env::set_var("ETERNAL_FAULT_ENCODER_AFTER", after.to_string()); + } +} + +/// A stream restart longer than the iPad's liveness (3 s) used to drop it: +/// nothing sent it heartbeats while the PC waited to restart, and it lost +/// the signal. The PC now keeps the session alive with heartbeats, and the +/// picture comes back on the same session with a keyframe. +#[test] +fn a_restart_longer_than_the_ipads_liveness_keeps_its_session() { + let host = Host::start_with( + GpuInfo::software_fallback(), + 4_000, + fail_the_encoder_once(240), + ); + let ipad = FakeIpad::new(&host, 1, NEW); + let ack = ipad.connect(1); + let id = ack.session_id; + // Until the stream has restarted: media from a second epoch. + let heard = ipad.stream(id, Duration::from_secs(60), |heard| { + heard + .media + .first() + .is_some_and(|(_, first, _)| heard.media.iter().any(|(_, epoch, _)| epoch != first)) + }); + assert!( + heard.goodbyes.is_empty(), + "the iPad was told to go: {:?}", + heard.goodbyes + ); + let first_epoch = heard.media.first().expect("a picture before the restart").1; + let last_before = heard + .media + .iter() + .filter(|(_, epoch, _)| *epoch == first_epoch) + .map(|(at, _, _)| *at) + .next_back() + .unwrap(); + let resumed = heard + .media + .iter() + .find(|(_, epoch, _)| *epoch != first_epoch) + .expect("the picture came back"); + let gap = resumed.0 - last_before; + assert!( + gap > Duration::from_secs(3), + "the restart took only {gap:?}" + ); + assert!(resumed.2, "the picture comes back with a keyframe"); + // Heartbeats kept coming through the gap, as the iPad's liveness needs. + let mut beats: Vec<_> = heard + .heartbeats + .iter() + .copied() + .filter(|at| *at > last_before && *at < resumed.0) + .collect(); + beats.insert(0, last_before); + beats.push(resumed.0); + let longest = beats + .windows(2) + .map(|pair| pair[1] - pair[0]) + .max() + .unwrap(); + assert!( + longest < Duration::from_millis(2_500), + "{longest:?} without a heartbeat in a {gap:?} restart" + ); + assert_eq!(host.shared.session.lock().session_id(), Some(id)); +} + +/// A restart that outlasts the hold limit ends the session with the goodbye +/// that tells the iPad to connect again; its HELLO2 then hears that the PC +/// is restarting. +#[test] +fn a_restart_past_the_hold_limit_tells_the_ipad_to_connect_again() { + eternal_host::transport::session::TEST_RESTART_HOLD_MS.store(2_000, Ordering::SeqCst); + let host = Host::start_with( + GpuInfo::software_fallback(), + 10_000, + fail_the_encoder_once(240), + ); + let ipad = FakeIpad::new(&host, 1, NEW); + let id = ipad.connect(1).session_id; + let heard = ipad.stream(id, Duration::from_secs(60), |heard| { + !heard.goodbyes.is_empty() + }); + assert_eq!( + heard.goodbyes.first(), + Some(&eternal_wire::v2::control::ByeReason::Error) + ); + assert!(heard.heartbeats.len() >= 2, "kept alive until the limit"); + assert_eq!(host.shared.session.lock().session_id(), None); + let ack = ipad.ask(2).expect("a restarting PC answers"); + assert_eq!(ack.status, HelloStatus::Restarting); +} From c70f270d62d7072fa3785d299b79cc0a844760ed Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:19:50 -0500 Subject: [PATCH 173/226] Say goodbye when the PC sleeps, shuts down or signs out The host said goodbye only when its window closed. A PC that slept, shut down, restarted or signed out mid-stream went silent: the iPad showed a lost signal, then dialled a PC that no longer answered, and no session summary was written. - A hidden window watches Windows' suspend (WM_POWERBROADCAST, registered for modern standby too), end-of-session (WM_ENDSESSION) and sign-out (WTS session change) messages. WM_QUERYENDSESSION only answers yes. - Session::power_down ends the session at once. Its summary is written first: through the log for sleep, and straight to the session log and upload outbox for shutdown and sign-out, which may end the process any moment. The summary says end=shutdown. - An iPad that sets the new FEATURE_POWER_BYE gets a goodbye that says the PC went to sleep (HostSleeping) or is powering off (HostPoweringOff). The transport is woken to send it at once, and the message waits for it at most 0.5 s (sleep) or 1.5 s (end of session); nothing asks Windows to wait. Older iPads drop a goodbye they can't read, so they get none, as before. - While the PC goes down, HELLO2s go unanswered, until it wakes, the shutdown is called off, or 30 s pass. The stream keeps running, so the PC streams again after waking, and the first pass after waking ends nothing a second time. - The iPad advertises the bit and shows "Your PC went to sleep" or "Your PC is shutting down or signing out", with Try again and no automatic reconnecting to a PC that won't answer. --- SETUP.md | 2 + host/Cargo.toml | 1 + host/src/control.rs | 4 + host/src/diagnostics.rs | 14 + host/src/lib.rs | 1 + host/src/main.rs | 3 + host/src/power.rs | 370 ++++++++++++++++++ host/src/startup/mod.rs | 6 +- host/src/transport/mod.rs | 9 + host/src/transport/session.rs | 248 +++++++++++- host/tests/power_goodbye.rs | 351 +++++++++++++++++ .../App/ConnectionManager.swift | 14 +- .../App/ConnectionProblem.swift | 14 +- ios/EternalMonitor/Network/WireProtocol.swift | 8 + .../ConnectionProblemTests.swift | 15 +- ios/EternalMonitorTests/GoodbyeTests.swift | 21 + .../WireProtocolGoldenTests.swift | 19 +- proto/src/v2/control.rs | 13 + proto/testdata/v2_vectors.txt | 2 + proto/tests/golden.rs | 12 + 20 files changed, 1110 insertions(+), 17 deletions(-) create mode 100644 host/src/power.rs create mode 100644 host/tests/power_goodbye.rs diff --git a/SETUP.md b/SETUP.md index e9db7f0a..2099dbdf 100644 --- a/SETUP.md +++ b/SETUP.md @@ -277,6 +277,8 @@ try fewer words. If your problem isn't here, see [Known issues](https://eternalm | **USB connections were turned off** | **Allow USB connections** is off in the iPad app's settings. | Turn on **Allow USB connections** to use the cable again. | | **Your PC ended the session** | Someone clicked **Stop** on the PC, changed a setting that restarts the stream, or closed EternalMonitor. | Tap **Try again** when the PC says **Ready for your iPad**. | | **The connection to your PC ended** | The connection dropped, for example in a Wi-Fi stall, the PC ran into a problem, or EternalMonitor restarted itself on the PC. | Tap **Try again**. With **Reconnect automatically** on, the iPad tries again by itself first. | +| **Your PC went to sleep** | Windows put the PC to sleep while it streamed. | Wake the PC, then tap **Try again**. | +| **Your PC is shutting down or signing out** | Windows shut down, restarted or signed out while the PC streamed. | Tap **Try again** once the PC is back on and EternalMonitor is open on it. | | **Your PC is already streaming to another device** | Only one iPad can connect at a time. | Disconnect the other iPad first, then tap **Try again**. | | **Your PC is updating EternalMonitor** | The PC is installing a new version. | Wait. With **Reconnect automatically** on, the iPad connects again by itself. Otherwise tap **Try again** once EternalMonitor has opened on the PC. | | **Your PC hasn't come back from its update yet** | The update is taking longer than expected. | Check that EternalMonitor opened again on the PC. Windows may be waiting for you to click **Yes** there. Then tap **Try again**. | diff --git a/host/Cargo.toml b/host/Cargo.toml index 5c26f83b..9b45e2c0 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -104,6 +104,7 @@ features = [ "Win32_Graphics_Gdi", "Win32_Globalization", "Win32_System_Registry", + "Win32_System_RemoteDesktop", "Win32_System_StationsAndDesktops", "Win32_System_SystemInformation", "Win32_System_IO", diff --git a/host/src/control.rs b/host/src/control.rs index 3c4b13f2..bb1237da 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -84,6 +84,9 @@ pub struct SharedControl { /// The transport loop's heartbeat and the step it was taking, which the /// supervisor watches (`transport::beat`). pub transport_beat: Arc, + /// Wakes the transport to send a goodbye the PC going down left, and + /// counts the goodbyes it sent (`crate::power`). + pub power: Arc, /// Desktop-space rectangle of the output currently being captured, set by /// the capture stage at (re)init. The input relay maps the client's /// normalized touch coordinates onto this rect. `None` until the first @@ -211,6 +214,7 @@ impl SharedControl { hb_capture_frame_ms: Arc::new(AtomicU64::new(0)), hb_encode_frame_ms: Arc::new(AtomicU64::new(0)), transport_beat: Default::default(), + power: Default::default(), capture_geometry: Arc::new(Mutex::new(None)), input: Arc::default(), hevc_enabled: Arc::new(AtomicBool::new(false)), diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 85dd7547..63651155 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -375,6 +375,20 @@ pub fn stop_session(shared: &SharedControl, why: StreamStop) { } } +/// [`log_session_end`] for a PC about to go down: the line goes straight to +/// the session log and the upload outbox on disk, so it survives the +/// process ending a moment later, and the next launch uploads it. +pub fn log_session_end_now(end: &SessionEnd, shared: &SharedControl) { + let second_screen = *shared.capture_target.lock() == CaptureTarget::VirtualExtended; + let fallback = end.session_id.and_then(take_fallback); + let line = + crate::startup::info_line(&session_summary(end, second_screen, fallback, host_facts())); + let written = crate::logging::append_now(&line).and(crate::telemetry::queue_now(&line, "info")); + if let Err(error) = written { + tracing::warn!(%error, "Could not save the session summary before the PC went down"); + } +} + /// The `session_summary v=1` line (b-session-summary contract): enumerated /// tokens, bounded numbers and the facts above, nothing else. pub fn session_summary( diff --git a/host/src/lib.rs b/host/src/lib.rs index 1e91fcbf..4b7be678 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -23,6 +23,7 @@ pub mod logging; pub mod network; pub mod pairing; pub mod pipeline; +pub mod power; pub mod relaunch; pub mod remote_config; pub mod settings; diff --git a/host/src/main.rs b/host/src/main.rs index fb76194a..c3735b1d 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -241,6 +241,9 @@ fn main() -> Result<(), Box> { shared: shared.clone(), supervisor_tx: supervisor_tx.clone(), }; + // Sleep, shutdown and sign-out end the iPad's session with a goodbye. + #[cfg(windows)] + eternal_host::power::watch(shared.clone()); /// Set by the termination-signal handler; the headless loop polls it. static SHUTDOWN_REQUESTED: std::sync::atomic::AtomicBool = diff --git a/host/src/power.rs b/host/src/power.rs new file mode 100644 index 00000000..cdea89cf --- /dev/null +++ b/host/src/power.rs @@ -0,0 +1,370 @@ +//! Windows putting the PC to sleep, shutting it down or signing the user +//! out while an iPad streams. +//! +//! The host said goodbye only when its window closed. A PC that slept or +//! shut down mid-stream went silent instead: the iPad showed a lost signal, +//! then tried an address that no longer answered, and no session summary +//! was written, so these drops never reached the diagnostics either. Now the +//! session ends when Windows says the PC is going down: its summary is +//! written first, and an iPad that understands it hears why. +//! +//! Windows waits for none of this for long: about two seconds after a +//! suspend message, and a few seconds after the end-of-session message before +//! it lists the app as holding up shutdown. Each step here is bounded well +//! inside that, and nothing ever asks Windows to wait. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use tracing::info; + +use crate::control::SharedControl; +use crate::transport::session::PowerEvent; + +/// Longest a suspend message waits for the goodbye to leave. +pub const SLEEP_WAIT: Duration = Duration::from_millis(500); +/// Longest the end-of-session message waits for it. +pub const END_SESSION_WAIT: Duration = Duration::from_millis(1500); + +/// Wakes the transport to send a goodbye the PC going down left, and counts +/// the goodbyes it sent, so the message can wait for its own. +#[derive(Debug, Default)] +pub struct Signal { + wake: tokio::sync::Notify, + said: AtomicU64, +} + +impl Signal { + /// The transport sent a goodbye. + pub(crate) fn said(&self) { + self.said.fetch_add(1, Ordering::SeqCst); + } + + /// Resolves when a goodbye waits for the transport; a wake that came + /// while the transport was busy is kept for its next look. + pub(crate) async fn woken(&self) { + self.wake.notified().await; + } + + fn count(&self) -> u64 { + self.said.load(Ordering::SeqCst) + } +} + +/// What Windows told the app, from its window messages. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Message { + /// WM_POWERBROADCAST PBT_APMSUSPEND. + Suspend, + /// WM_POWERBROADCAST PBT_APMRESUMEAUTOMATIC or PBT_APMRESUMESUSPEND. + Resume, + /// WM_QUERYENDSESSION: Windows asks whether the session may end. Always + /// yes, and nothing happens yet: another app can still call it off. + QueryEndSession, + /// WM_ENDSESSION: `ending` false when the shutdown was called off. + EndSession { ending: bool, logoff: bool }, + /// WM_WTSSESSION_CHANGE WTS_SESSION_LOGOFF. + SessionLogoff, +} + +/// Acts on `message` and returns what the window answers Windows: 1 (TRUE) +/// for WM_POWERBROADCAST and WM_QUERYENDSESSION, 0 otherwise. +pub fn handle(shared: &SharedControl, message: Message) -> isize { + match message { + Message::Suspend => { + going_down(shared, PowerEvent::Sleep, SLEEP_WAIT); + 1 + } + Message::Resume => { + back_up(shared, "The PC woke"); + 1 + } + Message::QueryEndSession => 1, + Message::EndSession { + ending: true, + logoff, + } => { + let event = if logoff { + PowerEvent::Logoff + } else { + PowerEvent::Shutdown + }; + going_down(shared, event, END_SESSION_WAIT); + 0 + } + Message::EndSession { ending: false, .. } => { + back_up(shared, "The shutdown was called off"); + 0 + } + Message::SessionLogoff => { + going_down(shared, PowerEvent::Logoff, END_SESSION_WAIT); + 0 + } + } +} + +/// Ends the session for `event`, writes its summary, and waits at most +/// `wait` for the transport to send the goodbye, if there is one. +fn going_down(shared: &SharedControl, event: PowerEvent, wait: Duration) { + let started = Instant::now(); + let before = shared.power.count(); + let (end, told) = { + let mut session = shared.session.lock(); + let end = session.power_down(started, event); + (end, session.power_goodbye_pending()) + }; + // Only a running transport can send it. + let sending = told && shared.transport_beat.read().is_some(); + if sending { + shared.power.wake.notify_one(); + } + let Some(end) = end else { + info!( + power = event.name(), + "Windows is taking the PC down; no iPad was streaming" + ); + return; + }; + info!( + power = event.name(), + told, "Windows is taking the PC down — ended the iPad's session" + ); + // A PC that sleeps comes back with this process and its log writers; + // one that shuts down or signs out may end it any moment now. + match event { + PowerEvent::Sleep => crate::diagnostics::log_session_end(&end, shared), + PowerEvent::Shutdown | PowerEvent::Logoff => { + crate::diagnostics::log_session_end_now(&end, shared) + } + } + if !sending { + return; + } + while shared.power.count() == before && started.elapsed() < wait { + std::thread::sleep(Duration::from_millis(5)); + } + let sent = shared.power.count() != before; + info!( + power = event.name(), + sent, + waited_ms = started.elapsed().as_millis() as u64, + "Goodbye before the PC goes down" + ); +} + +fn back_up(shared: &SharedControl, why: &'static str) { + shared.session.lock().power_up(); + info!("{why} — iPads may connect again"); +} + +/// Starts listening for Windows' sleep, shutdown and sign-out messages, on a +/// hidden window of its own. Later calls do nothing. +#[cfg(windows)] +pub fn watch(shared: SharedControl) { + windows_impl::watch(shared); +} + +/// The hidden window's handle, once it exists, for tests that send it +/// messages as Windows would. +#[cfg(windows)] +pub fn window() -> Option { + windows_impl::window() +} + +#[cfg(windows)] +mod windows_impl { + use std::sync::atomic::{AtomicIsize, Ordering}; + use std::sync::OnceLock; + + use windows::core::w; + use windows::Win32::Foundation::{HANDLE, HINSTANCE, HWND, LPARAM, LRESULT, WPARAM}; + use windows::Win32::System::LibraryLoader::GetModuleHandleW; + use windows::Win32::System::Power::RegisterSuspendResumeNotification; + use windows::Win32::System::RemoteDesktop::{ + WTSRegisterSessionNotification, NOTIFY_FOR_THIS_SESSION, + }; + use windows::Win32::UI::WindowsAndMessaging::{ + CreateWindowExW, DefWindowProcW, DispatchMessageW, GetMessageW, RegisterClassW, + TranslateMessage, DEVICE_NOTIFY_WINDOW_HANDLE, ENDSESSION_LOGOFF, MSG, + PBT_APMRESUMEAUTOMATIC, PBT_APMRESUMESUSPEND, PBT_APMSUSPEND, WINDOW_EX_STYLE, + WINDOW_STYLE, WM_ENDSESSION, WM_POWERBROADCAST, WM_QUERYENDSESSION, WM_WTSSESSION_CHANGE, + WNDCLASSW, WTS_SESSION_LOGOFF, + }; + + use super::Message; + use crate::control::SharedControl; + + static SHARED: OnceLock = OnceLock::new(); + static WINDOW: AtomicIsize = AtomicIsize::new(0); + + pub(super) fn watch(shared: SharedControl) { + if SHARED.set(shared).is_err() { + return; + } + let started = std::thread::Builder::new() + .name("power-watch".into()) + .spawn(|| { + if let Err(error) = unsafe { run() } { + tracing::warn!( + %error, + "Could not watch for sleep and shutdown; the iPad isn't told when the PC goes down" + ); + } + }); + if let Err(error) = started { + tracing::warn!(%error, "Could not start watching for sleep and shutdown"); + } + } + + pub(super) fn window() -> Option { + Some(WINDOW.load(Ordering::SeqCst)).filter(|&hwnd| hwnd != 0) + } + + unsafe fn run() -> windows::core::Result<()> { + let instance: HINSTANCE = GetModuleHandleW(None)?.into(); + let class = w!("EternalMonitorPowerWatch"); + let window_class = WNDCLASSW { + lpfnWndProc: Some(window_proc), + hInstance: instance, + lpszClassName: class, + ..Default::default() + }; + RegisterClassW(&window_class); + // A hidden top-level window: Windows sends the end-of-session + // messages to top-level windows only, never to message-only ones. + let hwnd = CreateWindowExW( + WINDOW_EX_STYLE::default(), + class, + w!("EternalMonitor power"), + WINDOW_STYLE::default(), + 0, + 0, + 0, + 0, + None, + None, + instance, + None, + )?; + // Sleep on a modern-standby PC reaches only registered windows. + if let Err(error) = + RegisterSuspendResumeNotification(HANDLE(hwnd.0), DEVICE_NOTIFY_WINDOW_HANDLE) + { + tracing::warn!(%error, "Could not register for sleep notices"); + } + if let Err(error) = WTSRegisterSessionNotification(hwnd, NOTIFY_FOR_THIS_SESSION) { + tracing::warn!(%error, "Could not register for sign-out notices"); + } + WINDOW.store(hwnd.0 as isize, Ordering::SeqCst); + let mut message = MSG::default(); + while GetMessageW(&mut message, None, 0, 0).as_bool() { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + Ok(()) + } + + /// The message Windows sent, if it is one of those [`super::handle`] acts on. + fn message(kind: u32, wparam: usize, lparam: isize) -> Option { + Some(match kind { + WM_POWERBROADCAST => match wparam as u32 { + PBT_APMSUSPEND => Message::Suspend, + PBT_APMRESUMEAUTOMATIC | PBT_APMRESUMESUSPEND => Message::Resume, + _ => return None, + }, + WM_QUERYENDSESSION => Message::QueryEndSession, + WM_ENDSESSION => Message::EndSession { + ending: wparam != 0, + logoff: lparam as u32 & ENDSESSION_LOGOFF != 0, + }, + WM_WTSSESSION_CHANGE if wparam as u32 == WTS_SESSION_LOGOFF => Message::SessionLogoff, + _ => return None, + }) + } + + unsafe extern "system" fn window_proc( + hwnd: HWND, + kind: u32, + wparam: WPARAM, + lparam: LPARAM, + ) -> LRESULT { + match (message(kind, wparam.0, lparam.0), SHARED.get()) { + (Some(message), Some(shared)) => LRESULT(super::handle(shared, message)), + _ => DefWindowProcW(hwnd, kind, wparam, lparam), + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn window_messages_map_to_what_windows_means() { + let cases = [ + ( + WM_POWERBROADCAST, + PBT_APMSUSPEND as usize, + 0, + Some(Message::Suspend), + ), + ( + WM_POWERBROADCAST, + PBT_APMRESUMEAUTOMATIC as usize, + 0, + Some(Message::Resume), + ), + ( + WM_POWERBROADCAST, + PBT_APMRESUMESUSPEND as usize, + 0, + Some(Message::Resume), + ), + // A power setting change is someone else's business. + (WM_POWERBROADCAST, 0x8013, 0, None), + (WM_QUERYENDSESSION, 0, 0, Some(Message::QueryEndSession)), + ( + WM_ENDSESSION, + 1, + 0, + Some(Message::EndSession { + ending: true, + logoff: false, + }), + ), + ( + WM_ENDSESSION, + 1, + ENDSESSION_LOGOFF as i32 as isize, + Some(Message::EndSession { + ending: true, + logoff: true, + }), + ), + ( + WM_ENDSESSION, + 0, + 0, + Some(Message::EndSession { + ending: false, + logoff: false, + }), + ), + ( + WM_WTSSESSION_CHANGE, + WTS_SESSION_LOGOFF as usize, + 1, + Some(Message::SessionLogoff), + ), + // A lock is not the PC going down. + (WM_WTSSESSION_CHANGE, 7, 1, None), + ]; + for (kind, wparam, lparam, expected) in cases { + assert_eq!( + message(kind, wparam, lparam), + expected, + "{kind:#x} {wparam:#x}" + ); + } + } + } +} diff --git a/host/src/startup/mod.rs b/host/src/startup/mod.rs index acbf96ba..5f36924c 100644 --- a/host/src/startup/mod.rs +++ b/host/src/startup/mod.rs @@ -408,9 +408,9 @@ fn log_line(summary: &str, detail: &str) -> String { timestamped("ERROR", &format!("{summary}: {detail}")) } -/// A line the reporter adds to the session log about its own work. -#[cfg(windows)] -fn info_line(text: &str) -> String { +/// An INFO line shaped like the tracing ones, for the session log written +/// directly: the reporter's own work, or a summary before the PC goes down. +pub(crate) fn info_line(text: &str) -> String { timestamped(" INFO", text) } diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 42667586..821c0afc 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -500,6 +500,13 @@ pub async fn start_sender( let actions = shared.session.lock().tick(&config, true, now()); act(actions, &mut state, &links, &shared, &config, &supervisor).await; } + // The PC is going down and left a goodbye: send it now, not at + // the next liveness tick, while Windows still waits. + _ = shared.power.woken() => { + beat.mark(Phase::Tick); + let actions = shared.session.lock().tick(&config, false, now()); + act(actions, &mut state, &links, &shared, &config, &supervisor).await; + } _ = network_check.tick(), if listen_on != crate::network::ListenOn::All => { beat.mark(Phase::Tick); if !shared.client_connected() && crate::network::listen_ip_now(&listen_on) != bind_ip { @@ -765,6 +772,7 @@ pub async fn start_sender( let goodbye = shared.session.lock().take_goodbye(); if let Some(goodbye) = goodbye { say_goodbye(&links, goodbye).await; + shared.power.said(); } park_stream_socket(&shared, links.into_udp_socket()); info!("NAL channel closed, transport sender shutting down"); @@ -1256,6 +1264,7 @@ async fn execute_actions( if let Some(goodbye) = actions.goodbye { say_goodbye(links, goodbye).await; + shared.power.said(); } if actions.force_idr { diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index c2399eac..902d1d10 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -14,7 +14,7 @@ use std::time::{Duration, Instant}; use eternal_wire::v2::control::{ ByeReason, ControlHeader, ControlMessage, FailedReason, HelloAck, HelloStatus, InputEvent, - KeyframeRequest, Nack, ReceiverReport, StreamConfig, FEATURE_MEDIA_SIZE, + KeyframeRequest, Nack, ReceiverReport, StreamConfig, FEATURE_MEDIA_SIZE, FEATURE_POWER_BYE, FEATURE_STANDBY_STATUS, FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_MEDIA_SIZE, HOSTCAP_NACK, MIN_MEDIA_DGRAM, }; @@ -32,6 +32,13 @@ pub const LIVENESS_TIMEOUT: Duration = Duration::from_millis(3000); /// 10 s is longer than an iPad waits for a silent PC (3 s, then 5 s more) /// and than the uplink stalls of a congested Wi-Fi channel. pub const HOLD_LIMIT: Duration = Duration::from_secs(10); + +/// How long the host stays closed to new clients after Windows said it is +/// going to sleep or shutting down ([`Session::power_down`]), unless it says +/// first that it woke or that the shutdown was cancelled. A suspend that +/// fails without a resume message, or a missed one, can't leave the PC +/// unreachable for longer. +pub const POWER_DOWN_HOLD: Duration = Duration::from_secs(30); const USB_TAKEOVER_TIMEOUT: Duration = Duration::from_secs(1); /// How long the late packets of a session ended for liveness get its /// goodbye again, at most one copy per [`ENDED_REPLY_GAP`]. @@ -208,6 +215,36 @@ impl StreamStop { } } +/// What Windows is doing to the PC ([`Session::power_down`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PowerEvent { + /// Sleep or hibernate. + Sleep, + /// Shut down or restart. + Shutdown, + /// The user signs out. + Logoff, +} + +impl PowerEvent { + /// Stable log value. + pub const fn name(self) -> &'static str { + match self { + Self::Sleep => "sleep", + Self::Shutdown => "shutdown", + Self::Logoff => "logoff", + } + } + + /// The goodbye for an iPad that understands it ([`FEATURE_POWER_BYE`]). + fn bye(self) -> ByeReason { + match self { + Self::Sleep => ByeReason::HostSleeping, + Self::Shutdown | Self::Logoff => ByeReason::HostPoweringOff, + } + } +} + /// How far a session got and how it ended, for its support summary. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SessionEnd { @@ -424,6 +461,8 @@ pub enum NotStreaming { enum Closed { /// Turned away with Busy, logged with this line. Busy(&'static str), + /// Not answered at all, logged with this line. + Silent(&'static str), /// Told why, once it passes the pairing check. NotStreaming(NotStreaming), } @@ -463,6 +502,13 @@ pub struct Session { /// has resumed since: new clients are turned away, as nothing would /// serve them. stream_stopped: bool, + /// Windows said the PC is going down, and when ([`Self::power_down`]). + powering_down: Option<(PowerEvent, Instant)>, + /// A session [`Self::power_down`] ended without a goodbye: the next tick + /// reports the client lost, as a goodbye would. + lost_unsaid: bool, + /// The queued goodbye is [`Self::power_down`]'s. + power_bye: bool, /// No stream runs: HELLO2s are told why instead of starting a session. not_streaming: Option, /// The goodbye for a stream an update or a stop ended, sent by the next @@ -496,6 +542,9 @@ impl Session { last_end: None, updating: false, stream_stopped: false, + powering_down: None, + lost_unsaid: false, + power_bye: false, not_streaming: None, goodbye: None, stopped: None, @@ -627,6 +676,7 @@ impl Session { let session = self.active.as_ref()?; let (peer, session_id) = (session.peer, session.session_id); let msg_seq = self.next_msg_seq(); + self.power_bye = false; self.goodbye = Some(( peer, eternal_wire::v2::control::encode_control( @@ -641,6 +691,7 @@ impl Session { /// The goodbye left for a stopped stream ([`Self::begin_update`], /// [`Self::shut_down`]), if any. pub fn take_goodbye(&mut self) -> Option<(PeerId, Vec)> { + self.power_bye = false; self.goodbye.take() } @@ -671,13 +722,55 @@ impl Session { self.stopped.take().or(active) } + /// Windows said the PC is going to sleep, shutting down or signing out. + /// The session ends now, and its summary is returned for the caller to + /// write while the PC is still up. An iPad that understands it + /// ([`FEATURE_POWER_BYE`]) is told why by the transport's next pass; an + /// older one is told nothing, as before, and notices the silence. Until + /// [`Self::power_up`] or [`POWER_DOWN_HOLD`], HELLO2s go unanswered, as + /// they would a moment later from a PC that is down. The stream keeps + /// running, so the PC streams again when it wakes. + pub fn power_down(&mut self, now: Instant, event: PowerEvent) -> Option { + self.powering_down = Some((event, now)); + let understands = + self.active.as_ref()?.accepted_hello.feature_caps & FEATURE_POWER_BYE != 0; + if understands { + let end = self.end_saying(event.bye(), EndReason::Shutdown, now); + self.power_bye = true; + end + } else { + self.lost_unsaid = true; + self.end(EndReason::Shutdown, now) + } + } + + /// A goodbye [`Self::power_down`] left waits for the transport. + pub fn power_goodbye_pending(&self) -> bool { + self.power_bye + } + + /// The PC woke, or the shutdown was called off: clients may connect + /// again. A goodbye no transport sent before the PC went down is for a + /// session long over, and is dropped. + pub fn power_up(&mut self) { + self.powering_down = None; + if std::mem::take(&mut self.power_bye) { + self.goodbye = None; + } + } + /// Why a HELLO2 can't start a session now, if it can't: an update is /// about to close the host, no stream runs (the supervisor says why), or /// the stream stopped and its transport is on its way out (the app /// quits). The first and last are turned away with Busy before anything /// else is looked at; a retry once the stream runs again works. - fn closed_to_clients(&self) -> Option { - if self.updating { + fn closed_to_clients(&self, now: Instant) -> Option { + if self + .powering_down + .is_some_and(|(_, since)| now.saturating_duration_since(since) < POWER_DOWN_HOLD) + { + Some(Closed::Silent("Client unanswered while the PC goes down")) + } else if self.updating { Some(Closed::Busy("Client turned away while an update starts")) } else if let Some(state) = self.not_streaming { Some(Closed::NotStreaming(state)) @@ -995,7 +1088,11 @@ impl Session { } // Don't start a stream nothing will serve. - let closed = self.closed_to_clients(); + let closed = self.closed_to_clients(now); + if let Some(Closed::Silent(why)) = closed { + info!(peer = %source, "{why}"); + return actions; + } if let Some(Closed::Busy(why)) = closed { info!(peer = %source, "{why}"); return self.refuse(source, HelloStatus::Busy, &hello, hello_len, config, now); @@ -1390,6 +1487,9 @@ impl Session { actions.goodbye = Some(goodbye); actions.client_lost = self.active.is_none(); } + if std::mem::take(&mut self.lost_unsaid) { + actions.client_lost |= self.active.is_none(); + } actions.ended = self.stopped.take(); let Some(session) = self.active.as_ref() else { return actions; @@ -4147,6 +4247,146 @@ mod tests { } /// The goodbye's destination, session id and reason. + fn power_hello(nonce: u32, feature_caps: u16) -> ControlMessage { + let ControlMessage::Hello2(mut hello) = hello(nonce, 50000) else { + unreachable!() + }; + hello.device_id = 77; + hello.feature_caps = feature_caps; + ControlMessage::Hello2(hello) + } + + /// Windows taking the PC down ends the session at once, with the summary + /// a quit writes. An iPad that understands it is told why by the next + /// pass; the reason says sleep or power off. + #[test] + fn the_pc_going_down_tells_an_ipad_that_understands_it() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + for (event, bye) in [ + (PowerEvent::Sleep, ByeReason::HostSleeping), + (PowerEvent::Shutdown, ByeReason::HostPoweringOff), + (PowerEvent::Logoff, ByeReason::HostPoweringOff), + ] { + let mut session = Session::new(500); + session.handle_control(peer, 0, power_hello(1, FEATURE_POWER_BYE), &TestConfig, now); + let id = session.session_id().unwrap(); + let end = session.power_down(now, event).expect("the session ended"); + assert_eq!((end.client_nonce, end.reason), (1, EndReason::Shutdown)); + assert!(!session.is_active()); + assert!(session.power_goodbye_pending()); + let next = session.tick(&TestConfig, false, now); + assert_eq!(goodbye_of(&next), (peer, id, bye), "{event:?}"); + assert!(next.client_lost); + assert_eq!(next.ended, None, "the summary was the caller's to write"); + assert!(!session.power_goodbye_pending()); + } + } + + /// An iPad from before these reasons drops a goodbye it can't read, so + /// it gets none, as before. Its session still ends with a summary, and + /// the next pass lets the transport forget it. + #[test] + fn an_older_ipad_gets_no_goodbye_when_the_pc_goes_down() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(500); + session.handle_control( + peer, + 0, + power_hello(1, FEATURE_SUPPORTS_NACK), + &TestConfig, + now, + ); + let end = session.power_down(now, PowerEvent::Sleep).unwrap(); + assert_eq!(end.reason, EndReason::Shutdown); + assert!(!session.power_goodbye_pending()); + let next = session.tick(&TestConfig, false, now); + assert_eq!(next.goodbye, None); + assert!(next.client_lost, "the transport forgets it"); + assert!(!session.tick(&TestConfig, false, now).client_lost, "once"); + } + + /// While the PC goes down a HELLO2 gets no answer, as it wouldn't a + /// moment later: a session started now would be cut without a word. + /// Waking opens it again, and so does a cancelled shutdown, or the hold + /// running out when Windows never said it woke. + #[test] + fn a_pc_going_down_answers_no_hello_until_it_is_back() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(500); + assert_eq!( + session.power_down(now, PowerEvent::Sleep), + None, + "nobody streamed" + ); + let unanswered = + session.handle_control(peer, 0, power_hello(2, FEATURE_POWER_BYE), &TestConfig, now); + assert!(unanswered.replies.is_empty() && unanswered.new_target.is_none()); + assert!(!session.is_active()); + session.power_up(); + let admitted = + session.handle_control(peer, 0, power_hello(3, FEATURE_POWER_BYE), &TestConfig, now); + assert_eq!(admitted.new_target, Some(peer)); + + let mut session = Session::new(500); + session.power_down(now, PowerEvent::Shutdown); + let later = now + POWER_DOWN_HOLD; + let admitted = session.handle_control( + peer, + 0, + power_hello(4, FEATURE_POWER_BYE), + &TestConfig, + later, + ); + assert_eq!(admitted.new_target, Some(peer), "the hold ran out"); + } + + /// A goodbye no transport sent before the PC slept is for a session + /// that is over by the time it wakes: dropped. Any other goodbye stays. + #[test] + fn waking_drops_only_the_goodbye_the_pc_going_down_left() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(500); + session.handle_control(peer, 0, power_hello(1, FEATURE_POWER_BYE), &TestConfig, now); + session.power_down(now, PowerEvent::Sleep).unwrap(); + session.power_up(); + assert_eq!(session.take_goodbye(), None); + + let mut session = Session::new(500); + session.power_down(now, PowerEvent::Sleep); + session.power_up(); + session.handle_control(peer, 0, power_hello(2, FEATURE_POWER_BYE), &TestConfig, now); + session.shut_down(now, StreamStop::Quit).unwrap(); + session.power_up(); + assert!(session.take_goodbye().is_some(), "a quit's goodbye stays"); + } + + /// The PC woke after hours: the session ended when it went to sleep, so + /// the first pass after waking ends nothing again. It used to expire the + /// session then, as a silence as long as the sleep. + #[test] + fn waking_ends_nothing_a_second_time() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + for caps in [FEATURE_POWER_BYE, 0] { + let mut session = Session::new(500); + session.handle_control(peer, 0, power_hello(1, caps), &TestConfig, now); + session.power_down(now, PowerEvent::Sleep).unwrap(); + session.tick(&TestConfig, false, now); + let woke = now + Duration::from_secs(3 * 3600); + session.power_up(); + session.transport_resumed(woke, false); + let first = session.tick(&TestConfig, true, woke); + assert_eq!( + (first.ended, first.goodbye, first.client_lost), + (None, None, false) + ); + } + } + fn goodbye_of(actions: &Actions) -> (PeerId, u32, ByeReason) { let (to, bytes) = actions.goodbye.clone().expect("a goodbye"); let (header, message) = eternal_wire::v2::control::parse_control(&bytes).unwrap(); diff --git a/host/tests/power_goodbye.rs b/host/tests/power_goodbye.rs new file mode 100644 index 00000000..6229b36f --- /dev/null +++ b/host/tests/power_goodbye.rs @@ -0,0 +1,351 @@ +//! When Windows puts the PC to sleep, shuts it down or signs the user out +//! while an iPad streams, the iPad that understands it hears why, an older +//! one hears nothing as before, and the session's summary is written first. +//! Drives the real transport loop with a fake iPad on loopback and gives it +//! the messages Windows would; nothing here sleeps or shuts down a PC. + +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use eternal_host::control::SharedControl; +use eternal_host::power::{self, Message, END_SESSION_WAIT, SLEEP_WAIT}; +use eternal_host::transport; +use eternal_wire::v2::control::{ + encode_control, parse_control, ByeReason, ControlMessage, Hello2, HelloStatus, CAP_DECODE_H264, + FEATURE_POWER_BYE, FEATURE_SUPPORTS_NACK, +}; +use tokio::sync::mpsc; + +/// The tests share the environment and the session log; one at a time. +static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Keeps the session log, which a summary written before a shutdown goes +/// to directly, out of the real app data folder. +fn private_app_data() -> std::path::PathBuf { + static DIR: std::sync::OnceLock = std::sync::OnceLock::new(); + DIR.get_or_init(|| { + let dir = std::env::temp_dir().join(format!("em-power-goodbye-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::env::set_var("APPDATA", &dir); + dir + }) + .clone() +} + +fn free_udp_port() -> u16 { + // The transport binds every address. + std::net::UdpSocket::bind("0.0.0.0:0") + .and_then(|s| s.local_addr()) + .map(|a| a.port()) + .unwrap() +} + +fn hello(nonce: u32, listen_port: u16, feature_caps: u16) -> Vec { + encode_control( + 0, + nonce, + &ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: nonce, + listen_port, + decoder_caps: CAP_DECODE_H264, + feature_caps, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "Power test iPad".into(), + device_id: 7, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + }), + ) +} + +/// A fake iPad streaming from a real transport on loopback. +struct Rig { + shared: SharedControl, + ipad: UdpSocket, + host: SocketAddr, + caps: u16, + nal_tx: Option>, + transport: Option>, +} + +impl Rig { + fn start(caps: u16) -> Self { + eternal_host::clock::init(); + private_app_data(); + // Never reach a real iPad over USB. + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + let port = free_udp_port(); + let shared = SharedControl::new(port, 10_000_000); + shared.pairing.lock().required = false; + shared.running.store(true, Ordering::SeqCst); + let (nal_tx, nal_rx) = mpsc::channel(4); + let control = shared.clone(); + let transport = std::thread::spawn(move || { + let (_audio_tx, audio_rx) = mpsc::channel(4); + let (supervisor_tx, _supervisor_rx) = std::sync::mpsc::channel(); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(transport::start_sender( + nal_rx, + audio_rx, + port, + control, + supervisor_tx, + 1, + )) + .unwrap(); + }); + let ipad = UdpSocket::bind("127.0.0.1:0").unwrap(); + ipad.set_read_timeout(Some(Duration::from_millis(50))) + .unwrap(); + let mut rig = Self { + shared, + ipad, + host: SocketAddr::from(([127, 0, 0, 1], port)), + caps, + nal_tx: Some(nal_tx), + transport: Some(transport), + }; + assert_eq!(rig.connect(1), Some(HelloStatus::Ok), "the iPad streams"); + rig + } + + /// Sends HELLO2 `nonce` until an answer comes, for at most `patience`. + fn ask(&self, nonce: u32, patience: Duration) -> Option { + let port = self.ipad.local_addr().unwrap().port(); + let until = Instant::now() + patience; + let mut bytes = [0; 2048]; + while Instant::now() < until { + assert!( + !self.transport.as_ref().unwrap().is_finished(), + "the transport ended early" + ); + self.ipad + .send_to(&hello(nonce, port, self.caps), self.host) + .unwrap(); + let asked = Instant::now(); + while asked.elapsed() < Duration::from_millis(200) { + if let Ok(len) = self.ipad.recv(&mut bytes) { + if let Ok((_, ControlMessage::HelloAck(ack))) = parse_control(&bytes[..len]) { + if ack.client_nonce == nonce { + return Some(ack.status); + } + } + } + } + } + None + } + + fn connect(&mut self, nonce: u32) -> Option { + // The transport binds its socket first. + self.ask(nonce, Duration::from_secs(30)) + } + + /// The goodbyes that arrive within `window`. + fn goodbyes(&self, window: Duration) -> Vec { + let until = Instant::now() + window; + let mut bytes = [0; 2048]; + let mut said = Vec::new(); + while Instant::now() < until { + if let Ok(len) = self.ipad.recv(&mut bytes) { + if let Ok((_, ControlMessage::Bye(reason))) = parse_control(&bytes[..len]) { + said.push(reason); + } + } + } + said + } + + /// Gives the host `message` as its window would, and how long that took. + fn tell(&self, message: Message) -> (isize, Duration) { + let started = Instant::now(); + let answer = power::handle(&self.shared, message); + (answer, started.elapsed()) + } +} + +impl Drop for Rig { + fn drop(&mut self) { + self.shared.stop(); + drop(self.nal_tx.take()); + if let Some(transport) = self.transport.take() { + let _ = transport.join(); + } + } +} + +fn session_log() -> String { + std::fs::read_to_string(eternal_host::logging::session_log_path()).unwrap_or_default() +} + +/// Sleep: the iPad that understands it hears that the PC went to sleep, in +/// the usual burst of three, before the message returns; then the PC answers +/// no HELLO2 until it wakes, and streams again after. +#[test] +fn sleep_tells_the_ipad_and_waking_lets_it_back() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let mut rig = Rig::start(FEATURE_POWER_BYE | FEATURE_SUPPORTS_NACK); + let (answer, took) = rig.tell(Message::Suspend); + assert_eq!(answer, 1, "never asks Windows to wait"); + assert!(took < SLEEP_WAIT + Duration::from_millis(100), "{took:?}"); + assert_eq!( + rig.goodbyes(Duration::from_millis(500)), + [ByeReason::HostSleeping; 3] + ); + assert!(!rig.shared.client_connected()); + + assert_eq!( + rig.ask(2, Duration::from_millis(600)), + None, + "no answer while it sleeps" + ); + assert!(!rig.shared.client_connected()); + assert_eq!(rig.tell(Message::Resume).0, 1); + assert_eq!( + rig.connect(3), + Some(HelloStatus::Ok), + "streams again after waking" + ); +} + +/// An iPad from before these goodbyes gets none, as before: it notices the +/// silence. Nothing waits for a goodbye that isn't coming. +#[test] +fn sleep_says_nothing_to_an_older_ipad() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let rig = Rig::start(FEATURE_SUPPORTS_NACK); + let (answer, took) = rig.tell(Message::Suspend); + assert_eq!(answer, 1); + assert!(took < SLEEP_WAIT, "{took:?}"); + assert!( + !rig.shared.client_connected(), + "the session ended all the same" + ); + let heard = rig.goodbyes(Duration::from_millis(500)); + assert!(heard.is_empty(), "{heard:?}"); +} + +/// Shutdown and sign-out: the iPad hears that the PC is powering off, and +/// the session's summary is in the session log on disk before the message +/// returns, ready for the next launch to upload. +#[test] +fn shutting_down_or_signing_out_says_so_and_saves_the_summary() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + for (nonce, message) in [ + ( + 11, + Message::EndSession { + ending: true, + logoff: false, + }, + ), + ( + 12, + Message::EndSession { + ending: true, + logoff: true, + }, + ), + (13, Message::SessionLogoff), + ] { + let mut rig = Rig::start(FEATURE_POWER_BYE); + // Asking is not going: another app may still call it off. + assert_eq!(rig.tell(Message::QueryEndSession).0, 1); + assert!(rig.shared.client_connected()); + assert_eq!( + rig.tell(Message::EndSession { + ending: false, + logoff: false + }) + .0, + 0 + ); + assert!( + rig.shared.client_connected(), + "a cancelled shutdown changes nothing" + ); + // The session to end, under its own nonce. + assert_eq!(rig.connect(nonce), Some(HelloStatus::Ok)); + + let (answer, took) = rig.tell(message); + assert_eq!(answer, 0, "{message:?}"); + assert!( + took < END_SESSION_WAIT + Duration::from_millis(100), + "{took:?}" + ); + let log = session_log(); + let summary = format!(" nonce={nonce:016x} "); + assert!( + log.lines() + .any(|line| line.contains("session_summary v=1 role=pc") + && line.contains(&summary) + && line.contains(" end=shutdown ")), + "{message:?}: no summary in the log" + ); + assert_eq!( + rig.goodbyes(Duration::from_millis(500)), + [ByeReason::HostPoweringOff; 3], + "{message:?}" + ); + } +} + +/// The real window, given the message Windows sends before sleep. Windows +/// only: it checks the window gets Windows' messages and answers them. +#[cfg(windows)] +#[test] +fn the_hidden_window_turns_windows_messages_into_goodbyes() { + use windows::Win32::Foundation::{HWND, LPARAM, WPARAM}; + use windows::Win32::UI::WindowsAndMessaging::{ + SendMessageW, PBT_APMRESUMEAUTOMATIC, PBT_APMSUSPEND, WM_POWERBROADCAST, + }; + + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let rig = Rig::start(FEATURE_POWER_BYE); + power::watch(rig.shared.clone()); + let started = Instant::now(); + let hwnd = loop { + if let Some(hwnd) = power::window() { + break HWND(hwnd as _); + } + assert!( + started.elapsed() < Duration::from_secs(10), + "the window opens" + ); + std::thread::sleep(Duration::from_millis(10)); + }; + let answer = unsafe { + SendMessageW( + hwnd, + WM_POWERBROADCAST, + WPARAM(PBT_APMSUSPEND as usize), + LPARAM(0), + ) + }; + assert_eq!(answer.0, 1); + assert_eq!( + rig.goodbyes(Duration::from_millis(500)), + [ByeReason::HostSleeping; 3] + ); + let answer = unsafe { + SendMessageW( + hwnd, + WM_POWERBROADCAST, + WPARAM(PBT_APMRESUMEAUTOMATIC as usize), + LPARAM(0), + ) + }; + assert_eq!(answer.0, 1); + assert_eq!(rig.ask(2, Duration::from_secs(10)), Some(HelloStatus::Ok)); +} diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 4e197d61..c96972c7 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -96,12 +96,13 @@ final class ConnectionManager: ObservableObject { /// HELLO2 feature bits. Every build says it checks for its own updates, /// so the PC tells only older iPad apps to update, and that it - /// understands why a PC isn't streaming. It draws the PC's pointer - /// itself, so pointer motion needs no video frame. + /// understands why a PC isn't streaming and why it went down. It draws + /// the PC's pointer itself, so pointer motion needs no video frame. nonisolated static func featureCaps(wantsInput: Bool, wantsAudio: Bool, isUSB: Bool) -> UInt16 { (wantsInput ? Hello2.featureWantsInput : 0) | (wantsAudio ? Hello2.featureWantsAudio : 0) | (isUSB ? 0 : Hello2.featureSupportsNack | Hello2.featureMediaSize) | Hello2.featureChecksUpdates - | Hello2.featureStandbyStatus | (E2E.drawsPointer ? Hello2.featureDrawsPointer : 0) + | Hello2.featureStandbyStatus | Hello2.featurePowerBye + | (E2E.drawsPointer ? Hello2.featureDrawsPointer : 0) } private var awaitingPairing = false /// The PC answered that it isn't streaming; the attempt waits for it. @@ -930,9 +931,10 @@ final class ConnectionManager: ObservableObject { } else { self.beginReconnect(.pcStoppedResponding, failure: failure, end: .pcGoodbye) } - case .userDisconnect, .superseded: - // Disconnected from the PC, or another device took over: - // stay away until the user connects again. + case .userDisconnect, .superseded, .hostSleeping, .hostPoweringOff: + // Disconnected from the PC, another device took over, or + // the PC is going down and won't answer: stay away until + // the user connects again. self.endReconnectCycle() self.disconnect(.pcGoodbye) self.problem = failure diff --git a/ios/EternalMonitor/App/ConnectionProblem.swift b/ios/EternalMonitor/App/ConnectionProblem.swift index ace092a3..dfeaaebd 100644 --- a/ios/EternalMonitor/App/ConnectionProblem.swift +++ b/ios/EternalMonitor/App/ConnectionProblem.swift @@ -37,6 +37,10 @@ enum ConnectionProblem: Equatable { /// The PC ended the session on an error: it stopped hearing this iPad (a /// Wi-Fi stall) or hit a problem of its own. Nobody stopped the stream. case pcEndedSessionOnError + /// The PC said it is going to sleep. + case pcAsleep + /// The PC said it is shutting down, restarting or signing out. + case pcPoweringOff /// The PC is streaming to another device, or still to this iPad's cable. case pcBusy /// The PC stopped the stream to install an update and restarts on it. @@ -126,6 +130,8 @@ enum ConnectionProblem: Equatable { case .pcEndedSession: return "Your PC ended the session" // Also what a PC that restarts by itself sends: true for both. case .pcEndedSessionOnError: return "The connection to your PC ended" + case .pcAsleep: return "Your PC went to sleep" + case .pcPoweringOff: return "Your PC is shutting down or signing out" case .pcBusy: return "Your PC is already streaming to another device" case .pcUpdating(.reconnecting), .pcUpdating(.manual): return "Your PC is updating EternalMonitor" case .pcUpdating(.notBack): return "Your PC hasn't come back from its update yet" @@ -183,6 +189,10 @@ enum ConnectionProblem: Equatable { return "Someone clicked Stop on the PC, changed a setting that restarts the stream, or closed EternalMonitor. Try again when the PC says Ready for your iPad." case .pcEndedSessionOnError: return "The connection to your PC dropped, or the PC ran into a problem. Tap Try again to reconnect." + case .pcAsleep: + return "Wake the PC, then tap Try again." + case .pcPoweringOff: + return "Tap Try again once the PC is back on and EternalMonitor is open on it." case .pcUpdating(.reconnecting): return "This iPad reconnects when it's back. If it doesn't, check that EternalMonitor opened again on the PC, then try again." case .pcUpdating(.manual): @@ -217,7 +227,7 @@ enum ConnectionProblem: Equatable { switch self { case .reconnecting, .pcUpdating(.reconnecting), .pcUpdating(.manual), .lost(.cableUnplugged), .lost(.usbTurnedOff), .pcBusy, .pairingCodeAsAddress, - .pcNotStreaming(.stopped), .pcNotStreaming(.restarting): + .pcNotStreaming(.stopped), .pcNotStreaming(.restarting), .pcAsleep, .pcPoweringOff: return false default: return true @@ -291,6 +301,8 @@ enum ConnectionProblem: Equatable { case .hostShuttingDown: return .pcUpdating(.reconnecting) case .error: return .pcEndedSessionOnError case .userDisconnect, .superseded, .appBackground: return .pcEndedSession + case .hostSleeping: return .pcAsleep + case .hostPoweringOff: return .pcPoweringOff } } diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index 4b87b593..1b6bbbaa 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -273,6 +273,9 @@ struct Hello2: Equatable { /// and wants heartbeats to count the frames the PC sent. Bit 5 is /// `featureDrawsPointer`. static let featureMediaSize: UInt16 = 1 << 6 + /// This app understands the goodbyes a PC sends as it goes to sleep or + /// shuts down. Older apps drop a goodbye they can't read. + static let featurePowerBye: UInt16 = 1 << 7 static let maxNameLength = 64 var protoMin: UInt8 = 2 @@ -370,6 +373,11 @@ enum ByeReason: UInt8 { case hostShuttingDown = 2 case error = 3 case superseded = 4 + /// The PC is going to sleep. Only to an iPad that sets `featurePowerBye`. + case hostSleeping = 5 + /// The PC is shutting down, restarting or signing out. Only to an iPad + /// that sets `featurePowerBye`. + case hostPoweringOff = 6 } enum KeyframeReason: UInt8 { diff --git a/ios/EternalMonitorTests/ConnectionProblemTests.swift b/ios/EternalMonitorTests/ConnectionProblemTests.swift index e29aff0a..3c604a3d 100644 --- a/ios/EternalMonitorTests/ConnectionProblemTests.swift +++ b/ios/EternalMonitorTests/ConnectionProblemTests.swift @@ -267,7 +267,7 @@ final class ConnectionProblemTests: XCTestCase { .noReply(usb: false), .noReply(usb: true), .noReplyAtNewAddress("192.0.2.31"), .noPicture, .receiverNotReady, .badAddress("pc"), .noNetwork, .wifiOff, .localNetworkBlocked, .couldNotConnect, .usbFailed, .lost(.pcStoppedResponding), .lost(.cableUnplugged), .lost(.usbTurnedOff), .reconnecting(attempt: 2, of: 6), .reconnectFailed, .pcEndedSession, - .pcEndedSessionOnError, .pcBusy, .pairingCodeAsAddress, .routerAddress, + .pcEndedSessionOnError, .pcAsleep, .pcPoweringOff, .pcBusy, .pairingCodeAsAddress, .routerAddress, .pcUpdating(.reconnecting), .pcUpdating(.manual), .pcUpdating(.notBack), ] for problem in all { @@ -289,7 +289,7 @@ final class ConnectionProblemTests: XCTestCase { /// someone stopped the stream; the other reasons keep saying so. func testEachGoodbyeFromThePCMapsToItsProblemAndText() { let reasons = (0...UInt8.max).compactMap(ByeReason.init(rawValue:)) - XCTAssertEqual(reasons.count, 5, "a new goodbye reason needs its card here") + XCTAssertEqual(reasons.count, 7, "a new goodbye reason needs its card here") let userAction = "Someone clicked Stop on the PC, changed a setting that restarts the stream, or closed EternalMonitor. " + "Try again when the PC says Ready for your iPad." let expected: [ByeReason: (ConnectionProblem, String)] = [ @@ -300,6 +300,8 @@ final class ConnectionProblemTests: XCTestCase { "The connection to your PC dropped, or the PC ran into a problem. Tap Try again to reconnect."), .hostShuttingDown: (.pcUpdating(.reconnecting), "This iPad reconnects when it's back. If it doesn't, check that EternalMonitor opened again on the PC, then try again."), + .hostSleeping: (.pcAsleep, "Wake the PC, then tap Try again."), + .hostPoweringOff: (.pcPoweringOff, "Tap Try again once the PC is back on and EternalMonitor is open on it."), ] for reason in reasons { let problem = ConnectionProblem.pcGoodbye(reason) @@ -316,6 +318,15 @@ final class ConnectionProblemTests: XCTestCase { XCTAssertTrue(error.canReport) XCTAssertEqual(error.withoutReconnecting, error) XCTAssertEqual(error.afterTheCycle, error) + // The PC going down is something that happened, not a fault to report. + for down: ConnectionProblem in [.pcAsleep, .pcPoweringOff] { + XCTAssertEqual(down.actions, [.retry], "\(down)") + XCTAssertFalse(down.canReport, "\(down)") + XCTAssertEqual(down.withoutReconnecting, down) + XCTAssertEqual(down.afterTheCycle, down) + } + XCTAssertEqual(ConnectionProblem.pcAsleep.title, "Your PC went to sleep") + XCTAssertEqual(ConnectionProblem.pcPoweringOff.title, "Your PC is shutting down or signing out") } func testAnUpdateStaysTheReasonWhileThePCRestarts() { diff --git a/ios/EternalMonitorTests/GoodbyeTests.swift b/ios/EternalMonitorTests/GoodbyeTests.swift index 210c861e..b70b2b87 100644 --- a/ios/EternalMonitorTests/GoodbyeTests.swift +++ b/ios/EternalMonitorTests/GoodbyeTests.swift @@ -235,6 +235,27 @@ final class GoodbyeTests: XCTestCase { XCTAssertEqual(manager.reconnectAttempt, 0) } + /// The PC said it is going to sleep or shutting down: the card says so, + /// and the iPad doesn't keep dialling a PC that won't answer. Without + /// these goodbyes it showed a lost signal, then "didn't answer". + func testAPCGoingDownSaysWhyAndStaysAway() async throws { + for (reason, problem) in [(ByeReason.hostSleeping, ConnectionProblem.pcAsleep), + (.hostPoweringOff, .pcPoweringOff)] { + let host = FakeUDPHost() + let manager = try await streaming(host) + let advertised = (host.lastHello?.featureCaps ?? 0) & Hello2.featurePowerBye + XCTAssertNotEqual(advertised, 0, "the PC sends these only to an iPad that asks") + host.sendBye(reason) + try await until("disconnected") { manager.state == .disconnected } + XCTAssertEqual(manager.problem, problem) + XCTAssertEqual(manager.problem?.actions, [.retry]) + let attempts = await Task.detached { host.helloAttempts(within: 1.5) }.value + XCTAssertEqual(attempts, 0, "no automatic reconnecting to a PC that went down") + XCTAssertEqual(manager.reconnectAttempt, 0) + manager.cancel() + } + } + /// The PC stopped hearing this iPad (a Wi-Fi stall) and ended the session /// with an error. With Auto-reconnect off the card says the connection /// dropped, not that someone stopped the stream on the PC. diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index 7eaf1040..6abf3d04 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -35,7 +35,7 @@ final class WireProtocolGoldenTests: XCTestCase { func testGoldenFileIsBundledAndComplete() throws { XCTAssertEqual( - Self.vectors.count, 47, + Self.vectors.count, 49, "golden vector count drifted — update both test suites together" ) } @@ -548,6 +548,23 @@ final class WireProtocolGoldenTests: XCTestCase { XCTAssertEqual(message, .bye(.appBackground)) } + /// The goodbyes a PC sends as it goes to sleep or powers off, only to an + /// iPad that sets `featurePowerBye`. An app without them drops the whole + /// message, as 0.3.4 does. + func testPowerGoodbyeVectors() throws { + for (name, reason) in [("bye_host_sleeping", ByeReason.hostSleeping), + ("bye_host_powering_off", .hostPoweringOff)] { + let data = try vector(name) + let (header, message) = try XCTUnwrap(Wire.parseControl(data)) + XCTAssertEqual(message, .bye(reason)) + XCTAssertEqual(Wire.encodeControl(sessionId: header.sessionId, msgSeq: header.msgSeq, message: message), data) + } + var unknown = try vector("bye_host_powering_off") + unknown[unknown.count - 1] = 7 + XCTAssertNil(Wire.parseControl(unknown)) + XCTAssertEqual(Hello2.featurePowerBye, 1 << 7) + } + func testKeyframeRequestVector() throws { let data = try vector("keyframe_request") let (_, message) = try XCTUnwrap(Wire.parseControl(data)) diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index bcd7351b..d651ee00 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -53,6 +53,10 @@ pub const FEATURE_DRAWS_POINTER: u16 = 1 << 5; /// to tell a PC that sends nothing from video lost on the way (0.3.5 and /// later). Bit 5 is [`FEATURE_DRAWS_POINTER`]. pub const FEATURE_MEDIA_SIZE: u16 = 1 << 6; +/// The iPad understands [`ByeReason::HostSleeping`] and +/// [`ByeReason::HostPoweringOff`] (0.3.5 and later). Older iPads drop a +/// goodbye whose reason they don't know, so the host doesn't send them one. +pub const FEATURE_POWER_BYE: u16 = 1 << 7; pub const HOSTCAP_NACK: u16 = 1 << 0; pub const HOSTCAP_AUDIO: u16 = 1 << 1; pub const HOSTCAP_USB: u16 = 1 << 2; @@ -308,6 +312,11 @@ pub enum ByeReason { HostShuttingDown = 2, Error = 3, Superseded = 4, + /// The PC is going to sleep. Only to an iPad with [`FEATURE_POWER_BYE`]. + HostSleeping = 5, + /// The PC is shutting down, restarting or signing out. Only to an iPad + /// with [`FEATURE_POWER_BYE`]. + HostPoweringOff = 6, } impl ByeReason { @@ -318,6 +327,8 @@ impl ByeReason { 2 => Self::HostShuttingDown, 3 => Self::Error, 4 => Self::Superseded, + 5 => Self::HostSleeping, + 6 => Self::HostPoweringOff, _ => return None, }) } @@ -1377,6 +1388,8 @@ mod tests { frames_sent: Some(0), }), ControlMessage::Bye(ByeReason::AppBackground), + ControlMessage::Bye(ByeReason::HostSleeping), + ControlMessage::Bye(ByeReason::HostPoweringOff), ControlMessage::KeyframeRequest(KeyframeRequest { stream_epoch: 3, last_complete_seq: 4111, diff --git a/proto/testdata/v2_vectors.txt b/proto/testdata/v2_vectors.txt index 62940b76..d5ef023c 100644 --- a/proto/testdata/v2_vectors.txt +++ b/proto/testdata/v2_vectors.txt @@ -22,6 +22,8 @@ hello_ack_busy 454d02110000270000000000020000000102efbeadde00000000e803f401b80b0 heartbeat 454d021200001800785634122a000000b168de3a0000000003000000000aa0053c000000c0e1e400 heartbeat_pc_locked 454d021200001900785634122a000000b168de3a0000000003000000000aa0053c000000c0e1e40001 bye_background 454d021300000100785634122b00000001 +bye_host_sleeping 454d021300000100785634123700000005 +bye_host_powering_off 454d021300000100785634123800000006 keyframe_request 454d021400000900785634122c000000030000000f10000000 keyframe_request_media_size 454d021400000b007856341233000000030000000000000002b004 heartbeat_frames_sent 454d021200001d007856341234000000b168de3a0000000003000000000aa0053c000000c0e1e40000d2040000 diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index 8e1f5546..ee270cd3 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -269,6 +269,18 @@ fn canonical_vectors() -> Vec<(&'static str, Vec)> { 43, ControlMessage::Bye(ByeReason::AppBackground), ), + ( + "bye_host_sleeping", + 0x1234_5678, + 55, + ControlMessage::Bye(ByeReason::HostSleeping), + ), + ( + "bye_host_powering_off", + 0x1234_5678, + 56, + ControlMessage::Bye(ByeReason::HostPoweringOff), + ), ( "keyframe_request", 0x1234_5678, From 3416d29dfca510e717ac329b1543a0eb88860eb4 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:17:32 -0500 Subject: [PATCH 174/226] Tell the iPad when the PC can't capture from its graphics card On laptops with two graphics cards, Windows can refuse to let EternalMonitor capture the screen from the card it runs on. The PC's window said so and offered the switch, but the iPad got no word: it showed Connecting for 10 s, then "no picture", and over a live stream a frozen picture. Heartbeats now carry a notice once the refusal has lasted 2 s (it comes and goes for a moment while the desktop changes mode): one for the screen, and one for Second screen mode after the second screen failed to start, when the main screen it fell back to can't be captured either. iPads that don't know the values ignore them, as before. The iPad says why on its Connecting card and over a held picture, and when its wait runs out it shows a card that points to the PC's window instead of "no picture". Both summaries name the session's failure capture_failed, which the backend's check already lists, instead of no_video. The two new iPad strings wait for their translations. The synthetic capture can refuse the card in debug builds (ETERNAL_SYNTH_REFUSE_GPU=1), for the end-to-end test. --- host/src/capture/recovery.rs | 110 ++++++++++++-- host/src/capture/synthetic.rs | 21 ++- host/src/control.rs | 4 + host/src/diagnostics.rs | 67 ++++++++- host/src/transport/mod.rs | 68 ++++++++- host/tests/e2e_synthetic.rs | 141 ++++++++++++++++++ ios/EternalMonitor/App/ConnectView.swift | 17 ++- .../App/ConnectionManager.swift | 20 ++- .../App/ConnectionProblem.swift | 41 ++++- ios/EternalMonitor/App/DisplayView.swift | 28 ++++ ios/EternalMonitor/App/SessionSummary.swift | 3 + ios/EternalMonitor/Network/WireProtocol.swift | 6 + .../Resources/Localizable.xcstrings | 24 +++ .../LocalizationTests.swift | 5 +- .../NoPictureTimeoutTests.swift | 41 +++++ .../SessionSummaryTests.swift | 13 ++ .../StreamRecoveryTests.swift | 24 +++ proto/src/v2/control.rs | 15 +- 18 files changed, 620 insertions(+), 28 deletions(-) diff --git a/host/src/capture/recovery.rs b/host/src/capture/recovery.rs index 6e4881dd..2ea6e3d6 100644 --- a/host/src/capture/recovery.rs +++ b/host/src/capture/recovery.rs @@ -485,15 +485,43 @@ fn access_lost_is_blocked( } } +/// How long Windows has to refuse the screen from this graphics card before +/// the iPad is told. The same refusal comes and goes for a moment while the +/// desktop changes mode, and the iPad's first-picture wait is 10 s. +pub const GPU_NOTICE_AFTER: Duration = Duration::from_secs(2); + /// The heartbeat's notice while capture is blocked. A secure screen says the /// PC is locked, and no display attached says so, so the iPad doesn't show a /// frozen picture without a word; it clears with the block when capture -/// works again. The other blocks keep the held picture as before. -pub fn heartbeat_notice(block: Option) -> u8 { - use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_NO_DISPLAY, NOTICE_PC_LOCKED}; +/// works again. A screen Windows won't share from this graphics card says so +/// once the refusal has lasted [`GPU_NOTICE_AFTER`] (`gpu_since` keeps when +/// it began), naming the failed second screen when `second_screen_failed`. +/// The other blocks keep the held picture as before. +pub fn heartbeat_notice( + block: Option, + second_screen_failed: bool, + gpu_since: &mut Option, + now: Instant, +) -> u8 { + use eternal_wire::v2::control::{ + NOTICE_NONE, NOTICE_NO_DISPLAY, NOTICE_PC_LOCKED, NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, + NOTICE_UNSUPPORTED_GPU, + }; + if block != Some(CaptureBlock::UnsupportedOnThisGpu) { + *gpu_since = None; + } match block { Some(CaptureBlock::SecureDesktop) => NOTICE_PC_LOCKED, Some(CaptureBlock::NoDisplay) => NOTICE_NO_DISPLAY, + Some(CaptureBlock::UnsupportedOnThisGpu) + if now.saturating_duration_since(*gpu_since.get_or_insert(now)) >= GPU_NOTICE_AFTER => + { + if second_screen_failed { + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU + } else { + NOTICE_UNSUPPORTED_GPU + } + } _ => NOTICE_NONE, } } @@ -618,31 +646,89 @@ mod tests { assert!(shared.gpu_switch.lock().as_ref().unwrap().failed); } + /// The notice for `block`, read once with no refusal before. + fn notice_now(block: Option) -> u8 { + heartbeat_notice(block, false, &mut None, Instant::now()) + } + #[test] - fn only_a_secure_screen_or_no_display_has_a_notice_for_the_ipad() { + fn only_a_secure_screen_no_display_or_the_wrong_card_has_a_notice_for_the_ipad() { use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_NO_DISPLAY, NOTICE_PC_LOCKED}; assert_eq!( - heartbeat_notice(Some(CaptureBlock::SecureDesktop)), + notice_now(Some(CaptureBlock::SecureDesktop)), NOTICE_PC_LOCKED ); // A closed lid or screens that are off: the field's PCs showed a // frozen picture without a word until a display came back. - assert_eq!( - heartbeat_notice(Some(CaptureBlock::NoDisplay)), - NOTICE_NO_DISPLAY - ); - assert_eq!(heartbeat_notice(None), NOTICE_NONE); + assert_eq!(notice_now(Some(CaptureBlock::NoDisplay)), NOTICE_NO_DISPLAY); + assert_eq!(notice_now(None), NOTICE_NONE); for block in [ CaptureBlock::DisplayChanging, - CaptureBlock::UnsupportedOnThisGpu, CaptureBlock::TooManyCaptureApps, CaptureBlock::SessionDisconnected, CaptureBlock::DriverNotResponding, ] { - assert_eq!(heartbeat_notice(Some(block)), NOTICE_NONE, "{block:?}"); + assert_eq!(notice_now(Some(block)), NOTICE_NONE, "{block:?}"); } } + /// The hybrid laptops whose iPad saw 10 s of black, then "no picture": + /// Windows refused the screen from the card EternalMonitor runs on, and + /// the PC's window said so, but the iPad got no word. Now it does, once + /// the refusal has lasted 2 s (it comes and goes for a moment while the + /// desktop changes mode), naming the second screen when it failed too. + /// It clears with the refusal, and a new one waits again. + #[test] + fn a_screen_refused_from_this_graphics_card_is_told_to_the_ipad_after_2_s() { + use eternal_wire::v2::control::{ + NOTICE_NONE, NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, NOTICE_UNSUPPORTED_GPU, + }; + let refused = Some(CaptureBlock::UnsupportedOnThisGpu); + let start = Instant::now(); + let at = |s: f64| start + Duration::from_secs_f64(s); + let mut since = None; + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(0.0)), + NOTICE_NONE + ); + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(1.9)), + NOTICE_NONE + ); + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(2.0)), + NOTICE_UNSUPPORTED_GPU + ); + // Second screen mode, after the second screen failed to start. + assert_eq!( + heartbeat_notice(refused, true, &mut since, at(3.0)), + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU + ); + // Capture works: no notice, and the next refusal waits again. + assert_eq!( + heartbeat_notice(None, false, &mut since, at(4.0)), + NOTICE_NONE + ); + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(5.0)), + NOTICE_NONE + ); + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(7.0)), + NOTICE_UNSUPPORTED_GPU + ); + // A display change in between restarts the wait too. + let changing = Some(CaptureBlock::DisplayChanging); + assert_eq!( + heartbeat_notice(changing, false, &mut since, at(8.0)), + NOTICE_NONE + ); + assert_eq!( + heartbeat_notice(refused, false, &mut since, at(9.0)), + NOTICE_NONE + ); + } + #[test] fn a_stuck_driver_call_is_waited_out_in_the_stage_not_restarted() { use crate::stall::{no_hooks, run_in, Call, Lanes, Stall, Wait}; diff --git a/host/src/capture/synthetic.rs b/host/src/capture/synthetic.rs index cf23baa5..de3c73ba 100644 --- a/host/src/capture/synthetic.rs +++ b/host/src/capture/synthetic.rs @@ -57,9 +57,17 @@ fn still_desktop() -> bool { std::env::var("ETERNAL_SYNTH_STILL").is_ok_and(|v| v.trim() == "1") } +/// `ETERNAL_SYNTH_REFUSE_GPU=1`, debug builds only: Windows refuses the +/// screen from the graphics card EternalMonitor runs on, as on the hybrid +/// laptops, for as long as it is set. Each attempt reads it again. +#[cfg(debug_assertions)] +fn gpu_refused() -> bool { + std::env::var("ETERNAL_SYNTH_REFUSE_GPU").is_ok_and(|v| v.trim() == "1") +} + /// Produce frames until `shared.running` clears or the channel closes. /// Mirrors the DXGI loop's contract: paced by `target_fps`, records capture -/// stats, ships BGRA `RawFrame`s. +/// stats, ships BGRA `RawFrame`s, and clears a block with its first one. pub fn run_capture_loop( slot: &FrameSlot, shared: SharedControl, @@ -76,8 +84,16 @@ pub fn run_capture_loop( if !proving { super::second_screen::fake::mirror(&shared)?; } + #[cfg(debug_assertions)] + if gpu_refused() { + return Err(Box::new(super::recovery::CaptureUnavailable { + block: super::recovery::CaptureBlock::UnsupportedOnThisGpu, + detail: "simulated: DXGI_ERROR_UNSUPPORTED (0x887A0004)".into(), + })); + } let (width, height) = synthetic_size(); let mut buffers = capture_buffers(width, height); + let mut resumed = false; info!(width, height, "Synthetic capture source active"); *shared.capture_geometry.lock() = Some(crate::input::CaptureGeometry { @@ -152,6 +168,9 @@ pub fn run_capture_loop( width, height, }); + if !std::mem::replace(&mut resumed, true) { + super::recovery::note_capture_resumed(&shared); + } #[cfg(debug_assertions)] if std::mem::take(&mut proving) { super::second_screen::note_image(&shared, generation); diff --git a/host/src/control.rs b/host/src/control.rs index 3c4b13f2..591e2e19 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -42,6 +42,9 @@ pub struct SharedControl { /// Set while Windows blocks screen capture (lock screen, UAC, a display /// it will not duplicate); the GUI explains it instead of a frozen picture. pub capture_block: Arc>>, + /// Since when the heartbeats have seen Windows refuse the screen from + /// this graphics card, for their notice (`recovery::heartbeat_notice`). + pub gpu_refused_since: Arc>>, /// The stream port between two owners (a generation's transport and the /// standby responder), bound and with its queue, so a HELLO2 sent while /// it changes hands waits for the next one. The supervisor closes it @@ -197,6 +200,7 @@ impl SharedControl { vdd_status: Arc::new(Mutex::new(VddStatus::Inactive)), vdd_problem: Arc::new(Mutex::new(None)), capture_block: Arc::new(Mutex::new(None)), + gpu_refused_since: Arc::new(Mutex::new(None)), stream_socket: Arc::new(Mutex::new(None)), gpu_diagnosis: Arc::new(Mutex::new(None)), screen_watch: Arc::new(Mutex::new(crate::gpu_preference::ScreenWatch::new())), diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 85dd7547..20c732fa 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -340,9 +340,11 @@ pub fn log_session_end(end: &SessionEnd, shared: &SharedControl) { // `mode=` is the mode at the end; the fallback was counted in Second // screen mode, perhaps before a switch to Mirror. let fallback = end.session_id.and_then(take_fallback); + let refused = *shared.capture_block.lock() + == Some(crate::capture::recovery::CaptureBlock::UnsupportedOnThisGpu); info!( "{}", - session_summary(end, second_screen, fallback, host_facts()) + session_summary(end, second_screen, fallback, refused, host_facts()) ); } @@ -376,11 +378,14 @@ pub fn stop_session(shared: &SharedControl, why: StreamStop) { } /// The `session_summary v=1` line (b-session-summary contract): enumerated -/// tokens, bounded numbers and the facts above, nothing else. +/// tokens, bounded numbers and the facts above, nothing else. `gpu_refused`: +/// Windows refuses the screen from the graphics card EternalMonitor runs on, +/// so a session that gets no picture failed in capture, not on the network. pub fn session_summary( end: &SessionEnd, second_screen: bool, fallback: Option<&'static str>, + gpu_refused: bool, host: &HostFacts, ) -> String { use eternal_wire::v2::control::{ByeReason, HelloStatus}; @@ -417,6 +422,9 @@ pub fn session_summary( EndReason::Refused(_) => "other", EndReason::LinkLost if matches!(end.link, LinkId::Usb { .. }) => "usb_lost", EndReason::LinkLost => "other", + EndReason::Liveness | EndReason::Goodbye(ByeReason::Error) if !streamed && gpu_refused => { + "capture_failed" + } EndReason::Liveness | EndReason::Goodbye(ByeReason::Error) if !streamed => "no_video", EndReason::Liveness => "signal_lost", // The iPad gave up on video that went out: none of it arrived, or @@ -739,6 +747,7 @@ pub(crate) mod tests { ), true, Some("attach_timeout"), + false, &pc(), ); assert_eq!( @@ -882,7 +891,7 @@ pub(crate) mod tests { ), ]; for (end, expected) in cases { - let line = session_summary(&end, false, None, &pc()); + let line = session_summary(&end, false, None, false, &pc()); assert_eq!(tokens(&line), expected, "{end:?}"); assert_contract(&line); } @@ -890,6 +899,7 @@ pub(crate) mod tests { &end(Refused(HelloStatus::Busy), None, 0), true, Some("no_frames"), + false, &pc(), ); assert!( @@ -900,6 +910,47 @@ pub(crate) mod tests { ); } + /// A hybrid laptop whose screen Windows won't share from the card + /// EternalMonitor runs on: the iPad gave up with no picture. That is a + /// capture failure, which the backend's check lists, not lost video. A + /// session that got its picture, or that the user ended, keeps its own + /// tokens. + #[test] + fn no_picture_from_a_refused_graphics_card_is_a_capture_failure() { + use EndReason::{Goodbye, Liveness}; + for (end, expected) in [ + ( + end(Goodbye(ByeReason::Error), None, 10_000), + ["accepted", "no_picture", "capture_failed", "goodbye"], + ), + ( + end(Liveness, None, 10_000), + ["accepted", "no_picture", "capture_failed", "liveness"], + ), + ( + end(Goodbye(ByeReason::UserDisconnect), None, 4_000), + ["accepted", "user_ended", "none", "goodbye"], + ), + ( + end(Liveness, Some(400), 60_000), + ["steady", "streamed", "signal_lost", "liveness"], + ), + ] { + let line = session_summary(&end, false, None, true, &pc()); + assert_eq!(tokens(&line), expected, "{end:?}"); + assert_contract(&line); + } + // Without the refusal, the same end is lost video, as before. + let line = session_summary( + &end(Goodbye(ByeReason::Error), None, 10_000), + false, + None, + false, + &pc(), + ); + assert_eq!(tokens(&line)[2], "no_video"); + } + #[test] fn summary_numbers_stay_in_the_contracts_range() { let long = SessionEnd { @@ -909,7 +960,7 @@ pub(crate) mod tests { duration: Duration::from_secs(40 * 86_400), ..end(EndReason::Liveness, None, 0) }; - let line = session_summary(&long, false, None, &pc()); + let line = session_summary(&long, false, None, false, &pc()); assert!(line.contains(" nonce=00000000ffffffff "), "{line}"); assert!( line.contains(" ttff_ms=600000 duration_ms=2592000000 "), @@ -921,7 +972,13 @@ pub(crate) mod tests { gpu: gpu_token(&[]), hybrid: None, }; - let line = session_summary(&end(EndReason::Shutdown, None, 0), false, None, &unknown); + let line = session_summary( + &end(EndReason::Shutdown, None, 0), + false, + None, + false, + &unknown, + ); assert!(line.ends_with(" os=win:0.0 hw=pc gpu=- hybrid=-"), "{line}"); assert_contract(&line); } diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 42667586..f208f6f6 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -274,7 +274,20 @@ impl ConfigSource for SharedConfigSource<'_> { } fn notice(&self) -> u8 { - crate::capture::recovery::heartbeat_notice(*self.shared.capture_block.lock()) + use crate::control::{CaptureTarget, VddStatus}; + let block = *self.shared.capture_block.lock(); + let second_screen_failed = *self.shared.capture_target.lock() + == CaptureTarget::VirtualExtended + && matches!( + *self.shared.vdd_status.lock(), + VddStatus::Failed | VddStatus::TaskFailed + ); + crate::capture::recovery::heartbeat_notice( + block, + second_screen_failed, + &mut self.shared.gpu_refused_since.lock(), + std::time::Instant::now(), + ) } fn host_caps(&self) -> u16 { @@ -2106,6 +2119,59 @@ mod tests { assert_eq!(notices(), [NOTICE_NONE]); } + /// A screen Windows won't share from this graphics card: the heartbeats + /// say so once it has lasted 2 s, and in Second screen mode, once the + /// second screen failed, that the main screen it fell back to can't be + /// shared either. While the second screen still starts, the mirror's + /// notice stays. + #[test] + fn heartbeats_say_which_screen_this_graphics_card_cannot_share() { + use crate::capture::recovery::CaptureBlock; + use crate::control::{CaptureTarget, VddStatus}; + use eternal_wire::v2::control::{ + NOTICE_NONE, NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, NOTICE_UNSUPPORTED_GPU, + }; + let shared = SharedControl::new(9876, 15_000_000); + shared.pairing.lock().required = false; + let now = Instant::now(); + repairable_session(&shared, "127.0.0.1:50000".parse().unwrap(), now); + let config = SharedConfigSource { + shared: &shared, + stream_epoch: 1, + pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), + }; + let notices = || { + let actions = shared.session.lock().tick(&config, true, now); + actions + .replies + .iter() + .filter_map(|(_, bytes)| match parse_control(bytes) { + Ok((_, ControlMessage::Heartbeat(heartbeat))) => Some(heartbeat.notice), + _ => None, + }) + .collect::>() + }; + *shared.capture_block.lock() = Some(CaptureBlock::UnsupportedOnThisGpu); + assert_eq!(notices(), [NOTICE_NONE], "a refusal that just began"); + *shared.gpu_refused_since.lock() = Some(Instant::now() - Duration::from_secs(3)); + assert_eq!(notices(), [NOTICE_UNSUPPORTED_GPU]); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + *shared.vdd_status.lock() = VddStatus::Attaching; + assert_eq!(notices(), [NOTICE_UNSUPPORTED_GPU], "still starting"); + for failed in [VddStatus::Failed, VddStatus::TaskFailed] { + *shared.vdd_status.lock() = failed; + assert_eq!( + notices(), + [NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU], + "{failed:?}" + ); + } + *shared.capture_block.lock() = None; + assert_eq!(notices(), [NOTICE_NONE]); + assert_eq!(*shared.gpu_refused_since.lock(), None); + } + /// A NACK of a session that ended for liveness, while another frame is /// paced, is answered with the goodbye, as any of its late packets is. #[tokio::test] diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index 431ceeaf..5a50b9eb 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -2078,6 +2078,147 @@ fn a_session_ends_with_one_summary_naming_the_clients_nonce() { } } +/// A hybrid laptop where Windows won't share the screen from the graphics +/// card EternalMonitor runs on (capture fails with DXGI_ERROR_UNSUPPORTED, +/// faked here). The iPad got 10 s of black, then "no picture", while only +/// the PC's window said why. Now the session's heartbeats tell it after the +/// refusal lasts 2 s, the summary names a capture failure, and once Windows +/// lets capture work, the next session streams with no notice. +#[cfg(debug_assertions)] +#[test] +fn a_screen_refused_from_this_graphics_card_is_told_to_the_ipad() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_UNSUPPORTED_GPU}; + let _env = test_env(); + init_test_tracing(); + let _ = ffmpeg_next::init(); + + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_REFUSE_GPU", "1"); + std::env::remove_var("ETERNAL_DROP"); + + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".to_string()); + let gpu_info = GpuInfo::software_fallback(); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor = { + let (shared, supervisor_tx) = (shared.clone(), supervisor_tx.clone()); + std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + shared, + gpu_info, + supervisor_tx, + supervisor_rx, + ); + }) + }; + + // The session the iPad gives up on: heartbeats, no video. + const REFUSED: u32 = 0x6B0_0001; + let mut receiver = FakeReceiver::connect_as(listen_port, CAP_DECODE_H264, 0, 0, REFUSED); + let accepted = Instant::now(); + let mut datagram = [0u8; 2048]; + let mut told_after = None; + while told_after.is_none() { + assert!( + accepted.elapsed() < Duration::from_secs(8), + "no notice within 8 s of the session" + ); + receiver.maybe_report(0, 0); + let Ok((len, _)) = receiver.socket.recv_from(&mut datagram) else { + continue; + }; + match classify(&datagram[..len]) { + Classified::Media { .. } => panic!("video went out from a refused screen"), + Classified::Control(_) => { + if let Ok((_, ControlMessage::Heartbeat(hb))) = parse_control(&datagram[..len]) { + if hb.notice == NOTICE_UNSUPPORTED_GPU { + told_after = Some(accepted.elapsed()); + } else { + assert_eq!(hb.notice, NOTICE_NONE); + } + } + } + _ => {} + } + } + let told_after = told_after.unwrap(); + assert!( + told_after >= Duration::from_millis(1500), + "told after {told_after:?}: a refusal that lasts a moment says nothing" + ); + // The iPad's first-picture wait runs out. + receiver.send(&ControlMessage::Bye(ByeReason::Error)); + receiver.send(&ControlMessage::Bye(ByeReason::Error)); + let deadline = Instant::now() + Duration::from_secs(3); + while summaries_with_nonce(REFUSED).is_empty() { + assert!( + Instant::now() < deadline, + "no summary within 3 s of the goodbye" + ); + std::thread::sleep(Duration::from_millis(20)); + } + let line = summaries_with_nonce(REFUSED)[0].clone(); + assert!( + line.contains(" stage=accepted outcome=no_picture failure=capture_failed "), + "{line}" + ); + + // Windows lets capture work again (the user switched the card). + std::env::remove_var("ETERNAL_SYNTH_REFUSE_GPU"); + const WORKS: u32 = 0x6B0_0002; + let mut receiver = FakeReceiver::connect_as(listen_port, CAP_DECODE_H264, 0, 0, WORKS); + let deadline = Instant::now() + DEADLINE; + let mut video = false; + let mut notices_after_video = Vec::new(); + while notices_after_video.len() < 2 { + assert!( + Instant::now() < deadline, + "no video and heartbeats within {DEADLINE:?}" + ); + receiver.maybe_report(0, 0); + let Ok((len, _)) = receiver.socket.recv_from(&mut datagram) else { + continue; + }; + match classify(&datagram[..len]) { + Classified::Media { .. } => video = true, + Classified::Control(_) if video => { + if let Ok((_, ControlMessage::Heartbeat(hb))) = parse_control(&datagram[..len]) { + notices_after_video.push(hb.notice); + } + } + _ => {} + } + } + assert_eq!(notices_after_video, [NOTICE_NONE, NOTICE_NONE]); + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + + shared.stop(); + supervisor_tx + .send(SupervisorCommand::Shutdown) + .expect("send shutdown"); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = supervisor.join(); + let _ = done_tx.send(()); + }); + done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("supervisor must shut down within 5s"); + if let Some(path) = std::env::var_os("ETERNAL_SUMMARY_DUMP") { + use std::io::Write; + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .unwrap(); + file.write_all(format!("{line}\n").as_bytes()).unwrap(); + } +} + /// Repair is negotiated and exercised through the real encoder and transport. #[test] fn lossy_stream_recovers_and_adapts() { diff --git a/ios/EternalMonitor/App/ConnectView.swift b/ios/EternalMonitor/App/ConnectView.swift index c2fe400f..2db3c58e 100644 --- a/ios/EternalMonitor/App/ConnectView.swift +++ b/ios/EternalMonitor/App/ConnectView.swift @@ -571,10 +571,19 @@ struct ConnectView: View { Text(L("Connecting…")) .font(.app(18, .semibold, relativeTo: .headline)) .foregroundStyle(Theme.text) - Text(normalizedHostIP.isEmpty ? L("Waiting for your PC") : L("Reaching \(normalizedHostIP)")) - .font(.app(14, relativeTo: .callout)) - .foregroundStyle(Theme.textMuted) - .lineLimit(1) + // The PC answered and said why no picture comes yet. + if let card = connectionManager.pcGraphicsCard { + Text(card.short) + .font(.app(14, relativeTo: .callout)) + .foregroundStyle(Theme.text) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("connect.pcGraphicsCard") + } else { + Text(normalizedHostIP.isEmpty ? L("Waiting for your PC") : L("Reaching \(normalizedHostIP)")) + .font(.app(14, relativeTo: .callout)) + .foregroundStyle(Theme.textMuted) + .lineLimit(1) + } } Spacer(minLength: 0) } diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 4e197d61..c23b868c 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -122,6 +122,9 @@ final class ConnectionManager: ObservableObject { /// The PC has no display turned on, so there is nothing to capture and /// the picture is held. Its heartbeats say so until one comes back. @Published var pcNoDisplay = false + /// The PC's heartbeats say Windows won't let it capture from its + /// graphics card, and which screen. + @Published var pcGraphicsCard: PCGraphicsCard? /// Nonzero while an automatic reconnect cycle is running. @Published private(set) var reconnectAttempt = 0 @@ -552,6 +555,7 @@ final class ConnectionManager: ObservableObject { signalLost = false pcLocked = false pcNoDisplay = false + pcGraphicsCard = nil degradedSinceUs = nil prevCounters = FrameAssembler.Counters() diagnostics.removeAll() @@ -967,6 +971,12 @@ final class ConnectionManager: ObservableObject { self.pcNoDisplay = noDisplay self.record(.info, "ctrl", noDisplay ? "PC has no active display: holding the picture" : "PC display back") } + let card = PCGraphicsCard(notice: heartbeat.notice) + if card != self.pcGraphicsCard { + self.pcGraphicsCard = card + self.record(.info, "ctrl", card.map { "PC can't capture from its graphics card screen=\($0)" } + ?? "PC captures from its graphics card again") + } } } channel.onStreamConfig = { [weak self, weak assembler] config in @@ -1462,7 +1472,11 @@ final class ConnectionManager: ObservableObject { if handshakeDone { self.noPictureStreak.endedWithoutPicture() } // A PC that said it isn't streaming and then went quiet (it // closed, or slept) didn't answer: its answers were no picture. - let problem: ConnectionProblem = self.waitingForStream ? .noReply(usb: isUSB) : self.noPictureStreak.shown(.timeout( + // A PC that said why (its graphics card, which only the PC + // can switch) gets that card instead of "no picture". + let card = handshakeDone ? self.pcGraphicsCard : nil + let problem: ConnectionProblem = self.waitingForStream ? .noReply(usb: isUSB) + : card.map { .pcGraphicsCard($0) } ?? self.noPictureStreak.shown(.timeout( listenerReady: debug.listenerReadyPort != nil, helloAttempts: debug.helloAttempts, handshakeDone: handshakeDone, datagramsReceived: debug.datagramsReceived, isUSB: isUSB, host: debug.host), at: isUSB ? nil : "\(debug.host):\(debug.port)") @@ -1724,6 +1738,7 @@ final class ConnectionManager: ObservableObject { pictureRestoring = false pcLocked = false pcNoDisplay = false + pcGraphicsCard = nil degradedSinceUs = nil prevCounters = FrameAssembler.Counters() hostInfo = nil @@ -1749,7 +1764,8 @@ final class ConnectionManager: ObservableObject { return SessionSummary( end: end, isUSB: mediaLink?.isUSB ?? (transportMode == "USB"), sessionId: hostInfo?.sessionId, nonce: controlChannel?.helloNonce ?? 0, awaitingPairing: awaitingPairing, - datagramsReceived: debugState.datagramsReceived, firstFrame: firstFrameAt.map { $0 - summaryStartedAt }, + datagramsReceived: debugState.datagramsReceived, captureRefused: pcGraphicsCard != nil, + firstFrame: firstFrameAt.map { $0 - summaryStartedAt }, duration: now - summaryStartedAt) } diff --git a/ios/EternalMonitor/App/ConnectionProblem.swift b/ios/EternalMonitor/App/ConnectionProblem.swift index ace092a3..30ff42e5 100644 --- a/ios/EternalMonitor/App/ConnectionProblem.swift +++ b/ios/EternalMonitor/App/ConnectionProblem.swift @@ -44,6 +44,9 @@ enum ConnectionProblem: Equatable { /// The PC answered but isn't streaming. This iPad keeps asking and /// connects as soon as it is. case pcNotStreaming(StreamWait) + /// The PC answered, but Windows won't let it capture its screen from the + /// graphics card EternalMonitor runs on, as on some laptops with two. + case pcGraphicsCard(PCGraphicsCard) enum StreamWait: Equatable { /// Someone clicked Stop on the PC. @@ -132,6 +135,8 @@ enum ConnectionProblem: Equatable { case .pcNotStreaming(.stopped): return "Your PC isn't streaming" case .pcNotStreaming(.restarting): return "Your PC is restarting the stream…" case .pcNotStreaming(.failed): return "Your PC stopped streaming after repeated errors" + case .pcGraphicsCard(.screen): return "Your PC can't share its screen from its current graphics card" + case .pcGraphicsCard(.secondScreen): return "The second screen didn't start, and your PC can't share its main screen" } } @@ -198,6 +203,12 @@ enum ConnectionProblem: Equatable { case .pcNotStreaming(.failed(let token)): let start = "Click Start streaming in EternalMonitor on the PC to try again. This iPad connects as soon as it starts." return StreamWait.failedReason(token).map { "\($0). \(start)" } ?? start + case .pcGraphicsCard(.screen): + return "Windows won't let EternalMonitor share the screen from the graphics card it runs on. " + + "The EternalMonitor window on the PC says how to switch it. Then try again." + case .pcGraphicsCard(.secondScreen): + return "Windows won't let EternalMonitor share the PC's main screen from the graphics card it runs on, " + + "so it can't mirror it instead. The EternalMonitor window on the PC says how to switch the card. Then try again." } } @@ -266,7 +277,7 @@ enum ConnectionProblem: Equatable { static func afterReconnecting(lastAttempt: ConnectionProblem?) -> ConnectionProblem { switch lastAttempt { case .localNetworkBlocked, .noNetwork, .wifiOff, .badAddress, .usbFailed, .noPicture, .pcBusy, .pcEndedSession, - .pcEndedSessionOnError, .pcUpdating: + .pcEndedSessionOnError, .pcUpdating, .pcGraphicsCard: return lastAttempt ?? .reconnectFailed default: return .reconnectFailed @@ -438,3 +449,31 @@ enum USBStatusText { } } } + +/// Which screen Windows won't let the PC capture from its graphics card, as +/// its heartbeats say. +enum PCGraphicsCard: Equatable { + /// The screen it mirrors. + case screen + /// The main screen, in Second screen mode after the second screen + /// failed to start. + case secondScreen + + /// From a heartbeat's notice; nil for any other. + init?(notice: UInt8) { + switch notice { + case Heartbeat.noticeUnsupportedGPU: self = .screen + case Heartbeat.noticeSecondScreenUnsupportedGPU: self = .secondScreen + default: return nil + } + } + + /// What the iPad says while it waits, and over a held picture. + var short: String { + switch self { + case .screen: return L("Your PC can't share its screen from its current graphics card. Check the PC.") + case .secondScreen: + return L("The second screen didn't start, and your PC can't share its main screen from its current graphics card. Check the PC.") + } + } +} diff --git a/ios/EternalMonitor/App/DisplayView.swift b/ios/EternalMonitor/App/DisplayView.swift index bb557073..d37f939c 100644 --- a/ios/EternalMonitor/App/DisplayView.swift +++ b/ios/EternalMonitor/App/DisplayView.swift @@ -59,6 +59,10 @@ struct DisplayView: View { noDisplayPill .transition(.opacity) .allowsHitTesting(false) + } else if let card = connectionManager.pcGraphicsCard { + graphicsCardPill(card) + .transition(.opacity) + .allowsHitTesting(false) } else if connectionManager.pictureRestoring { restoringPill .transition(.opacity) @@ -123,6 +127,7 @@ struct DisplayView: View { .animation(.easeInOut(duration: 0.25), value: connectionManager.pictureRestoring) .animation(.easeInOut(duration: 0.25), value: connectionManager.pcLocked) .animation(.easeInOut(duration: 0.25), value: connectionManager.pcNoDisplay) + .animation(.easeInOut(duration: 0.25), value: connectionManager.pcGraphicsCard) .animation(reduceMotion ? nil : .easeInOut(duration: 0.35), value: appeared) .onAppear { appeared = true @@ -460,6 +465,29 @@ struct DisplayView: View { .accessibilityIdentifier("display.pcNoDisplay") } + /// Over the held picture while Windows won't let the PC capture from its + /// graphics card, until it can again. + private func graphicsCardPill(_ card: PCGraphicsCard) -> some View { + HStack(spacing: 10) { + Image(systemName: "display") + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(Theme.text) + .accessibilityHidden(true) + Text(card.short) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + .multilineTextAlignment(.center) + } + .padding(.horizontal, 18) + .padding(.vertical, 8) + .frame(minHeight: 44) + .background(glass(Capsule())) + .padding(.horizontal, 24) + .accessibilityElement(children: .ignore) + .accessibilityLabel(card.short) + .accessibilityIdentifier("display.pcGraphicsCard") + } + /// The PC is still connected but a lost frame left the picture waiting /// for a fresh one; usually gone in well under a second. private var restoringPill: some View { diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index e2af7146..b4760816 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -73,6 +73,8 @@ struct SessionSummary: Equatable { /// The PC asked for the pairing code and the attempt ended before a session. var awaitingPairing = false var datagramsReceived = 0 + /// The PC said it can't capture its screen from its graphics card. + var captureRefused = false /// Seconds from the start to the first decoded frame. var firstFrame: TimeInterval? var duration: TimeInterval @@ -115,6 +117,7 @@ struct SessionSummary: Equatable { private var failure: String { switch end { case .connectTimeout where sessionId == nil, .pcMoved: return "no_reply" + case .connectTimeout where captureRefused: return "capture_failed" case .connectTimeout: return datagramsReceived == 0 ? "no_video" : "no_frame" case .noVideoPath: return datagramsReceived == 0 ? "no_video" : "no_frame" case .pcStoppedResponding: return "signal_lost" diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index 4b87b593..0b3b1c31 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -358,6 +358,12 @@ struct Heartbeat: Equatable { static let noticePCLocked: UInt8 = 1 /// No display is on at the PC (a closed lid, screens off or unplugged). static let noticeNoDisplay: UInt8 = 2 + /// Windows won't let EternalMonitor capture the PC's screen from the + /// graphics card it runs on. The PC's window says how to switch. + static let noticeUnsupportedGPU: UInt8 = 3 + /// The same in Second screen mode, after the second screen failed to + /// start: the main screen it fell back to can't be captured either. + static let noticeSecondScreenUnsupportedGPU: UInt8 = 4 /// The video frames the PC sent this session, from a PC that offered /// `HelloAck.hostCapMediaSize`; nil from others. Appended after /// `notice`, which is then always written. diff --git a/ios/EternalMonitor/Resources/Localizable.xcstrings b/ios/EternalMonitor/Resources/Localizable.xcstrings index 7c85e091..874ab5e2 100644 --- a/ios/EternalMonitor/Resources/Localizable.xcstrings +++ b/ios/EternalMonitor/Resources/Localizable.xcstrings @@ -8188,6 +8188,18 @@ } } }, + "The second screen didn't start, and your PC can't share its main screen from its current graphics card. Check the PC." : { + "comment" : "On the connecting card and over the held picture, in Second screen mode after the second screen failed to start: Windows won't let the PC app capture its main screen from its graphics card either, so it can't mirror it.", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "The second screen didn't start, and your PC can't share its main screen from its current graphics card. Check the PC." + } + } + } + }, "Try again in %lld s" : { "comment" : "Button countdown after wrong codes; %lld is seconds", "extractionState" : "manual", @@ -9187,6 +9199,18 @@ } } }, + "Your PC can't share its screen from its current graphics card. Check the PC." : { + "comment" : "On the connecting card and over the held picture: Windows won't let the PC app capture its screen from the graphics card it runs on (laptops with two graphics cards). The PC app's window says how to switch.", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Your PC can't share its screen from its current graphics card. Check the PC." + } + } + } + }, "Your PC has no active display. Turn on a screen to continue." : { "comment" : "Over the held picture while no display is on at the PC: a closed laptop lid, or screens that are off or unplugged", "extractionState" : "manual", diff --git a/ios/EternalMonitorTests/LocalizationTests.swift b/ios/EternalMonitorTests/LocalizationTests.swift index 0db3ce8d..15cebc0a 100644 --- a/ios/EternalMonitorTests/LocalizationTests.swift +++ b/ios/EternalMonitorTests/LocalizationTests.swift @@ -22,7 +22,10 @@ final class LocalizationTests: XCTestCase { /// English strings whose translations come in a separate pass. Until /// then every language shows them in English. - private let awaitingTranslation: Set = [] + private let awaitingTranslation: Set = [ + "Your PC can't share its screen from its current graphics card. Check the PC.", + "The second screen didn't start, and your PC can't share its main screen from its current graphics card. Check the PC.", + ] /// Every language has exactly English's strings, none empty, each with /// the same placeholders. diff --git a/ios/EternalMonitorTests/NoPictureTimeoutTests.swift b/ios/EternalMonitorTests/NoPictureTimeoutTests.swift index a205845a..aee60587 100644 --- a/ios/EternalMonitorTests/NoPictureTimeoutTests.swift +++ b/ios/EternalMonitorTests/NoPictureTimeoutTests.swift @@ -73,6 +73,47 @@ final class NoPictureTimeoutTests: XCTestCase { XCTAssertNil(logged(manager, "Disconnect state=connecting reason=userDisconnect")) } + /// The hybrid laptops: the PC accepted, then its heartbeats said + /// Windows won't let it capture from its graphics card. The iPad showed + /// "Connecting…" for 10 s, then "no picture". Now it says why while it + /// waits, its card says what to do on the PC when the wait runs out, and + /// its summary names a capture failure, as the PC's does. + func testAPCThatCantCaptureFromItsGraphicsCardSaysWhyInsteadOfNoPicture() async throws { + let host = FakeUDPHost() + var timing = fast + timing.connectTimeoutSeconds = 3 + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + host.sendHeartbeat(notice: Heartbeat.noticeUnsupportedGPU) + try await until("the PC says why") { manager.pcGraphicsCard == .screen } + XCTAssertEqual(manager.state, .connecting, "still waiting for a picture") + try await until("timed out", seconds: 15) { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .pcGraphicsCard(.screen)) + XCTAssertEqual(manager.problem?.actions, [.retry]) + let summary = logged(manager, "session_summary v=1 role=ipad") ?? "" + XCTAssertTrue(summary.contains(" stage=accepted outcome=no_picture failure=capture_failed "), summary) + XCTAssertNil(manager.pcGraphicsCard, "the next attempt starts without it") + } + + /// The same in Second screen mode, after the second screen failed. + func testThePCsSecondScreenModeKeepsItsOwnCard() async throws { + let host = FakeUDPHost() + var timing = fast + timing.connectTimeoutSeconds = 3 + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + host.sendHeartbeat(notice: Heartbeat.noticeSecondScreenUnsupportedGPU) + try await until("the PC says why") { manager.pcGraphicsCard == .secondScreen } + try await until("timed out", seconds: 15) { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .pcGraphicsCard(.secondScreen)) + } + func testTheFirstFrameLineNamesTheSessionAndTheWait() async throws { let host = FakeUDPHost() let manager = try await streaming(host) diff --git a/ios/EternalMonitorTests/SessionSummaryTests.swift b/ios/EternalMonitorTests/SessionSummaryTests.swift index 0ae21b58..cf04ba93 100644 --- a/ios/EternalMonitorTests/SessionSummaryTests.swift +++ b/ios/EternalMonitorTests/SessionSummaryTests.swift @@ -107,6 +107,19 @@ final class SessionSummaryTests: XCTestCase { XCTAssertTrue(summary(.cableUnplugged, usb: true).line.contains(" transport=usb ")) } + /// The PC said Windows won't let it capture from its graphics card, and + /// the wait for a picture ran out: a capture failure, as the PC's own + /// summary says, not lost video. A picture that came later keeps its tokens. + func testATimeoutAfterThePCSaidItCantCaptureIsACaptureFailure() { + var refused = summary(.connectTimeout, duration: 10) + refused.captureRefused = true + XCTAssertEqual(tokens(refused.line), ["accepted", "no_picture", "capture_failed", "timeout"]) + Self.assertContract(refused.line) + var streamed = summary(.pcStoppedResponding, firstFrame: 0.3, duration: 90) + streamed.captureRefused = true + XCTAssertEqual(tokens(streamed.line), ["steady", "streamed", "signal_lost", "liveness"]) + } + func testOnlyTheUsersOwnActionSaysUserDisconnectToThePC() { XCTAssertEqual(AttemptEnd.userDisconnect.bye, .userDisconnect) XCTAssertEqual(AttemptEnd.usbTurnedOff.bye, .userDisconnect) diff --git a/ios/EternalMonitorTests/StreamRecoveryTests.swift b/ios/EternalMonitorTests/StreamRecoveryTests.swift index c0317ecc..e4d491ba 100644 --- a/ios/EternalMonitorTests/StreamRecoveryTests.swift +++ b/ios/EternalMonitorTests/StreamRecoveryTests.swift @@ -133,6 +133,30 @@ final class StreamRecoveryTests: XCTestCase { XCTAssertFalse(manager.pcNoDisplay, "the next session starts without it") } + /// Windows moved the PC's screen to the graphics card EternalMonitor + /// doesn't run on, mid-stream. Its heartbeats say so, which screen, and + /// the iPad says so over the held picture until the PC captures again. + /// A locked PC's notice still comes first. + func testAPCThatCantCaptureFromItsGraphicsCardIsShownOverTheHeldPicture() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + XCTAssertNil(manager.pcGraphicsCard) + host.sendHeartbeat(notice: Heartbeat.noticeUnsupportedGPU) + try await until("the PC says why") { manager.pcGraphicsCard == .screen } + XCTAssertEqual(manager.state, .connected) + XCTAssertFalse(manager.pcLocked || manager.pcNoDisplay) + host.sendHeartbeat(notice: Heartbeat.noticeSecondScreenUnsupportedGPU) + try await until("the second screen failed too") { manager.pcGraphicsCard == .secondScreen } + host.sendHeartbeat(notice: Heartbeat.noticePCLocked) + try await until("locked instead") { manager.pcLocked && manager.pcGraphicsCard == nil } + host.sendHeartbeat() + try await until("the PC captures again") { !manager.pcLocked && manager.pcGraphicsCard == nil } + host.sendHeartbeat(notice: Heartbeat.noticeUnsupportedGPU) + try await until("again") { manager.pcGraphicsCard == .screen } + manager.cancel() + XCTAssertNil(manager.pcGraphicsCard, "the next session starts without it") + } + func testContiguousNumberingRevealsALostReopenKeyframe() async throws { // The PC reopened its encoder and its new keyframe (seq 2) was lost; // the new encoder's first P-frame continues the codec's count. A PC diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index bcd7351b..1a04fdfe 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -286,6 +286,13 @@ pub const NOTICE_PC_LOCKED: u8 = 1; /// No display is attached to the PC's desktop (a closed lid, screens off or /// unplugged), so there is nothing to capture until one comes back. pub const NOTICE_NO_DISPLAY: u8 = 2; +/// Windows won't let EternalMonitor capture the screen from the graphics +/// card it runs on, as on a laptop whose screen is wired to its other card. +/// The PC's window says how to switch; nothing changes until someone does. +pub const NOTICE_UNSUPPORTED_GPU: u8 = 3; +/// [`NOTICE_UNSUPPORTED_GPU`] in Second screen mode, once the second screen +/// failed to start: the main screen it fell back to can't be captured either. +pub const NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU: u8 = 4; /// Host → client, every heartbeat interval while a session is active. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1612,7 +1619,13 @@ mod tests { // A notice is one byte more. Such a body, cut back to the old length // as an older iPad reads it, is the plain heartbeat; an unknown value // reaches the iPad as it is, for it to ignore. - for notice in [NOTICE_PC_LOCKED, NOTICE_NO_DISPLAY, 0xC8] { + for notice in [ + NOTICE_PC_LOCKED, + NOTICE_NO_DISPLAY, + NOTICE_UNSUPPORTED_GPU, + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, + 0xC8, + ] { let datagram = encode_control(7, 1, &heartbeat(notice)); assert_eq!(datagram.len(), plain.len() + 1); assert_eq!(datagram[..plain.len()][8..], plain[8..]); From 4a800ec96174e8f5c3a50eff928815cb81fa3e1e Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:22:38 -0500 Subject: [PATCH 175/226] Hold a restarting session only until the limit, even once the stream resumes The limit was checked only while the PC waited to restart. The next generation could start and then take its time to send a frame, and its own tick kept the frozen picture going. The stream's tick now ends the session at the limit too, counted from the last frame, until a frame goes out. The integration test sets its shortened limit under the harness's lock. --- host/src/transport/session.rs | 88 +++++++++++++++++++++++++---------- host/tests/hello_standby.rs | 12 ++--- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 7860b096..4a32e358 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -330,8 +330,11 @@ struct ActiveSession { /// while the PC restarts carry again. last_stream_config: Option, /// Since when it has been kept alive through restarts without a frame - /// ([`RESTART_HOLD_LIMIT`]). + /// ([`RESTART_HOLD_LIMIT`]): its last frame, or the restart if none. + /// Cleared by the next frame. restart_held_since: Option, + /// When the last frame was let through to it. + last_frame_at: Option, } /// What a client sent last, for the liveness log lines. @@ -588,6 +591,7 @@ impl Session { return true; }; session.restart_held_since = None; + session.last_frame_at = Some(now); if session.answered { session.frames_sent = session.frames_sent.saturating_add(1); return true; @@ -1194,6 +1198,7 @@ impl Session { withheld: false, last_stream_config: None, restart_held_since: None, + last_frame_at: None, }); // Only a new pairing carries the token. iPads keep the token they @@ -1436,6 +1441,27 @@ impl Session { return actions; } + // Kept alive through a restart and no frame since: past the limit, + // even once the next generation runs, tell the iPad to connect again. + if let Some(since) = session + .restart_held_since + .filter(|since| now.saturating_duration_since(*since) >= restart_hold_limit()) + { + info!( + session_id = session.session_id, + held_ms = now.saturating_duration_since(since).as_millis() as u64, + "The stream is taking too long to restart: telling the iPad to connect again" + ); + actions.ended = self.end_saying( + ByeReason::Error, + EndReason::Failed(StageFailure::Other), + now, + ); + actions.goodbye = self.take_goodbye(); + actions.client_lost = true; + return actions; + } + if now >= session.liveness_deadline { // A client that had a picture may only have lost its uplink: it // keeps the session, on hold, until the hold limit. One that @@ -1504,10 +1530,11 @@ impl Session { /// standby responder holds the port. A session kept across the restart /// stays alive: its iPad gets heartbeats as from a running stream, so it /// holds its last picture instead of losing the signal, and its silence - /// still expires it. Only an iPad that has answered gets them. After - /// [`RESTART_HOLD_LIMIT`] without a frame, the session ends with the - /// goodbye that tells its iPad to connect again. Does nothing unless the - /// PC is restarting. + /// still expires it. Only an iPad that has answered gets them. Once + /// [`RESTART_HOLD_LIMIT`] has passed since its last frame, the session + /// ends with the goodbye that tells its iPad to connect again, on this + /// tick or the next stream's ([`Self::tick`]) until a frame goes out. + /// Does nothing unless the PC is restarting. pub fn restart_tick(&mut self, config: &impl ConfigSource, now: Instant) -> Actions { if self.not_streaming != Some(NotStreaming::Restarting) { return Actions::default(); @@ -1515,25 +1542,9 @@ impl Session { let Some(session) = self.active.as_mut() else { return self.tick(config, false, now); }; - let since = *session.restart_held_since.get_or_insert(now); - if now.saturating_duration_since(since) >= restart_hold_limit() { - info!( - session_id = session.session_id, - held_ms = now.saturating_duration_since(since).as_millis() as u64, - "The stream is taking too long to restart: telling the iPad to connect again" - ); - let ended = self.end_saying( - ByeReason::Error, - EndReason::Failed(StageFailure::Other), - now, - ); - return Actions { - goodbye: self.take_goodbye(), - ended, - client_lost: true, - ..Actions::default() - }; - } + // Counted from the last frame, so a restart ladder keeps counting. + let last_frame = session.last_frame_at.unwrap_or(now); + session.restart_held_since.get_or_insert(last_frame); let answered = session.answered; self.tick(config, answered, now) } @@ -5726,4 +5737,33 @@ mod tests { HelloStatus::Restarting ); } + + /// The supervisor clears the restart before the new generation sends a + /// frame. A slow start still ends at the hold limit, counted from the + /// last frame, on the stream's own tick. + #[test] + fn the_hold_limit_holds_after_the_stream_resumes_without_a_frame() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let (mut session, id) = restarting_session(peer, now); + assert!(session.may_send_media(id, 1000, now)); + session.restart_tick(&RestartConfig, now + Duration::from_secs(1)); + // The next generation starts; it hasn't sent a frame yet. + session.set_not_streaming(None); + session.transport_resumed(now + Duration::from_secs(2), false); + let before = now + restart_hold_limit() - Duration::from_millis(1); + session.handle_control_authed(peer, report(10), &TestConfig, before); + assert!(!session.tick(&TestConfig, true, before).client_lost); + let at = now + restart_hold_limit(); + session.handle_control_authed(peer, report(10), &TestConfig, at); + let actions = session.tick(&TestConfig, true, at); + assert!( + actions.client_lost && !session.is_active(), + "still held past the limit" + ); + assert!(matches!( + eternal_wire::v2::control::parse_control(&actions.goodbye.expect("a goodbye").1), + Ok((_, ControlMessage::Bye(ByeReason::Error))) + )); + } } diff --git a/host/tests/hello_standby.rs b/host/tests/hello_standby.rs index 43a24709..d8b126af 100644 --- a/host/tests/hello_standby.rs +++ b/host/tests/hello_standby.rs @@ -1088,12 +1088,12 @@ fn a_restart_longer_than_the_ipads_liveness_keeps_its_session() { /// is restarting. #[test] fn a_restart_past_the_hold_limit_tells_the_ipad_to_connect_again() { - eternal_host::transport::session::TEST_RESTART_HOLD_MS.store(2_000, Ordering::SeqCst); - let host = Host::start_with( - GpuInfo::software_fallback(), - 10_000, - fail_the_encoder_once(240), - ); + // Set under the lock Host takes: another test's Host resets it on drop. + let fail = fail_the_encoder_once(240); + let host = Host::start_with(GpuInfo::software_fallback(), 10_000, |shared| { + eternal_host::transport::session::TEST_RESTART_HOLD_MS.store(2_000, Ordering::SeqCst); + fail(shared); + }); let ipad = FakeIpad::new(&host, 1, NEW); let id = ipad.connect(1).session_id; let heard = ipad.stream(id, Duration::from_secs(60), |heard| { From 6b0eda17f1865fcfb322ddb642d177702cb28727 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:27:20 -0500 Subject: [PATCH 176/226] Stop answering the cable while the PC turns the iPad down Over USB the PC dials the iPad again about a second after each closed link. When it turned the iPad down as Busy, or as too old, the iPad took every dial and was turned down again, for as long as the cable stayed in. Now the iPad stops answering the cable after such a refusal and keeps the card for that status up. After Busy it waits 30 s, doubling with each Busy in a row up to 5 minutes, then answers again; a session resets the wait. After a version refusal, or a v0.1.x PC, it waits for Try again or Connect USB. Wi-Fi keeps its bounded reconnect cycle. Each attempt's summary names the refusal: failure=pc_busy or failure=old_pc instead of failure=other. --- ios/EternalMonitor/App/CableHold.swift | 28 +++ .../App/ConnectionManager.swift | 65 ++++++- ios/EternalMonitor/App/SessionSummary.swift | 9 +- .../SessionSummaryTests.swift | 8 +- .../USBLinkFailureTests.swift | 164 ++++++++++++++++++ 5 files changed, 267 insertions(+), 7 deletions(-) create mode 100644 ios/EternalMonitor/App/CableHold.swift diff --git a/ios/EternalMonitor/App/CableHold.swift b/ios/EternalMonitor/App/CableHold.swift new file mode 100644 index 00000000..aa2d2b81 --- /dev/null +++ b/ios/EternalMonitor/App/CableHold.swift @@ -0,0 +1,28 @@ +import Foundation + +/// A PC that turns this iPad down over the cable dials it again about a +/// second after each refusal, so the refusal would repeat for as long as the +/// cable stays in. Until something changes, the iPad stops answering the +/// cable: after Busy for a wait that grows with each Busy in a row, and after +/// a version refusal until the user taps Try again or Connect USB. +struct CableHold: Equatable { + enum Reason: Equatable { + case busy + case version + } + + let reason: Reason + /// When the iPad answers the cable again; nil waits for the user. + let until: Date? + + /// The wait after `count` Busy refusals in a row: 30 s, doubling each + /// time, at most 5 minutes. + static func busySeconds(_ count: Int) -> Double { + min(30 * pow(2, Double(min(max(count, 1), 16) - 1)), 300) + } + + /// Whether the cable still goes unanswered at `now`. + func holds(at now: Date) -> Bool { + until.map { now < $0 } ?? true + } +} diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 4e197d61..aff1c2ed 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -178,6 +178,11 @@ final class ConnectionManager: ObservableObject { /// Disconnect, coming back to the app keeps it stopped until Try again /// or Connect USB. private var usbStoppedForNoPicture = false + /// The PC turned the cable down: it isn't answered until the hold ends + /// (see `CableHold`). Busy refusals in a row set the wait. + private var cableHold: CableHold? + private var cableBusyInARow = 0 + private var cableHoldTask: Task? private let currentAttempt = OSAllocatedUnfairLock(initialState: nil) private var frameAssembler: FrameAssembler? private var controlChannel: ControlChannel? @@ -249,6 +254,9 @@ final class ConnectionManager: ObservableObject { /// How long video the PC says it sent may go missing before the /// next no-video step (see `NoVideoFallback`). var noVideoStepSeconds = 1.5 + /// How long the cable goes unanswered after the given number of + /// Busy refusals in a row (see `CableHold`). + var cableBusyHoldSeconds: (Int) -> Double = CableHold.busySeconds } var timing: Timing /// The Wi-Fi link to a host's port. Tests pass a fake one, or replace it @@ -348,10 +356,46 @@ final class ConnectionManager: ObservableObject { func resumeUSBConnections() { noPictureStreak.restartCount() usbStoppedForNoPicture = false + endCableHold() + cableBusyInARow = 0 usbPaused = false refreshUSBAvailability() } + /// The PC turned this iPad down over the cable. Stop answering the cable + /// for the hold's wait, or until the user acts after a version refusal, + /// instead of taking the PC's next dial a second later. + private func holdCable(_ reason: CableHold.Reason) { + var wait: Double? + if reason == .busy { + cableBusyInARow += 1 + wait = timing.cableBusyHoldSeconds(cableBusyInARow) + } + let hold = CableHold(reason: reason, until: wait.map { now().addingTimeInterval($0) }) + endCableHold() + cableHold = hold + record(.info, "usb", "The PC turned the cable down (\(reason)); " + (wait.map { "answering it again in \(Int($0.rounded())) s" } + ?? "answering it again after Try again or Connect USB")) + usbPaused = true + usbListener?.stop() + usbListener = nil + usbStatus = "USB: disconnected" + guard let wait else { return } + cableHoldTask = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(wait)) + guard let self, !Task.isCancelled, self.cableHold == hold else { return } + self.endCableHold() + self.usbPaused = self.usbStoppedForNoPicture + self.refreshUSBAvailability() + } + } + + private func endCableHold() { + cableHoldTask?.cancel() + cableHoldTask = nil + cableHold = nil + } + // MARK: - Frame slot (thread-safe single-slot) var latestFrame: CVPixelBuffer? { @@ -823,7 +867,10 @@ final class ConnectionManager: ObservableObject { do { try self.pairingStore.save(token: info.authToken, hostName: info.hostName, address: pairingAddress) } catch { self.record(.warning, "pairing", "Pairing was not saved on this iPad. Pair again next time.") } } - if isUSB { self.usbStatus = "USB: connected to \(info.hostName)" } + if isUSB { + self.usbStatus = "USB: connected to \(info.hostName)" + self.cableBusyInARow = 0 + } self.hostStreamConfig = info.streamConfig self.record( .info, "ctrl", @@ -874,7 +921,16 @@ final class ConnectionManager: ObservableObject { self.connectionError = "Your PC turned down the connection." } self.record(.error, "ctrl", self.connectionError ?? self.problem?.title ?? "rejected") - self.disconnect(.pcRefused) + switch status { + case .busy: self.disconnect(.pcBusy) + case .versionUnsupported where mismatch == .pcOlder: self.disconnect(.oldPC) + default: self.disconnect(.pcRefused) + } + // Over the cable the PC dials again a second later, and would + // be turned down again: wait for a change instead. + if isUSB, status == .busy || status == .versionUnsupported { + self.holdCable(status == .busy ? .busy : .version) + } } } channel.onPCNotStreaming = { [weak self] status, reason in @@ -904,6 +960,7 @@ final class ConnectionManager: ObservableObject { self.connectionError = VersionMismatch.pcLegacy.message self.record(.error, "ctrl", "Legacy v1 host detected") self.disconnect(.legacyPC) + if isUSB { self.holdCable(.version) } } else { self.record(.warning, "ctrl", "HELLO2 handshake got no reply yet") if self.leaveRememberedAddress(.pcMoved, result: "no_reply") { @@ -1641,7 +1698,9 @@ final class ConnectionManager: ObservableObject { endBackgroundGoodbye() isForeground = true recheckAppleDevicesSighting() - usbPaused = usbStoppedForNoPicture + // A hold whose wait ran out in the background ends here. + if let hold = cableHold, !hold.holds(at: now()) { endCableHold() } + usbPaused = usbStoppedForNoPicture || cableHold != nil refreshUSBAvailability() guard resumeOnForeground else { return } resumeOnForeground = false diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index e2af7146..ce7a74d1 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -13,6 +13,11 @@ enum AttemptEnd: Equatable { case pcGoodbye /// The PC turned the connection down. case pcRefused + /// The PC turned it down as Busy: it streams to another device. + case pcBusy + /// The PC turned it down as too old for this app. + case oldPC + /// A v0.1.x PC, too old to answer HELLO2. case legacyPC /// The connect watchdog ran out before a picture. case connectTimeout @@ -120,7 +125,9 @@ struct SessionSummary: Equatable { case .pcStoppedResponding: return "signal_lost" case .cableUnplugged, .usbLinkFailed, .closedBeforePicture: return "usb_lost" case .noNetwork: return "no_network" - case .pcRefused, .legacyPC, .linkFailed, .receiverFailed, .pictureStalled, .otherPC: return "other" + case .pcBusy: return "pc_busy" + case .oldPC, .legacyPC: return "old_pc" + case .pcRefused, .linkFailed, .receiverFailed, .pictureStalled, .otherPC: return "other" default: return "none" } } diff --git a/ios/EternalMonitorTests/SessionSummaryTests.swift b/ios/EternalMonitorTests/SessionSummaryTests.swift index 0ae21b58..de020d40 100644 --- a/ios/EternalMonitorTests/SessionSummaryTests.swift +++ b/ios/EternalMonitorTests/SessionSummaryTests.swift @@ -14,7 +14,7 @@ final class SessionSummaryTests: XCTestCase { + #" transport=(usb|wifi) mode=(mirror|second_screen|unknown)"# + #" stage=(started|hello_sent|accepted|video_sent|first_frame|steady)"# + #" outcome=(streamed|no_reply|no_picture|pairing|user_ended|app_ended|error)"# - + #" failure=(none|no_reply|no_video|no_frame|vdd_enable_failed|vdd_attach_timeout|vdd_not_attached|capture_failed|encoder_failed|usb_lost|signal_lost|no_network|other)"# + + #" failure=(none|no_reply|no_video|no_frame|vdd_enable_failed|vdd_attach_timeout|vdd_not_attached|capture_failed|encoder_failed|usb_lost|signal_lost|no_network|pc_busy|old_pc|other)"# + #" fallback=(none|attach_timeout|enable_failed|no_driver|no_frames|duplication_failed|remembered|other)"# + #" end=(goodbye|liveness|superseded|shutdown|timeout|user|background|error)"# + #" ttff_ms=(0|[1-9][0-9]{0,5}|-) duration_ms=(0|[1-9][0-9]{0,9})"# @@ -78,7 +78,9 @@ final class SessionSummaryTests: XCTestCase { (summary(.pcGoodbye), ["accepted", "app_ended", "none", "goodbye"]), (summary(.pcRefused, session: nil), ["hello_sent", "error", "other", "error"]), (pairing, ["hello_sent", "pairing", "none", "user"]), - (summary(.legacyPC, session: nil), ["hello_sent", "error", "other", "error"]), + (summary(.pcBusy, session: nil), ["hello_sent", "error", "pc_busy", "error"]), + (summary(.oldPC, session: nil), ["hello_sent", "error", "old_pc", "error"]), + (summary(.legacyPC, session: nil), ["hello_sent", "error", "old_pc", "error"]), (summary(.connectTimeout, session: nil, duration: 10), ["hello_sent", "no_reply", "no_reply", "timeout"]), (summary(.pcMoved, session: nil, duration: 3), ["hello_sent", "no_reply", "no_reply", "superseded"]), (summary(.otherPC), ["accepted", "error", "other", "error"]), @@ -112,7 +114,7 @@ final class SessionSummaryTests: XCTestCase { XCTAssertEqual(AttemptEnd.usbTurnedOff.bye, .userDisconnect) XCTAssertEqual(AttemptEnd.appBackground.bye, .appBackground) XCTAssertEqual(AttemptEnd.superseded.bye, .superseded) - for end in [AttemptEnd.pcGoodbye, .pcRefused, .legacyPC, .connectTimeout, .pcStoppedResponding, + for end in [AttemptEnd.pcGoodbye, .pcRefused, .pcBusy, .oldPC, .legacyPC, .connectTimeout, .pcStoppedResponding, .pictureStalled, .cableUnplugged, .usbLinkFailed, .closedBeforePicture, .linkFailed, .noNetwork, .receiverFailed, .retriedWithCode] { XCTAssertEqual(end.bye, .error, "\(end)") diff --git a/ios/EternalMonitorTests/USBLinkFailureTests.swift b/ios/EternalMonitorTests/USBLinkFailureTests.swift index a8ffaf14..ad84980a 100644 --- a/ios/EternalMonitorTests/USBLinkFailureTests.swift +++ b/ios/EternalMonitorTests/USBLinkFailureTests.swift @@ -618,6 +618,170 @@ final class USBLinkFailureTests: XCTestCase { XCTAssertEqual(manager.reconnectAttempt, 0) } + // MARK: - A PC that turns the cable down + + /// Answers the iPad's HELLO2 on the cable's far end `fd` with `status`, + /// reporting protocol `version`. False after five quiet seconds. + nonisolated private static func refuseSession(on fd: Int32, status: HelloStatus, version: UInt8 = 2) -> Bool { + var timeout = timeval(tv_sec: 0, tv_usec: 300_000) + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, socklen_t(MemoryLayout.size)) + var framer = EMLinkFramer(expectsPreamble: false) + var buffer = [UInt8](repeating: 0, count: 4096) + let deadline = Date().addingTimeInterval(5) + while Date() < deadline { + let count = recv(fd, &buffer, buffer.count, 0) + guard count > 0, let datagrams = try? framer.append(Data(buffer.prefix(count))) else { continue } + for datagram in datagrams { + guard case .hello2(let hello) = Wire.parseControl(datagram)?.message, + let framed = try? EMLinkFramer.encode(FakeUDPHost.ack(nonce: hello.clientNonce, status: status, + acceptedVersion: version)) else { continue } + return framed.withUnsafeBytes { Darwin.send(fd, $0.baseAddress, $0.count, 0) } == framed.count + } + } + return false + } + + /// The newest cable's PC turns the iPad down with `status`. + private func refuse(_ status: HelloStatus, version: UInt8 = 2) async { + let fd = peer + let refused = await Task.detached { Self.refuseSession(on: fd, status: status, version: version) }.value + XCTAssertTrue(refused, "the iPad never sent HELLO2") + } + + /// The summary lines recorded so far. + private func summaries(_ manager: ConnectionManager) -> [String] { + manager.diagnostics.map(\.message).filter { $0.hasPrefix("session_summary ") } + } + + /// Offers the PC's dial every 100 ms until the iPad takes one; the + /// seconds from `since` until it did. + private func dialUntilTaken(_ manager: ConnectionManager, since: Date, cables: inout [FramedLink]) async throws -> Double { + let deadline = Date().addingTimeInterval(30) + while Date() < deadline { + cables.append(try await plugIn(manager)) + if manager.state == .connecting { return Date().timeIntervalSince(since) } + try await Task.sleep(nanoseconds: 100_000_000) + } + XCTFail("no dial was taken in 30 s") + return .infinity + } + + /// The wait after each Busy in a row: 30 s, doubling, at most 5 minutes. + func testTheBusyWaitDoublesUpToFiveMinutes() { + XCTAssertEqual((1...8).map(CableHold.busySeconds), [30, 60, 120, 240, 300, 300, 300, 300]) + XCTAssertEqual(CableHold.busySeconds(0), 30) + XCTAssertEqual(CableHold.busySeconds(.max), 300) + let at = Date(timeIntervalSince1970: 1_800_000_000) + let hold = CableHold(reason: .busy, until: at.addingTimeInterval(30)) + XCTAssertTrue(hold.holds(at: at.addingTimeInterval(29.9))) + XCTAssertFalse(hold.holds(at: at.addingTimeInterval(30))) + XCTAssertTrue(CableHold(reason: .version, until: nil).holds(at: .distantFuture)) + } + + /// A PC streaming to another device turns the cable down as Busy, then + /// dials it again a second later. The iPad shows why and doesn't take + /// the cable again until the wait ends, a longer wait after each Busy in + /// a row, back to the first after a session. Each attempt's summary + /// names the refusal. + func testABusyPCOnTheCableIsAnsweredOnlyAfterAGrowingWait() async throws { + let previousUSB = UserDefaults.standard.object(forKey: "allowUSB") + defer { UserDefaults.standard.set(previousUSB, forKey: "allowUSB") } + // The end of a wait must not open the app's real USB listener port. + UserDefaults.standard.set(false, forKey: "allowUSB") + var waits: [Int] = [] + var timing = answering + timing.cableBusyHoldSeconds = { count in waits.append(count); return Double(count) * 0.8 } + let (manager, first) = try await connectOverCable(timing: timing) + var cables = [first] + defer { cables.forEach { $0.stop() }; manager.cancel() } + + await refuse(.busy) + try await until("the refusal shows") { manager.state == .disconnected && manager.problem == .pcBusy } + let refusedAt = Date() + XCTAssertEqual(manager.usbStatus, "USB: disconnected") + XCTAssertTrue(summaries(manager).last?.contains(" stage=hello_sent outcome=error failure=pc_busy fallback=none end=error ") == true, + summaries(manager).joined(separator: "\n")) + // The PC dials again and again within the wait: none is taken. + for _ in 1...5 { + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .disconnected, "the cable isn't taken during the wait") + } + XCTAssertEqual(manager.problem, .pcBusy, "the reason stays up") + XCTAssertEqual(summaries(manager).count, 1, "one attempt") + + // The PC dials every 100 ms: the first taken comes once the wait + // ended, and the next Busy waits longer. + let firstWait = try await dialUntilTaken(manager, since: refusedAt, cables: &cables) + XCTAssertGreaterThanOrEqual(firstWait, 0.7, "the 0.8 s wait, less the time to see the refusal") + await refuse(.busy) + try await until("the second refusal") { manager.state == .disconnected } + let secondWait = try await dialUntilTaken(manager, since: Date(), cables: &cables) + XCTAssertGreaterThanOrEqual(secondWait, 1.5, "the 1.6 s wait") + + // A session resets the count: the next Busy waits the first wait. + await answer() + try await until("the session") { manager.hostInfo != nil } + manager.cancel() + manager.resumeUSBConnections() + cables.append(try await plugIn(manager)) + await refuse(.busy) + try await until("the third refusal") { manager.state == .disconnected } + XCTAssertEqual(waits, [1, 2, 1]) + XCTAssertEqual(summaries(manager).filter { $0.contains(" failure=pc_busy ") }.count, 1, + "the list holds the latest attempt's lines") + } + + /// Try again during the wait takes the PC's next dial at once. + func testTryAgainEndsTheBusyWait() async throws { + let previousUSB = UserDefaults.standard.object(forKey: "allowUSB") + defer { UserDefaults.standard.set(previousUSB, forKey: "allowUSB") } + UserDefaults.standard.set(false, forKey: "allowUSB") + let (manager, first) = try await connectOverCable(timing: answering) + var cables = [first] + defer { cables.forEach { $0.stop() }; manager.cancel() } + await refuse(.busy) + try await until("the refusal shows") { manager.problem == .pcBusy } + XCTAssertEqual(manager.problem?.actions, [.retry]) + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .disconnected) + manager.retry() + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .connecting, "Try again takes the cable at once") + } + + /// A PC too old for this app, or one so old it doesn't answer HELLO2, + /// can't connect until someone updates it: the iPad shows that, and + /// takes the cable again only after Try again or Connect USB, not when + /// it comes back to the foreground. Each names an old PC in its summary. + func testAnOldPCOnTheCableWaitsForTheUser() async throws { + let previousUSB = UserDefaults.standard.object(forKey: "allowUSB") + defer { UserDefaults.standard.set(previousUSB, forKey: "allowUSB") } + UserDefaults.standard.set(false, forKey: "allowUSB") + let (manager, first) = try await connectOverCable(timing: answering) + var cables = [first] + defer { cables.forEach { $0.stop() }; manager.cancel() } + await refuse(.versionUnsupported, version: 1) + try await until("the refusal shows") { manager.state == .disconnected && manager.versionMismatch == .pcOlder } + XCTAssertEqual(manager.connectionError, VersionMismatch.pcOlder.message) + XCTAssertEqual(manager.usbStatus, "USB: disconnected", "Connect USB is offered") + XCTAssertTrue(summaries(manager).last?.contains(" stage=hello_sent outcome=error failure=old_pc fallback=none end=error ") == true, + summaries(manager).joined(separator: "\n")) + try await Task.sleep(nanoseconds: 1_000_000_000) + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .disconnected, "no wait ends it") + manager.handleAppBackgrounded() + manager.handleAppForegrounded() + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .disconnected, "coming back to the app doesn't end it") + + manager.resumeUSBConnections() + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .connecting, "Connect USB takes the cable again") + // The PC was updated: it connects as any other. + await answer() + try await until("the session") { manager.hostInfo != nil } + } + /// The PC still holds this iPad's cable session and answers the Wi-Fi /// fallback with Busy, then frees up. func testBusyDuringTheFallbackRetriesUntilThePCIsFree() async throws { From ed30efad5bc04cb359a07b0bb494e9ad84317341 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:33:00 -0500 Subject: [PATCH 177/226] Log why a USB tunnel closed, and how it opened Some USB sessions end 9.5-9.8 s after they start, often right after the iPad moved from Wi-Fi to the cable. The tunnel closes on both sides and neither side says why. No host timer explains it: a session that times out ends with a goodbye and keeps its tunnel, and the tunnel itself has no timer. That leaves the other end or Apple's device service closing the tunnel, or the host's device poll dropping its worker. None of these was logged. - "USB tunnel closed" now says which end closed it first (cause=peer_eof, peer_reset, read_error, bad_length, write_error or host), how long it was open and how long the iPad had been quiet. - "USB tunnel connected" says how long the service's Connect and the preamble took, how long the worker waited for the tunnel, and how many connects failed meanwhile, with the last error. - "USB device worker stopped" says when the device poll stops a worker and why: the device left the service's list, or the list failed. Either way that closes its tunnel. Tests: a paused-clock run of the real transport loop where the iPad says goodbye on Wi-Fi and takes over over USB, in either order, keeps its session and tunnel for 30 s; link tests for each close cause and for a tunnel with no lifetime limit; the connect bookkeeping against a fake device service. --- host/src/transport/link.rs | 197 ++++++++++++++++++++++++++-- host/src/transport/usb.rs | 216 +++++++++++++++++++++++++++++-- host/tests/liveness_hold.rs | 250 ++++++++++++++++++++++++++++++++++++ 3 files changed, 639 insertions(+), 24 deletions(-) diff --git a/host/src/transport/link.rs b/host/src/transport/link.rs index 221c397f..af1dc9ec 100644 --- a/host/src/transport/link.rs +++ b/host/src/transport/link.rs @@ -402,10 +402,65 @@ pub fn tunnel_congestion_drops() -> u64 { TUNNEL_CONGESTION.load(Ordering::Relaxed) } +/// What ended a tunnel first. Neither side used to say: a tunnel that +/// closed on both sides at once looked the same whoever closed it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CloseCause { + /// The other end (the iPad, or Apple's device service) ended the stream. + PeerEof, + /// Reading failed, e.g. a reset. + ReadFailed(io::ErrorKind), + /// The other end sent a length no datagram has. + BadLength, + /// Writing failed. + WriteFailed(io::ErrorKind), + /// This host let go of it: its device worker stopped, or the link was + /// dropped. + Host, +} + +impl CloseCause { + pub fn as_str(self) -> &'static str { + match self { + Self::PeerEof => "peer_eof", + Self::ReadFailed(io::ErrorKind::ConnectionReset) => "peer_reset", + Self::ReadFailed(_) => "read_error", + Self::BadLength => "bad_length", + Self::WriteFailed(_) => "write_error", + Self::Host => "host", + } + } + + fn of_read(error: &io::Error) -> Self { + match error.kind() { + io::ErrorKind::UnexpectedEof => Self::PeerEof, + io::ErrorKind::InvalidData => Self::BadLength, + kind => Self::ReadFailed(kind), + } + } +} + +/// How a closed tunnel ended, for its close line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CloseReport { + pub cause: CloseCause, + /// From the preamble to the close. + pub open: Duration, + /// From the last datagram read (or the preamble) to the close. + pub quiet: Duration, +} + +struct Lifetime { + opened: Instant, + last_read: Instant, + closed: Option<(CloseCause, Instant)>, +} + struct WriterState { queue: Mutex, ready: Notify, closed: watch::Sender, + lifetime: Mutex, /// Control datagrams fully written to the stream, in queue order. controls_written: watch::Sender, force_idr: Arc, @@ -426,6 +481,15 @@ impl WriterState { audio.map(|audio| (audio, false)) } + /// Closes the tunnel. Only the first cause counts. + fn close(&self, cause: CloseCause) { + self.lifetime + .lock() + .closed + .get_or_insert((cause, Instant::now())); + let _ = self.closed.send(true); + } + fn wrote(&self, control: bool) { if control { self.controls_written.send_modify(|written| *written += 1); @@ -490,37 +554,48 @@ impl FramedLink { let (mut reader, writer) = tokio::io::split(stream); let (inbound_tx, inbound_rx) = mpsc::channel(128); let (closed, mut read_closed) = watch::channel(false); + let opened = Instant::now(); let state = Arc::new(WriterState { queue: Mutex::new(Outbound::default()), ready: Notify::new(), closed, + lifetime: Mutex::new(Lifetime { + opened, + last_read: opened, + closed: None, + }), controls_written: watch::channel(0).0, force_idr, stats: Arc::new(FramedStats::default()), }); let read_state = Arc::clone(&state); let reading = tokio::spawn(async move { - loop { + let cause = loop { let packet = tokio::select! { biased; - _ = read_closed.changed() => break, + _ = read_closed.changed() => break CloseCause::Host, packet = read_datagram(&mut reader) => packet, }; - let Ok(bytes) = packet else { - break; + let bytes = match packet { + Ok(bytes) => bytes, + Err(error) => break CloseCause::of_read(&error), }; + read_state.lifetime.lock().last_read = Instant::now(); tokio::select! { biased; - _ = read_closed.changed() => break, - sent = inbound_tx.send((peer, bytes)) => if sent.is_err() { break; }, + _ = read_closed.changed() => break CloseCause::Host, + sent = inbound_tx.send((peer, bytes)) => if sent.is_err() { break CloseCause::Host; }, } - } - let _ = read_state.closed.send(true); + }; + read_state.close(cause); }); let write_state = Arc::clone(&state); let writing = tokio::spawn(async move { - let _ = write_loop(writer, &write_state).await; - let _ = write_state.closed.send(true); + let cause = match write_loop(writer, &write_state).await { + Ok(()) => CloseCause::Host, + Err(error) => CloseCause::WriteFailed(error.kind()), + }; + write_state.close(cause); }); Self { peer, @@ -545,7 +620,19 @@ impl FramedLink { } pub fn close_guard(&self) -> CloseGuard { - CloseGuard(self.state.closed.clone()) + CloseGuard(Arc::clone(&self.state)) + } + + /// What closed the tunnel, how long it was open and how long the other + /// end had been quiet. `None` while it is open. + pub fn close_report(&self) -> Option { + let lifetime = self.state.lifetime.lock(); + let (cause, at) = lifetime.closed?; + Some(CloseReport { + cause, + open: at.saturating_duration_since(lifetime.opened), + quiet: at.saturating_duration_since(lifetime.last_read), + }) } /// Waits until every control datagram queued so far has been written to @@ -585,10 +672,10 @@ impl FramedLink { /// A device worker owns this guard after handing the link to the transport. /// Removing the device or stopping its supervisor closes both tunnel halves. -pub struct CloseGuard(watch::Sender); +pub struct CloseGuard(Arc); impl Drop for CloseGuard { fn drop(&mut self) { - let _ = self.0.send(true); + self.0.close(CloseCause::Host); } } @@ -647,7 +734,7 @@ impl Link for FramedLink { impl Drop for FramedLink { fn drop(&mut self) { - let _ = self.state.closed.send(true); + self.state.close(CloseCause::Host); for task in &self.tasks { task.abort(); } @@ -1053,6 +1140,88 @@ mod tests { ); } + /// A tunnel says which end closed it first. In the field some closed on + /// both sides 9.5-9.8 s after they opened, and neither side said why. + #[tokio::test] + async fn a_closed_tunnel_reports_which_end_closed_it() { + let force_idr = || Arc::new(AtomicBool::new(false)); + let open = || async { + let (host, mut ipad) = tokio::io::duplex(1024); + let link = FramedLink::connect(host, PeerId::usb(7), force_idr()) + .await + .unwrap(); + read_preamble(&mut ipad).await.unwrap(); + (link, ipad) + }; + let closed = |link: &FramedLink| { + let mut closed = link.closed_signal(); + async move { + tokio::time::timeout(Duration::from_secs(1), closed.wait_for(|closed| *closed)) + .await + .unwrap() + .unwrap(); + } + }; + + let (link, ipad) = open().await; + assert_eq!(link.close_report(), None); + drop(ipad); + closed(&link).await; + assert_eq!(link.close_report().unwrap().cause, CloseCause::PeerEof); + + let (link, mut ipad) = open().await; + ipad.write_all(&[0, 0]).await.unwrap(); + closed(&link).await; + assert_eq!(link.close_report().unwrap().cause, CloseCause::BadLength); + + let (link, _ipad) = open().await; + drop(link.close_guard()); + closed(&link).await; + assert_eq!(link.close_report().unwrap().cause, CloseCause::Host); + // A later end changes nothing: the first cause stands. + link.state.close(CloseCause::PeerEof); + assert_eq!(link.close_report().unwrap().cause, CloseCause::Host); + + assert_eq!( + CloseCause::ReadFailed(io::ErrorKind::ConnectionReset).as_str(), + "peer_reset" + ); + assert_eq!( + CloseCause::ReadFailed(io::ErrorKind::ConnectionAborted).as_str(), + "read_error" + ); + } + + /// Nothing in the link times a tunnel out: one whose iPad pings every + /// 2 s stays open for a minute, and when it closes it says how long it + /// was open and how long the iPad had been quiet. + #[tokio::test(start_paused = true)] + async fn a_tunnel_stays_open_until_an_end_closes_it_and_reports_its_timings() { + let (host, mut ipad) = tokio::io::duplex(64 * 1024); + let mut link = FramedLink::connect(host, PeerId::usb(7), Arc::new(AtomicBool::new(false))) + .await + .unwrap(); + read_preamble(&mut ipad).await.unwrap(); + let mut inbound = link.take_inbound().unwrap(); + let start = Instant::now(); + let ping = encode_control(1, 1, &ControlMessage::Ping(Ping { t1_us: 1 })); + while start.elapsed() < Duration::from_secs(60) { + write_datagram(&mut ipad, &ping).await.unwrap(); + assert!(inbound.recv().await.is_some()); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(!link.is_closed(), "closed {:?} in", start.elapsed()); + } + tokio::time::sleep(Duration::from_millis(1_500)).await; + drop(ipad); + tokio::time::timeout(Duration::from_secs(1), inbound.recv()) + .await + .unwrap(); + let report = link.close_report().unwrap(); + assert_eq!(report.cause, CloseCause::PeerEof); + assert_eq!(report.open, Duration::from_millis(61_500)); + assert_eq!(report.quiet, Duration::from_millis(3_500)); + } + /// The frame being written is finished however many newer frames /// overflow behind it. Cutting it short sent the iPad a partial keyframe /// over a lossless cable: lost fragments in its report, and another diff --git a/host/src/transport/usb.rs b/host/src/transport/usb.rs index a82ba985..099a95de 100644 --- a/host/src/transport/usb.rs +++ b/host/src/transport/usb.rs @@ -15,7 +15,7 @@ use tracing::{debug, info, warn}; use super::apple_usb::{self, AppleUsbState}; use super::congestion::{is_no_buffer_space, SendCongestion}; -use super::link::{FramedLink, Link, LinkId, PeerId, UdpLink}; +use super::link::{CloseReport, FramedLink, Link, LinkId, PeerId, UdpLink}; use super::usbmuxd::{self, BoxedTunnel, Client}; use crate::stats::PIPELINE_STATS; @@ -31,6 +31,26 @@ pub enum LinkEvent { struct UsbEvent { generation: u64, link: FramedLink, + opened: Opened, +} + +/// How a tunnel came to be, for its connected line. +#[derive(Clone, Debug, Default)] +struct Opened { + /// The service's Connect and the preamble. + connect: Duration, + /// Since the device was listed, or since its last tunnel closed. + waited: Duration, + /// Connects that failed in that time, and the last one's error. + failed: u32, + last_error: Option, +} + +impl Opened { + fn failed_with(&mut self, error: String) { + self.failed += 1; + self.last_error = Some(error); + } } /// What the hub reports to the pipeline generation that currently owns it. @@ -38,6 +58,7 @@ enum HubEvent { Connected { generation: u64, link: Arc, + opened: Opened, }, Datagram { generation: u64, @@ -166,6 +187,7 @@ async fn run_hub(mut events: mpsc::Receiver, state: Arc, state: Arc event, }; match event { - Some(HubEvent::Connected { generation, link }) => { - info!(id = ?link.id(), generation, "USB tunnel connected"); + Some(HubEvent::Connected { + generation, + link, + opened, + }) => { + info!( + id = ?link.id(), + generation, + connect_ms = opened.connect.as_millis() as u64, + waited_ms = opened.waited.as_millis() as u64, + failed = opened.failed, + last_error = opened.last_error.as_deref().unwrap_or("none"), + "USB tunnel connected" + ); self.adopt(generation, link); } Some(HubEvent::Datagram { @@ -365,12 +405,15 @@ impl TransportLinks { .get(&id) .is_some_and(|active| active.generation == generation) { - self.usb.remove(&id); + let report = self + .usb + .remove(&id) + .and_then(|active| active.link.close_report()); if self.usb.is_empty() { PIPELINE_STATS.lock().usb_link_state = "Waiting for the iPad app".into(); } - info!(?id, generation, "USB tunnel closed"); + log_closed(id, generation, report); return Ok(LinkEvent::Closed(id)); } } @@ -391,6 +434,20 @@ impl TransportLinks { } } +/// Which end closed the tunnel first, after how long, and how long the +/// iPad had been quiet: a timer shows as the same `open_ms` every time. +fn log_closed(id: LinkId, generation: u64, report: Option) { + let ms = |duration: Duration| duration.as_millis() as u64; + info!( + ?id, + generation, + cause = %report.map_or("unknown", |report| report.cause.as_str()), + open_ms = report.map_or(0, |report| ms(report.open)), + quiet_ms = report.map_or(0, |report| ms(report.quiet)), + "USB tunnel closed" + ); +} + #[derive(Clone)] enum Source { Direct(String), @@ -428,14 +485,28 @@ impl Source { async fn device_worker(source: Source, events: mpsc::Sender, force_idr: Arc) { let peer = source.peer(); let mut backoff = 1; + let mut opened = Opened::default(); + let mut waiting_since = tokio::time::Instant::now(); loop { + let started = tokio::time::Instant::now(); match source.connect().await { Ok(stream) => match FramedLink::connect(stream, peer, Arc::clone(&force_idr)).await { Ok(link) => { let mut closed = link.closed_signal(); let _close_on_exit = link.close_guard(); let generation = NEXT_GENERATION.fetch_add(1, Ordering::Relaxed); - if events.send(UsbEvent { generation, link }).await.is_err() { + opened.connect = started.elapsed(); + opened.waited = started.saturating_duration_since(waiting_since); + let opened = std::mem::take(&mut opened); + if events + .send(UsbEvent { + generation, + link, + opened, + }) + .await + .is_err() + { return; } loop { @@ -447,10 +518,17 @@ async fn device_worker(source: Source, events: mpsc::Sender, force_idr } } backoff = 1; + waiting_since = tokio::time::Instant::now(); + } + Err(error) => { + debug!(%peer, %error, "USB framing handshake failed"); + opened.failed_with(format!("framing handshake: {error}")); } - Err(error) => debug!(%peer, %error, "USB framing handshake failed"), }, - Err(error) => debug!(%peer, %error, backoff, "USB app unavailable; retrying"), + Err(error) => { + debug!(%peer, %error, backoff, "USB app unavailable; retrying"); + opened.failed_with(error.to_string()); + } } tokio::time::sleep(Duration::from_secs(backoff)).await; backoff = (backoff + 1).min(5); @@ -504,6 +582,7 @@ async fn supervise(events: mpsc::Sender, force_idr: Arc) { _ = poll.tick() => { let devices = client.list_devices().await; let available = devices.is_ok(); + let list_error = devices.as_ref().err().map(ToString::to_string); let devices = devices.unwrap_or_default(); let ids: HashSet<_> = devices.iter().map(|device| device.device_id).collect(); // Without the service the cable is invisible to us; ask @@ -532,7 +611,17 @@ async fn supervise(events: mpsc::Sender, force_idr: Arc) { else if stats.usb_link_state != "Connected" { stats.usb_link_state = "Waiting for the iPad app".into(); } } handles.retain(|id, task| { - if !ids.contains(id) { task.abort(); false } else { !task.is_finished() } + if ids.contains(id) { + return !task.is_finished(); + } + // Stopping the worker closes its tunnel, if it has one. + info!( + reason = %if available { "device_gone" } else { "list_failed" }, + error = list_error.as_deref().unwrap_or("none"), + "USB device worker stopped" + ); + task.abort(); + false }); for id in ids { handles.entry(id).or_insert_with(|| workers.spawn(device_worker( @@ -569,6 +658,113 @@ mod tests { assert_eq!(direct_value(true, || None), None); } + /// The close line says which end closed the tunnel first and when, so a + /// daily check can tell a timer (the same `open_ms` every time) from an + /// unplugged cable or Apple's service letting go. + #[test] + fn the_close_line_says_who_closed_the_tunnel_and_when() { + use super::super::link::CloseCause; + use tracing_subscriber::prelude::*; + #[derive(Clone, Default)] + struct Lines(Arc>>); + impl io::Write for Lines { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let lines = Lines::default(); + let writer = lines.clone(); + let subscriber = tracing_subscriber::registry().with( + tracing_subscriber::fmt::layer() + .with_ansi(false) + .without_time() + .with_target(false) + .with_writer(move || writer.clone()), + ); + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, || { + let report = CloseReport { + cause: CloseCause::ReadFailed(io::ErrorKind::ConnectionReset), + open: Duration::from_millis(9_612), + quiet: Duration::from_millis(4), + }; + log_closed(PeerId::usb(3).link, 12, Some(report)); + log_closed(PeerId::usb(3).link, 13, None); + }); + let text = String::from_utf8(lines.0.lock().clone()).unwrap(); + let lines: Vec<&str> = text.lines().map(str::trim).collect(); + assert_eq!( + lines, + [ + "INFO USB tunnel closed id=Usb { device_id: 3 } generation=12 \ + cause=peer_reset open_ms=9612 quiet_ms=4", + "INFO USB tunnel closed id=Usb { device_id: 3 } generation=13 \ + cause=unknown open_ms=0 quiet_ms=0", + ] + ); + } + + /// The connected line says how long the worker waited for the tunnel and + /// how many connects failed meanwhile: Apple's service refuses a Connect + /// while the app isn't listening, and a tunnel the iPad closed starts + /// the count again. + #[tokio::test] + async fn a_tunnel_says_how_it_was_opened() { + let service = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = Arc::new(Client::new(usbmuxd::Endpoint::Tcp( + service.local_addr().unwrap(), + ))); + // Answers each Connect with `result`, and returns the connection. + let answer = |result: u64| { + let service = &service; + async move { + let (mut stream, _) = service.accept().await.unwrap(); + let (tag, _) = usbmuxd::read_message(&mut stream).await.unwrap(); + let reply = plist::Value::Dictionary(plist::Dictionary::from_iter([ + ("MessageType".to_string(), plist::Value::from("Result")), + ("Number".to_string(), plist::Value::from(result)), + ])); + tokio::io::AsyncWriteExt::write_all( + &mut stream, + &usbmuxd::encode_message(tag, &reply).unwrap(), + ) + .await + .unwrap(); + stream + } + }; + let (events, mut opened) = mpsc::channel(4); + let worker = tokio::spawn(device_worker( + Source::Device(client, 3), + events, + Arc::new(AtomicBool::new(false)), + )); + answer(3).await; + let mut ipad = answer(0).await; + let first = opened.recv().await.unwrap(); + assert_eq!(first.opened.failed, 1, "{:?}", first.opened); + assert_eq!(first.opened.last_error.as_deref(), Some("usbmuxd Result 3")); + assert!(first.opened.waited >= Duration::from_secs(1)); + + super::super::link::read_preamble(&mut ipad).await.unwrap(); + drop(ipad); + let _ipad = answer(0).await; + let second = opened.recv().await.unwrap(); + assert_eq!( + first.link.close_report().map(|report| report.cause), + Some(super::super::link::CloseCause::PeerEof) + ); + assert_eq!(second.opened.failed, 0, "{:?}", second.opened); + assert_eq!(second.opened.last_error, None); + assert!(second.opened.waited >= Duration::from_secs(1)); + assert!(second.generation > first.generation); + worker.abort(); + } + /// `cargo test --release` runs this; other builds may use the endpoint. #[cfg(not(any(debug_assertions, feature = "usb-direct")))] #[test] diff --git a/host/tests/liveness_hold.rs b/host/tests/liveness_hold.rs index dbea1b48..28a519d9 100644 --- a/host/tests/liveness_hold.rs +++ b/host/tests/liveness_hold.rs @@ -777,3 +777,253 @@ fn another_ipad_takes_the_slot_of_a_silent_one() { sim.stop().await; }); } + +/// The same iPad over the cable: the tunnel the host's USB worker opened to +/// the test's listener, read and written as the app's link does. Its pings +/// wait for their pong without moving the clock, so the host hears them +/// when they are sent. +struct UsbIpad { + stream: std::net::TcpStream, + buffer: Vec, + msg_seq: u32, + nonce: u32, + session_id: Option, + received: Vec<(Duration, Heard)>, + pongs: usize, + /// When the host's end of the tunnel closed, if it did. + closed: Option, + next_report: Duration, + next_ping: Duration, +} + +impl UsbIpad { + async fn accept(listener: &std::net::TcpListener) -> Self { + let mut accepted = None; + settle("the host's USB tunnel", || { + accepted = listener.accept().ok(); + accepted.is_some() + }) + .await; + let (stream, _) = accepted.unwrap(); + stream.set_nonblocking(true).unwrap(); + stream.set_nodelay(true).unwrap(); + let mut ipad = Self { + stream, + buffer: Vec::new(), + msg_seq: 0, + nonce: 100, + session_id: None, + received: Vec::new(), + pongs: 0, + closed: None, + next_report: Duration::MAX, + next_ping: Duration::MAX, + }; + let preamble = eternal_host::transport::link::PREAMBLE; + settle("the EMLINK preamble", || { + ipad.fill(Duration::ZERO); + ipad.buffer.len() >= preamble.len() || ipad.closed.is_some() + }) + .await; + assert_eq!(&ipad.buffer[..preamble.len()], preamble); + ipad.buffer.drain(..preamble.len()); + ipad + } + + fn fill(&mut self, now: Duration) { + use std::io::Read; + let mut bytes = [0; 64 * 1024]; + loop { + match self.stream.read(&mut bytes) { + Ok(0) => { + self.closed.get_or_insert(now); + return; + } + Ok(len) => self.buffer.extend_from_slice(&bytes[..len]), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => return, + Err(_) => { + self.closed.get_or_insert(now); + return; + } + } + } + } + + fn drain(&mut self, now: Duration) { + self.fill(now); + while self.buffer.len() >= 2 { + let len = usize::from(u16::from_le_bytes([self.buffer[0], self.buffer[1]])); + if self.buffer.len() < 2 + len { + break; + } + let packet: Vec = self.buffer.drain(..2 + len).skip(2).collect(); + if matches!(parse_control(&packet), Ok((_, ControlMessage::Pong(_)))) { + self.pongs += 1; + } + let heard = heard(&packet); + if let (None, Heard::Ack(HelloStatus::Ok, id)) = (self.session_id, heard) { + self.session_id = Some(id); + self.next_report = now + Duration::from_millis(500); + self.next_ping = now + Duration::from_secs(2); + } + self.received.push((now, heard)); + } + } + + fn send(&mut self, message: &ControlMessage) { + use std::io::Write; + self.msg_seq += 1; + let bytes = encode_control(self.session_id.unwrap_or(0), self.msg_seq, message); + let mut framed = (bytes.len() as u16).to_le_bytes().to_vec(); + framed.extend_from_slice(&bytes); + self.stream.write_all(&framed).unwrap(); + } + + /// HELLO2 as the iPad that streamed over Wi-Fi, and its ACK. + async fn hello(&mut self, device_id: u64, now: Duration) -> u32 { + self.nonce += 1; + self.session_id = None; + self.send(&ControlMessage::Hello2(Hello2 { + proto_min: 2, + proto_max: 2, + client_nonce: self.nonce, + listen_port: 0, + decoder_caps: CAP_DECODE_H264, + feature_caps: FEATURE_WANTS_INPUT, + screen_px_w: 1280, + screen_px_h: 720, + screen_pt_w: 1280, + screen_pt_h: 720, + refresh_hz: 60, + device_name: "Stall test iPad".into(), + device_id, + preferred_fps: 0, + auth_token: [0; 16], + pairing_code: 0, + })); + settle("the ACK over USB", || { + self.drain(now); + self.session_id.is_some() || self.closed.is_some() + }) + .await; + self.session_id.expect("an ACK over USB") + } + + async fn step(&mut self, now: Duration) { + self.drain(now); + let Some(id) = self.session_id else { + return; + }; + if self.closed.is_some() { + return; + } + if now >= self.next_report { + self.next_report += Duration::from_millis(500); + let frames = self + .received + .iter() + .filter(|(_, heard)| *heard == Heard::Media(id)) + .count() as u32; + self.send(&ControlMessage::ReceiverReport(ReceiverReport { + stream_epoch: 1, + highest_seq: frames, + frames_complete: frames, + ..ReceiverReport::default() + })); + } + if now >= self.next_ping { + self.next_ping += Duration::from_secs(2); + let pongs = self.pongs; + self.send(&ControlMessage::Ping(Ping { t1_us: 1 })); + // A session that ended answers with a goodbye instead. + settle("a pong over USB", || { + self.drain(now); + self.pongs > pongs || self.closed.is_some() || !byes(&self.received).is_empty() + }) + .await; + } + } +} + +/// The iPad moves from Wi-Fi to the cable: it says goodbye on Wi-Fi +/// (`superseded`) and sends HELLO2 over USB, in either order. In the field +/// some of these tunnels closed on both sides 9.5-9.8 s later, with no +/// reason on either side. Nothing in the host times that: for 30 s the +/// session and the tunnel stay up and video flows over the cable, and +/// neither link hears a goodbye. A deadline left from the Wi-Fi session or +/// the takeover wait (1 s), the liveness hold (10 s) or the tunnel would +/// show here. +#[test] +fn a_usb_takeover_from_wifi_keeps_its_tunnel_and_session() { + for bye_first in [true, false] { + run(async { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + std::env::set_var( + "ETERNAL_USB_DIRECT", + listener.local_addr().unwrap().to_string(), + ); + let mut sim = Sim::start().await; + let wifi = sim.add_ipad(77).await; + sim.ipads[wifi].hello(); + sim.run_until(secs(2.0)).await; + let (Some(wifi_id), None) = sim.session() else { + panic!("no session: {:?}", sim.session()) + }; + assert!(sim.ipads[wifi].video_after(wifi_id, secs(1.5))); + let mut usb = UsbIpad::accept(&listener).await; + + let took_over = sim.start.elapsed(); + let bye = ControlMessage::Bye(ByeReason::Superseded); + if bye_first { + sim.ipads[wifi].send_now(wifi_id, &bye).await; + } + let usb_id = usb.hello(77, took_over).await; + if !bye_first { + // The Wi-Fi goodbye arrives late, on a session already gone. + sim.ipads[wifi].send_now(wifi_id, &bye).await; + } + let wifi = sim.ipads.remove(wifi); + assert_ne!(usb_id, wifi_id); + + let end = took_over + secs(30.0); + let mut next = took_over; + while next < end { + next += Duration::from_millis(50); + sim.run_until(next).await; + usb.step(sim.start.elapsed()).await; + } + + eprintln!( + "USB_TAKEOVER_SIM bye_first={bye_first} took_over={took_over:?} closed={:?} \ + session={:?} usb_byes={:?}", + usb.closed, + sim.session(), + byes(&usb.received) + ); + assert_eq!(usb.closed, None, "the host closed the tunnel"); + assert_eq!( + sim.session(), + (Some(usb_id), Some((wifi_id, EndReason::Superseded))) + ); + assert!(byes(&usb.received).is_empty(), "{:?}", byes(&usb.received)); + assert!( + byes(&wifi.received()).is_empty(), + "{:?}", + byes(&wifi.received()) + ); + let last_video = usb + .received + .iter() + .filter(|(_, heard)| *heard == Heard::Media(usb_id)) + .map(|(at, _)| *at) + .max(); + assert!( + last_video.is_some_and(|at| at >= end - secs(0.5)), + "{last_video:?}" + ); + sim.stop().await; + std::env::set_var("ETERNAL_USB_DIRECT", "127.0.0.1:0"); + }); + } +} From 4283c55fba7dcc88aef2a6179848d38929cec57e Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:37:33 -0500 Subject: [PATCH 178/226] Treat a capture error as the stream stops as the stop Tearing the display down can fail the capture call in flight ("The parameter is incorrect"). Once the stream is stopping, or a newer generation has replaced this one, the capture stage now logs that at info as a stop and ends as completed, so the supervisor neither counts it as a capture failure nor restarts for it. An error while the stream runs is still a failure, logged as before. --- host/src/capture/mod.rs | 109 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 5 deletions(-) diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 5ee270de..2c89dafa 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -10,9 +10,9 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use parking_lot::{Condvar, Mutex}; -use tracing::error; #[cfg(any(windows, test, debug_assertions))] -use tracing::{info, warn}; +use tracing::warn; +use tracing::{error, info}; use crate::control::SharedControl; #[cfg(any(windows, test, debug_assertions))] @@ -1176,10 +1176,34 @@ pub fn run_capture_stage( }) }, ); - if let Err(ref e) = result { - error!(error = %e, "Capture loop exited with error"); + stage_exit(result, stopping(&shared, generation)) +} + +/// The stream is stopping, or a newer generation has replaced this one. +fn stopping(shared: &SharedControl, generation: u64) -> bool { + !shared.running.load(std::sync::atomic::Ordering::SeqCst) + || !crate::supervisor::generation_is_current(generation) +} + +/// How the capture stage ends. An error once the stream is stopping is the +/// stop: tearing the display down can fail the call in flight ("The +/// parameter is incorrect"), and that is no capture failure for the +/// supervisor to restart or count. +fn stage_exit( + result: Result<(), Box>, + stopping: bool, +) -> Result<(), Box> { + match result { + Err(e) if stopping => { + info!(error = %e, "Capture loop stopped with the stream"); + Ok(()) + } + Err(e) => { + error!(error = %e, "Capture loop exited with error"); + Err(e) + } + Ok(()) => Ok(()), } - result } /// What a capture attempt's error says about a second screen proving @@ -1333,6 +1357,81 @@ fn pick_new_virtual_output<'a>(before: &[String], now: &'a [OutputInfo]) -> Opti #[cfg(test)] mod tests { + use std::sync::Arc; + + use parking_lot::Mutex; + + use crate::control::SharedControl; + + /// Log lines `run` writes on this thread. + fn logged(run: impl FnOnce()) -> String { + #[derive(Clone)] + struct Captured(Arc>>); + impl std::io::Write for Captured { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let buffer = Arc::new(Mutex::new(Vec::new())); + let writer = Captured(buffer.clone()); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, run); + let text = String::from_utf8_lossy(&buffer.lock()).into_owned(); + text + } + + fn failed() -> Result<(), Box> { + Err("The parameter is incorrect. (0x80070057)".into()) + } + + /// A call that fails as the stream stops is the stop: logged at info, + /// and the stage ends as completed, so the supervisor neither restarts + /// nor counts it. + #[test] + fn an_error_as_the_stream_stops_is_a_stop() { + let mut result = None; + let text = logged(|| result = Some(super::stage_exit(failed(), true))); + assert!(result.unwrap().is_ok()); + assert!(text.contains(" INFO ") && !text.contains("ERROR"), "{text}"); + assert!( + text.contains("Capture loop stopped with the stream"), + "{text}" + ); + } + + /// While the stream runs, the same error is a capture failure, as before. + #[test] + fn an_error_while_streaming_is_still_a_failure() { + let mut result = None; + let text = logged(|| result = Some(super::stage_exit(failed(), false))); + assert!(result.unwrap().is_err()); + assert!( + text.contains("ERROR") && text.contains("Capture loop exited with error"), + "{text}" + ); + let text = logged(|| assert!(super::stage_exit(Ok(()), true).is_ok())); + assert_eq!(text, ""); + } + + /// Stopping is `running` off, or a newer generation. + #[test] + fn a_stage_is_stopping_once_its_stream_stops_or_is_replaced() { + use std::sync::atomic::Ordering; + let shared = SharedControl::new(0, 1); + let live = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); + shared.running.store(false, Ordering::SeqCst); + assert!(super::stopping(&shared, live)); + shared.running.store(true, Ordering::SeqCst); + assert!(super::stopping(&shared, live.wrapping_add(1_000_000))); + } #[test] fn a_timed_take_returns_a_frame_nothing_or_the_close() { From 7c8e2283dca212b8eb2b36b510a43ac65dc9ab3f Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:50:12 -0500 Subject: [PATCH 179/226] End the attach wait early only on a confirmed absence of monitors Review r1: the device read the attach wait used counted uncertain states as "no monitor": a failed child query, a child status that couldn't be read, and a child that hadn't started yet. It also looked at the first child only. Ten seconds of any of those could end a start whose monitor was still arriving, or the wait for Extend. The attach wait now uses a stricter read. "No monitor" means the device runs and Windows listed no child at all. A child that hasn't started, or children or a status that couldn't be read, count as uncertain and never end the wait early; the 10 s count starts at the first confirmed absence and starts over when it stops. Every child is examined, through its siblings, and a started one anywhere counts as a monitor. The snapshot and notice after a timeout name the device as before. Tests: the child list's classification (empty, pending, unread, a monitor among the siblings), the confirmed read across the driver's devices, and attach waits that go from pending to a monitor, from uncertain to no monitor, through a broken count, and from no monitor to a monitor before 10 s. --- host/src/capture/second_screen.rs | 87 +++++++++- host/src/vdd_device.rs | 264 ++++++++++++++++++++++++++---- 2 files changed, 316 insertions(+), 35 deletions(-) diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 9c82b5f2..56459387 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -1503,8 +1503,8 @@ pub(crate) trait Driver { /// Say why the display did not appear or showed no picture, while it is /// still on. fn report(&self, _failure: Failure) {} - /// The device's state while the attach waits, `None` when it can't be - /// read. + /// The device's state while the attach waits. `None` when it can't be + /// read, or a monitor may still be arriving. fn device(&self) -> Option { None } @@ -1935,7 +1935,7 @@ impl Driver for WindowsDriver<'_> { } fn device(&self) -> Option { - Some(crate::vdd_device::query()) + crate::vdd_device::query_confirmed() } fn turned_on(&self) -> TurnedOn { @@ -2089,6 +2089,9 @@ mod tests { /// What the device reads while it is on and no display has attached /// yet; `None` when it can't be read. Attached, it reads ready. device_until_attached: Option, + /// When set, what the device reads that long after it was turned on, + /// in place of `device_until_attached`: the last entry not later. + device_since_on: Vec<(Duration, Option)>, calls: RefCell>, } @@ -2110,6 +2113,7 @@ mod tests { enable_fails: None, turned_on: TurnedOn::Beside, device_until_attached: None, + device_since_on: Vec::new(), calls: RefCell::new(Vec::new()), } } @@ -2127,7 +2131,17 @@ mod tests { if self.outputs().iter().any(|output| !output.is_primary) { return Some(DeviceState::Ready); } - self.device_until_attached + let since_on = self.elapsed.get() - self.on_since.get().unwrap_or_default(); + match self + .device_since_on + .iter() + .rev() + .find(|(at, _)| *at <= since_on) + { + Some((_, device)) => *device, + None if self.device_since_on.is_empty() => self.device_until_attached, + None => None, + } } fn outputs(&self) -> Vec { @@ -2269,6 +2283,71 @@ mod tests { assert_eq!(remembered_failure(&shared), None); } + /// Review r1: only a confirmed absence counts. A monitor that hasn't + /// started, or a read that isn't certain (`None`), never ends the wait, + /// however long it lasts; the 10 s count starts at the first confirmed + /// absence and starts over when it stops. + #[test] + fn the_early_end_counts_only_confirmed_absence_and_starts_over() { + let run = |schedule: Vec<(Duration, Option)>, + attach_after: Option| { + let shared = second_screen_selected(); + connect_viewer(&shared); + assert_eq!(next_step(&shared, Instant::now()), Step::Attach); + let on_sleep = |_: Duration| {}; + let mut driver = FakeDriver::new(secs(2.0), attach_after, &on_sleep); + driver.device_since_on = schedule; + attach(&shared, &driver, ATTACH_TIMEOUT); + let toggles = driver.toggles.borrow().clone(); + let waited = toggles.get(1).map(|(at, _)| *at - toggles[0].0); + let status = *shared.vdd_status.lock(); + (status, waited) + }; + // A monitor still on its way for longer than 10 s, then attached. + let (status, _) = run(vec![(secs(0.0), None)], Some(secs(14.0))); + assert_eq!( + status, + VddStatus::Active, + "a pending monitor ended the wait" + ); + // Uncertain all along: the whole wait, as before. + let (status, waited) = run(vec![(secs(0.0), None)], None); + assert_eq!(status, VddStatus::Failed); + assert!(waited.unwrap() >= ATTACH_TIMEOUT, "{waited:?}"); + // Uncertain for 5 s, then a confirmed absence: 10 s from then. Reads + // come once a second, so a change is seen up to one read late, and + // the count ends up to one read late. + let (_, waited) = run( + vec![(secs(0.0), None), (secs(5.0), Some(DeviceState::NoMonitor))], + None, + ); + let waited = waited.unwrap(); + assert!( + waited >= secs(15.0) && waited < secs(15.0) + DEVICE_READ_EVERY * 2 + secs(0.5), + "{waited:?}" + ); + // A confirmed absence broken by an uncertain read starts over. + let (_, waited) = run( + vec![ + (secs(0.0), Some(DeviceState::NoMonitor)), + (secs(8.0), None), + (secs(9.0), Some(DeviceState::NoMonitor)), + ], + None, + ); + let waited = waited.unwrap(); + assert!( + waited >= secs(19.0) && waited < secs(19.0) + DEVICE_READ_EVERY * 2 + secs(0.5), + "{waited:?}" + ); + // No monitor, then a monitor arrives before 10 s. + let (status, _) = run( + vec![(secs(0.0), Some(DeviceState::NoMonitor))], + Some(secs(8.0)), + ); + assert_eq!(status, VddStatus::Active); + } + /// A 0.3.2 log and the 0.3.1 field data: the display never attaches, /// the iPad sends its goodbye at 10 s, and USB dials again 1 s later. /// Here no transport step restarts the pipeline while the display diff --git a/host/src/vdd_device.rs b/host/src/vdd_device.rs index d7fd9787..4bc76bfe 100644 --- a/host/src/vdd_device.rs +++ b/host/src/vdd_device.rs @@ -51,6 +51,42 @@ pub enum VddProblem { NotExtended, } +/// A device's monitor children, as Configuration Manager listed them. +#[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Monitors { + /// The list was read and is empty. + None, + /// At least one child started without a problem. + Started, + /// Every child was read, and none has started yet. + Pending, + /// The list, or a child's status, couldn't be read. + Unread, +} + +impl Monitors { + /// From every child's status flags (`None` when a child's couldn't be + /// read), or `None` when the children couldn't be listed. A started + /// child counts even when another couldn't be read. + #[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. + fn from_children(children: Option<&[Option]>) -> Self { + let Some(children) = children else { + return Self::Unread; + }; + let started = |flags: &u32| flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0; + if children.iter().flatten().any(started) { + Self::Started + } else if children.is_empty() { + Self::None + } else if children.iter().any(Option::is_none) { + Self::Unread + } else { + Self::Pending + } + } +} + impl DeviceState { /// The Enable task's result, as vdd-toggle.ps1 exits. pub fn from_task_result(code: i64) -> Option { @@ -65,23 +101,37 @@ impl DeviceState { } /// The device as Configuration Manager reports it: its status flags and - /// problem number, whether a driver is bound, and whether its monitor - /// child is started (`None` without a child). + /// problem number, whether a driver is bound, and its monitor children. + /// A running device without a started monitor reads as `NoMonitor`, for + /// the snapshot and the notice once the display did not appear. #[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. - fn from_devnode(status: u32, problem: u32, has_driver: bool, monitor: Option) -> Self { + fn from_devnode(status: u32, problem: u32, has_driver: bool, monitors: Monitors) -> Self { if status & DN_HAS_PROBLEM != 0 { Self::Problem(problem) } else if !has_driver { Self::NoDriver } else if status & DN_STARTED == 0 { Self::NotStarted - } else if monitor == Some(true) { + } else if monitors == Monitors::Started { Self::Ready } else { Self::NoMonitor } } + /// As `from_devnode`, for a decision made while the display may still be + /// arriving: `NoMonitor` only when the device runs and Windows reported + /// no child at all. A child that has not started, or children that could + /// not all be read, are `None`: the monitor may be on its way. + #[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. + fn confirmed(status: u32, problem: u32, has_driver: bool, monitors: Monitors) -> Option { + let state = Self::from_devnode(status, problem, has_driver, monitors); + match (state, monitors) { + (Self::NoMonitor, Monitors::Pending | Monitors::Unread) => None, + (state, _) => Some(state), + } + } + /// With more than one device, the one closest to showing a display. #[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. fn best(states: impl IntoIterator) -> Self { @@ -140,29 +190,100 @@ pub fn from_task_stderr(stderr: &str) -> Option { /// Read the driver's device from Configuration Manager. #[cfg(windows)] pub fn query() -> DeviceState { - use windows::Win32::Devices::DeviceAndDriverInstallation::{ - CM_Get_Child, CM_DRP_DRIVER, CR_SUCCESS, - }; + DeviceState::best(read_devices().into_iter().filter_map(|read| { + let (flags, problem, has_driver, monitors) = read?; + Some(DeviceState::from_devnode( + flags, problem, has_driver, monitors, + )) + })) +} +/// As `query`, for the attach wait: `NoMonitor` only when every running +/// device of the driver has confirmed no monitor child (see +/// `DeviceState::confirmed`). `None` when that isn't certain: a device or +/// its children couldn't be read, or a monitor hasn't started yet. +#[cfg(windows)] +pub fn query_confirmed() -> Option { + confirmed_from(read_devices()) +} + +/// `query_confirmed` from each driver device's read (`None` when its status +/// couldn't be read). A device with a monitor wins; otherwise anything +/// uncertain makes the whole read uncertain. +#[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. +fn confirmed_from(reads: Vec>) -> Option { + let states: Vec> = reads + .into_iter() + .map(|read| { + let (flags, problem, has_driver, monitors) = read?; + DeviceState::confirmed(flags, problem, has_driver, monitors) + }) + .collect(); + if states.contains(&Some(DeviceState::Ready)) { + return Some(DeviceState::Ready); + } + let states: Option> = states.into_iter().collect(); + Some(DeviceState::best(states?)) +} + +/// Each present device of the driver: its status flags and problem code, +/// whether a driver is bound, and its monitor children; `None` for one whose +/// status couldn't be read. +#[cfg(windows)] +fn read_devices() -> Vec> { + use windows::Win32::Devices::DeviceAndDriverInstallation::CM_DRP_DRIVER; let Some(devnodes) = root_devnodes() else { - return DeviceState::Missing; + return Vec::new(); }; - let states = devnodes.into_iter().filter_map(|devnode| { - let Devnode::Driver { devinst, status } = devnode else { + devnodes + .into_iter() + .filter_map(|devnode| match devnode { + Devnode::Driver { devinst, status } => Some((devinst, status)), + _ => None, + }) + .map(|(devinst, status)| { + let (flags, problem) = status?; + let has_driver = matches!(registry_property(devinst, CM_DRP_DRIVER), Ok(Some(_))); + let children = children(devinst); + Some(( + flags, + problem, + has_driver, + Monitors::from_children(children.as_deref()), + )) + }) + .collect() +} + +/// The status flags of every child of `devinst` (`None` for one whose +/// status couldn't be read), or `None` when the children couldn't be listed. +#[cfg(windows)] +fn children(devinst: u32) -> Option>> { + use windows::Win32::Devices::DeviceAndDriverInstallation::{ + CM_Get_Child, CM_Get_Sibling, CR_NO_SUCH_DEVNODE, CR_SUCCESS, + }; + /// More children than a virtual display ever has: a list this long is + /// not one to trust. + const MOST: usize = 64; + let mut statuses = Vec::new(); + let mut child = 0u32; + match unsafe { CM_Get_Child(&mut child, devinst, 0) } { + CR_NO_SUCH_DEVNODE => return Some(statuses), + CR_SUCCESS => {} + _ => return None, + } + loop { + statuses.push(devnode_status(child).map(|(flags, _)| flags)); + if statuses.len() > MOST { return None; - }; - let (flags, problem) = status?; - let mut child = 0u32; - let monitor = (unsafe { CM_Get_Child(&mut child, devinst, 0) } == CR_SUCCESS).then(|| { - devnode_status(child) - .is_some_and(|(flags, _)| flags & DN_STARTED != 0 && flags & DN_HAS_PROBLEM == 0) - }); - let has_driver = matches!(registry_property(devinst, CM_DRP_DRIVER), Ok(Some(_))); - Some(DeviceState::from_devnode( - flags, problem, has_driver, monitor, - )) - }); - DeviceState::best(states) + } + let mut next = 0u32; + match unsafe { CM_Get_Sibling(&mut next, child, 0) } { + CR_NO_SUCH_DEVNODE => return Some(statuses), + CR_SUCCESS => child = next, + _ => return None, + } + } } /// One present root device, as Configuration Manager read it. @@ -572,39 +693,120 @@ mod tests { let started = DN_STARTED; let problem = DN_HAS_PROBLEM; assert_eq!( - DeviceState::from_devnode(problem, 22, true, None), + DeviceState::from_devnode(problem, 22, true, Monitors::None), DeviceState::Problem(22) ); assert_eq!( - DeviceState::from_devnode(problem, 43, true, None), + DeviceState::from_devnode(problem, 43, true, Monitors::None), DeviceState::Problem(43) ); assert_eq!( - DeviceState::from_devnode(problem, 28, false, None), + DeviceState::from_devnode(problem, 28, false, Monitors::None), DeviceState::Problem(28) ); assert_eq!( - DeviceState::from_devnode(started, 0, false, None), + DeviceState::from_devnode(started, 0, false, Monitors::None), DeviceState::NoDriver ); assert_eq!( - DeviceState::from_devnode(0, 0, true, None), + DeviceState::from_devnode(0, 0, true, Monitors::None), DeviceState::NotStarted ); assert_eq!( - DeviceState::from_devnode(started, 0, true, None), + DeviceState::from_devnode(started, 0, true, Monitors::None), DeviceState::NoMonitor ); assert_eq!( - DeviceState::from_devnode(started, 0, true, Some(false)), + DeviceState::from_devnode(started, 0, true, Monitors::Pending), DeviceState::NoMonitor ); assert_eq!( - DeviceState::from_devnode(started, 0, true, Some(true)), + DeviceState::from_devnode(started, 0, true, Monitors::Started), DeviceState::Ready ); } + /// Review r1 of #233: the attach wait ends early on `NoMonitor`, so only + /// a confirmed absence may read as one. Every child counts, not only the + /// first: Windows lists further children as its siblings. + #[test] + fn only_a_confirmed_absence_of_monitors_reads_as_no_monitor_while_waiting() { + let (started, problem) = (DN_STARTED, DN_HAS_PROBLEM); + let starting = 0; + // The children as listed. + assert_eq!(Monitors::from_children(None), Monitors::Unread); + assert_eq!(Monitors::from_children(Some(&[])), Monitors::None); + assert_eq!( + Monitors::from_children(Some(&[Some(starting)])), + Monitors::Pending + ); + assert_eq!(Monitors::from_children(Some(&[None])), Monitors::Unread); + assert_eq!( + Monitors::from_children(Some(&[Some(started | problem)])), + Monitors::Pending + ); + assert_eq!( + Monitors::from_children(Some(&[Some(starting), Some(started)])), + Monitors::Started, + "a monitor among the siblings" + ); + assert_eq!( + Monitors::from_children(Some(&[None, Some(started)])), + Monitors::Started + ); + assert_eq!( + Monitors::from_children(Some(&[Some(starting), None])), + Monitors::Unread + ); + + // A running device: only no children at all is a confirmed absence. + let running = |monitors| DeviceState::confirmed(started, 0, true, monitors); + assert_eq!(running(Monitors::None), Some(DeviceState::NoMonitor)); + assert_eq!(running(Monitors::Started), Some(DeviceState::Ready)); + assert_eq!(running(Monitors::Pending), None); + assert_eq!(running(Monitors::Unread), None); + // The snapshot after a timeout still names the device as before. + for monitors in [Monitors::None, Monitors::Pending, Monitors::Unread] { + assert_eq!( + DeviceState::from_devnode(started, 0, true, monitors), + DeviceState::NoMonitor + ); + } + // States that don't depend on the children are certain. + assert_eq!( + DeviceState::confirmed(0, 0, true, Monitors::Unread), + Some(DeviceState::NotStarted) + ); + assert_eq!( + DeviceState::confirmed(problem, 22, true, Monitors::Unread), + Some(DeviceState::Problem(22)) + ); + + // Across the driver's devices. + let confirmed_none = Some((started, 0, true, Monitors::None)); + assert_eq!( + confirmed_from(vec![confirmed_none]), + Some(DeviceState::NoMonitor) + ); + assert_eq!( + confirmed_from(vec![None]), + None, + "a status that couldn't be read" + ); + assert_eq!( + confirmed_from(vec![ + confirmed_none, + Some((started, 0, true, Monitors::Pending)) + ]), + None + ); + assert_eq!( + confirmed_from(vec![None, Some((started, 0, true, Monitors::Started))]), + Some(DeviceState::Ready) + ); + assert_eq!(confirmed_from(vec![]), Some(DeviceState::Missing)); + } + #[test] fn the_device_closest_to_a_display_wins() { assert_eq!(DeviceState::best([]), DeviceState::Missing); From c541fea3eed9350f5a8c7b0f94f0b5529c4016a5 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:50:12 -0500 Subject: [PATCH 180/226] Keep USB tunnels through a failed device list The host asks Apple's device service for its list of devices every 2 s and stops the worker of every device the list leaves out, which closes its tunnel. A list that failed, for example because the service was slow to answer, left out every device, so a single failed poll closed every live USB tunnel even though the service and the tunnels were fine. A failed list now keeps the workers that are running. Only a list that answers can show that a device is gone. If the list keeps failing for 10 s, the service counts as down and the workers stop, as before. Each step is logged at info with the number of failed polls: tunnels kept, tunnels closed after the outage, and the list answering again. --- host/src/transport/usb.rs | 297 +++++++++++++++++++++++++++++++++++++- 1 file changed, 296 insertions(+), 1 deletion(-) diff --git a/host/src/transport/usb.rs b/host/src/transport/usb.rs index a82ba985..ac272032 100644 --- a/host/src/transport/usb.rs +++ b/host/src/transport/usb.rs @@ -494,18 +494,80 @@ async fn supervise(events: mpsc::Sender, force_idr: Arc) { } let client = Arc::new(Client::new(usbmuxd::Endpoint::default())); + supervise_devices(client, events, force_idr).await; +} + +/// How long Apple's device list may keep failing before the service counts +/// as down and the devices' workers stop, closing their tunnels. A single +/// failed poll used to close every live tunnel, though the service and the +/// tunnels through it were fine. +const LIST_OUTAGE: Duration = Duration::from_secs(10); + +/// Device-list polls that failed in a row, and when the first one did. +#[derive(Debug, Default)] +struct ListFailures { + count: u32, + since: Option, +} + +impl ListFailures { + /// Whether a poll keeps the running workers, whatever it listed: its + /// list failed, but the failures have not lasted [`LIST_OUTAGE`] yet. + /// Only a list that answers can show a device gone. + fn keep_workers(&mut self, listed: bool, workers: usize, now: tokio::time::Instant) -> bool { + if listed { + if self.count > 0 { + info!( + failed_polls = self.count, + "Apple's device list answers again" + ); + } + *self = Self::default(); + return false; + } + self.count = self.count.saturating_add(1); + let since = *self.since.get_or_insert(now); + if workers == 0 { + return false; + } + if now.saturating_duration_since(since) < LIST_OUTAGE { + info!( + failed_polls = self.count, + "Apple's device list failed — keeping the USB tunnels" + ); + return true; + } + info!( + failed_polls = self.count, + failing_ms = now.saturating_duration_since(since).as_millis() as u64, + "Apple's device list keeps failing — closing the USB tunnels" + ); + false + } +} + +async fn supervise_devices( + client: Arc, + events: mpsc::Sender, + force_idr: Arc, +) { let mut workers = JoinSet::new(); let mut handles: HashMap = HashMap::new(); let mut poll = tokio::time::interval(Duration::from_secs(2)); poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); let mut unreachable = AppleUsbState::default(); + let mut failures = ListFailures::default(); loop { tokio::select! { _ = poll.tick() => { let devices = client.list_devices().await; let available = devices.is_ok(); let devices = devices.unwrap_or_default(); - let ids: HashSet<_> = devices.iter().map(|device| device.device_id).collect(); + let ids: HashSet<_> = if failures.keep_workers(available, handles.len(), tokio::time::Instant::now()) { + handles.keys().copied().collect() + } else { + devices.iter().map(|device| device.device_id).collect() + }; // Without the service the cable is invisible to us; ask // Windows whether an iPad is attached so the GUI can say why. let cabled = if available { @@ -577,4 +639,237 @@ mod tests { assert_eq!(direct_endpoint(), None); std::env::remove_var("ETERNAL_USB_DIRECT"); } + + /// A failed device list keeps the workers that run, until the failures + /// last [`LIST_OUTAGE`]: then the service counts as down and they stop, + /// as every failed poll used to stop them. Each step is logged with the + /// count of failed polls. + #[test] + fn failed_device_lists_keep_the_workers_for_a_while() { + use tracing_subscriber::prelude::*; + #[derive(Clone, Default)] + struct Lines(Arc>>); + impl io::Write for Lines { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let lines = Lines::default(); + let writer = lines.clone(); + let subscriber = tracing_subscriber::registry().with( + tracing_subscriber::fmt::layer() + .with_ansi(false) + .without_time() + .with_target(false) + .with_writer(move || writer.clone()), + ); + let start = tokio::time::Instant::now(); + let at = |secs: u64| start + Duration::from_secs(secs); + let mut failures = ListFailures::default(); + crate::logging::prepare_log_capture(); + tracing::subscriber::with_default(subscriber, || { + // One failed poll between good ones. + assert!(!failures.keep_workers(true, 1, at(0))); + assert!(failures.keep_workers(false, 1, at(2))); + assert!(!failures.keep_workers(true, 1, at(4))); + // Failures in a row, until they have lasted the outage. + for secs in [6, 8, 10, 12, 14] { + assert!(failures.keep_workers(false, 1, at(secs)), "{secs} s"); + } + assert!(!failures.keep_workers(false, 1, at(16))); + // Nothing left to keep, nothing more to say. + assert!(!failures.keep_workers(false, 0, at(18))); + assert!(!failures.keep_workers(true, 0, at(20))); + // Without workers a failure keeps nothing, but still counts. + assert!(!failures.keep_workers(false, 0, at(22))); + assert!(!failures.keep_workers(false, 1, at(32))); + }); + let text = String::from_utf8(lines.0.lock().clone()).unwrap(); + let lines: Vec<&str> = text.lines().map(str::trim).collect(); + let keeping = |n: u32| { + format!("INFO Apple's device list failed — keeping the USB tunnels failed_polls={n}") + }; + assert_eq!( + lines, + [ + keeping(1), + "INFO Apple's device list answers again failed_polls=1".into(), + keeping(1), + keeping(2), + keeping(3), + keeping(4), + keeping(5), + "INFO Apple's device list keeps failing — closing the USB tunnels \ + failed_polls=6 failing_ms=10000" + .into(), + "INFO Apple's device list answers again failed_polls=7".into(), + "INFO Apple's device list keeps failing — closing the USB tunnels \ + failed_polls=2 failing_ms=10000" + .into(), + ] + ); + } + + /// Apple's device service whose list fails on demand: it drops the + /// request unanswered. It lists one USB iPad otherwise, and hands every + /// Connect's stream to the test as that iPad's end of a tunnel. + struct FlakyService { + failing: Arc, + fail_next: Arc, + listed: Arc, + failed: Arc, + tunnels: mpsc::Receiver, + client: Arc, + } + + impl FlakyService { + async fn start() -> Self { + use std::sync::atomic::AtomicU32; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = Arc::new(Client::new(usbmuxd::Endpoint::Tcp( + listener.local_addr().unwrap(), + ))); + let failing = Arc::new(AtomicBool::new(false)); + let fail_next = Arc::new(AtomicU32::new(0)); + let (listed, failed) = (Arc::new(AtomicU64::new(0)), Arc::new(AtomicU64::new(0))); + let (tunnel_tx, tunnels) = mpsc::channel(4); + let state = ( + failing.clone(), + fail_next.clone(), + listed.clone(), + failed.clone(), + ); + tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + let (failing, fail_next, listed, failed) = state.clone(); + let tunnel_tx = tunnel_tx.clone(); + tokio::spawn(async move { + let (tag, request) = usbmuxd::read_message(&mut stream).await.unwrap(); + let kind = request.as_dictionary().unwrap()["MessageType"] + .as_string() + .unwrap() + .to_string(); + let reply = if kind == "ListDevices" { + let once = fail_next + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| { + n.checked_sub(1) + }) + .is_ok(); + if once || failing.load(Ordering::SeqCst) { + failed.fetch_add(1, Ordering::SeqCst); + return; + } + listed.fetch_add(1, Ordering::SeqCst); + let device = plist::Dictionary::from_iter([ + ("DeviceID".to_string(), plist::Value::from(3u64)), + ( + "Properties".to_string(), + plist::Value::Dictionary(plist::Dictionary::from_iter([( + "ConnectionType".to_string(), + plist::Value::from("USB"), + )])), + ), + ]); + plist::Dictionary::from_iter([( + "DeviceList".to_string(), + plist::Value::Array(vec![plist::Value::Dictionary(device)]), + )]) + } else { + plist::Dictionary::from_iter([ + ("MessageType".to_string(), plist::Value::from("Result")), + ("Number".to_string(), plist::Value::from(0u64)), + ]) + }; + let bytes = + usbmuxd::encode_message(tag, &plist::Value::Dictionary(reply)).unwrap(); + tokio::io::AsyncWriteExt::write_all(&mut stream, &bytes) + .await + .unwrap(); + if kind == "Connect" { + let _ = tunnel_tx.send(stream).await; + } + }); + } + }); + Self { + failing, + fail_next, + listed, + failed, + tunnels, + client, + } + } + + /// The iPad's end of the next tunnel, past its preamble. + async fn tunnel(&mut self) -> tokio::net::TcpStream { + let mut ipad = tokio::time::timeout(Duration::from_secs(20), self.tunnels.recv()) + .await + .expect("a tunnel") + .unwrap(); + super::super::link::read_preamble(&mut ipad).await.unwrap(); + ipad + } + + async fn until(&self, what: &str, done: impl Fn(&Self) -> bool) { + tokio::time::timeout(Duration::from_secs(30), async { + while !done(self) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("{what}")); + } + } + + /// A device list that fails once leaves the live tunnel open; one that + /// keeps failing closes it after [`LIST_OUTAGE`], and once the list + /// answers again the iPad gets a new tunnel. A single failed list used + /// to close the tunnel at once. + #[tokio::test] + async fn a_failed_device_list_keeps_the_tunnel_until_the_service_is_down() { + let mut service = FlakyService::start().await; + let (events, mut opened) = mpsc::channel(4); + let supervisor = tokio::spawn(supervise_devices( + Arc::clone(&service.client), + events, + Arc::new(AtomicBool::new(false)), + )); + let _ipad = service.tunnel().await; + let first = opened.recv().await.unwrap(); + + service.fail_next.store(1, Ordering::SeqCst); + let listed = service.listed.load(Ordering::SeqCst); + service + .until("a failed list, then a good one", |service| { + service.failed.load(Ordering::SeqCst) == 1 + && service.listed.load(Ordering::SeqCst) > listed + 1 + }) + .await; + assert!(!first.link.is_closed(), "one failed list closed the tunnel"); + assert!(service.tunnels.try_recv().is_err(), "a second tunnel"); + + service.failing.store(true, Ordering::SeqCst); + let failing_from = tokio::time::Instant::now(); + let mut closed = first.link.closed_signal(); + tokio::time::timeout(Duration::from_secs(30), closed.wait_for(|closed| *closed)) + .await + .expect("the tunnel closes once the service is down") + .unwrap(); + let failing_for = failing_from.elapsed(); + assert!(failing_for >= LIST_OUTAGE, "{failing_for:?}"); + assert!(service.failed.load(Ordering::SeqCst) >= 6); + + service.failing.store(false, Ordering::SeqCst); + let _ipad = service.tunnel().await; + let second = opened.recv().await.unwrap(); + assert!(second.generation > first.generation); + assert!(!second.link.is_closed()); + supervisor.abort(); + } } From fbfeea996dfdea316a16086a08cde20b25371e40 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:51:54 -0500 Subject: [PATCH 181/226] Count only frames that go out, and keep an iPad's own goodbye, in the restart hold - A frame held back for an iPad that hasn't answered never reached it, so it no longer resets the hold limit or counts as the session's last frame. - An iPad that said goodbye to hand over to the cable during the pause ends on its takeover deadline, as it said, instead of with the hold limit's goodbye. --- host/src/transport/session.rs | 70 ++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 4a32e358..9ad787c0 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -590,10 +590,10 @@ impl Session { else { return true; }; - session.restart_held_since = None; - session.last_frame_at = Some(now); if session.answered { session.frames_sent = session.frames_sent.saturating_add(1); + session.restart_held_since = None; + session.last_frame_at = Some(now); return true; } if session.withheld @@ -603,6 +603,9 @@ impl Session { session.withheld = true; return false; } + // Only a frame that goes out ends a hold through a restart. + session.restart_held_since = None; + session.last_frame_at = Some(now); session.unanswered_sent += bytes; session.frames_sent = session.frames_sent.saturating_add(1); self.unanswered_media = (host_spent + bytes, now); @@ -1443,9 +1446,12 @@ impl Session { // Kept alive through a restart and no frame since: past the limit, // even once the next generation runs, tell the iPad to connect again. + // Not one that said goodbye to hand over to the cable: its takeover + // deadline below ends it as it said. if let Some(since) = session .restart_held_since .filter(|since| now.saturating_duration_since(*since) >= restart_hold_limit()) + .filter(|_| session.bye.is_none()) { info!( session_id = session.session_id, @@ -5766,4 +5772,64 @@ mod tests { Ok((_, ControlMessage::Bye(ByeReason::Error))) )); } + + /// A frame held back for an iPad that hasn't answered never went out, + /// so it neither resets the hold limit nor counts as the last frame. + #[test] + fn a_held_back_frame_does_not_reset_the_hold_limit() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + let id = session.session_id().unwrap(); + session.set_not_streaming(Some(NotStreaming::Restarting)); + session.restart_tick(&RestartConfig, now); + let held = |session: &Session| { + let active = session.active.as_ref().unwrap(); + (active.restart_held_since, active.last_frame_at) + }; + let before = held(&session); + assert!(before.0.is_some()); + let later = now + Duration::from_millis(500); + assert!(!session.may_send_media(id, UNANSWERED_MEDIA_BUDGET + 1, later)); + assert_eq!( + held(&session), + before, + "a frame that never went out counted" + ); + } + + /// An iPad that said goodbye to hand over to the cable during the pause + /// ends as it said, on its takeover deadline, not with the hold limit's + /// goodbye. + #[test] + fn an_ipads_own_goodbye_during_the_pause_ends_its_session_as_it_said() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + session.handle_control_authed(peer, identified_hello(1, 77), &TestConfig, now); + session.handle_control_authed(peer, report(10), &TestConfig, now); + session.set_not_streaming(Some(NotStreaming::Restarting)); + session.restart_tick(&RestartConfig, now); + let limit = now + restart_hold_limit(); + let bye_at = limit - Duration::from_millis(500); + session.handle_control_authed(peer, report(10), &RestartConfig, bye_at); + session.handle_control_authed( + peer, + ControlMessage::Bye(ByeReason::Superseded), + &RestartConfig, + bye_at, + ); + let mut ended = None; + for step in 0..20 { + let at = limit + Duration::from_millis(100 * step); + let actions = session.restart_tick(&RestartConfig, at); + assert!(actions.goodbye.is_none(), "told to connect again at {step}"); + if let Some(end) = actions.ended { + ended = Some(end.reason); + break; + } + } + assert_eq!(ended, Some(EndReason::Goodbye(ByeReason::Superseded))); + } } From 8dc190dec0ea701422ee008e144b43df8500f1ea Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:51:59 -0500 Subject: [PATCH 182/226] Connect by itself when a PC that didn't answer comes back The iPad sends HELLO2 for two seconds, then waits silently until the connect timeout. A PC app opened after that never heard it, and the "didn't answer" card stayed up until the user tapped Try again. Once the HELLO2s go unanswered, the iPad now watches Bonjour for that PC: its service by name when the PC is saved, or the service at the address tried. When the card is up and a listing of the PC appears or changes, it connects once, after a one-second debounce. What Bonjour already listed when the watch began doesn't count. The watch ends when the card is dismissed or replaced (a busy, old or stopped PC included), when the user connects elsewhere, on a stream, or three minutes after the card first went up. In the background the search stops and no try starts; a PC that came back meanwhile gets its try on return. The card says it is waiting for the PC. Over USB nothing changes: the PC connects to the iPad's cable listener by itself when its app opens. Tests: PCReturnTests with a fake Bonjour browser and an injected clock; scripts/e2e_pc_returns.sh starts the real host after the card is up. --- SETUP.md | 2 +- ios/EternalMonitor/App/ConnectView.swift | 8 + .../App/ConnectionManager.swift | 163 +++++++++- ios/EternalMonitor/App/PCReturn.swift | 119 +++++++ .../Network/NetworkScanner.swift | 3 + ios/EternalMonitorTests/PCReturnTests.swift | 293 ++++++++++++++++++ scripts/e2e_pc_returns.sh | 185 +++++++++++ 7 files changed, 771 insertions(+), 2 deletions(-) create mode 100644 ios/EternalMonitor/App/PCReturn.swift create mode 100644 ios/EternalMonitorTests/PCReturnTests.swift create mode 100755 scripts/e2e_pc_returns.sh diff --git a/SETUP.md b/SETUP.md index e9db7f0a..5618e8e1 100644 --- a/SETUP.md +++ b/SETUP.md @@ -255,7 +255,7 @@ try fewer words. If your problem isn't here, see [Known issues](https://eternalm | **EternalMonitor can't look for PCs** | The iPad isn't allowed to search your local network. | Tap **Open Settings** and turn on **Local Network**, or go to **Settings > Privacy & Security > Local Network** and turn on **EternalMonitor**. Then tap **Search again**. A USB cable works without this setting. | | **Couldn't search for PCs** | The iPad's search for PCs failed for a reason other than Local Network access. | Tap **Search again**, or scan the QR code or type the **Address** shown in the PC window. If it keeps failing, turn on **Local Network** for EternalMonitor in the iPad's Settings. | | **Couldn't get its address. Type the Address shown on the PC.** under a found PC | The iPad saw the PC's name but not its address. | Type the **Address** shown in the PC window, then tap **Connect to PC**. | -| **Your PC didn't answer** | Nothing came back from the PC. It may be closed, asleep, on another network, or blocked by a firewall. | Check that EternalMonitor is open on the PC and says **Ready for your iPad**, that the PC is awake, and that both are on the same Wi-Fi, not a guest network. If all of that is right, Windows Firewall or security software is probably blocking EternalMonitor: run EternalMonitor-Setup.exe again, or use a USB cable. Then tap **Try again**. | +| **Your PC didn't answer** | Nothing came back from the PC. It may be closed, asleep, on another network, or blocked by a firewall. | Check that EternalMonitor is open on the PC and says **Ready for your iPad**, that the PC is awake, and that both are on the same Wi-Fi, not a guest network. If all of that is right, Windows Firewall or security software is probably blocking EternalMonitor: run EternalMonitor-Setup.exe again, or use a USB cable. Then tap **Try again**. If you open EternalMonitor on the PC within a few minutes of the message appearing, the iPad connects by itself. | | **Your PC didn't answer over the cable** | The cable is connected, but EternalMonitor on the PC didn't reply through it. | Check that EternalMonitor and Apple Devices are both open on the PC, then unplug the cable and plug it in again. | | **Your PC answered, but no picture arrived** | The connection worked, but the PC's second screen didn't start, or the PC couldn't send its screen. | On the PC, choose **Mirror my screen** (**Main display (mirror)** in version 0.3.1), then tap **Try again** on the iPad. If the PC already mirrors, click **Restart** there first. After two attempts in a row end this way, the iPad stops reconnecting by itself until you tap **Try again**. | | **That looks like the pairing code, not the PC's address** | A six-digit number was typed where the address goes. | Type the **Address** from the PC window, for example 192.168.1.20. The iPad asks for the code after it connects. | diff --git a/ios/EternalMonitor/App/ConnectView.swift b/ios/EternalMonitor/App/ConnectView.swift index c2fe400f..fc968654 100644 --- a/ios/EternalMonitor/App/ConnectView.swift +++ b/ios/EternalMonitor/App/ConnectView.swift @@ -352,6 +352,14 @@ struct ConnectView: View { .foregroundStyle(Theme.textMuted) .fixedSize(horizontal: false, vertical: true) .accessibilityIdentifier("connect.problemDetail") + if connectionManager.watchingForPC { + // Problem cards stay English. + Text(verbatim: "Waiting for your PC: this iPad connects by itself when EternalMonitor opens there.") + .font(.app(14, .medium, relativeTo: .callout)) + .foregroundStyle(Theme.text) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("connect.problemWaiting") + } if !problem.actions.isEmpty { HStack(spacing: 10) { ForEach(problem.actions, id: \.self) { action in diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 4e197d61..18c209af 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -86,7 +86,12 @@ final class ConnectionManager: ObservableObject { @Published var connectionError: String? /// A connection problem in plain words, with its next action. Shown in /// place of `connectionError` when set. - @Published var problem: ConnectionProblem? + @Published var problem: ConnectionProblem? { + didSet { problemChanged() } + } + /// The "didn't answer" card is up and this iPad connects by itself when + /// the PC shows up on the network (see `PCReturn`). + @Published private(set) var watchingForPC = false /// The link failure classified for the current attempt. A link's close /// callback that follows its own failure reports this, not a removed cable. private var linkFailure: ConnectionProblem? @@ -249,6 +254,12 @@ final class ConnectionManager: ObservableObject { /// How long video the PC says it sent may go missing before the /// next no-video step (see `NoVideoFallback`). var noVideoStepSeconds = 1.5 + /// How long after the PC that didn't answer shows up on the network + /// the one automatic try waits, so its stream can start and a few + /// Bonjour updates count as one. + var pcReturnDebounceSeconds = 1.0 + /// How long the "didn't answer" card watches for the PC. + var pcReturnLimitSeconds: TimeInterval = 180 } var timing: Timing /// The Wi-Fi link to a host's port. Tests pass a fake one, or replace it @@ -293,6 +304,17 @@ final class ConnectionManager: ObservableObject { /// the step whose result isn't known yet, as its log line names it. private var noVideoFallback: NoVideoFallback? private var noVideoStep: String? + /// Watches Bonjour for the PC that didn't answer. Tests pass a fake. + var makePCBrowser: () -> any PCBrowser = { BonjourPCBrowser() } + /// The PC watched for, its search (stopped in the background), the + /// automatic try waiting out its debounce, and the time limit. + private var pcReturn: PCReturn? + private var pcReturnBrowser: (any PCBrowser)? + private var pcReturnPaused = false + private var pcReturnTry: Task? + private var pcReturnLimit: Task? + /// The automatic try is starting: the attempt it starts doesn't end the watch. + private var startingPCReturnTry = false init(timing: Timing = Timing(), udpReceiver: @escaping (UInt16) -> any MediaLink = { UDPReceiver(port: $0) }, pairingStore: PairingStore? = nil) { @@ -382,6 +404,7 @@ final class ConnectionManager: ObservableObject { /// may be looked up by name again if the address changes. func connect(host: String, port: UInt16, token: Data? = nil, found: Bool = false) { guard state == .disconnected else { return } + if !startingPCReturnTry { endPCReturn() } explicitPairingToken = token if found { foundAddresses.insert(PairingStore.address(host: host, port: port)) } if let token { @@ -489,6 +512,8 @@ final class ConnectionManager: ObservableObject { guard state == .disconnected else { return } if case .udp(let host, let port) = target, host.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || port == 0 { return } + // The user picked a PC, or a cable took over. + if !startingPCReturnTry { endPCReturn() } endReconnectCycle() if case .udp(let host, _) = target, let typed = ConnectionProblem.typedAddress(host) { connectionError = nil @@ -630,6 +655,7 @@ final class ConnectionManager: ObservableObject { self.noVideoFallback = nil self.timeoutTask?.cancel() self.noPictureStreak = NoPictureStreak() + self.endPCReturn() self.state = .connected self.streamingSince = Date() // A reconnect attempt that streams ends its cycle. @@ -906,6 +932,11 @@ final class ConnectionManager: ObservableObject { self.disconnect(.legacyPC) } else { self.record(.warning, "ctrl", "HELLO2 handshake got no reply yet") + // No more HELLO2s go out: a PC app that opens from now + // on is watched for. + if !isUSB, self.attemptMove == nil, self.attemptRedirect == nil, self.reconnectCycleID == nil { + self.watchForPC(host: normalizedHost, port: port) + } if self.leaveRememberedAddress(.pcMoved, result: "no_reply") { // Back to the address tapped. } else if !isUSB { @@ -1433,6 +1464,129 @@ final class ConnectionManager: ObservableObject { reconnectAttempt = 0 } + // MARK: - The PC that didn't answer comes back + + private var showsNoReplyOverWiFi: Bool { + switch problem { + case .noReply(usb: false)?, .noReplyAtNewAddress?: return true + default: return false + } + } + + /// Starts watching Bonjour for the PC at `host:port`, by its service + /// when this iPad knows its name. A watch for that PC goes on as it was. + private func watchForPC(host: String, port: UInt16) { + if let watch = pcReturn, watch.host == host, watch.port == port { return } + endPCReturn() + let address = PairingStore.address(host: host, port: port) + let name = RecentConnectionStore.shared.entry(host: host, port: port)?.name ?? pairingStore.pcName(address: address) + pcReturn = PCReturn(service: name.flatMap { $0.isEmpty ? nil : MovedPC.serviceName(pcName: $0) }, + host: host, port: port, at: now()) + let browser = makePCBrowser() + browser.onHosts = { [weak self] hosts in self?.pcReturnHeard(hosts) } + pcReturnBrowser = browser + pcReturnPaused = !isForeground + if isForeground { browser.start() } + } + + /// The "didn't answer" card is up for the PC at `host:port`: it connects + /// once the PC shows up, until the time limit. + private func armPCReturn(host: String, port: UInt16) { + watchForPC(host: host, port: port) + guard pcReturn?.armedAt == nil else { return considerPCReturn() } + pcReturn?.arm(at: now()) + watchingForPC = true + record(.info, "connection", "pc_return result=watching") + let limit = timing.pcReturnLimitSeconds + pcReturnLimit = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(limit)) + guard !Task.isCancelled, let self, self.pcReturn != nil else { return } + self.record(.info, "connection", "pc_return result=expired") + self.endPCReturn() + } + considerPCReturn() + } + + private func pcReturnHeard(_ hosts: [DiscoveredHost]) { + guard pcReturn != nil else { return } + pcReturn?.heard(hosts, at: now()) + considerPCReturn() + } + + /// The PC showed up while the card is up, in the foreground: one try, + /// after the debounce. Updates meanwhile count as the same appearance. + private func considerPCReturn() { + guard let watch = pcReturn, watch.armedAt != nil, watch.appeared else { return } + guard !watch.expired(at: now(), limit: timing.pcReturnLimitSeconds) else { + record(.info, "connection", "pc_return result=expired") + return endPCReturn() + } + guard isForeground, state == .disconnected, showsNoReplyOverWiFi, pcReturnTry == nil else { return } + let wait = timing.pcReturnDebounceSeconds + pcReturnTry = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(wait)) + guard !Task.isCancelled, let self else { return } + self.pcReturnTry = nil + guard self.isForeground, self.state == .disconnected, self.showsNoReplyOverWiFi, + let watch = self.pcReturn, !watch.expired(at: self.now(), limit: self.timing.pcReturnLimitSeconds), + self.pcReturn?.takeAppearance() == true else { return } + self.tryPCAgain(host: watch.host, port: watch.port) + } + } + + /// Connects as Try again does, without ending the watch: if the PC + /// still doesn't answer, the card comes back and waits for it again. + private func tryPCAgain(host: String, port: UInt16) { + record(.info, "connection", "pc_return result=tried") + startingPCReturnTry = true + defer { startingPCReturnTry = false } + problem = nil + connectionError = nil + let target = lastTarget ?? (host: host, port: port) + connect(host: target.host, port: target.port) + } + + /// Dismissed, or replaced by another card: the watch ends. An attempt + /// that starts clears the card too; a user's own ends the watch where + /// it starts. + private func problemChanged() { + guard pcReturn != nil, !startingPCReturnTry, !showsNoReplyOverWiFi else { return } + if problem == nil && state != .disconnected { return } + endPCReturn() + } + + /// The app left the foreground: the search stops and no try starts. + private func pausePCReturn() { + pcReturnTry?.cancel() + pcReturnTry = nil + guard let browser = pcReturnBrowser, !pcReturnPaused else { return } + pcReturnPaused = true + browser.stop() + } + + /// Back in the foreground: the search starts again, and a PC that + /// showed up meanwhile gets its try. + private func resumePCReturn() { + guard let browser = pcReturnBrowser, pcReturnPaused else { return } + pcReturnPaused = false + pcReturn?.restarted(at: now()) + browser.start() + considerPCReturn() + } + + private func endPCReturn() { + pcReturnTry?.cancel() + pcReturnTry = nil + pcReturnLimit?.cancel() + pcReturnLimit = nil + pcReturnBrowser?.onHosts = nil + pcReturnBrowser?.stop() + pcReturnBrowser = nil + pcReturnPaused = false + pcReturn = nil + if watchingForPC { watchingForPC = false } + } + /// The session and how long the current attempt has taken, appended to /// the support log's first-frame and timeout lines. private var attemptTimings: String { @@ -1471,6 +1625,9 @@ final class ConnectionManager: ObservableObject { // The app gave up, not the user: the PC's log tells them apart. self.disconnect(.connectTimeout) if handshakeDone && self.noPictureStreak.stopsReconnecting { self.stopAutomaticAttempts(isUSB: isUSB) } + if self.showsNoReplyOverWiFi && self.reconnectCycleID == nil { + self.armPCReturn(host: debug.host, port: debug.port) + } } } } @@ -1590,6 +1747,7 @@ final class ConnectionManager: ObservableObject { isForeground = false usbListener?.stop() usbListener = nil + pausePCReturn() // A suspended app can't reconnect. Backgrounding ends an automatic // cycle where running out would, and foregrounding resumes it as an // interrupted session. @@ -1643,6 +1801,7 @@ final class ConnectionManager: ObservableObject { recheckAppleDevicesSighting() usbPaused = usbStoppedForNoPicture refreshUSBAvailability() + resumePCReturn() guard resumeOnForeground else { return } resumeOnForeground = false guard UserDefaults.standard.flag(forKey: "autoResumeOnForeground"), @@ -1675,6 +1834,8 @@ final class ConnectionManager: ObservableObject { state = .disconnected return } + // Only an attempt that got no answer leaves the PC watched for. + if ![.connectTimeout, .pcMoved, .otherPC].contains(end) { endPCReturn() } if state != .disconnected { endNoVideoStep("no_video") recordStreamSummary(reason: "disconnect=\(end.name)") diff --git a/ios/EternalMonitor/App/PCReturn.swift b/ios/EternalMonitor/App/PCReturn.swift new file mode 100644 index 00000000..8ae086b4 --- /dev/null +++ b/ios/EternalMonitor/App/PCReturn.swift @@ -0,0 +1,119 @@ +import Combine +import Foundation + +/// Bonjour's list of PCs, kept up to date until stopped. Tests pass a fake. +@MainActor +protocol PCBrowser: AnyObject { + /// Called with the whole list each time it changes. + var onHosts: (([DiscoveredHost]) -> Void)? { get set } + func start() + func stop() +} + +/// A Bonjour search that runs until stopped. +@MainActor +final class BonjourPCBrowser: PCBrowser { + var onHosts: (([DiscoveredHost]) -> Void)? + private let scanner = NetworkScanner() + private var watch: AnyCancellable? + + init() { + scanner.retriesWhenEmpty = false + scanner.searchesUntilStopped = true + } + + func start() { + // One update replaces a PC's entry in steps; only the list it ends on counts. + watch = scanner.$hosts.debounce(for: .milliseconds(100), scheduler: RunLoop.main).sink { [weak self] _ in + guard let self else { return } + self.onHosts?(self.scanner.hosts) + } + scanner.startScan() + } + + func stop() { + watch = nil + scanner.stopScan() + } +} + +/// The PC that didn't answer, watched for on Bonjour. Its HELLO2s go out +/// for two seconds only, so a PC app that opens after that never hears this +/// iPad: the PC's service turning up, or changing, is the sign to try once +/// more. What Bonjour already listed when the search began is no sign; the +/// PC was there and didn't answer. +struct PCReturn { + /// The first second of a search lists what Bonjour already knew. + static let settleSeconds = 1.0 + + /// The PC's Bonjour service, when this iPad knows the PC's name. The + /// service at the address tried is the PC too. + let service: String? + let host: String + let port: UInt16 + /// Until when the latest search lists what Bonjour already knew. + private var settlesAt: Date + /// A search ran before this one: what it listed is what was known. + private var resumed = false + /// The PC's listings, by name, address, port and version. + private var listed = Set() + /// The latest list while settling, which replaces `listed` after it. + private var settled: Set? + /// A listing turned up that has had no try yet. + private(set) var appeared = false + /// When the card went up and the watch's time limit started. + private(set) var armedAt: Date? + + init(service: String?, host: String, port: UInt16, at now: Date) { + self.service = service + self.host = host + self.port = port + settlesAt = now.addingTimeInterval(Self.settleSeconds) + } + + func matches(_ pc: DiscoveredHost) -> Bool { + guard let address = pc.address else { return false } + return (service != nil && pc.name == service) || (address == host && pc.port == port) + } + + /// Bonjour's list now. A listing of the PC that wasn't there before is + /// an appearance; a new address, port or version is one too. + mutating func heard(_ hosts: [DiscoveredHost], at now: Date) { + let current = Set(hosts.filter(matches).map { "\($0.name)|\($0.address ?? "")|\($0.port)|\($0.version ?? "")" }) + if now < settlesAt { + // A search that starts again lists only what changed meanwhile as new. + if resumed, !current.subtracting(listed).isEmpty { appeared = true } + listed.formUnion(current) + settled = current + return + } + if let settled { + listed = settled + self.settled = nil + } + if !current.subtracting(listed).isEmpty { appeared = true } + // A listing that left and comes back is a new appearance. + listed = current + } + + /// The search starts again, as when the app comes back to the foreground. + mutating func restarted(at now: Date) { + settlesAt = now.addingTimeInterval(Self.settleSeconds) + resumed = true + } + + /// The card is up: the time limit starts, once. + mutating func arm(at now: Date) { + if armedAt == nil { armedAt = now } + } + + func expired(at now: Date, limit: TimeInterval) -> Bool { + armedAt.map { now.timeIntervalSince($0) >= limit } ?? false + } + + /// Takes the appearance for its one try. + mutating func takeAppearance() -> Bool { + defer { appeared = false } + return appeared + } +} diff --git a/ios/EternalMonitor/Network/NetworkScanner.swift b/ios/EternalMonitor/Network/NetworkScanner.swift index bd65a5a5..99056438 100644 --- a/ios/EternalMonitor/Network/NetworkScanner.swift +++ b/ios/EternalMonitor/Network/NetworkScanner.swift @@ -94,6 +94,8 @@ final class NetworkScanner: NSObject, ObservableObject { /// Search once more when a search finds nothing. Off for a one-off /// lookup that stops the search itself. var retriesWhenEmpty = true + /// Keep searching until `stopScan`, for a watch rather than a lookup. + var searchesUntilStopped = false override init() { super.init() @@ -122,6 +124,7 @@ final class NetworkScanner: NSObject, ObservableObject { } #endif browser.searchForServices(ofType: "_eternaldisplay._udp.", inDomain: "local.") + guard !searchesUntilStopped else { return } scanTask = Task { @MainActor in // 10s timeout — many routers are slow to forward the first mDNS diff --git a/ios/EternalMonitorTests/PCReturnTests.swift b/ios/EternalMonitorTests/PCReturnTests.swift new file mode 100644 index 00000000..233a2b26 --- /dev/null +++ b/ios/EternalMonitorTests/PCReturnTests.swift @@ -0,0 +1,293 @@ +import Combine +import XCTest +@testable import EternalMonitor + +/// Bonjour's list as a test hands it over. +@MainActor +private final class FakePCBrowser: PCBrowser { + var onHosts: (([DiscoveredHost]) -> Void)? + private(set) var starts = 0 + private(set) var stops = 0 + var running: Bool { starts > stops } + func start() { starts += 1 } + func stop() { stops += 1 } + func list(_ hosts: [DiscoveredHost]) { onHosts?(hosts) } +} + +/// The iPad shows "didn't answer" because the PC app wasn't running yet. +/// When the PC's service then shows up on the network, it connects once by +/// itself, while the card is up, in the foreground, for a few minutes. +@MainActor +final class PCReturnTests: XCTestCase { + private let start = Date(timeIntervalSince1970: 1_000_000) + + private func pc(_ name: String = "EternalMonitor (Test PC)", _ address: String? = "127.0.0.1", port: UInt16, + version: String? = "0.3.5") -> DiscoveredHost { + DiscoveredHost(name: name, address: address, port: port, version: version) + } + + // MARK: - What counts as the PC showing up + + func testOnlyAListingThatWasNotThereBeforeCounts() { + var watch = PCReturn(service: "EternalMonitor (Test PC)", host: "192.0.2.20", port: 9876, at: start) + let listed = pc(port: 9876) + // What Bonjour already knew when the search began. + watch.heard([listed], at: start.addingTimeInterval(0.2)) + watch.heard([listed], at: start.addingTimeInterval(PCReturn.settleSeconds + 1)) + XCTAssertFalse(watch.appeared, "the PC was listed and didn't answer") + // It leaves (the app closed) and comes back. + watch.heard([], at: start.addingTimeInterval(5)) + XCTAssertFalse(watch.appeared) + watch.heard([listed], at: start.addingTimeInterval(9)) + XCTAssertTrue(watch.takeAppearance()) + XCTAssertFalse(watch.takeAppearance(), "one try per appearance") + // The same listing again is no new appearance; a change is. + watch.heard([listed], at: start.addingTimeInterval(10)) + XCTAssertFalse(watch.appeared) + watch.heard([pc(port: 9877)], at: start.addingTimeInterval(11)) + XCTAssertTrue(watch.takeAppearance(), "a new port") + watch.heard([pc(port: 9877, version: "0.3.6")], at: start.addingTimeInterval(12)) + XCTAssertTrue(watch.takeAppearance(), "a new version") + } + + func testAPCMissingAtTheStartCountsAsSoonAsItShowsUp() { + var watch = PCReturn(service: nil, host: "192.0.2.20", port: 9876, at: start) + watch.heard([], at: start.addingTimeInterval(0.1)) + watch.heard([pc("EternalMonitor (Test PC)", "192.0.2.20", port: 9876)], at: start.addingTimeInterval(PCReturn.settleSeconds + 0.5)) + XCTAssertTrue(watch.appeared) + } + + func testOnlyThisPCCounts() { + var named = PCReturn(service: "EternalMonitor (Test PC)", host: "192.0.2.20", port: 9876, at: start) + var unnamed = PCReturn(service: nil, host: "192.0.2.20", port: 9876, at: start) + let later = start.addingTimeInterval(5) + let others = [pc("EternalMonitor (Other PC)", "192.0.2.30", port: 9876), + pc("EternalMonitor (Test PC 2)", "192.0.2.31", port: 9876), + pc("EternalMonitor (Other PC)", "192.0.2.20", port: 9900), + pc("EternalMonitor (Test PC)", nil, port: 9876)] + named.heard(others, at: later) + unnamed.heard(others, at: later) + XCTAssertFalse(named.appeared) + XCTAssertFalse(unnamed.appeared) + // By name at another address, or by the address tried. + named.heard(others + [pc("EternalMonitor (Test PC)", "192.0.2.40", port: 9876)], at: later) + XCTAssertTrue(named.appeared) + unnamed.heard(others + [pc("EternalMonitor (Any)", "192.0.2.20", port: 9876)], at: later) + XCTAssertTrue(unnamed.appeared) + } + + /// Back from the background, the new search lists what changed while + /// the app was away as new, and what was there before as known. + func testASearchStartedAgainKeepsWhatItKnew() { + var watch = PCReturn(service: "EternalMonitor (Test PC)", host: "192.0.2.20", port: 9876, at: start) + watch.heard([pc(port: 9876)], at: start.addingTimeInterval(0.1)) + watch.restarted(at: start.addingTimeInterval(30)) + watch.heard([], at: start.addingTimeInterval(30.1)) + watch.heard([pc(port: 9876)], at: start.addingTimeInterval(30.2)) + XCTAssertFalse(watch.appeared, "still the listing that didn't answer") + + var away = PCReturn(service: "EternalMonitor (Test PC)", host: "192.0.2.20", port: 9876, at: start) + away.heard([], at: start.addingTimeInterval(0.1)) + away.restarted(at: start.addingTimeInterval(30)) + away.heard([pc(port: 9876)], at: start.addingTimeInterval(30.2)) + XCTAssertTrue(away.appeared, "it showed up while the app was away") + } + + func testTheTimeLimitStartsWhenTheCardGoesUp() { + var watch = PCReturn(service: nil, host: "192.0.2.20", port: 9876, at: start) + XCTAssertFalse(watch.expired(at: start.addingTimeInterval(1_000), limit: 180), "not armed yet") + watch.arm(at: start.addingTimeInterval(10)) + watch.arm(at: start.addingTimeInterval(100)) + XCTAssertFalse(watch.expired(at: start.addingTimeInterval(189), limit: 180)) + XCTAssertTrue(watch.expired(at: start.addingTimeInterval(190), limit: 180), "a second card keeps the first limit") + } + + // MARK: - Connecting + + private struct Setup { + let manager: ConnectionManager + let browser: FakePCBrowser + let host: FakeUDPHost + let lines: () -> [String] + let log: AnyCancellable + /// Moves the manager's clock. + let advance: (TimeInterval) -> Void + + var tries: Int { lines().filter { $0 == "pc_return result=tried" }.count } + var attempts: Int { lines().filter { $0.hasPrefix("Starting connection attempt=") }.count } + var listing: DiscoveredHost { DiscoveredHost(name: "EternalMonitor (Test PC)", address: "127.0.0.1", port: host.port) } + } + + override func setUp() async throws { + let store = RecentConnectionStore.shared + let saved = store.connections + let defaults = ["recentConnections", "lastHost", "lastPort", "allowUSB", "autoReconnect"] + .map { ($0, UserDefaults.standard.object(forKey: $0)) } + // No cable listener in these tests. + UserDefaults.standard.set(false, forKey: "allowUSB") + addTeardownBlock { @MainActor in + store.connections = saved + for (key, value) in defaults { UserDefaults.standard.set(value, forKey: key) } + } + } + + /// A PC that doesn't answer at loopback, saved under its name so the + /// watch knows its service. The "didn't answer" card is up when this + /// returns, and the watch armed. + private func noReply(named: Bool = true) async throws -> Setup { + let host = FakeUDPHost() + if named { + RecentConnectionStore.shared.connections = [ + RecentConnection(host: "127.0.0.1", port: host.port, lastUsed: .now, isUSB: false, name: "Test PC", found: false), + ] + } + let manager = ConnectionManager(timing: .init(connectTimeoutSeconds: 4, reconnectGraceSeconds: 0.1, + reconnectBackoffSeconds: { _ in 0.05 }, pcReturnDebounceSeconds: 0.1), + pairingStore: PairingStore(storage: MemoryPairingStorage())) + var clock = start + manager.now = { clock } + let browser = FakePCBrowser() + manager.makePCBrowser = { browser } + var lines: [String] = [] + let log = manager.$diagnostics.sink { if let line = $0.last?.message { lines.append(line) } } + manager.connect(host: "127.0.0.1", port: host.port) + try await until("didn't answer", seconds: 15) { manager.problem == .noReply(usb: false) } + XCTAssertTrue(manager.watchingForPC) + XCTAssertTrue(browser.running, "the search runs while the card is up") + // Bonjour has nothing for the PC: its app isn't open. + browser.list([]) + clock = start.addingTimeInterval(PCReturn.settleSeconds + 1) + // Drop the unanswered HELLO2s, so only a new attempt's count. + _ = await Task.detached { host.helloAttempts(within: 0.4) }.value + return Setup(manager: manager, browser: browser, host: host, lines: { lines }, log: log, + advance: { clock = clock.addingTimeInterval($0) }) + } + + private func until(_ description: String, seconds: Double, _ condition: () -> Bool) async throws { + let deadline = Date().addingTimeInterval(seconds) + while !condition(), Date() < deadline { try await Task.sleep(nanoseconds: 10_000_000) } + XCTAssertTrue(condition(), description) + } + + private func settle(_ seconds: Double = 1) async throws { + try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) + } + + /// No answer; the PC app opens and its service shows up; the iPad + /// connects once, by itself, and streams. + func testThePCShowingUpConnectsOnceByItself() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + XCTAssertEqual(setup.attempts, 1) + setup.browser.list([setup.listing]) + // Updates while it waits count as the same appearance. + setup.browser.list([setup.listing]) + let accepted = await Task.detached { [host = setup.host] in host.acceptSession() }.value + XCTAssertTrue(accepted, "no automatic attempt reached the PC") + try await until("handshake", seconds: 15) { setup.manager.hostInfo != nil } + setup.host.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming", seconds: 30) { setup.manager.state == .connected } + XCTAssertEqual(setup.tries, 1) + XCTAssertEqual(setup.attempts, 2) + XCTAssertFalse(setup.manager.watchingForPC) + XCTAssertFalse(setup.browser.running, "streaming ends the watch") + } + + /// The automatic try gets no answer either: the card comes back and + /// waits for the PC to show up again. The same listing never brings + /// another try. + func testOneTryPerAppearance() async throws { + let setup = try await noReply(named: false) + defer { setup.manager.cancel() } + let listing = DiscoveredHost(name: "EternalMonitor (Unknown)", address: "127.0.0.1", port: setup.host.port) + setup.browser.list([listing]) + try await until("tried", seconds: 5) { setup.tries == 1 } + try await until("didn't answer again", seconds: 15) { + setup.manager.state == .disconnected && setup.manager.problem == .noReply(usb: false) + } + XCTAssertTrue(setup.manager.watchingForPC, "still waiting for the PC") + setup.browser.list([listing]) + try await settle() + XCTAssertEqual(setup.tries, 1) + XCTAssertEqual(setup.attempts, 2) + // It leaves and comes back: one more try. + setup.browser.list([]) + setup.browser.list([listing]) + try await until("tried again", seconds: 5) { setup.tries == 2 } + } + + func testNoTryAfterTheCardIsDismissed() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + setup.manager.problem = nil + XCTAssertFalse(setup.manager.watchingForPC) + XCTAssertFalse(setup.browser.running) + setup.browser.list([setup.listing]) + try await settle() + XCTAssertEqual(setup.attempts, 1) + } + + func testNoTryAfterTheTimeLimit() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + setup.advance(ConnectionManager.Timing().pcReturnLimitSeconds) + setup.browser.list([setup.listing]) + try await settle() + XCTAssertEqual(setup.attempts, 1) + XCTAssertFalse(setup.manager.watchingForPC) + XCTAssertFalse(setup.browser.running) + XCTAssertTrue(setup.lines().contains("pc_return result=expired")) + } + + /// Nothing starts in the background. Back in the foreground, a PC that + /// showed up meanwhile gets its try. + func testNoTryInTheBackground() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + setup.manager.handleAppBackgrounded() + XCTAssertFalse(setup.browser.running, "no search in the background") + setup.browser.list([setup.listing]) + try await settle() + XCTAssertEqual(setup.attempts, 1) + setup.manager.handleAppForegrounded() + XCTAssertTrue(setup.browser.running) + try await until("tried on return", seconds: 5) { setup.tries == 1 } + } + + func testPickingAnotherPCEndsTheWatch() async throws { + let setup = try await noReply() + let other = FakeUDPHost() + setup.manager.connect(host: "127.0.0.1", port: other.port) + defer { setup.manager.cancel() } + XCTAssertFalse(setup.browser.running) + setup.browser.list([setup.listing]) + try await settle() + XCTAssertEqual(setup.tries, 0) + } + + /// A PC that turns the automatic try down (busy here) ends the watch: + /// it never loops. + func testARefusalEndsTheWatch() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + setup.browser.list([setup.listing]) + let refused = await Task.detached { [host = setup.host] in host.acceptSession(status: .busy) }.value + XCTAssertTrue(refused) + try await until("busy", seconds: 15) { setup.manager.problem == .pcBusy } + XCTAssertFalse(setup.browser.running) + setup.browser.list([]) + setup.browser.list([setup.listing]) + try await settle() + XCTAssertEqual(setup.tries, 1) + } +} + +/// Pairings kept in memory. +private final class MemoryPairingStorage: PairingStorage { + var values: [String: Data] = [:] + func records() throws -> [PairingRecord] { values.map { PairingRecord(account: $0.key, token: $0.value) } } + func save(account: String, token: Data) throws { values[account] = token } + func remove(account: String?) throws { + if let account { values.removeValue(forKey: account) } else { values.removeAll() } + } +} diff --git a/scripts/e2e_pc_returns.sh b/scripts/e2e_pc_returns.sh new file mode 100755 index 00000000..be3106bf --- /dev/null +++ b/scripts/e2e_pc_returns.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# The PC app isn't open when the iPad connects, and opens afterwards. The real +# app streams once from the real host, so it knows the PC's name and pairing. +# The host is then stopped and the app opened again: it connects to the saved +# PC, which doesn't answer, and shows "didn't answer". The host then starts +# again and advertises with Bonjour, and the app should connect and stream by +# itself, with no tap. +# +# On a shared Mac this is an iOS e2e run: hold the machine's e2e lock and use +# your own simulator (EM_SIM_UDID). The PC's port is picked free when the run +# starts; EM_PORT chooses it instead, and the run stops before anything starts +# if it is taken. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +export DEVELOPER_DIR="${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer}" +export PATH="$HOME/.cargo/bin:$PATH" +export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:-/opt/homebrew/opt/ffmpeg@7/lib/pkgconfig}" + +UDID="${EM_SIM_UDID:?set EM_SIM_UDID to your own simulator}" +TIMEOUT_SECS="${EM_TIMEOUT:-90}" +# How long the card is up before the PC app opens. +CLOSED_SECS="${EM_CLOSED_SECS:-8}" +OUT="${EM_OUTPUT_DIR:-$ROOT/build/e2e/pc-returns}" +HOST_BIN="${EM_HOST_BIN:-$ROOT/target/release/eternal-host}" +# The PC's name: its Bonjour service and HELLO_ACK both carry it. A new one +# each run, so no earlier run's Bonjour record stands in for it. +PC_NAME="EMReturnPC$$" +# A test pairing token, written to the host's settings and handed to the +# app in a pairing link, as the PC's QR code would. +TOKEN="0123456789abcdef0123456789abcdef" +mkdir -p "$OUT" +rm -f "$OUT/result.json" +APP_LOG="$OUT/app.log" +HOST_LOG="$OUT/host.log" + +# This Mac's address on its home network, as an iPad on the same Wi-Fi dials it. +IFACE=$(route -n get default 2>/dev/null | awk '/interface:/ {print $2}') +LAN=$(ipconfig getifaddr "$IFACE" 2>/dev/null || true) +[ -n "$LAN" ] || { echo "FAIL: no address on the default interface" >&2; exit 1; } + +PORT=$(python3 - "${EM_PORT:-0}" <<'PY' +import socket,sys +wanted=int(sys.argv[1]) +try: + udp=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); udp.bind(('0.0.0.0',wanted)) + port=udp.getsockname()[1] + tcp=socket.socket(socket.AF_INET,socket.SOCK_STREAM); tcp.bind(('0.0.0.0',port)) +except OSError: + print(f"port {wanted} is in use"); sys.exit(1) +print(port) +PY +) || { echo "FAIL: $PORT" >&2; exit 1; } + +HOST_PID="" +cleanup() { + status=$? + trap - EXIT + [ -n "$HOST_PID" ] && kill -TERM "$HOST_PID" 2>/dev/null || true + [ -n "$HOST_PID" ] && wait "$HOST_PID" 2>/dev/null || true + xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true + python3 - "$OUT/result.json" "$status" <<'PY' +import json,pathlib,sys +p=pathlib.Path(sys.argv[1]) +r=json.loads(p.read_text()) if p.exists() else {} +r.setdefault('status', 'PASS' if sys.argv[2]=='0' else 'FAIL') +p.write_text(json.dumps(r, indent=2)+'\n') +PY + [ "$status" -eq 0 ] || echo "FAIL: evidence at $OUT" >&2 + exit "$status" +} +trap cleanup EXIT + +if [ "${EM_SKIP_BUILD:-0}" != 1 ]; then + echo "==> Building app" + (cd "$ROOT/ios" && xcodegen generate >/dev/null) + xcodebuild build -project "$ROOT/ios/EternalMonitor.xcodeproj" -scheme EternalMonitor -configuration Release \ + -destination "platform=iOS Simulator,id=$UDID" -derivedDataPath "$ROOT/ios/build/e2e" \ + CODE_SIGNING_ALLOWED=YES CODE_SIGN_IDENTITY=- -quiet + if [ -z "${EM_HOST_BIN:-}" ]; then + echo "==> Building host" + (cd "$ROOT" && cargo build -q --release -p eternal-host --locked) + fi +fi +APP="$ROOT/ios/build/e2e/Build/Products/Release-iphonesimulator/EternalMonitor.app" +[ -d "$APP" ] || { echo "FAIL: app bundle not found at $APP"; exit 1; } +[ -x "$HOST_BIN" ] || { echo "FAIL: host binary not found"; exit 1; } + +echo "==> Installing app with nothing saved" +xcrun simctl bootstatus "$UDID" -b >/dev/null +xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true +xcrun simctl uninstall "$UDID" com.eternal.monitor 2>/dev/null || true +# Pairings live in the simulator's Keychain and outlast the app. +xcrun simctl keychain "$UDID" reset +STAGE="$(mktemp -d /tmp/em-pc-returns-install.XXXXXX)" +ditto "$APP" "$STAGE/EternalMonitor.app" +xcrun simctl install "$UDID" "$STAGE/EternalMonitor.app" +rm -rf "$STAGE" +APP_DATA=$(xcrun simctl get_app_container "$UDID" com.eternal.monitor data) +SIM_LOG="$APP_DATA/tmp/eternal-e2e.log" +mkdir -p "$(dirname "$SIM_LOG")" + +mkdir -p "$OUT/state/EternalMonitor" +echo "{\"bitrate_mbps\": 8, \"target_fps\": 30, \"start_on_boot\": false, \"stream_audio\": false, \"require_pairing\": true, \"auth_token_hex\": \"$TOKEN\"}" \ + > "$OUT/state/EternalMonitor/settings.json" +start_host() { + APPDATA="$OUT/state" COMPUTERNAME="$PC_NAME" ETERNAL_HEADLESS=1 ETERNAL_E2E_LOG=1 ETERNAL_USB_DIRECT= \ + ETERNAL_CAPTURE=synthetic ETERNAL_AUDIO=synthetic ETERNAL_SYNTH_SIZE=640x360 ETERNAL_ENCODER=libx264 ETERNAL_FPS=30 \ + "$HOST_BIN" "$PORT" >>"$HOST_LOG" 2>&1 & + HOST_PID=$! +} +stop_host() { + # A stopped app says goodbye on Bonjour, as closing it on the PC does. + kill -TERM "$HOST_PID" + wait "$HOST_PID" 2>/dev/null || true + HOST_PID="" +} +# The app writes its log afresh at each launch. +launch_app() { + rm -f "$SIM_LOG" + touch "$SIM_LOG" + SIMCTL_CHILD_EM_AUTOCONNECT="$1" SIMCTL_CHILD_EM_E2E_LOG=1 SIMCTL_CHILD_EM_USB_PORT=0 \ + xcrun simctl launch "$UDID" com.eternal.monitor -didSeeOnboarding YES -allowUSB NO -autoReconnect NO \ + -autoResumeOnForeground NO -playPCaudio NO >/dev/null +} +seen() { grep -q "$1" "$SIM_LOG"; } +wait_for() { + local what=$1 limit=$2 elapsed=0 + shift 2 + until "$@"; do + sleep 1 + elapsed=$((elapsed + 1)) + if [ "$elapsed" -ge "$limit" ]; then + echo "FAIL: $what after ${limit}s" + echo "----- app milestones -----"; grep -v E2E_STATS "$SIM_LOG" | tail -40 + cp "$SIM_LOG" "$APP_LOG" 2>/dev/null || true + exit 1 + fi + done +} + +: > "$HOST_LOG" +echo "==> Pairing with $PC_NAME at port $PORT and streaming once" +start_host +launch_app "eternaldisplay://$LAN:$PORT?t=$TOKEN" +wait_for "no first stream" "$TIMEOUT_SECS" seen E2E_FIRST_FRAME +cp "$SIM_LOG" "$OUT/app-first.log" + +echo "==> Closing the PC app, then opening the iPad app on the saved PC" +stop_host +xcrun simctl terminate "$UDID" com.eternal.monitor 2>/dev/null || true +launch_app "$LAN:$PORT" +wait_for "no \"didn't answer\" card" "$TIMEOUT_SECS" seen "pc_return result=watching" +sleep "$CLOSED_SECS" +xcrun simctl io "$UDID" screenshot "$OUT/didnt-answer.png" >/dev/null 2>&1 || true +seen E2E_FIRST_FRAME && { echo "FAIL: streamed with the PC app closed"; exit 1; } + +echo "==> Opening the PC app; the iPad should connect by itself (timeout ${TIMEOUT_SECS}s)" +OPENED_AT=$(wc -l < "$SIM_LOG") +start_host +wait_for "no stream after the PC app opened" "$TIMEOUT_SECS" seen E2E_FIRST_FRAME +sleep 2 +xcrun simctl io "$UDID" screenshot "$OUT/streaming.png" >/dev/null 2>&1 || true +cp "$SIM_LOG" "$APP_LOG" + +python3 - "$APP_LOG" "$OPENED_AT" "$OUT/result.json" <<'PY' +import json,pathlib,sys +log,opened_at,result=sys.argv[1:4] +lines=pathlib.Path(log).read_text(errors='replace').splitlines() +before,after=lines[:int(opened_at)],lines[int(opened_at):] +attempt=lambda l: 'Starting connection attempt=' in l and 'transport=WiFi' in l +errors=[] +if not any('outcome=no_reply' in l for l in before): errors.append('no no_reply summary before the PC app opened') +if sum(map(attempt, before))!=1: errors.append('more than one attempt while the PC app was closed') +tries=sum('pc_return result=tried' in l for l in after) +if tries!=1: errors.append(f'{tries} automatic tries after the PC app opened, not 1') +if not any('E2E_FIRST_FRAME' in l for l in after): errors.append('no stream after the PC app opened') +data=dict(scenario='pc-returns', status='FAIL' if errors else 'PASS', errors=errors, + attempts_while_closed=sum(map(attempt, before)), automatic_tries=tries, + attempts_after_opening=sum(map(attempt, after))) +pathlib.Path(result).write_text(json.dumps(data, indent=2)+'\n') +print(json.dumps(data)) +sys.exit(1 if errors else 0) +PY +echo "PASS. Evidence: $OUT" From f4395183436c666549954f9be105fbf5d5140a7c Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:52:58 -0500 Subject: [PATCH 183/226] Make the moving-desktop test need the wait's half frame of margin The jitter in the re-submit tests gave gaps of 0.75 to 1.25 frames, so the moving-desktop test passed with a wait of one frame too. The pattern now has a 1.5-frame gap once per 12 pictures, rarer than the 8 gaps the pace remembers, and the test fails at 1.0x. The `cached` doc now lists the recovery resend. --- host/src/capture/mod.rs | 7 ++++--- host/src/encoder/mod.rs | 15 +++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 097dedc8..4597805f 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -258,9 +258,10 @@ pub struct RawFrame { pub data: Arc>, pub width: u32, pub height: u32, - /// A picture capture had already, sent again for a still desktop: the - /// keepalive, a keyframe request's resend, or the cached picture - /// redrawn with or without the pointer. The desktop gave no new one. + /// A picture capture already had, sent again because the desktop gave + /// no new one: the keepalive, a keyframe request's resend, the cached + /// picture redrawn with or without the pointer, or a resend during + /// duplication recovery. pub cached: bool, } diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index 99cfde2f..1a5e5839 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -6405,17 +6405,20 @@ mod tests { again } - /// Real pictures every `frame`, each up to a quarter frame early or late, - /// so gaps run from half a frame to a frame and a half. + /// Real pictures every `frame`, off by up to a quarter frame (offsets in + /// eighths of a frame). Gaps run from 0.75 to 1.5 frames. The 1.25- and + /// 1.5-frame gaps come once per 12 pictures, rarer than the 8 gaps the + /// pace remembers, so the 1.5-frame gap is longer than the pace when it + /// comes: the wait needs its half frame of margin (1.0x fails here). fn moving(start: Instant, frame: Duration, count: u32) -> Vec<(Instant, bool)> { - let quarter = frame / 4; - let offsets = [0i32, 1, 0, -1, -2]; + let eighth = frame / 8; + let offsets = [0i32, 0, 2, 0, 0, -1, 0, 0, 0, 0, -2, 2]; (0..count) .map(|n| { let at = start + frame * n; let at = match offsets[n as usize % offsets.len()] { - k if k >= 0 => at + quarter * k as u32, - k => at - quarter * (-k) as u32 / 2, + k if k >= 0 => at + eighth * k as u32, + k => at - eighth * (-k) as u32, }; (at, false) }) From d166809f2c3ab7bb7444e8324e8a6313b1dd2cee Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 07:58:51 -0500 Subject: [PATCH 184/226] Time the cable-hold test on a sleeper it controls The Busy test now ends each hold through an injected sleeper and checks the requested waits, instead of timing them on the wall clock from the moment it saw the refusal. It also lets the PC end the accepted session by closing the cable, so the reset of the Busy count on an accepted session is what brings the next wait back to 30 s. --- .../App/ConnectionManager.swift | 6 +- .../USBLinkFailureTests.swift | 67 +++++++++++++------ 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index aff1c2ed..e1633f66 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -257,6 +257,9 @@ final class ConnectionManager: ObservableObject { /// How long the cable goes unanswered after the given number of /// Busy refusals in a row (see `CableHold`). var cableBusyHoldSeconds: (Int) -> Double = CableHold.busySeconds + /// Waits out a cable hold. Tests replace it to end the wait when + /// they choose. + var cableHoldSleep: (Double) async -> Void = { try? await Task.sleep(for: .seconds($0)) } } var timing: Timing /// The Wi-Fi link to a host's port. Tests pass a fake one, or replace it @@ -381,8 +384,9 @@ final class ConnectionManager: ObservableObject { usbListener = nil usbStatus = "USB: disconnected" guard let wait else { return } + let sleep = timing.cableHoldSleep cableHoldTask = Task { @MainActor [weak self] in - try? await Task.sleep(for: .seconds(wait)) + await sleep(wait) guard let self, !Task.isCancelled, self.cableHold == hold else { return } self.endCableHold() self.usbPaused = self.usbStoppedForNoPicture diff --git a/ios/EternalMonitorTests/USBLinkFailureTests.swift b/ios/EternalMonitorTests/USBLinkFailureTests.swift index ad84980a..146868fa 100644 --- a/ios/EternalMonitorTests/USBLinkFailureTests.swift +++ b/ios/EternalMonitorTests/USBLinkFailureTests.swift @@ -653,17 +653,15 @@ final class USBLinkFailureTests: XCTestCase { manager.diagnostics.map(\.message).filter { $0.hasPrefix("session_summary ") } } - /// Offers the PC's dial every 100 ms until the iPad takes one; the - /// seconds from `since` until it did. - private func dialUntilTaken(_ manager: ConnectionManager, since: Date, cables: inout [FramedLink]) async throws -> Double { + /// Offers the PC's dial every 100 ms until the iPad takes one. + private func dialUntilTaken(_ manager: ConnectionManager, cables: inout [FramedLink]) async throws { let deadline = Date().addingTimeInterval(30) while Date() < deadline { cables.append(try await plugIn(manager)) - if manager.state == .connecting { return Date().timeIntervalSince(since) } + if manager.state == .connecting { return } try await Task.sleep(nanoseconds: 100_000_000) } XCTFail("no dial was taken in 30 s") - return .infinity } /// The wait after each Busy in a row: 30 s, doubling, at most 5 minutes. @@ -681,26 +679,27 @@ final class USBLinkFailureTests: XCTestCase { /// A PC streaming to another device turns the cable down as Busy, then /// dials it again a second later. The iPad shows why and doesn't take /// the cable again until the wait ends, a longer wait after each Busy in - /// a row, back to the first after a session. Each attempt's summary - /// names the refusal. + /// a row, back to the first after a session the PC accepted. Each + /// attempt's summary names the refusal. The waits run on a sleeper the + /// test ends, so the timing of this machine doesn't matter. func testABusyPCOnTheCableIsAnsweredOnlyAfterAGrowingWait() async throws { let previousUSB = UserDefaults.standard.object(forKey: "allowUSB") defer { UserDefaults.standard.set(previousUSB, forKey: "allowUSB") } // The end of a wait must not open the app's real USB listener port. UserDefaults.standard.set(false, forKey: "allowUSB") - var waits: [Int] = [] + let sleeper = ManualSleeper() var timing = answering - timing.cableBusyHoldSeconds = { count in waits.append(count); return Double(count) * 0.8 } + timing.cableHoldSleep = { await sleeper.sleep($0) } let (manager, first) = try await connectOverCable(timing: timing) var cables = [first] defer { cables.forEach { $0.stop() }; manager.cancel() } await refuse(.busy) try await until("the refusal shows") { manager.state == .disconnected && manager.problem == .pcBusy } - let refusedAt = Date() XCTAssertEqual(manager.usbStatus, "USB: disconnected") XCTAssertTrue(summaries(manager).last?.contains(" stage=hello_sent outcome=error failure=pc_busy fallback=none end=error ") == true, summaries(manager).joined(separator: "\n")) + try await until("the first wait starts") { sleeper.requested == [30] } // The PC dials again and again within the wait: none is taken. for _ in 1...5 { cables.append(try await plugIn(manager)) @@ -709,26 +708,31 @@ final class USBLinkFailureTests: XCTestCase { XCTAssertEqual(manager.problem, .pcBusy, "the reason stays up") XCTAssertEqual(summaries(manager).count, 1, "one attempt") - // The PC dials every 100 ms: the first taken comes once the wait - // ended, and the next Busy waits longer. - let firstWait = try await dialUntilTaken(manager, since: refusedAt, cables: &cables) - XCTAssertGreaterThanOrEqual(firstWait, 0.7, "the 0.8 s wait, less the time to see the refusal") + // Once the wait is over the next dial is taken; the next Busy in a + // row waits twice as long. + sleeper.advance() + try await dialUntilTaken(manager, cables: &cables) await refuse(.busy) try await until("the second refusal") { manager.state == .disconnected } - let secondWait = try await dialUntilTaken(manager, since: Date(), cables: &cables) - XCTAssertGreaterThanOrEqual(secondWait, 1.5, "the 1.6 s wait") + try await until("the second wait starts") { sleeper.requested == [30, 60] } + cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .disconnected, "the cable isn't taken during the second wait") + sleeper.advance() + try await dialUntilTaken(manager, cables: &cables) - // A session resets the count: the next Busy waits the first wait. + // The PC accepts a session, then ends it by closing the cable. Its + // next dial is taken as usual, and the next Busy waits the first + // wait again. await answer() try await until("the session") { manager.hostInfo != nil } - manager.cancel() - manager.resumeUSBConnections() + shutdown(peer, SHUT_RDWR) + try await until("the closed cable ends it") { manager.state == .disconnected } cables.append(try await plugIn(manager)) + XCTAssertEqual(manager.state, .connecting, "the next dial is taken") await refuse(.busy) try await until("the third refusal") { manager.state == .disconnected } - XCTAssertEqual(waits, [1, 2, 1]) - XCTAssertEqual(summaries(manager).filter { $0.contains(" failure=pc_busy ") }.count, 1, - "the list holds the latest attempt's lines") + try await until("the third wait starts") { sleeper.requested.count == 3 } + XCTAssertEqual(sleeper.requested, [30, 60, 30]) } /// Try again during the wait takes the PC's next dial at once. @@ -851,3 +855,22 @@ private final class UnresolvableLink: MediaLink { func send(_ data: Data) {} func stop() {} } + +/// Ends each wait when the test says so, and remembers how long each was. +@MainActor +private final class ManualSleeper { + private(set) var requested: [Double] = [] + private var waiting: [CheckedContinuation] = [] + + func sleep(_ seconds: Double) async { + requested.append(seconds) + await withCheckedContinuation { waiting.append($0) } + } + + /// Ends every wait that has started. + func advance() { + let ended = waiting + waiting = [] + ended.forEach { $0.resume() } + } +} From a68319e9ef5583ee36f013fa07c26e09e8cb28ba Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:02:51 -0500 Subject: [PATCH 185/226] Log each USB tunnel once in the hub, without device ids The open and close lines were written only while a pipeline took the hub's events, so a tunnel that opened or closed during a pipeline restart went unlogged. The hub now logs both itself, once per tunnel, whether or not a pipeline is subscribed. The lifecycle lines carry the tunnel generation, not the device id. The close line's field is now seen_first: the first end this host saw, which is not necessarily what started it. --- host/src/transport/link.rs | 9 +- host/src/transport/usb.rs | 171 ++++++++++++++++++++++++------------- 2 files changed, 117 insertions(+), 63 deletions(-) diff --git a/host/src/transport/link.rs b/host/src/transport/link.rs index af1dc9ec..2c5fb4df 100644 --- a/host/src/transport/link.rs +++ b/host/src/transport/link.rs @@ -402,8 +402,9 @@ pub fn tunnel_congestion_drops() -> u64 { TUNNEL_CONGESTION.load(Ordering::Relaxed) } -/// What ended a tunnel first. Neither side used to say: a tunnel that -/// closed on both sides at once looked the same whoever closed it. +/// What this host saw end a tunnel first. Neither side used to say: a +/// tunnel that closed on both sides at once looked the same whoever closed +/// it. It is the first end seen here, not necessarily what started it. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CloseCause { /// The other end (the iPad, or Apple's device service) ended the stream. @@ -623,8 +624,8 @@ impl FramedLink { CloseGuard(Arc::clone(&self.state)) } - /// What closed the tunnel, how long it was open and how long the other - /// end had been quiet. `None` while it is open. + /// What this host saw end the tunnel first, how long it was open and + /// how long the other end had been quiet. `None` while it is open. pub fn close_report(&self) -> Option { let lifetime = self.state.lifetime.lock(); let (cause, at) = lifetime.closed?; diff --git a/host/src/transport/usb.rs b/host/src/transport/usb.rs index 099a95de..b62bc8bf 100644 --- a/host/src/transport/usb.rs +++ b/host/src/transport/usb.rs @@ -58,7 +58,6 @@ enum HubEvent { Connected { generation: u64, link: Arc, - opened: Opened, }, Datagram { generation: u64, @@ -183,6 +182,8 @@ fn subscriber(state: &Mutex) -> Option> { /// Owns tunnels for the hub: forwards their datagrams to whichever pipeline /// generation is subscribed and drops them while none is (a restart gap). +/// Each tunnel's opening and close are logged here, once, whether or not a +/// pipeline is subscribed. async fn run_hub(mut events: mpsc::Receiver, state: Arc>) { while let Some(UsbEvent { generation, @@ -190,21 +191,17 @@ async fn run_hub(mut events: mpsc::Receiver, state: Arc, state: Arc event, }; match event { - Some(HubEvent::Connected { - generation, - link, - opened, - }) => { - info!( - id = ?link.id(), - generation, - connect_ms = opened.connect.as_millis() as u64, - waited_ms = opened.waited.as_millis() as u64, - failed = opened.failed, - last_error = opened.last_error.as_deref().unwrap_or("none"), - "USB tunnel connected" - ); + Some(HubEvent::Connected { generation, link }) => { self.adopt(generation, link); } Some(HubEvent::Datagram { @@ -405,15 +391,11 @@ impl TransportLinks { .get(&id) .is_some_and(|active| active.generation == generation) { - let report = self - .usb - .remove(&id) - .and_then(|active| active.link.close_report()); + self.usb.remove(&id); if self.usb.is_empty() { PIPELINE_STATS.lock().usb_link_state = "Waiting for the iPad app".into(); } - log_closed(id, generation, report); return Ok(LinkEvent::Closed(id)); } } @@ -434,14 +416,28 @@ impl TransportLinks { } } -/// Which end closed the tunnel first, after how long, and how long the -/// iPad had been quiet: a timer shows as the same `open_ms` every time. -fn log_closed(id: LinkId, generation: u64, report: Option) { +/// How long the service's Connect and the preamble took, how long the +/// worker waited for the tunnel, and the connects that failed meanwhile. +fn log_connected(generation: u64, opened: &Opened) { + info!( + generation, + connect_ms = opened.connect.as_millis() as u64, + waited_ms = opened.waited.as_millis() as u64, + failed = opened.failed, + last_error = opened.last_error.as_deref().unwrap_or("none"), + "USB tunnel connected" + ); +} + +/// What the host saw end the tunnel first, after how long, and how long +/// the iPad had been quiet: a timer shows as the same `open_ms` every time. +/// `seen_first` is the first end this host saw, not necessarily what +/// started it: the iPad or Apple's service closing shows as `peer_eof`. +fn log_closed(generation: u64, report: Option) { let ms = |duration: Duration| duration.as_millis() as u64; info!( - ?id, generation, - cause = %report.map_or("unknown", |report| report.cause.as_str()), + seen_first = %report.map_or("unknown", |report| report.cause.as_str()), open_ms = report.map_or(0, |report| ms(report.open)), quiet_ms = report.map_or(0, |report| ms(report.quiet)), "USB tunnel closed" @@ -658,12 +654,11 @@ mod tests { assert_eq!(direct_value(true, || None), None); } - /// The close line says which end closed the tunnel first and when, so a - /// daily check can tell a timer (the same `open_ms` every time) from an - /// unplugged cable or Apple's service letting go. - #[test] - fn the_close_line_says_who_closed_the_tunnel_and_when() { - use super::super::link::CloseCause; + /// Every tunnel's opening and close are logged once, by the hub, whether + /// or not a pipeline is subscribed: with none (a restart gap) they used + /// to go unlogged. The lines carry the tunnel generation, not the device. + #[tokio::test(start_paused = true)] + async fn the_hub_logs_each_tunnel_once_with_or_without_a_pipeline() { use tracing_subscriber::prelude::*; #[derive(Clone, Default)] struct Lines(Arc>>); @@ -678,32 +673,90 @@ mod tests { } let lines = Lines::default(); let writer = lines.clone(); - let subscriber = tracing_subscriber::registry().with( - tracing_subscriber::fmt::layer() - .with_ansi(false) - .without_time() - .with_target(false) - .with_writer(move || writer.clone()), - ); crate::logging::prepare_log_capture(); - tracing::subscriber::with_default(subscriber, || { - let report = CloseReport { - cause: CloseCause::ReadFailed(io::ErrorKind::ConnectionReset), - open: Duration::from_millis(9_612), - quiet: Duration::from_millis(4), + let _capture = tracing::subscriber::set_default( + tracing_subscriber::registry().with( + tracing_subscriber::fmt::layer() + .with_ansi(false) + .without_time() + .with_target(false) + .with_writer(move || writer.clone()), + ), + ); + let state = Arc::new(Mutex::new(HubState::default())); + let (events, hub_events) = mpsc::channel(4); + tokio::spawn(run_hub(hub_events, Arc::clone(&state))); + let open = |generation| async move { + let (host, mut ipad) = tokio::io::duplex(64 * 1024); + let link = FramedLink::connect(host, PeerId::usb(3), Arc::new(AtomicBool::new(false))) + .await + .unwrap(); + super::super::link::read_preamble(&mut ipad).await.unwrap(); + let opened = Opened { + connect: Duration::from_millis(5), + waited: Duration::from_millis(1_200), + failed: 2, + last_error: Some("usbmuxd Result 3".into()), }; - log_closed(PeerId::usb(3).link, 12, Some(report)); - log_closed(PeerId::usb(3).link, 13, None); - }); + let event = UsbEvent { + generation, + link, + opened, + }; + (event, ipad) + }; + let gone = |state: Arc>| async move { + for _ in 0..1000 { + if state.lock().links.is_empty() { + return; + } + tokio::task::yield_now().await; + } + panic!("the hub kept a closed tunnel"); + }; + + // No pipeline: a restart gap. + let (event, ipad) = open(41).await; + events.send(event).await.unwrap(); + tokio::time::sleep(Duration::from_millis(9_600)).await; + drop(ipad); + gone(Arc::clone(&state)).await; + + // A pipeline gets the events, and the lines are not logged again. + let (pipeline, mut hub) = mpsc::channel(8); + state.lock().subscriber = Some((1, pipeline)); + let (event, ipad) = open(42).await; + events.send(event).await.unwrap(); + assert!(matches!( + hub.recv().await, + Some(HubEvent::Connected { generation: 42, .. }) + )); + tokio::time::sleep(Duration::from_millis(1_500)).await; + drop(ipad); + assert!(matches!( + hub.recv().await, + Some(HubEvent::Closed { generation: 42, .. }) + )); + let text = String::from_utf8(lines.0.lock().clone()).unwrap(); let lines: Vec<&str> = text.lines().map(str::trim).collect(); + let connected = |generation| { + format!( + "INFO USB tunnel connected generation={generation} connect_ms=5 waited_ms=1200 \ + failed=2 last_error=\"usbmuxd Result 3\"" + ) + }; assert_eq!( lines, [ - "INFO USB tunnel closed id=Usb { device_id: 3 } generation=12 \ - cause=peer_reset open_ms=9612 quiet_ms=4", - "INFO USB tunnel closed id=Usb { device_id: 3 } generation=13 \ - cause=unknown open_ms=0 quiet_ms=0", + connected(41), + "INFO USB tunnel closed generation=41 seen_first=peer_eof open_ms=9600 \ + quiet_ms=9600" + .into(), + connected(42), + "INFO USB tunnel closed generation=42 seen_first=peer_eof open_ms=1500 \ + quiet_ms=1500" + .into(), ] ); } From 6b598c66032fdb50c6501766c0769e60b8e3ad60 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:06:05 -0500 Subject: [PATCH 186/226] Mark the GPU fallback test's pictures as new ones --- host/tests/gpu_detect_fallback.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/host/tests/gpu_detect_fallback.rs b/host/tests/gpu_detect_fallback.rs index ed24700f..97fd72cc 100644 --- a/host/tests/gpu_detect_fallback.rs +++ b/host/tests/gpu_detect_fallback.rs @@ -47,6 +47,7 @@ fn notice_after_stage(gpu: GpuInfo, chosen: Option<&str>) -> (bool, bool) { data: Arc::new(data), width, height, + cached: false, }); std::thread::sleep(Duration::from_millis(20)); } From 7f4691e449c31f25879f0d3bc6187aedbaf7cca1 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:15:40 -0500 Subject: [PATCH 187/226] Tell the iPad when the PC is restarting its stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An iPad held through a stream restart kept its last picture for up to 20 seconds with no word about why. - While a session is held through a restart, until its first new frame, its heartbeats carry a restarting notice (value 5). Only iPads that set FEATURE_STANDBY_STATUS get it; older ones see the same heartbeats as before. - The iPad shows a short pill, "Your PC is restarting the stream…", once the picture has been held 1.5 s since its last frame, so a quick restart shows nothing. The first new frame clears it, as does a heartbeat without the notice. - The string is English, in the catalog and awaiting translation. --- host/src/transport/session.rs | 78 ++++++++++++++++++- host/tests/hello_standby.rs | 26 ++++++- .../App/ConnectionManager.swift | 44 +++++++++++ ios/EternalMonitor/App/DisplayView.swift | 24 ++++++ ios/EternalMonitor/Network/WireProtocol.swift | 4 + .../Resources/Localizable.xcstrings | 12 +++ .../LocalizationTests.swift | 4 +- .../StreamRecoveryTests.swift | 44 +++++++++++ proto/src/v2/control.rs | 5 ++ 9 files changed, 237 insertions(+), 4 deletions(-) diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 9ad787c0..06b5b867 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -1507,6 +1507,15 @@ impl Session { let session_id = session.session_id; let frames_sent = (session.info.feature_caps & FEATURE_MEDIA_SIZE != 0) .then_some(session.frames_sent); + // Held through a restart, until the first new frame: an iPad + // that understands it says the picture comes back by itself. + let notice = if session.restart_held_since.is_some() + && session.info.feature_caps & FEATURE_STANDBY_STATUS != 0 + { + eternal_wire::v2::control::NOTICE_RESTARTING + } else { + config.notice() + }; let stream_config = match self.not_streaming { // No stream: the config of the one the iPad last had. Some(_) => session @@ -1521,7 +1530,7 @@ impl Session { let heartbeat = ControlMessage::Heartbeat(eternal_wire::v2::control::Heartbeat { host_time_us: crate::clock::host_now_us(), stream_config, - notice: config.notice(), + notice, frames_sent, }); actions.replies.push(( @@ -5832,4 +5841,71 @@ mod tests { } assert_eq!(ended, Some(EndReason::Goodbye(ByeReason::Superseded))); } + + fn notices(actions: &Actions) -> Vec { + actions + .replies + .iter() + .filter_map( + |(_, bytes)| match eternal_wire::v2::control::parse_control(bytes) { + Ok((_, ControlMessage::Heartbeat(hb))) => Some(hb.notice), + _ => None, + }, + ) + .collect() + } + + /// While a session is held through a restart, its heartbeats say so to + /// an iPad that understands it, on the standby's ticks and on the next + /// stream's until its first frame; then they stop saying it. + #[test] + fn a_held_session_hears_that_the_pc_is_restarting() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_RESTARTING}; + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + let ControlMessage::Hello2(mut hello) = hello(1, 50000) else { + unreachable!() + }; + hello.feature_caps |= FEATURE_STANDBY_STATUS; + session.handle_control_authed(peer, ControlMessage::Hello2(hello), &TestConfig, now); + session.handle_control_authed(peer, report(10), &TestConfig, now); + let id = session.session_id().unwrap(); + assert_eq!( + notices(&session.tick(&TestConfig, true, now)), + [NOTICE_NONE] + ); + + session.set_not_streaming(Some(NotStreaming::Restarting)); + let at = now + Duration::from_secs(1); + assert_eq!( + notices(&session.restart_tick(&RestartConfig, at)), + [NOTICE_RESTARTING] + ); + // The next stream runs, with no frame yet. + session.set_not_streaming(None); + let at = now + Duration::from_secs(2); + session.handle_control_authed(peer, report(10), &TestConfig, at); + assert_eq!( + notices(&session.tick(&TestConfig, true, at)), + [NOTICE_RESTARTING] + ); + // Its first frame. + assert!(session.may_send_media(id, 1000, at)); + assert_eq!(notices(&session.tick(&TestConfig, true, at)), [NOTICE_NONE]); + } + + /// A 0.3.4 iPad, without the feature bit, gets the heartbeats with no + /// notice, as from a running stream. + #[test] + fn an_old_ipad_held_through_a_restart_gets_no_notice() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let (mut session, _) = restarting_session(peer, now); + let at = now + Duration::from_secs(1); + assert_eq!( + notices(&session.restart_tick(&RestartConfig, at)), + [eternal_wire::v2::control::NOTICE_NONE] + ); + } } diff --git a/host/tests/hello_standby.rs b/host/tests/hello_standby.rs index d8b126af..f671e9eb 100644 --- a/host/tests/hello_standby.rs +++ b/host/tests/hello_standby.rs @@ -945,6 +945,8 @@ mod identify { struct Heard { /// When each heartbeat for the session came. heartbeats: Vec, + /// Each heartbeat's notice, with when it came. + notices: Vec<(Instant, u8)>, /// Each media datagram for the session: when, its epoch, and whether /// it was part of a keyframe. media: Vec<(Instant, u32, bool)>, @@ -980,10 +982,11 @@ impl FakeIpad { let at = Instant::now(); match eternal_wire::v2::classify(bytes) { eternal_wire::v2::Classified::Control(_) => match parse_control(bytes) { - Ok((header, ControlMessage::Heartbeat(_))) + Ok((header, ControlMessage::Heartbeat(hb))) if header.session_id == session_id => { - heard.heartbeats.push(at) + heard.heartbeats.push(at); + heard.notices.push((at, hb.notice)); } Ok((header, ControlMessage::Bye(reason))) if header.session_id == session_id => @@ -1081,6 +1084,25 @@ fn a_restart_longer_than_the_ipads_liveness_keeps_its_session() { "{longest:?} without a heartbeat in a {gap:?} restart" ); assert_eq!(host.shared.session.lock().session_id(), Some(id)); + // This iPad understands it: the heartbeats said the PC was restarting + // until the picture came back. + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_RESTARTING}; + assert!( + heard.notices.iter().any(|(at, notice)| *at > last_before + && *at < resumed.0 + && *notice == NOTICE_RESTARTING), + "{:?}", + heard.notices + ); + let after = ipad.stream(id, Duration::from_secs(5), |heard| heard.notices.len() >= 2); + assert!( + after + .notices + .iter() + .all(|(_, notice)| *notice == NOTICE_NONE), + "still restarting after the picture came back: {:?}", + after.notices + ); } /// A restart that outlasts the hold limit ends the session with the goodbye diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 4e197d61..d1c05857 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -122,6 +122,14 @@ final class ConnectionManager: ObservableObject { /// The PC has no display turned on, so there is nothing to capture and /// the picture is held. Its heartbeats say so until one comes back. @Published var pcNoDisplay = false + /// The PC is restarting its stream and kept this session, and no new + /// frame has come for `Timing.restartNoticeSeconds`. The first new frame + /// clears it. + @Published private(set) var pcRestarting = false + /// When the last frame was decoded, by the system uptime clock. + private var lastFrameAt: TimeInterval? + /// Shows the restarting pill once the picture has been held long enough. + private var restartNoticeTask: Task? /// Nonzero while an automatic reconnect cycle is running. @Published private(set) var reconnectAttempt = 0 @@ -249,6 +257,9 @@ final class ConnectionManager: ObservableObject { /// How long video the PC says it sent may go missing before the /// next no-video step (see `NoVideoFallback`). var noVideoStepSeconds = 1.5 + /// How long the picture is held through a PC's stream restart before + /// the iPad says so. A quick restart shows nothing. + var restartNoticeSeconds = 1.5 } var timing: Timing /// The Wi-Fi link to a host's port. Tests pass a fake one, or replace it @@ -552,6 +563,7 @@ final class ConnectionManager: ObservableObject { signalLost = false pcLocked = false pcNoDisplay = false + clearRestartNotice() degradedSinceUs = nil prevCounters = FrameAssembler.Counters() diagnostics.removeAll() @@ -623,6 +635,9 @@ final class ConnectionManager: ObservableObject { guard self.currentAttempt.withLock({ $0 == connectionID }) else { return } self.debugState.decodedFrames += 1 self.debugState.lastDecodedTimestampUs = ts + self.lastFrameAt = ProcessInfo.processInfo.systemUptime + if self.pcRestarting { self.record(.info, "ctrl", "Picture back after the PC's stream restart") } + self.clearRestartNotice() if self.debugState.decodedFrames == 1 { self.firstFrameAt = ProcessInfo.processInfo.systemUptime self.record(.info, "decode", "First frame decoded successfully" + self.attemptTimings) @@ -962,6 +977,11 @@ final class ConnectionManager: ObservableObject { self.pcLocked = locked self.record(.info, "ctrl", locked ? "PC locked: holding the picture" : "PC unlocked") } + if heartbeat.notice == Heartbeat.noticeRestarting { + self.noteRestartNotice() + } else { + self.clearRestartNotice() + } let noDisplay = heartbeat.notice == Heartbeat.noticeNoDisplay if noDisplay != self.pcNoDisplay { self.pcNoDisplay = noDisplay @@ -1204,6 +1224,29 @@ final class ConnectionManager: ObservableObject { } } + /// The PC said it is restarting its stream and holds this session. Once + /// the picture has been held `restartNoticeSeconds` since its last + /// frame, say so; a frame first means a quick restart, and nothing shows. + private func noteRestartNotice() { + guard state == .connected, !pcRestarting, restartNoticeTask == nil else { return } + let heldFor = ProcessInfo.processInfo.systemUptime - (lastFrameAt ?? ProcessInfo.processInfo.systemUptime) + let wait = max(0, timing.restartNoticeSeconds - heldFor) + let frameAt = lastFrameAt + restartNoticeTask = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(wait)) + guard !Task.isCancelled, let self, self.state == .connected, self.lastFrameAt == frameAt else { return } + self.restartNoticeTask = nil + self.pcRestarting = true + self.record(.info, "ctrl", "PC is restarting its stream: holding the picture") + } + } + + private func clearRestartNotice() { + restartNoticeTask?.cancel() + restartNoticeTask = nil + pcRestarting = false + } + /// Ends the session that `reason` interrupted (`end` when it was not /// the link: the PC said goodbye) and reconnects if that is on. private func beginReconnect(_ reason: ConnectionProblem.LostReason, failure: ConnectionProblem? = nil, @@ -1724,6 +1767,7 @@ final class ConnectionManager: ObservableObject { pictureRestoring = false pcLocked = false pcNoDisplay = false + clearRestartNotice() degradedSinceUs = nil prevCounters = FrameAssembler.Counters() hostInfo = nil diff --git a/ios/EternalMonitor/App/DisplayView.swift b/ios/EternalMonitor/App/DisplayView.swift index bb557073..10219521 100644 --- a/ios/EternalMonitor/App/DisplayView.swift +++ b/ios/EternalMonitor/App/DisplayView.swift @@ -59,6 +59,10 @@ struct DisplayView: View { noDisplayPill .transition(.opacity) .allowsHitTesting(false) + } else if connectionManager.pcRestarting { + pcRestartingPill + .transition(.opacity) + .allowsHitTesting(false) } else if connectionManager.pictureRestoring { restoringPill .transition(.opacity) @@ -123,6 +127,7 @@ struct DisplayView: View { .animation(.easeInOut(duration: 0.25), value: connectionManager.pictureRestoring) .animation(.easeInOut(duration: 0.25), value: connectionManager.pcLocked) .animation(.easeInOut(duration: 0.25), value: connectionManager.pcNoDisplay) + .animation(.easeInOut(duration: 0.25), value: connectionManager.pcRestarting) .animation(reduceMotion ? nil : .easeInOut(duration: 0.35), value: appeared) .onAppear { appeared = true @@ -460,6 +465,25 @@ struct DisplayView: View { .accessibilityIdentifier("display.pcNoDisplay") } + /// Over the held picture while the PC restarts its stream, until the + /// first new frame. + private var pcRestartingPill: some View { + HStack(spacing: 10) { + ProgressView() + .tint(Theme.text) + .accessibilityHidden(true) + Text(L("Your PC is restarting the stream…")) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + } + .padding(.horizontal, 18) + .frame(minHeight: 44) + .background(glass(Capsule())) + .accessibilityElement(children: .ignore) + .accessibilityLabel(L("Your PC is restarting the stream…")) + .accessibilityIdentifier("display.pcRestarting") + } + /// The PC is still connected but a lost frame left the picture waiting /// for a fresh one; usually gone in well under a second. private var restoringPill: some View { diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index 4b87b593..df178dc0 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -358,6 +358,10 @@ struct Heartbeat: Equatable { static let noticePCLocked: UInt8 = 1 /// No display is on at the PC (a closed lid, screens off or unplugged). static let noticeNoDisplay: UInt8 = 2 + /// The PC is restarting its stream and keeps this session: the picture + /// comes back by itself. Sent only to an iPad that sets + /// `Hello2.featureStandbyStatus`. (3 and 4 are the graphics card notices.) + static let noticeRestarting: UInt8 = 5 /// The video frames the PC sent this session, from a PC that offered /// `HelloAck.hostCapMediaSize`; nil from others. Appended after /// `notice`, which is then always written. diff --git a/ios/EternalMonitor/Resources/Localizable.xcstrings b/ios/EternalMonitor/Resources/Localizable.xcstrings index 7c85e091..137ed2c8 100644 --- a/ios/EternalMonitor/Resources/Localizable.xcstrings +++ b/ios/EternalMonitor/Resources/Localizable.xcstrings @@ -9331,6 +9331,18 @@ } } }, + "Your PC is restarting the stream…" : { + "comment" : "Over the held picture while the PC restarts its stream and keeps the session", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Your PC is restarting the stream…" + } + } + } + }, "⌘ key acts as" : { "comment" : "Values are Ctrl or Win", "extractionState" : "manual", diff --git a/ios/EternalMonitorTests/LocalizationTests.swift b/ios/EternalMonitorTests/LocalizationTests.swift index 0db3ce8d..92d5e809 100644 --- a/ios/EternalMonitorTests/LocalizationTests.swift +++ b/ios/EternalMonitorTests/LocalizationTests.swift @@ -22,7 +22,9 @@ final class LocalizationTests: XCTestCase { /// English strings whose translations come in a separate pass. Until /// then every language shows them in English. - private let awaitingTranslation: Set = [] + private let awaitingTranslation: Set = [ + "Your PC is restarting the stream…", + ] /// Every language has exactly English's strings, none empty, each with /// the same placeholders. diff --git a/ios/EternalMonitorTests/StreamRecoveryTests.swift b/ios/EternalMonitorTests/StreamRecoveryTests.swift index c0317ecc..c997199a 100644 --- a/ios/EternalMonitorTests/StreamRecoveryTests.swift +++ b/ios/EternalMonitorTests/StreamRecoveryTests.swift @@ -109,6 +109,50 @@ final class StreamRecoveryTests: XCTestCase { XCTAssertFalse(manager.pcLocked, "the next session starts without it") } + /// The PC restarts its stream and holds this session: once the picture + /// has been held a moment, the iPad says so, still connected, and the + /// first new frame clears it. + func testARestartingPCIsShownOnceThePictureIsHeldAWhile() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + let lastFrame = Date() + host.sendHeartbeat(notice: Heartbeat.noticeRestarting) + try await Task.sleep(nanoseconds: 700_000_000) + host.sendHeartbeat(notice: Heartbeat.noticeRestarting) + XCTAssertFalse(manager.pcRestarting, "shown before the picture was held a moment") + try await until("the PC says it is restarting") { manager.pcRestarting } + XCTAssertGreaterThanOrEqual(Date().timeIntervalSince(lastFrame), 1.3) + XCTAssertEqual(manager.state, .connected) + // The restart is over: the first new frame clears it. + host.sendHeartbeat(notice: Heartbeat.noticeRestarting) + let units = try twoGOPs() + host.sendFrame(units[4], seq: 5, isKeyframe: true) + try await until("the picture is back") { !manager.pcRestarting } + XCTAssertEqual(manager.state, .connected) + manager.cancel() + XCTAssertFalse(manager.pcRestarting, "the next session starts without it") + } + + /// A quick restart, whose new frame comes before the picture was held a + /// moment, shows nothing; nor does a notice that stops. + func testAQuickRestartShowsNothing() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + host.sendHeartbeat(notice: Heartbeat.noticeRestarting) + try await Task.sleep(nanoseconds: 400_000_000) + let units = try twoGOPs() + host.sendFrame(units[4], seq: 5, isKeyframe: true) + let deadline = Date().addingTimeInterval(2.5) + while Date() < deadline { + XCTAssertFalse(manager.pcRestarting, "a quick restart showed the pill") + try await Task.sleep(nanoseconds: 50_000_000) + } + host.sendHeartbeat(notice: Heartbeat.noticeRestarting) + host.sendHeartbeat() + try await Task.sleep(nanoseconds: 2_500_000_000) + XCTAssertFalse(manager.pcRestarting, "a notice that stopped still showed") + } + /// A closed lid or screens that are off leave the PC nothing to /// capture. Its heartbeats say so, and the iPad says so over the held /// picture, still connected, until a display comes back. diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index bcd7351b..e096bc17 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -286,6 +286,11 @@ pub const NOTICE_PC_LOCKED: u8 = 1; /// No display is attached to the PC's desktop (a closed lid, screens off or /// unplugged), so there is nothing to capture until one comes back. pub const NOTICE_NO_DISPLAY: u8 = 2; +/// The PC is restarting its stream and keeps the session meanwhile: the +/// picture comes back by itself. Sent only to iPads that set +/// [`FEATURE_STANDBY_STATUS`]. (3 and 4 are taken by the graphics card +/// notices.) +pub const NOTICE_RESTARTING: u8 = 5; /// Host → client, every heartbeat interval while a session is active. #[derive(Debug, Clone, Copy, PartialEq, Eq)] From 5e1fb9d7d1007ea695fc51667f090ab51212104f Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:22:39 -0500 Subject: [PATCH 188/226] Prove the PC before an automatic try, and go where Bonjour has it An automatic try now needs the PC's saved pairing: an unpaired PC isn't watched for. The PC at the address Bonjour gives first proves it holds the pairing, as a moved PC does, so nothing is sent to another PC that took the address; an address other than the one tried must also be on this iPad's own network. The attempt then carries the pairing, and a session that doesn't give it back, or a pairing refusal, is dropped before any picture and leaves the saved pairing alone. The try dials the address and port Bonjour has the PC at now. A PC at a new address or port moves its saved entry once it shows a picture. The watch starts with the attempt, so its first second, which lists what Bonjour already knew, ends before the attempt's HELLO2s do: a PC opened just after them is a return, not part of what was known. --- .../App/ConnectionManager.swift | 146 +++++++--- ios/EternalMonitor/App/PCReturn.swift | 52 ++-- ios/EternalMonitorTests/PCReturnTests.swift | 250 +++++++++++++----- 3 files changed, 321 insertions(+), 127 deletions(-) diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 18c209af..809a84f3 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -313,6 +313,12 @@ final class ConnectionManager: ObservableObject { private var pcReturnPaused = false private var pcReturnTry: Task? private var pcReturnLimit: Task? + /// The automatic try checking that the PC is the same one. + private var pcReturnCheck: UUID? + /// The pairing an automatic try's session must give back, for the next + /// attempt and the current one. + private var pendingPairing: Data? + private var attemptPairing: Data? /// The automatic try is starting: the attempt it starts doesn't end the watch. private var startingPCReturnTry = false @@ -561,6 +567,8 @@ final class ConnectionManager: ObservableObject { pendingMove = nil attemptRedirect = isUSB ? nil : pendingRedirect pendingRedirect = nil + attemptPairing = isUSB ? nil : pendingPairing + pendingPairing = nil noVideoFallback = nil noVideoStep = nil let connectionID = UUID() @@ -595,6 +603,11 @@ final class ConnectionManager: ObservableObject { sessionWantsAudio = wantsAudio audioStats = AudioStats() debugState = ConnectionDebugState(host: normalizedHost, port: port) + // Watched for from the start, so the search knows what Bonjour + // already listed before this attempt's HELLO2s stop. + if !isUSB, attemptMove == nil, attemptRedirect == nil, reconnectCycleID == nil { + watchForPC(host: normalizedHost, port: port) + } record(.info, "connection", "Starting connection attempt=\(connectionID) transport=\(transportMode) to \(normalizedHost):\(port) drawing=\(drawing.enabled) input=\(wantsInput) audio=\(wantsAudio)") if attemptMove != nil { record(.info, "connection", attemptMove?.noVideo == true @@ -810,6 +823,14 @@ final class ConnectionManager: ObservableObject { let acceptedAt = ProcessInfo.processInfo.systemUptime Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } + // An automatic try streams only from the PC that gives the + // saved pairing back, whatever answered at that address. + if let pairing = self.attemptPairing, info.authToken != pairing { + self.record(.warning, "connection", "pc_return result=other_pc") + self.disconnect(.otherPC) + self.problem = .noReply(usb: false) + return + } if let move = self.attemptMove { guard Self.isSamePC(move, info) else { self.record(.warning, "connection", "The PC at the new address isn't the saved one") @@ -862,6 +883,14 @@ final class ConnectionManager: ObservableObject { let mismatch = channel?.lastMismatch Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } + // Nor did the PC an automatic try reached: no pairing prompt, + // and the saved pairing stays. + if self.attemptPairing != nil, self.attemptMove == nil, status == .unauthorized || status == .rateLimited { + self.record(.warning, "connection", "pc_return result=other_pc") + self.disconnect(.otherPC) + self.problem = .noReply(usb: false) + return + } // A PC that wants a code didn't take the saved PC's pairing: // it is another PC, and the saved pairing stays. if self.attemptMove != nil, status == .unauthorized || status == .rateLimited { @@ -932,11 +961,6 @@ final class ConnectionManager: ObservableObject { self.disconnect(.legacyPC) } else { self.record(.warning, "ctrl", "HELLO2 handshake got no reply yet") - // No more HELLO2s go out: a PC app that opens from now - // on is watched for. - if !isUSB, self.attemptMove == nil, self.attemptRedirect == nil, self.reconnectCycleID == nil { - self.watchForPC(host: normalizedHost, port: port) - } if self.leaveRememberedAddress(.pcMoved, result: "no_reply") { // Back to the address tapped. } else if !isUSB { @@ -1473,15 +1497,23 @@ final class ConnectionManager: ObservableObject { } } - /// Starts watching Bonjour for the PC at `host:port`, by its service - /// when this iPad knows its name. A watch for that PC goes on as it was. + /// The pairing saved for the PC at `host:port`, if a valid one. + private func savedPairing(host: String, port: UInt16) -> Data? { + let token = (try? pairingStore.token(address: PairingStore.address(host: host, port: port))) ?? nil + return token.flatMap { PairingStore.valid($0) ? $0 : nil } + } + + /// Starts watching Bonjour for the paired PC at `host:port`, by its + /// service when this iPad knows its name, as an attempt to it starts. An + /// automatic try must prove it is the same PC with the pairing, so an + /// unpaired PC isn't watched for. A watch for that PC goes on as it was. private func watchForPC(host: String, port: UInt16) { if let watch = pcReturn, watch.host == host, watch.port == port { return } endPCReturn() - let address = PairingStore.address(host: host, port: port) - let name = RecentConnectionStore.shared.entry(host: host, port: port)?.name ?? pairingStore.pcName(address: address) - pcReturn = PCReturn(service: name.flatMap { $0.isEmpty ? nil : MovedPC.serviceName(pcName: $0) }, - host: host, port: port, at: now()) + guard savedPairing(host: host, port: port) != nil else { return } + let name = RecentConnectionStore.shared.entry(host: host, port: port)?.name + ?? pairingStore.pcName(address: PairingStore.address(host: host, port: port)) + pcReturn = PCReturn(name: name, host: host, port: port, at: now()) let browser = makePCBrowser() browser.onHosts = { [weak self] hosts in self?.pcReturnHeard(hosts) } pcReturnBrowser = browser @@ -1489,10 +1521,10 @@ final class ConnectionManager: ObservableObject { if isForeground { browser.start() } } - /// The "didn't answer" card is up for the PC at `host:port`: it connects + /// The "didn't answer" card is up for the PC watched for: it connects /// once the PC shows up, until the time limit. - private func armPCReturn(host: String, port: UInt16) { - watchForPC(host: host, port: port) + private func armPCReturn() { + guard pcReturn != nil else { return } guard pcReturn?.armedAt == nil else { return considerPCReturn() } pcReturn?.arm(at: now()) watchingForPC = true @@ -1513,37 +1545,81 @@ final class ConnectionManager: ObservableObject { considerPCReturn() } + /// The card is up, in the foreground, with no try under way. + private var pcReturnMayTry: Bool { + guard let watch = pcReturn else { return false } + return isForeground && state == .disconnected && showsNoReplyOverWiFi && pcReturnCheck == nil + && !watch.expired(at: now(), limit: timing.pcReturnLimitSeconds) + } + /// The PC showed up while the card is up, in the foreground: one try, /// after the debounce. Updates meanwhile count as the same appearance. private func considerPCReturn() { - guard let watch = pcReturn, watch.armedAt != nil, watch.appeared else { return } + guard let watch = pcReturn, watch.armedAt != nil, watch.appeared != nil else { return } guard !watch.expired(at: now(), limit: timing.pcReturnLimitSeconds) else { record(.info, "connection", "pc_return result=expired") return endPCReturn() } - guard isForeground, state == .disconnected, showsNoReplyOverWiFi, pcReturnTry == nil else { return } + guard pcReturnMayTry, pcReturnTry == nil else { return } let wait = timing.pcReturnDebounceSeconds pcReturnTry = Task { @MainActor [weak self] in try? await Task.sleep(for: .seconds(wait)) guard !Task.isCancelled, let self else { return } self.pcReturnTry = nil - guard self.isForeground, self.state == .disconnected, self.showsNoReplyOverWiFi, - let watch = self.pcReturn, !watch.expired(at: self.now(), limit: self.timing.pcReturnLimitSeconds), - self.pcReturn?.takeAppearance() == true else { return } - self.tryPCAgain(host: watch.host, port: watch.port) - } - } - - /// Connects as Try again does, without ending the watch: if the PC - /// still doesn't answer, the card comes back and waits for it again. - private func tryPCAgain(host: String, port: UInt16) { + guard self.pcReturnMayTry, let found = self.pcReturn?.takeAppearance() else { return } + self.tryPCAgain(at: found) + } + } + + /// One try at where Bonjour has the PC now. First the PC there shows it + /// holds the saved pairing, as a moved PC does, so nothing goes to + /// another PC that took the address; an address other than the one + /// tried must also be on this iPad's own network. Then the attempt + /// carries the pairing, and only a session that gives it back counts. + /// If the PC still doesn't answer, the card comes back and waits again. + private func tryPCAgain(at found: DiscoveredHost) { + guard let watch = pcReturn, let host = found.address, + let token = savedPairing(host: watch.host, port: watch.port) else { return } + let port = found.port + let moved = host != watch.host || port != watch.port record(.info, "connection", "pc_return result=tried") - startingPCReturnTry = true - defer { startingPCReturnTry = false } - problem = nil - connectionError = nil - let target = lastTarget ?? (host: host, port: port) - connect(host: target.host, port: target.port) + let check = UUID() + pcReturnCheck = check + // The card may go, or another attempt start, while the PC is asked. + let current = { [weak self] () -> Bool in + guard let self, self.pcReturnCheck == check, self.pcReturn != nil else { return false } + return self.isForeground && self.state == .disconnected && self.showsNoReplyOverWiFi + } + Task { @MainActor [weak self] in + guard let self else { return } + let done = { (result: String) in + if self.pcReturnCheck == check { self.pcReturnCheck = nil } + self.record(.info, "connection", "pc_return result=\(result)") + } + if host != watch.host { + guard self.homeAddress(host) else { return done("not_local") } + let route = await self.localRoute(host, port) + guard current() else { return } + guard let route, MovedPC.isLocal(host, over: route) else { return done("not_local") } + } + let same = await MovedPC.confirm(host: host, port: port, token: token, link: self.makeUDPLink(port)) + guard current() else { return } + guard same else { return done("not_confirmed") } + self.pcReturnCheck = nil + self.startingPCReturnTry = true + defer { self.startingPCReturnTry = false } + self.problem = nil + self.connectionError = nil + // A PC at a new address or port moves the saved entry once it shows a picture. + if moved, let name = watch.name { + self.pendingMove = Move(from: (watch.host, watch.port), name: name, token: token) + } + self.pendingPairing = token + self.explicitPairingToken = token + self.connect(target: .udp(host: host, port: port)) + self.pendingMove = nil + self.pendingPairing = nil + } } /// Dismissed, or replaced by another card: the watch ends. An attempt @@ -1559,6 +1635,7 @@ final class ConnectionManager: ObservableObject { private func pausePCReturn() { pcReturnTry?.cancel() pcReturnTry = nil + pcReturnCheck = nil guard let browser = pcReturnBrowser, !pcReturnPaused else { return } pcReturnPaused = true browser.stop() @@ -1577,6 +1654,7 @@ final class ConnectionManager: ObservableObject { private func endPCReturn() { pcReturnTry?.cancel() pcReturnTry = nil + pcReturnCheck = nil pcReturnLimit?.cancel() pcReturnLimit = nil pcReturnBrowser?.onHosts = nil @@ -1625,9 +1703,7 @@ final class ConnectionManager: ObservableObject { // The app gave up, not the user: the PC's log tells them apart. self.disconnect(.connectTimeout) if handshakeDone && self.noPictureStreak.stopsReconnecting { self.stopAutomaticAttempts(isUSB: isUSB) } - if self.showsNoReplyOverWiFi && self.reconnectCycleID == nil { - self.armPCReturn(host: debug.host, port: debug.port) - } + if self.showsNoReplyOverWiFi && self.reconnectCycleID == nil { self.armPCReturn() } } } } diff --git a/ios/EternalMonitor/App/PCReturn.swift b/ios/EternalMonitor/App/PCReturn.swift index 8ae086b4..5d190b88 100644 --- a/ios/EternalMonitor/App/PCReturn.swift +++ b/ios/EternalMonitor/App/PCReturn.swift @@ -37,18 +37,19 @@ final class BonjourPCBrowser: PCBrowser { } } -/// The PC that didn't answer, watched for on Bonjour. Its HELLO2s go out -/// for two seconds only, so a PC app that opens after that never hears this -/// iPad: the PC's service turning up, or changing, is the sign to try once -/// more. What Bonjour already listed when the search began is no sign; the -/// PC was there and didn't answer. +/// The PC that didn't answer, watched for on Bonjour from the start of the +/// attempt. Its HELLO2s go out for two seconds only, so a PC app that opens +/// after that never hears this iPad: the PC's service turning up, or +/// changing, is the sign to try once more. What Bonjour already listed in +/// the search's first second is no sign; the PC was there and didn't answer. struct PCReturn { - /// The first second of a search lists what Bonjour already knew. + /// The first second of a search lists what Bonjour already knew. It ends + /// before the attempt's HELLO2s do. static let settleSeconds = 1.0 - /// The PC's Bonjour service, when this iPad knows the PC's name. The - /// service at the address tried is the PC too. - let service: String? + /// The PC's name, when this iPad knows it: its Bonjour service is the + /// PC, wherever it is. The service at the address tried is the PC too. + let name: String? let host: String let port: UInt16 /// Until when the latest search lists what Bonjour already knew. @@ -59,18 +60,20 @@ struct PCReturn { private var listed = Set() /// The latest list while settling, which replaces `listed` after it. private var settled: Set? - /// A listing turned up that has had no try yet. - private(set) var appeared = false + /// The listing that turned up last and has had no try yet. + private(set) var appeared: DiscoveredHost? /// When the card went up and the watch's time limit started. private(set) var armedAt: Date? - init(service: String?, host: String, port: UInt16, at now: Date) { - self.service = service + init(name: String?, host: String, port: UInt16, at now: Date) { + self.name = name.flatMap { $0.isEmpty ? nil : $0 } self.host = host self.port = port settlesAt = now.addingTimeInterval(Self.settleSeconds) } + var service: String? { name.map(MovedPC.serviceName) } + func matches(_ pc: DiscoveredHost) -> Bool { guard let address = pc.address else { return false } return (service != nil && pc.name == service) || (address == host && pc.port == port) @@ -79,21 +82,26 @@ struct PCReturn { /// Bonjour's list now. A listing of the PC that wasn't there before is /// an appearance; a new address, port or version is one too. mutating func heard(_ hosts: [DiscoveredHost], at now: Date) { - let current = Set(hosts.filter(matches).map { "\($0.name)|\($0.address ?? "")|\($0.port)|\($0.version ?? "")" }) + var current: [String: DiscoveredHost] = [:] + for pc in hosts where matches(pc) { + current["\(pc.name)|\(pc.address ?? "")|\(pc.port)|\(pc.version ?? "")"] = pc + } + let keys = Set(current.keys) + let new = { (known: Set) in keys.subtracting(known).sorted().compactMap { current[$0] }.first } if now < settlesAt { // A search that starts again lists only what changed meanwhile as new. - if resumed, !current.subtracting(listed).isEmpty { appeared = true } - listed.formUnion(current) - settled = current + if resumed, let pc = new(listed) { appeared = pc } + listed.formUnion(keys) + settled = keys return } if let settled { listed = settled self.settled = nil } - if !current.subtracting(listed).isEmpty { appeared = true } + if let pc = new(listed) { appeared = pc } // A listing that left and comes back is a new appearance. - listed = current + listed = keys } /// The search starts again, as when the app comes back to the foreground. @@ -111,9 +119,9 @@ struct PCReturn { armedAt.map { now.timeIntervalSince($0) >= limit } ?? false } - /// Takes the appearance for its one try. - mutating func takeAppearance() -> Bool { - defer { appeared = false } + /// Takes the appearance for its one try: where the PC is now. + mutating func takeAppearance() -> DiscoveredHost? { + defer { appeared = nil } return appeared } } diff --git a/ios/EternalMonitorTests/PCReturnTests.swift b/ios/EternalMonitorTests/PCReturnTests.swift index 233a2b26..f8f6e5e2 100644 --- a/ios/EternalMonitorTests/PCReturnTests.swift +++ b/ios/EternalMonitorTests/PCReturnTests.swift @@ -2,23 +2,34 @@ import Combine import XCTest @testable import EternalMonitor -/// Bonjour's list as a test hands it over. +/// Bonjour's list as a test hands it over. A search that starts lists +/// what Bonjour knows already, as a real one does in its first moments. @MainActor private final class FakePCBrowser: PCBrowser { var onHosts: (([DiscoveredHost]) -> Void)? + /// What Bonjour has now. + var current: [DiscoveredHost] = [] private(set) var starts = 0 private(set) var stops = 0 var running: Bool { starts > stops } - func start() { starts += 1 } + func start() { + starts += 1 + onHosts?(current) + } func stop() { stops += 1 } - func list(_ hosts: [DiscoveredHost]) { onHosts?(hosts) } + func list(_ hosts: [DiscoveredHost]) { + current = hosts + onHosts?(hosts) + } } /// The iPad shows "didn't answer" because the PC app wasn't running yet. /// When the PC's service then shows up on the network, it connects once by -/// itself, while the card is up, in the foreground, for a few minutes. +/// itself, while the card is up, in the foreground, for a few minutes, and +/// only to the PC that proves it holds the saved pairing. @MainActor final class PCReturnTests: XCTestCase { + nonisolated private static let token = Data(17...32) private let start = Date(timeIntervalSince1970: 1_000_000) private func pc(_ name: String = "EternalMonitor (Test PC)", _ address: String? = "127.0.0.1", port: UInt16, @@ -29,37 +40,39 @@ final class PCReturnTests: XCTestCase { // MARK: - What counts as the PC showing up func testOnlyAListingThatWasNotThereBeforeCounts() { - var watch = PCReturn(service: "EternalMonitor (Test PC)", host: "192.0.2.20", port: 9876, at: start) + var watch = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876, at: start) let listed = pc(port: 9876) // What Bonjour already knew when the search began. watch.heard([listed], at: start.addingTimeInterval(0.2)) watch.heard([listed], at: start.addingTimeInterval(PCReturn.settleSeconds + 1)) - XCTAssertFalse(watch.appeared, "the PC was listed and didn't answer") + XCTAssertNil(watch.appeared, "the PC was listed and didn't answer") // It leaves (the app closed) and comes back. watch.heard([], at: start.addingTimeInterval(5)) - XCTAssertFalse(watch.appeared) + XCTAssertNil(watch.appeared) watch.heard([listed], at: start.addingTimeInterval(9)) - XCTAssertTrue(watch.takeAppearance()) - XCTAssertFalse(watch.takeAppearance(), "one try per appearance") - // The same listing again is no new appearance; a change is. + XCTAssertEqual(watch.takeAppearance(), listed) + XCTAssertNil(watch.takeAppearance(), "one try per appearance") + // The same listing again is no new appearance; a change is, and the + // try goes where the PC is now. watch.heard([listed], at: start.addingTimeInterval(10)) - XCTAssertFalse(watch.appeared) + XCTAssertNil(watch.appeared) watch.heard([pc(port: 9877)], at: start.addingTimeInterval(11)) - XCTAssertTrue(watch.takeAppearance(), "a new port") + XCTAssertEqual(watch.takeAppearance()?.port, 9877, "a new port") watch.heard([pc(port: 9877, version: "0.3.6")], at: start.addingTimeInterval(12)) - XCTAssertTrue(watch.takeAppearance(), "a new version") + XCTAssertEqual(watch.takeAppearance()?.version, "0.3.6", "a new version") } func testAPCMissingAtTheStartCountsAsSoonAsItShowsUp() { - var watch = PCReturn(service: nil, host: "192.0.2.20", port: 9876, at: start) + var watch = PCReturn(name: nil, host: "192.0.2.20", port: 9876, at: start) watch.heard([], at: start.addingTimeInterval(0.1)) - watch.heard([pc("EternalMonitor (Test PC)", "192.0.2.20", port: 9876)], at: start.addingTimeInterval(PCReturn.settleSeconds + 0.5)) - XCTAssertTrue(watch.appeared) + watch.heard([pc("EternalMonitor (Test PC)", "192.0.2.20", port: 9876)], + at: start.addingTimeInterval(PCReturn.settleSeconds + 0.5)) + XCTAssertNotNil(watch.appeared) } func testOnlyThisPCCounts() { - var named = PCReturn(service: "EternalMonitor (Test PC)", host: "192.0.2.20", port: 9876, at: start) - var unnamed = PCReturn(service: nil, host: "192.0.2.20", port: 9876, at: start) + var named = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876, at: start) + var unnamed = PCReturn(name: nil, host: "192.0.2.20", port: 9876, at: start) let later = start.addingTimeInterval(5) let others = [pc("EternalMonitor (Other PC)", "192.0.2.30", port: 9876), pc("EternalMonitor (Test PC 2)", "192.0.2.31", port: 9876), @@ -67,34 +80,34 @@ final class PCReturnTests: XCTestCase { pc("EternalMonitor (Test PC)", nil, port: 9876)] named.heard(others, at: later) unnamed.heard(others, at: later) - XCTAssertFalse(named.appeared) - XCTAssertFalse(unnamed.appeared) + XCTAssertNil(named.appeared) + XCTAssertNil(unnamed.appeared) // By name at another address, or by the address tried. named.heard(others + [pc("EternalMonitor (Test PC)", "192.0.2.40", port: 9876)], at: later) - XCTAssertTrue(named.appeared) + XCTAssertEqual(named.appeared?.address, "192.0.2.40") unnamed.heard(others + [pc("EternalMonitor (Any)", "192.0.2.20", port: 9876)], at: later) - XCTAssertTrue(unnamed.appeared) + XCTAssertEqual(unnamed.appeared?.name, "EternalMonitor (Any)") } /// Back from the background, the new search lists what changed while /// the app was away as new, and what was there before as known. func testASearchStartedAgainKeepsWhatItKnew() { - var watch = PCReturn(service: "EternalMonitor (Test PC)", host: "192.0.2.20", port: 9876, at: start) + var watch = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876, at: start) watch.heard([pc(port: 9876)], at: start.addingTimeInterval(0.1)) watch.restarted(at: start.addingTimeInterval(30)) watch.heard([], at: start.addingTimeInterval(30.1)) watch.heard([pc(port: 9876)], at: start.addingTimeInterval(30.2)) - XCTAssertFalse(watch.appeared, "still the listing that didn't answer") + XCTAssertNil(watch.appeared, "still the listing that didn't answer") - var away = PCReturn(service: "EternalMonitor (Test PC)", host: "192.0.2.20", port: 9876, at: start) + var away = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876, at: start) away.heard([], at: start.addingTimeInterval(0.1)) away.restarted(at: start.addingTimeInterval(30)) away.heard([pc(port: 9876)], at: start.addingTimeInterval(30.2)) - XCTAssertTrue(away.appeared, "it showed up while the app was away") + XCTAssertNotNil(away.appeared, "it showed up while the app was away") } func testTheTimeLimitStartsWhenTheCardGoesUp() { - var watch = PCReturn(service: nil, host: "192.0.2.20", port: 9876, at: start) + var watch = PCReturn(name: nil, host: "192.0.2.20", port: 9876, at: start) XCTAssertFalse(watch.expired(at: start.addingTimeInterval(1_000), limit: 180), "not armed yet") watch.arm(at: start.addingTimeInterval(10)) watch.arm(at: start.addingTimeInterval(100)) @@ -104,17 +117,25 @@ final class PCReturnTests: XCTestCase { // MARK: - Connecting - private struct Setup { + private final class Setup { let manager: ConnectionManager let browser: FakePCBrowser let host: FakeUDPHost - let lines: () -> [String] - let log: AnyCancellable - /// Moves the manager's clock. - let advance: (TimeInterval) -> Void + let pairings: PairingStore + var lines: [String] = [] + var log: AnyCancellable? + var clock: Date - var tries: Int { lines().filter { $0 == "pc_return result=tried" }.count } - var attempts: Int { lines().filter { $0.hasPrefix("Starting connection attempt=") }.count } + init(manager: ConnectionManager, browser: FakePCBrowser, host: FakeUDPHost, pairings: PairingStore, clock: Date) { + self.manager = manager + self.browser = browser + self.host = host + self.pairings = pairings + self.clock = clock + } + + var tries: Int { lines.filter { $0 == "pc_return result=tried" }.count } + var attempts: Int { lines.filter { $0.hasPrefix("Starting connection attempt=") }.count } var listing: DiscoveredHost { DiscoveredHost(name: "EternalMonitor (Test PC)", address: "127.0.0.1", port: host.port) } } @@ -131,36 +152,46 @@ final class PCReturnTests: XCTestCase { } } - /// A PC that doesn't answer at loopback, saved under its name so the - /// watch knows its service. The "didn't answer" card is up when this - /// returns, and the watch armed. - private func noReply(named: Bool = true) async throws -> Setup { + /// A paired PC at loopback, saved under its name unless `named` is + /// false, that doesn't answer. Loopback counts as a home address here. + private func paired(named: Bool = true) throws -> Setup { let host = FakeUDPHost() - if named { - RecentConnectionStore.shared.connections = [ - RecentConnection(host: "127.0.0.1", port: host.port, lastUsed: .now, isUSB: false, name: "Test PC", found: false), - ] - } + let name = named ? "Test PC" : "" + RecentConnectionStore.shared.connections = [ + RecentConnection(host: "127.0.0.1", port: host.port, lastUsed: .now, isUSB: false, + name: named ? name : nil, found: false), + ] + let pairings = PairingStore(storage: MemoryPairingStorage()) + try pairings.save(token: Self.token, hostName: name, address: PairingStore.address(host: "127.0.0.1", port: host.port)) let manager = ConnectionManager(timing: .init(connectTimeoutSeconds: 4, reconnectGraceSeconds: 0.1, reconnectBackoffSeconds: { _ in 0.05 }, pcReturnDebounceSeconds: 0.1), - pairingStore: PairingStore(storage: MemoryPairingStorage())) - var clock = start - manager.now = { clock } + pairingStore: pairings) + manager.homeAddress = { _ in true } + manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "127.0.0.1", netmask: "255.0.0.0") } + manager.findPCs = { _ in [] } let browser = FakePCBrowser() manager.makePCBrowser = { browser } - var lines: [String] = [] - let log = manager.$diagnostics.sink { if let line = $0.last?.message { lines.append(line) } } - manager.connect(host: "127.0.0.1", port: host.port) - try await until("didn't answer", seconds: 15) { manager.problem == .noReply(usb: false) } - XCTAssertTrue(manager.watchingForPC) - XCTAssertTrue(browser.running, "the search runs while the card is up") + let setup = Setup(manager: manager, browser: browser, host: host, pairings: pairings, clock: start) + manager.now = { [unowned setup] in setup.clock } + setup.log = manager.$diagnostics.sink { [unowned setup] in + if let line = $0.last?.message { setup.lines.append(line) } + } + return setup + } + + /// The "didn't answer" card is up for `paired()`'s PC, and its watch armed. + private func noReply(named: Bool = true) async throws -> Setup { + let setup = try paired(named: named) + setup.manager.connect(host: "127.0.0.1", port: setup.host.port) + XCTAssertTrue(setup.browser.running, "the search runs from the attempt's start") // Bonjour has nothing for the PC: its app isn't open. - browser.list([]) - clock = start.addingTimeInterval(PCReturn.settleSeconds + 1) + setup.clock = start.addingTimeInterval(PCReturn.settleSeconds + 1) + try await until("didn't answer", seconds: 15) { setup.manager.problem == .noReply(usb: false) } + XCTAssertTrue(setup.manager.watchingForPC) + XCTAssertTrue(setup.browser.running, "the search runs while the card is up") // Drop the unanswered HELLO2s, so only a new attempt's count. - _ = await Task.detached { host.helloAttempts(within: 0.4) }.value - return Setup(manager: manager, browser: browser, host: host, lines: { lines }, log: log, - advance: { clock = clock.addingTimeInterval($0) }) + _ = await Task.detached { [host = setup.host] in host.helloAttempts(within: 0.4) }.value + return setup } private func until(_ description: String, seconds: Double, _ condition: () -> Bool) async throws { @@ -173,6 +204,16 @@ final class PCReturnTests: XCTestCase { try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) } + /// The PC proves the pairing, then accepts and streams. + private func stream(_ setup: Setup, from pc: FakeUDPHost) async throws { + let accepted = await Task.detached { pc.answerIdentify(token: Self.token) && pc.acceptSession(authToken: Self.token) }.value + XCTAssertTrue(accepted, "no automatic attempt reached the PC") + XCTAssertFalse(pc.helloBeforeIdentify, "nothing with the pairing before the PC proved it holds it") + try await until("handshake", seconds: 15) { setup.manager.hostInfo != nil } + pc.sendKeyframe(try FakeUDPHost.keyframe()) + try await until("streaming", seconds: 30) { setup.manager.state == .connected } + } + /// No answer; the PC app opens and its service shows up; the iPad /// connects once, by itself, and streams. func testThePCShowingUpConnectsOnceByItself() async throws { @@ -182,17 +223,70 @@ final class PCReturnTests: XCTestCase { setup.browser.list([setup.listing]) // Updates while it waits count as the same appearance. setup.browser.list([setup.listing]) - let accepted = await Task.detached { [host = setup.host] in host.acceptSession() }.value - XCTAssertTrue(accepted, "no automatic attempt reached the PC") - try await until("handshake", seconds: 15) { setup.manager.hostInfo != nil } - setup.host.sendKeyframe(try FakeUDPHost.keyframe()) - try await until("streaming", seconds: 30) { setup.manager.state == .connected } + try await stream(setup, from: setup.host) XCTAssertEqual(setup.tries, 1) XCTAssertEqual(setup.attempts, 2) XCTAssertFalse(setup.manager.watchingForPC) XCTAssertFalse(setup.browser.running, "streaming ends the watch") } + /// The PC opened just after the attempt's HELLO2s stopped, before the + /// card: its listing is a return, not what Bonjour already knew. + func testAPCOpenedJustAfterTheHellosIsNotMissed() async throws { + let setup = try paired() + defer { setup.manager.cancel() } + setup.manager.connect(host: "127.0.0.1", port: setup.host.port) + setup.clock = start.addingTimeInterval(PCReturn.settleSeconds + 0.5) + setup.browser.list([setup.listing]) + try await until("didn't answer", seconds: 15) { setup.manager.problem == .noReply(usb: false) } + _ = await Task.detached { [host = setup.host] in host.helloAttempts(within: 0.2) }.value + try await stream(setup, from: setup.host) + XCTAssertEqual(setup.tries, 1) + } + + /// Bonjour has the saved PC at a new port: the try goes there, after the + /// PC there proves the pairing, and the saved entry moves with it. + func testAPCBackAtANewPortIsTriedThere() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + let moved = FakeUDPHost() + setup.browser.list([DiscoveredHost(name: "EternalMonitor (Test PC)", address: "127.0.0.1", port: moved.port)]) + try await stream(setup, from: moved) + let atOld = await Task.detached { [host = setup.host] in host.helloAttempts(within: 0.3) }.value + XCTAssertEqual(atOld, 0, "nothing went to the old port") + XCTAssertEqual(RecentConnectionStore.shared.connections.first?.port, moved.port) + XCTAssertEqual(try setup.pairings.token(address: PairingStore.address(host: "127.0.0.1", port: moved.port)), Self.token) + } + + /// Another PC took the saved address, with pairing off: it can't prove + /// the pairing, so nothing is sent to it. One that got past that and + /// accepts without giving the pairing back is dropped before a picture. + func testADifferentPCAtTheSavedAddressGetsNoAutomaticSession() async throws { + let setup = try await noReply(named: false) + defer { setup.manager.cancel() } + let other = DiscoveredHost(name: "EternalMonitor (Other PC)", address: "127.0.0.1", port: setup.host.port) + setup.browser.list([other]) + let types = await Task.detached { [host = setup.host] in host.received(within: 4) }.value + XCTAssertTrue(types.contains(.identify), "\(types)") + XCTAssertFalse(types.contains(.hello2), "\(types)") + try await until("not confirmed", seconds: 5) { setup.lines.contains("pc_return result=not_confirmed") } + XCTAssertEqual(setup.attempts, 1) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + + setup.browser.list([]) + setup.browser.list([other]) + let accepted = await Task.detached { [host = setup.host] in + host.answerIdentify(token: Self.token) && host.acceptSession(hostName: "Other PC", authToken: Data(repeating: 0, count: 16)) + }.value + XCTAssertTrue(accepted) + try await until("dropped", seconds: 15) { setup.lines.contains("pc_return result=other_pc") } + XCTAssertEqual(setup.manager.state, .disconnected) + XCTAssertNil(setup.manager.hostInfo) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + XCTAssertEqual(try setup.pairings.token(address: PairingStore.address(host: "127.0.0.1", port: setup.host.port)), Self.token) + XCTAssertFalse(setup.lines.contains { $0.hasPrefix("Connected to") }) + } + /// The automatic try gets no answer either: the card comes back and /// waits for the PC to show up again. The same listing never brings /// another try. @@ -201,7 +295,9 @@ final class PCReturnTests: XCTestCase { defer { setup.manager.cancel() } let listing = DiscoveredHost(name: "EternalMonitor (Unknown)", address: "127.0.0.1", port: setup.host.port) setup.browser.list([listing]) - try await until("tried", seconds: 5) { setup.tries == 1 } + let proved = await Task.detached { [host = setup.host] in host.answerIdentify(token: Self.token) }.value + XCTAssertTrue(proved) + try await until("tried", seconds: 5) { setup.attempts == 2 } try await until("didn't answer again", seconds: 15) { setup.manager.state == .disconnected && setup.manager.problem == .noReply(usb: false) } @@ -224,19 +320,19 @@ final class PCReturnTests: XCTestCase { XCTAssertFalse(setup.browser.running) setup.browser.list([setup.listing]) try await settle() - XCTAssertEqual(setup.attempts, 1) + XCTAssertEqual(setup.tries, 0) } func testNoTryAfterTheTimeLimit() async throws { let setup = try await noReply() defer { setup.manager.cancel() } - setup.advance(ConnectionManager.Timing().pcReturnLimitSeconds) + setup.clock = setup.clock.addingTimeInterval(ConnectionManager.Timing().pcReturnLimitSeconds) setup.browser.list([setup.listing]) try await settle() - XCTAssertEqual(setup.attempts, 1) + XCTAssertEqual(setup.tries, 0) XCTAssertFalse(setup.manager.watchingForPC) XCTAssertFalse(setup.browser.running) - XCTAssertTrue(setup.lines().contains("pc_return result=expired")) + XCTAssertTrue(setup.lines.contains("pc_return result=expired")) } /// Nothing starts in the background. Back in the foreground, a PC that @@ -246,14 +342,26 @@ final class PCReturnTests: XCTestCase { defer { setup.manager.cancel() } setup.manager.handleAppBackgrounded() XCTAssertFalse(setup.browser.running, "no search in the background") - setup.browser.list([setup.listing]) + setup.browser.current = [setup.listing] try await settle() - XCTAssertEqual(setup.attempts, 1) + XCTAssertEqual(setup.tries, 0) setup.manager.handleAppForegrounded() XCTAssertTrue(setup.browser.running) try await until("tried on return", seconds: 5) { setup.tries == 1 } } + /// An unpaired PC can't prove it is the same one: it isn't watched for. + func testAnUnpairedPCIsNotWatchedFor() async throws { + let setup = try paired() + defer { setup.manager.cancel() } + try setup.pairings.forgetAll() + setup.manager.connect(host: "127.0.0.1", port: setup.host.port) + XCTAssertEqual(setup.browser.starts, 0) + try await until("didn't answer", seconds: 15) { setup.manager.problem == .noReply(usb: false) } + XCTAssertFalse(setup.manager.watchingForPC) + XCTAssertEqual(setup.browser.starts, 0) + } + func testPickingAnotherPCEndsTheWatch() async throws { let setup = try await noReply() let other = FakeUDPHost() @@ -271,7 +379,9 @@ final class PCReturnTests: XCTestCase { let setup = try await noReply() defer { setup.manager.cancel() } setup.browser.list([setup.listing]) - let refused = await Task.detached { [host = setup.host] in host.acceptSession(status: .busy) }.value + let refused = await Task.detached { [host = setup.host] in + host.answerIdentify(token: Self.token) && host.acceptSession(status: .busy) + }.value XCTAssertTrue(refused) try await until("busy", seconds: 15) { setup.manager.problem == .pcBusy } XCTAssertFalse(setup.browser.running) From 2d9c27d12dcf1d239c3b4c05d7018d92b528b9df Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:26:45 -0500 Subject: [PATCH 189/226] Keep a root device that couldn't be read as uncertain in the attach wait Review r2: the stricter read left out root devices whose hardware IDs couldn't be read. Such a device may be the driver's, with a monitor that has started or is on its way. Next to another device of the driver that runs with no children, the read then said "no monitor", and the wait ended after 10 s. A root device that couldn't be read now counts as a device of the driver whose read isn't certain, so the whole read stays uncertain and the wait goes on. Only devices known to belong to another driver are left out. Test: an unread root device next to a running device with no children reads as uncertain, as does a device of the driver whose status couldn't be read; the running device alone still reads as no monitor. --- host/src/vdd_device.rs | 89 +++++++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/host/src/vdd_device.rs b/host/src/vdd_device.rs index 4bc76bfe..2129ea44 100644 --- a/host/src/vdd_device.rs +++ b/host/src/vdd_device.rs @@ -235,22 +235,37 @@ fn read_devices() -> Vec> { let Some(devnodes) = root_devnodes() else { return Vec::new(); }; + reads_from(&devnodes, |devinst, (flags, problem)| { + let has_driver = matches!(registry_property(devinst, CM_DRP_DRIVER), Ok(Some(_))); + let children = children(devinst); + ( + flags, + problem, + has_driver, + Monitors::from_children(children.as_deref()), + ) + }) +} + +/// `read_devices` from the present root devices, with `read` reading one +/// device of the driver whose status was read. A root device that couldn't +/// be read may be the driver's, with a monitor of its own: it stays, as a +/// read that isn't certain. Only devices known to be another driver's are +/// left out. +#[cfg(any(windows, test))] +fn reads_from( + devnodes: &[Devnode], + mut read: impl FnMut(u32, (u32, u32)) -> (u32, u32, bool, Monitors), +) -> Vec> { devnodes - .into_iter() - .filter_map(|devnode| match devnode { - Devnode::Driver { devinst, status } => Some((devinst, status)), - _ => None, - }) - .map(|(devinst, status)| { - let (flags, problem) = status?; - let has_driver = matches!(registry_property(devinst, CM_DRP_DRIVER), Ok(Some(_))); - let children = children(devinst); - Some(( - flags, - problem, - has_driver, - Monitors::from_children(children.as_deref()), - )) + .iter() + .filter_map(|devnode| match *devnode { + Devnode::Other => None, + Devnode::Unread | Devnode::Driver { status: None, .. } => Some(None), + Devnode::Driver { + devinst, + status: Some(status), + } => Some(Some(read(devinst, status))), }) .collect() } @@ -807,6 +822,50 @@ mod tests { assert_eq!(confirmed_from(vec![]), Some(DeviceState::Missing)); } + /// Review r2 of #233: a root device that couldn't be read may be the + /// driver's, with a monitor still arriving. Next to a running device + /// with no children, the read isn't a confirmed absence. + #[test] + fn an_unread_root_device_keeps_the_read_uncertain() { + let running_childless = |_, (flags, problem)| (flags, problem, true, Monitors::None); + let alone = reads_from( + &[ + Devnode::Other, + Devnode::Driver { + devinst: 7, + status: Some((DN_STARTED, 0)), + }, + ], + running_childless, + ); + assert_eq!(confirmed_from(alone), Some(DeviceState::NoMonitor)); + let with_unread = reads_from( + &[ + Devnode::Unread, + Devnode::Driver { + devinst: 7, + status: Some((DN_STARTED, 0)), + }, + ], + running_childless, + ); + assert_eq!(confirmed_from(with_unread), None); + let unread_status = reads_from( + &[ + Devnode::Driver { + devinst: 8, + status: None, + }, + Devnode::Driver { + devinst: 7, + status: Some((DN_STARTED, 0)), + }, + ], + running_childless, + ); + assert_eq!(confirmed_from(unread_status), None); + } + #[test] fn the_device_closest_to_a_display_wins() { assert_eq!(DeviceState::best([]), DeviceState::Missing); From 58d0932125ab4c74e86b3c0768f9cfb14882b54a Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:28:35 -0500 Subject: [PATCH 190/226] Restart the notice wait for each refusal, and keep lost video lost The 2 s wait before the iPad hears that this graphics card is refused reset only when a heartbeat saw another block. Capture that worked and was refused again between two heartbeats took the old refusal's start, and the notice came at once. The refusal's start is now set where the block is recorded and cleared, so each new refusal waits its own 2 s. The iPad's summary named a timeout after the refusal a capture failure even when the PC had sent video that gave no picture. As on the PC, it is one only when no video went out: the PC's heartbeats counted none, and none arrived. Otherwise it stays no_video or no_frame. --- host/src/capture/dxgi.rs | 2 +- host/src/capture/recovery.rs | 87 +++++++++++++++++-- .../App/ConnectionManager.swift | 5 ++ ios/EternalMonitor/App/SessionSummary.swift | 6 +- ios/EternalMonitorTests/FakeUDPHost.swift | 7 +- .../NoPictureTimeoutTests.swift | 21 +++++ .../SessionSummaryTests.swift | 15 ++++ 7 files changed, 130 insertions(+), 13 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index f72ea660..7e32739c 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -865,7 +865,7 @@ pub fn run_capture_loop( } // Tell the GUI why while the retries continue. if let Some(block) = CaptureBlock::from_hresult(last_code.0) { - *shared.capture_block.lock() = Some(block); + super::recovery::note_capture_blocked(&shared, block); } // The output may be gone: look at the whole layout. let allowed = keepalive_allowed(picture.real(), second_screen); diff --git a/host/src/capture/recovery.rs b/host/src/capture/recovery.rs index 2ea6e3d6..aaeeabf2 100644 --- a/host/src/capture/recovery.rs +++ b/host/src/capture/recovery.rs @@ -446,12 +446,7 @@ pub(crate) fn run_until_unblocked_on( }; // The first working frame clears the block (see // [`note_capture_resumed`]). - if shared - .capture_block - .lock() - .replace(unavailable.block) - .is_none() - { + if note_capture_blocked(shared, unavailable.block).is_none() { retry.resumed(); } let (delay, changed) = retry.blocked(unavailable.block); @@ -495,7 +490,9 @@ pub const GPU_NOTICE_AFTER: Duration = Duration::from_secs(2); /// frozen picture without a word; it clears with the block when capture /// works again. A screen Windows won't share from this graphics card says so /// once the refusal has lasted [`GPU_NOTICE_AFTER`] (`gpu_since` keeps when -/// it began), naming the failed second screen when `second_screen_failed`. +/// it began: [`note_capture_blocked`] sets it, and a heartbeat that finds it +/// unset starts it), naming the failed second screen when +/// `second_screen_failed`. /// The other blocks keep the held picture as before. pub fn heartbeat_notice( block: Option, @@ -526,9 +523,26 @@ pub fn heartbeat_notice( } } +/// Records that Windows blocks capture with `block`, returning the block +/// before. A refusal from this graphics card that begins now, after capture +/// worked or another block, starts its own wait for the iPad's notice, so a +/// refusal that came back between two heartbeats never inherits an earlier +/// one's. +pub fn note_capture_blocked(shared: &SharedControl, block: CaptureBlock) -> Option { + let mut current = shared.capture_block.lock(); + let before = current.replace(block); + if before != Some(block) { + *shared.gpu_refused_since.lock() = + (block == CaptureBlock::UnsupportedOnThisGpu).then(Instant::now); + } + before +} + /// Clears the blocked state once frames flow again. pub fn note_capture_resumed(shared: &SharedControl) { - if let Some(block) = shared.capture_block.lock().take() { + let resumed = shared.capture_block.lock().take(); + *shared.gpu_refused_since.lock() = None; + if let Some(block) = resumed { info!(block = ?block, "Screen capture resumed"); } } @@ -678,6 +692,63 @@ mod tests { /// the refusal has lasted 2 s (it comes and goes for a moment while the /// desktop changes mode), naming the second screen when it failed too. /// It clears with the refusal, and a new one waits again. + /// Review r1: capture worked for a moment and Windows refused the card + /// again, all between two heartbeats. The heartbeat never saw the gap, + /// so the new refusal took the old one's start and the iPad was told at + /// once. Now the refusal's start is kept where the block is set and + /// cleared: a new refusal waits its own 2 s, after capture worked or + /// after another block, while the same refusal going on keeps its start. + #[test] + fn a_refusal_that_comes_back_between_heartbeats_waits_its_own_2_s() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_UNSUPPORTED_GPU}; + let shared = SharedControl::new(9876, 15_000_000); + let notice = |shared: &SharedControl| { + let block = *shared.capture_block.lock(); + heartbeat_notice( + block, + false, + &mut shared.gpu_refused_since.lock(), + Instant::now(), + ) + }; + let long_ago = || Some(Instant::now() - Duration::from_secs(3)); + note_capture_blocked(&shared, CaptureBlock::UnsupportedOnThisGpu); + assert_eq!(notice(&shared), NOTICE_NONE, "a refusal that just began"); + // The same refusal, retried 3 s on: its start stays. + *shared.gpu_refused_since.lock() = long_ago(); + note_capture_blocked(&shared, CaptureBlock::UnsupportedOnThisGpu); + assert_eq!(notice(&shared), NOTICE_UNSUPPORTED_GPU); + // Capture works, then is refused again, with no heartbeat between. + note_capture_resumed(&shared); + note_capture_blocked(&shared, CaptureBlock::UnsupportedOnThisGpu); + assert_eq!(notice(&shared), NOTICE_NONE, "a fresh refusal waits again"); + // Another block in between restarts it too. + *shared.gpu_refused_since.lock() = long_ago(); + note_capture_blocked(&shared, CaptureBlock::DisplayChanging); + note_capture_blocked(&shared, CaptureBlock::UnsupportedOnThisGpu); + assert_eq!(notice(&shared), NOTICE_NONE); + // Through the capture stage's own retries: worked, then refused. + *shared.gpu_refused_since.lock() = long_ago(); + note_capture_resumed(&shared); + let mut attempts = 0; + let result = run_until_unblocked( + &shared, + |_| { + attempts += 1; + if attempts == 1 { + Err(blocked(CaptureBlock::UnsupportedOnThisGpu)) + } else { + Ok(()) + } + }, + |_| { + assert_eq!(notice(&shared), NOTICE_NONE); + true + }, + ); + assert!(result.is_ok()); + } + #[test] fn a_screen_refused_from_this_graphics_card_is_told_to_the_ipad_after_2_s() { use eternal_wire::v2::control::{ diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index c23b868c..f6c7cb2c 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -125,6 +125,8 @@ final class ConnectionManager: ObservableObject { /// The PC's heartbeats say Windows won't let it capture from its /// graphics card, and which screen. @Published var pcGraphicsCard: PCGraphicsCard? + /// The PC's heartbeats counted video frames sent this attempt. + private var pcSentVideo = false /// Nonzero while an automatic reconnect cycle is running. @Published private(set) var reconnectAttempt = 0 @@ -556,6 +558,7 @@ final class ConnectionManager: ObservableObject { pcLocked = false pcNoDisplay = false pcGraphicsCard = nil + pcSentVideo = false degradedSinceUs = nil prevCounters = FrameAssembler.Counters() diagnostics.removeAll() @@ -960,6 +963,7 @@ final class ConnectionManager: ObservableObject { self.hostStreamConfig = heartbeat.streamConfig if let frames = heartbeat.framesSent { self.noVideoFallback?.heard(framesSent: frames, at: ProcessInfo.processInfo.systemUptime) + if frames > 0 { self.pcSentVideo = true } } let locked = heartbeat.notice == Heartbeat.noticePCLocked if locked != self.pcLocked { @@ -1765,6 +1769,7 @@ final class ConnectionManager: ObservableObject { end: end, isUSB: mediaLink?.isUSB ?? (transportMode == "USB"), sessionId: hostInfo?.sessionId, nonce: controlChannel?.helloNonce ?? 0, awaitingPairing: awaitingPairing, datagramsReceived: debugState.datagramsReceived, captureRefused: pcGraphicsCard != nil, + pcSentVideo: pcSentVideo, firstFrame: firstFrameAt.map { $0 - summaryStartedAt }, duration: now - summaryStartedAt) } diff --git a/ios/EternalMonitor/App/SessionSummary.swift b/ios/EternalMonitor/App/SessionSummary.swift index b4760816..4bf4e399 100644 --- a/ios/EternalMonitor/App/SessionSummary.swift +++ b/ios/EternalMonitor/App/SessionSummary.swift @@ -75,6 +75,8 @@ struct SessionSummary: Equatable { var datagramsReceived = 0 /// The PC said it can't capture its screen from its graphics card. var captureRefused = false + /// The PC's heartbeats counted video frames sent this attempt. + var pcSentVideo = false /// Seconds from the start to the first decoded frame. var firstFrame: TimeInterval? var duration: TimeInterval @@ -117,7 +119,9 @@ struct SessionSummary: Equatable { private var failure: String { switch end { case .connectTimeout where sessionId == nil, .pcMoved: return "no_reply" - case .connectTimeout where captureRefused: return "capture_failed" + // Only when no video went out, as the PC's summary has it: video + // that went out and gave no picture was lost or didn't decode. + case .connectTimeout where captureRefused && !pcSentVideo && datagramsReceived == 0: return "capture_failed" case .connectTimeout: return datagramsReceived == 0 ? "no_video" : "no_frame" case .noVideoPath: return datagramsReceived == 0 ? "no_video" : "no_frame" case .pcStoppedResponding: return "signal_lost" diff --git a/ios/EternalMonitorTests/FakeUDPHost.swift b/ios/EternalMonitorTests/FakeUDPHost.swift index 8bc148e9..0eb98d71 100644 --- a/ios/EternalMonitorTests/FakeUDPHost.swift +++ b/ios/EternalMonitorTests/FakeUDPHost.swift @@ -218,10 +218,11 @@ final class FakeUDPHost: @unchecked Sendable { send(Wire.encodeControl(sessionId: 41, msgSeq: 2, message: .bye(reason))) } - /// A session 41 heartbeat, as the PC sends one each second. - func sendHeartbeat(notice: UInt8 = Heartbeat.noticeNone) { + /// A session 41 heartbeat, as the PC sends one each second, counting + /// `framesSent` video frames when it is given. + func sendHeartbeat(notice: UInt8 = Heartbeat.noticeNone, framesSent: UInt32? = nil) { send(Wire.encodeControl(sessionId: 41, msgSeq: 3, message: .heartbeat( - Heartbeat(hostTimeUs: 0, streamConfig: StreamConfig(), notice: notice)))) + Heartbeat(hostTimeUs: 0, streamConfig: StreamConfig(), notice: notice, framesSent: framesSent)))) } /// The 64x64 H.264 keyframe fixture. diff --git a/ios/EternalMonitorTests/NoPictureTimeoutTests.swift b/ios/EternalMonitorTests/NoPictureTimeoutTests.swift index aee60587..90b773a1 100644 --- a/ios/EternalMonitorTests/NoPictureTimeoutTests.swift +++ b/ios/EternalMonitorTests/NoPictureTimeoutTests.swift @@ -98,6 +98,27 @@ final class NoPictureTimeoutTests: XCTestCase { XCTAssertNil(manager.pcGraphicsCard, "the next attempt starts without it") } + /// The PC counted video it sent before Windows refused its card, and no + /// picture came: the card still says why, and the summary keeps the + /// lost video, as the PC's does. + func testVideoSentBeforeTheRefusalIsStillCountedAsLost() async throws { + let host = FakeUDPHost() + var timing = fast + timing.connectTimeoutSeconds = 3 + let manager = ConnectionManager(timing: timing) + defer { manager.cancel() } + manager.connect(host: "127.0.0.1", port: host.port) + let accepted = await Task.detached { host.acceptSession() }.value + XCTAssertTrue(accepted, "the iPad never sent HELLO2") + host.sendHeartbeat(framesSent: 5) + host.sendHeartbeat(notice: Heartbeat.noticeUnsupportedGPU, framesSent: 5) + try await until("the PC says why") { manager.pcGraphicsCard == .screen } + try await until("timed out", seconds: 15) { manager.state == .disconnected } + XCTAssertEqual(manager.problem, .pcGraphicsCard(.screen)) + let summary = logged(manager, "session_summary v=1 role=ipad") ?? "" + XCTAssertTrue(summary.contains(" outcome=no_picture failure=no_video "), summary) + } + /// The same in Second screen mode, after the second screen failed. func testThePCsSecondScreenModeKeepsItsOwnCard() async throws { let host = FakeUDPHost() diff --git a/ios/EternalMonitorTests/SessionSummaryTests.swift b/ios/EternalMonitorTests/SessionSummaryTests.swift index cf04ba93..77887186 100644 --- a/ios/EternalMonitorTests/SessionSummaryTests.swift +++ b/ios/EternalMonitorTests/SessionSummaryTests.swift @@ -120,6 +120,21 @@ final class SessionSummaryTests: XCTestCase { XCTAssertEqual(tokens(streamed.line), ["steady", "streamed", "signal_lost", "liveness"]) } + /// Review r1: the PC sent video before Windows refused its card, and + /// none of it gave a picture. That is lost or undecodable video, as the + /// PC's summary says (it names a capture failure only when no video went + /// out), and the refusal mustn't hide it. + func testVideoThePCSentBeforeTheRefusalKeepsItsDeliveryFailure() { + var counted = summary(.connectTimeout, duration: 10) + (counted.captureRefused, counted.pcSentVideo) = (true, true) + XCTAssertEqual(tokens(counted.line), ["accepted", "no_picture", "no_video", "timeout"]) + var arrived = summary(.connectTimeout, duration: 10) + (arrived.captureRefused, arrived.datagramsReceived) = (true, 12) + XCTAssertEqual(tokens(arrived.line), ["accepted", "no_picture", "no_frame", "timeout"]) + Self.assertContract(counted.line) + Self.assertContract(arrived.line) + } + func testOnlyTheUsersOwnActionSaysUserDisconnectToThePC() { XCTAssertEqual(AttemptEnd.userDisconnect.bye, .userDisconnect) XCTAssertEqual(AttemptEnd.usbTurnedOff.bye, .userDisconnect) From 950076ba7c9eb6f11c7150c878dedfd96f895393 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:29:01 -0500 Subject: [PATCH 191/226] Show USB as it was while a failed list keeps the tunnels A poll whose list failed but kept the tunnels still marked Apple's device service unreachable and the device count zero. The PC window then told the user to open Apple Devices while their USB stream was live, and new HELLO_ACKs left out USB. While the tunnels are kept, the poll now leaves the USB status as it was and skips the check for an attached iPad. The supervisor now takes its poll times and the stats it writes as arguments, so its test hands it each poll's time and reads the status the window shows. The test no longer depends on how many polls a busy machine gets through in 10 s. --- host/src/transport/usb.rs | 170 ++++++++++++++++++++++++++------------ 1 file changed, 116 insertions(+), 54 deletions(-) diff --git a/host/src/transport/usb.rs b/host/src/transport/usb.rs index ac272032..d38e3515 100644 --- a/host/src/transport/usb.rs +++ b/host/src/transport/usb.rs @@ -17,7 +17,7 @@ use super::apple_usb::{self, AppleUsbState}; use super::congestion::{is_no_buffer_space, SendCongestion}; use super::link::{FramedLink, Link, LinkId, PeerId, UdpLink}; use super::usbmuxd::{self, BoxedTunnel, Client}; -use crate::stats::PIPELINE_STATS; +use crate::stats::{PipelineStats, PIPELINE_STATS}; static NEXT_GENERATION: AtomicU64 = AtomicU64::new(1); static HUB: OnceLock = OnceLock::new(); @@ -494,7 +494,22 @@ async fn supervise(events: mpsc::Sender, force_idr: Arc) { } let client = Arc::new(Client::new(usbmuxd::Endpoint::default())); - supervise_devices(client, events, force_idr).await; + let mut poll = tokio::time::interval(Duration::from_secs(2)); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + supervise_devices(client, events, force_idr, &PIPELINE_STATS, poll).await; +} + +/// When the supervisor polls Apple's device list next, and the time it +/// judges that poll by. +trait PollClock { + fn next(&mut self) -> impl std::future::Future + Send; +} + +impl PollClock for tokio::time::Interval { + async fn next(&mut self) -> tokio::time::Instant { + self.tick().await; + tokio::time::Instant::now() + } } /// How long Apple's device list may keep failing before the service counts @@ -546,31 +561,35 @@ impl ListFailures { } } +/// Polls Apple's device list and runs a worker for each listed device. What +/// a poll finds goes to `stats`, which the PC window reads. async fn supervise_devices( client: Arc, events: mpsc::Sender, force_idr: Arc, + stats: &Mutex, + mut poll: impl PollClock, ) { let mut workers = JoinSet::new(); let mut handles: HashMap = HashMap::new(); - let mut poll = tokio::time::interval(Duration::from_secs(2)); - poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); let mut unreachable = AppleUsbState::default(); let mut failures = ListFailures::default(); loop { tokio::select! { - _ = poll.tick() => { + now = poll.next() => { let devices = client.list_devices().await; let available = devices.is_ok(); let devices = devices.unwrap_or_default(); - let ids: HashSet<_> = if failures.keep_workers(available, handles.len(), tokio::time::Instant::now()) { + let kept = failures.keep_workers(available, handles.len(), now); + let ids: HashSet<_> = if kept { handles.keys().copied().collect() } else { devices.iter().map(|device| device.device_id).collect() }; // Without the service the cable is invisible to us; ask // Windows whether an iPad is attached so the GUI can say why. - let cabled = if available { + // A poll that keeps the tunnels shows nothing new: they work. + let cabled = if available || kept { AppleUsbState::default() } else { tokio::task::spawn_blocking(apple_usb::probe).await.unwrap_or_default() @@ -583,8 +602,8 @@ async fn supervise_devices( } } unreachable = cabled; - { - let mut stats = PIPELINE_STATS.lock(); + if !kept { + let mut stats = stats.lock(); stats.usb_service_reachable = available; stats.usb_devices = devices.len(); stats.usb_cabled_devices = cabled.cabled_devices; @@ -719,8 +738,6 @@ mod tests { /// Connect's stream to the test as that iPad's end of a tunnel. struct FlakyService { failing: Arc, - fail_next: Arc, - listed: Arc, failed: Arc, tunnels: mpsc::Receiver, client: Arc, @@ -728,25 +745,18 @@ mod tests { impl FlakyService { async fn start() -> Self { - use std::sync::atomic::AtomicU32; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let client = Arc::new(Client::new(usbmuxd::Endpoint::Tcp( listener.local_addr().unwrap(), ))); let failing = Arc::new(AtomicBool::new(false)); - let fail_next = Arc::new(AtomicU32::new(0)); - let (listed, failed) = (Arc::new(AtomicU64::new(0)), Arc::new(AtomicU64::new(0))); + let failed = Arc::new(AtomicU64::new(0)); let (tunnel_tx, tunnels) = mpsc::channel(4); - let state = ( - failing.clone(), - fail_next.clone(), - listed.clone(), - failed.clone(), - ); + let state = (failing.clone(), failed.clone()); tokio::spawn(async move { loop { let (mut stream, _) = listener.accept().await.unwrap(); - let (failing, fail_next, listed, failed) = state.clone(); + let (failing, failed) = state.clone(); let tunnel_tx = tunnel_tx.clone(); tokio::spawn(async move { let (tag, request) = usbmuxd::read_message(&mut stream).await.unwrap(); @@ -755,16 +765,10 @@ mod tests { .unwrap() .to_string(); let reply = if kind == "ListDevices" { - let once = fail_next - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| { - n.checked_sub(1) - }) - .is_ok(); - if once || failing.load(Ordering::SeqCst) { + if failing.load(Ordering::SeqCst) { failed.fetch_add(1, Ordering::SeqCst); return; } - listed.fetch_add(1, Ordering::SeqCst); let device = plist::Dictionary::from_iter([ ("DeviceID".to_string(), plist::Value::from(3u64)), ( @@ -798,8 +802,6 @@ mod tests { }); Self { failing, - fail_next, - listed, failed, tunnels, client, @@ -815,61 +817,121 @@ mod tests { super::super::link::read_preamble(&mut ipad).await.unwrap(); ipad } + } - async fn until(&self, what: &str, done: impl Fn(&Self) -> bool) { - tokio::time::timeout(Duration::from_secs(30), async { - while !done(self) { - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .unwrap_or_else(|_| panic!("{what}")); + /// Poll times a test hands the supervisor one at a time. It hears when + /// each poll has finished: the supervisor asks for the next one. + struct Polls { + finished: mpsc::Sender<()>, + at: mpsc::Receiver, + } + + impl PollClock for Polls { + async fn next(&mut self) -> tokio::time::Instant { + let _ = self.finished.send(()).await; + self.at.recv().await.expect("the test's next poll") + } + } + + /// The test's side of [`Polls`]. + struct Poller { + at: mpsc::Sender, + polled: mpsc::Receiver<()>, + start: tokio::time::Instant, + } + + impl Poller { + /// Polls `secs` after the start, and returns once that poll has + /// finished. + async fn poll(&mut self, secs: u64) { + let at = self.start + Duration::from_secs(secs); + self.at.send(at).await.unwrap(); + self.polled.recv().await.unwrap(); } } /// A device list that fails once leaves the live tunnel open; one that - /// keeps failing closes it after [`LIST_OUTAGE`], and once the list - /// answers again the iPad gets a new tunnel. A single failed list used - /// to close the tunnel at once. + /// keeps failing closes it once the failures have lasted + /// [`LIST_OUTAGE`], and once the list answers again the iPad gets a new + /// tunnel. While the tunnel is kept, the PC window goes on showing USB + /// as it was, not "Open Apple Devices". A single failed list used to + /// close the tunnel at once. Each poll's time is the test's, so how the + /// machine schedules the test changes nothing. #[tokio::test] async fn a_failed_device_list_keeps_the_tunnel_until_the_service_is_down() { let mut service = FlakyService::start().await; + let stats: &'static Mutex = + Box::leak(Box::new(Mutex::new(PipelineStats::default()))); + let (finished, mut polled) = mpsc::channel(1); + let (at, polls) = mpsc::channel(1); let (events, mut opened) = mpsc::channel(4); let supervisor = tokio::spawn(supervise_devices( Arc::clone(&service.client), events, Arc::new(AtomicBool::new(false)), + stats, + Polls { + finished, + at: polls, + }, )); + polled.recv().await.unwrap(); + let mut poller = Poller { + at, + polled, + start: tokio::time::Instant::now(), + }; + let shown = || { + let stats = stats.lock(); + ( + stats.usb_service_reachable, + stats.usb_devices, + stats.usb_link_state.clone(), + ) + }; + let streaming = (true, 1, "Waiting for the iPad app".to_string()); + + poller.poll(0).await; let _ipad = service.tunnel().await; let first = opened.recv().await.unwrap(); + assert_eq!(shown(), streaming); - service.fail_next.store(1, Ordering::SeqCst); - let listed = service.listed.load(Ordering::SeqCst); - service - .until("a failed list, then a good one", |service| { - service.failed.load(Ordering::SeqCst) == 1 - && service.listed.load(Ordering::SeqCst) > listed + 1 - }) - .await; + // One failed list between good ones. + service.failing.store(true, Ordering::SeqCst); + poller.poll(2).await; + assert_eq!(service.failed.load(Ordering::SeqCst), 1); assert!(!first.link.is_closed(), "one failed list closed the tunnel"); + assert_eq!(shown(), streaming, "a kept tunnel shown as unavailable"); + service.failing.store(false, Ordering::SeqCst); + poller.poll(4).await; + assert!(!first.link.is_closed()); assert!(service.tunnels.try_recv().is_err(), "a second tunnel"); + // Failures in a row: kept until they have lasted the outage. service.failing.store(true, Ordering::SeqCst); - let failing_from = tokio::time::Instant::now(); + for secs in [6, 8, 10, 12, 14] { + poller.poll(secs).await; + assert!(!first.link.is_closed(), "closed at {secs} s"); + assert_eq!(shown(), streaming, "at {secs} s"); + } + poller.poll(16).await; let mut closed = first.link.closed_signal(); - tokio::time::timeout(Duration::from_secs(30), closed.wait_for(|closed| *closed)) + tokio::time::timeout(Duration::from_secs(20), closed.wait_for(|closed| *closed)) .await .expect("the tunnel closes once the service is down") .unwrap(); - let failing_for = failing_from.elapsed(); - assert!(failing_for >= LIST_OUTAGE, "{failing_for:?}"); - assert!(service.failed.load(Ordering::SeqCst) >= 6); + assert_eq!( + shown(), + (false, 0, "Apple device service unavailable".to_string()) + ); service.failing.store(false, Ordering::SeqCst); + poller.poll(18).await; let _ipad = service.tunnel().await; let second = opened.recv().await.unwrap(); assert!(second.generation > first.generation); assert!(!second.link.is_closed()); + assert_eq!(shown(), streaming); supervisor.abort(); } } From 6e5f72575c7e5109bb4909c3456b0d1656fac291 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:31:47 -0500 Subject: [PATCH 192/226] Name the tunnel tests after what they check The link test is renamed to a_closed_tunnel_reports_the_first_end_seen, since the close cause is the first end this host saw, not necessarily the one that started the close. The hub test's comment now says it checks the hub only: its subscriber stands in for a pipeline. --- host/src/transport/link.rs | 7 ++++--- host/src/transport/usb.rs | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/host/src/transport/link.rs b/host/src/transport/link.rs index 2c5fb4df..a980e43c 100644 --- a/host/src/transport/link.rs +++ b/host/src/transport/link.rs @@ -1141,10 +1141,11 @@ mod tests { ); } - /// A tunnel says which end closed it first. In the field some closed on - /// both sides 9.5-9.8 s after they opened, and neither side said why. + /// A tunnel says which end this host saw close it first, which is not + /// necessarily the end that started the close. In the field some closed + /// on both sides 9.5-9.8 s after they opened, and neither side said why. #[tokio::test] - async fn a_closed_tunnel_reports_which_end_closed_it() { + async fn a_closed_tunnel_reports_the_first_end_seen() { let force_idr = || Arc::new(AtomicBool::new(false)); let open = || async { let (host, mut ipad) = tokio::io::duplex(1024); diff --git a/host/src/transport/usb.rs b/host/src/transport/usb.rs index 4961a12a..f3288a13 100644 --- a/host/src/transport/usb.rs +++ b/host/src/transport/usb.rs @@ -803,7 +803,9 @@ mod tests { drop(ipad); gone(Arc::clone(&state)).await; - // A pipeline gets the events, and the lines are not logged again. + // A subscriber (standing in for a pipeline) gets the events, and the + // hub does not log the lines again. This checks the hub only: the + // pipeline's TransportLinks no longer logs these lines at all. let (pipeline, mut hub) = mpsc::channel(8); state.lock().subscriber = Some((1, pipeline)); let (event, ipad) = open(42).await; From 86a8a109069843a34a7c719ac73d363a7ca49a92 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:32:12 -0500 Subject: [PATCH 193/226] Keep the capture's own notice over the restarting one A stream that came back into a locked PC, or one with no display, before its first frame said it was restarting instead, and the iPad dropped the locked or no-display pill for the restarting one. The restarting notice now goes out only when the capture has no notice of its own. --- host/src/transport/session.rs | 66 +++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 06b5b867..fe34cff1 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -1508,13 +1508,16 @@ impl Session { let frames_sent = (session.info.feature_caps & FEATURE_MEDIA_SIZE != 0) .then_some(session.frames_sent); // Held through a restart, until the first new frame: an iPad - // that understands it says the picture comes back by itself. - let notice = if session.restart_held_since.is_some() - && session.info.feature_caps & FEATURE_STANDBY_STATUS != 0 - { - eternal_wire::v2::control::NOTICE_RESTARTING - } else { - config.notice() + // that understands it says the picture comes back by itself, + // unless the capture has more to say (a locked PC, no display). + let notice = match config.notice() { + eternal_wire::v2::control::NOTICE_NONE + if session.restart_held_since.is_some() + && session.info.feature_caps & FEATURE_STANDBY_STATUS != 0 => + { + eternal_wire::v2::control::NOTICE_RESTARTING + } + notice => notice, }; let stream_config = match self.not_streaming { // No stream: the config of the one the iPad last had. @@ -5908,4 +5911,53 @@ mod tests { [eternal_wire::v2::control::NOTICE_NONE] ); } + + /// The capture's own notice says more than "restarting": a stream that + /// comes back into a locked PC, or one with no display, says that, and + /// "restarting" shows only when nothing else does. + #[test] + fn a_capture_notice_comes_before_the_restarting_one() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_PC_LOCKED, NOTICE_RESTARTING}; + struct Locked; + impl ConfigSource for Locked { + fn stream_config(&self) -> StreamConfig { + TestConfig.stream_config() + } + fn host_name(&self) -> String { + TestConfig.host_name() + } + fn notice(&self) -> u8 { + NOTICE_PC_LOCKED + } + } + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(1234); + let ControlMessage::Hello2(mut hello) = hello(1, 50000) else { + unreachable!() + }; + hello.feature_caps |= FEATURE_STANDBY_STATUS; + session.handle_control_authed(peer, ControlMessage::Hello2(hello), &TestConfig, now); + session.handle_control_authed(peer, report(10), &TestConfig, now); + session.set_not_streaming(Some(NotStreaming::Restarting)); + let at = now + Duration::from_secs(1); + assert_eq!( + notices(&session.restart_tick(&RestartConfig, at)), + [NOTICE_RESTARTING] + ); + // The next stream runs into a locked PC, before its first frame. + session.set_not_streaming(None); + let at = now + Duration::from_secs(2); + session.handle_control_authed(peer, report(10), &Locked, at); + assert_eq!( + notices(&session.tick(&Locked, true, at)), + [NOTICE_PC_LOCKED] + ); + // Unlocked, still no frame: restarting again. + assert_eq!( + notices(&session.tick(&TestConfig, true, at)), + [NOTICE_RESTARTING] + ); + assert_ne!(NOTICE_PC_LOCKED, NOTICE_NONE); + } } From 088a34a93a8a6f6daeee7f6f00735a5023181788 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:36:07 -0500 Subject: [PATCH 194/226] Make three host lib tests independent of the machine's load updates: pending_saved named each test's result folder by process id and the clock, which ticks in microseconds on macOS. Two tests starting in the same microsecond shared a folder, so one read or removed the other's setup_exit result. Each call now takes the next number from a counter and clears any folder a crashed earlier process with the same id left. input::injector: the hop test counted every one of its 2000 events, but a thread the machine starved for a moment built a backlog, and from 32 waiting commands hover moves merge by design. The test now uses limits that never merge or drop, so every event's hop is timed; the 50 ms p99 bound is unchanged. transport::link: the two writer tests waited on the real clock (100 ms and 50 ms reads, a 1 s wait for the deferred keyframe request), which a starved test thread outran. They now run on a paused tokio clock, as the link's other writer tests do; the link keeps all its time in tokio's Instant, so the backoff passes in virtual time. --- host/src/input/injector.rs | 9 ++++++++- host/src/transport/link.rs | 4 ++-- host/src/updates/tests.rs | 11 +++++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/host/src/input/injector.rs b/host/src/input/injector.rs index ca324cc9..64e7782d 100644 --- a/host/src/input/injector.rs +++ b/host/src/input/injector.rs @@ -1213,7 +1213,14 @@ mod tests { const EVENTS: u32 = 2000; let taken = Arc::new(Mutex::new(Vec::new())); let recorded = Arc::clone(&taken); - let thread = InputThread::with_sink(Limits::default(), move || Timed(recorded)); + // Every event is timed: a thread the machine starves for a moment + // gets a backlog, which would merge moves or drop them at the bound. + let limits = Limits { + capacity: EVENTS as usize, + merge_from: usize::MAX, + ..Limits::default() + }; + let thread = InputThread::with_sink(limits, move || Timed(recorded)); let injector = thread.begin(geometry()); let mut queued = Vec::new(); let mut next = Instant::now(); diff --git a/host/src/transport/link.rs b/host/src/transport/link.rs index 221c397f..0f909745 100644 --- a/host/src/transport/link.rs +++ b/host/src/transport/link.rs @@ -1057,7 +1057,7 @@ mod tests { /// overflow behind it. Cutting it short sent the iPad a partial keyframe /// over a lossless cable: lost fragments in its report, and another /// keyframe request. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn bounded_writer_finishes_the_frame_in_flight_and_recovers_with_a_keyframe() { let (host, mut client) = tokio::io::duplex(64); let force_idr = Arc::new(AtomicBool::new(false)); @@ -1109,7 +1109,7 @@ mod tests { /// A link that overflows again right after a keyframe waits before it /// asks for the next one, and still asks once the wait is over, even if /// no other frame comes. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn overflow_right_after_a_keyframe_defers_the_next_request() { let (host, mut client) = tokio::io::duplex(64); let force_idr = Arc::new(AtomicBool::new(false)); diff --git a/host/src/updates/tests.rs b/host/src/updates/tests.rs index 407e9822..87911b82 100644 --- a/host/src/updates/tests.rs +++ b/host/src/updates/tests.rs @@ -1829,14 +1829,17 @@ fn the_failure_message_check_is_validated_and_bounded() { } fn pending_saved(from: &str, to: &str, result: Option<&str>) -> (Saved, PathBuf) { + // Each call its own folder: the clock alone repeats for calls in the same + // microsecond, and one test's update then reads or removes another's + // result. + static FOLDERS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let dir = std::env::temp_dir().join(format!( "eternal-update-result-{}-{}", std::process::id(), - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() + FOLDERS.fetch_add(1, std::sync::atomic::Ordering::Relaxed) )); + // A failed run of an earlier process with this id may have left one. + let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join(eternal_updater::RESULT_FILE); if let Some(text) = result { From 2013b686121351ed17fb8110e63946798c451a9f Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 08:37:38 -0500 Subject: [PATCH 195/226] Treat an installer closing the app as a quit, and bound the shutdown summary Review round 1. - WM_ENDSESSION with ENDSESSION_CLOSEAPP (and no logoff) is Restart Manager closing the app for an installer, as Setup does on updates: the PC stays up. It now ends the stream as a quit, with the goodbye every iPad reads (connect again when an update hands over), instead of telling the iPad the PC is shutting down and ignoring HELLOs for 30 s. - The summary written straight to disk before shutdown, sign-out or that close runs on a thread of its own; the message waits for it at most 3 s from when it came, while the goodbye's wait runs alongside. - Session::power_down reports a repeat (PowerDown::Repeat): the session change that follows a sign-out's end of session, or sleep's broadcast plus registration, no longer logs "no iPad was streaming" or restarts the 30 s hold. - The sign-out card says to try again once the PC is on, someone has signed in, and EternalMonitor is open. --- SETUP.md | 2 +- host/src/diagnostics.rs | 35 +-- host/src/power.rs | 200 +++++++++++++++--- host/src/transport/session.rs | 113 ++++++++-- host/tests/power_goodbye.rs | 72 +++++-- .../App/ConnectionProblem.swift | 2 +- .../ConnectionProblemTests.swift | 2 +- 7 files changed, 347 insertions(+), 79 deletions(-) diff --git a/SETUP.md b/SETUP.md index 2099dbdf..5fba065b 100644 --- a/SETUP.md +++ b/SETUP.md @@ -278,7 +278,7 @@ try fewer words. If your problem isn't here, see [Known issues](https://eternalm | **Your PC ended the session** | Someone clicked **Stop** on the PC, changed a setting that restarts the stream, or closed EternalMonitor. | Tap **Try again** when the PC says **Ready for your iPad**. | | **The connection to your PC ended** | The connection dropped, for example in a Wi-Fi stall, the PC ran into a problem, or EternalMonitor restarted itself on the PC. | Tap **Try again**. With **Reconnect automatically** on, the iPad tries again by itself first. | | **Your PC went to sleep** | Windows put the PC to sleep while it streamed. | Wake the PC, then tap **Try again**. | -| **Your PC is shutting down or signing out** | Windows shut down, restarted or signed out while the PC streamed. | Tap **Try again** once the PC is back on and EternalMonitor is open on it. | +| **Your PC is shutting down or signing out** | Windows shut down, restarted or signed out while the PC streamed. | Tap **Try again** once the PC is on, someone has signed in, and EternalMonitor is open. | | **Your PC is already streaming to another device** | Only one iPad can connect at a time. | Disconnect the other iPad first, then tap **Try again**. | | **Your PC is updating EternalMonitor** | The PC is installing a new version. | Wait. With **Reconnect automatically** on, the iPad connects again by itself. Otherwise tap **Try again** once EternalMonitor has opened on the PC. | | **Your PC hasn't come back from its update yet** | The update is taking longer than expected. | Check that EternalMonitor opened again on the PC. Windows may be waiting for you to click **Yes** there. Then tap **Try again**. | diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 63651155..f71d891d 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -350,31 +350,38 @@ pub fn log_session_end(end: &SessionEnd, shared: &SharedControl) { /// told the PC ended the session, or, when a restart or an update hands over /// to a new copy, to connect again. pub fn log_session_end_at_quit(shared: &SharedControl) { - let why = if shared.handoff.holder().is_some() { + stop_session(shared, quit_stop(shared)); +} + +/// Why the app quits: for good, or for a restart or an update that hands +/// over to a new copy. +pub fn quit_stop(shared: &SharedControl) -> StreamStop { + if shared.handoff.holder().is_some() { StreamStop::Handoff } else { StreamStop::Quit - }; - stop_session(shared, why); + } } /// The stream stops for good: ends the session, which the transport tells -/// why on its way out, stops the stream, and logs the summary. The goodbye -/// and `running` change under one session lock, so a transport starting -/// meanwhile never takes the goodbye for a stale one -/// (`Session::transport_resumed`). +/// why on its way out, stops the stream, and logs the summary. pub fn stop_session(shared: &SharedControl, why: StreamStop) { - let end = { - let mut session = shared.session.lock(); - let end = session.shut_down(std::time::Instant::now(), why); - shared.stop(); - end - }; - if let Some(end) = end { + if let Some(end) = end_stream(shared, why) { log_session_end(&end, shared); } } +/// [`stop_session`] without the summary, which the caller writes. The +/// goodbye and `running` change under one session lock, so a transport +/// starting meanwhile never takes the goodbye for a stale one +/// (`Session::transport_resumed`). +pub fn end_stream(shared: &SharedControl, why: StreamStop) -> Option { + let mut session = shared.session.lock(); + let end = session.shut_down(std::time::Instant::now(), why); + shared.stop(); + end +} + /// [`log_session_end`] for a PC about to go down: the line goes straight to /// the session log and the upload outbox on disk, so it survives the /// process ending a moment later, and the next launch uploads it. diff --git a/host/src/power.rs b/host/src/power.rs index cdea89cf..57e02251 100644 --- a/host/src/power.rs +++ b/host/src/power.rs @@ -16,10 +16,10 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; -use tracing::info; +use tracing::{debug, info, warn}; use crate::control::SharedControl; -use crate::transport::session::PowerEvent; +use crate::transport::session::{PowerDown, PowerEvent, SessionEnd}; /// Longest a suspend message waits for the goodbye to leave. pub const SLEEP_WAIT: Duration = Duration::from_millis(500); @@ -62,11 +62,24 @@ pub enum Message { /// yes, and nothing happens yet: another app can still call it off. QueryEndSession, /// WM_ENDSESSION: `ending` false when the shutdown was called off. - EndSession { ending: bool, logoff: bool }, + /// `close_app` (ENDSESSION_CLOSEAPP) without `logoff`: the PC stays up, + /// and Restart Manager closes the app because an installer needs its + /// files, as Setup does on every update. + EndSession { + ending: bool, + logoff: bool, + close_app: bool, + }, /// WM_WTSSESSION_CHANGE WTS_SESSION_LOGOFF. SessionLogoff, } +/// Longest the end-of-session message waits for the session summary to +/// reach the disk, counted from when the message came. Windows lists the +/// app as holding up shutdown after about five seconds; the goodbye's own +/// wait runs at the same time. +pub const SUMMARY_WAIT: Duration = Duration::from_secs(3); + /// Acts on `message` and returns what the window answers Windows: 1 (TRUE) /// for WM_POWERBROADCAST and WM_QUERYENDSESSION, 0 otherwise. pub fn handle(shared: &SharedControl, message: Message) -> isize { @@ -80,9 +93,18 @@ pub fn handle(shared: &SharedControl, message: Message) -> isize { 1 } Message::QueryEndSession => 1, + Message::EndSession { + ending: true, + logoff: false, + close_app: true, + } => { + closing_app(shared); + 0 + } Message::EndSession { ending: true, logoff, + .. } => { let event = if logoff { PowerEvent::Logoff @@ -108,22 +130,27 @@ pub fn handle(shared: &SharedControl, message: Message) -> isize { fn going_down(shared: &SharedControl, event: PowerEvent, wait: Duration) { let started = Instant::now(); let before = shared.power.count(); - let (end, told) = { + let (down, told) = { let mut session = shared.session.lock(); - let end = session.power_down(started, event); - (end, session.power_goodbye_pending()) + let down = session.power_down(started, event); + (down, session.power_goodbye_pending()) }; - // Only a running transport can send it. - let sending = told && shared.transport_beat.read().is_some(); - if sending { - shared.power.wake.notify_one(); - } - let Some(end) = end else { - info!( - power = event.name(), - "Windows is taking the PC down; no iPad was streaming" - ); - return; + let end = match down { + PowerDown::Repeat => { + debug!( + power = event.name(), + "Windows repeated that the PC is going down" + ); + return; + } + PowerDown::NoSession => { + info!( + power = event.name(), + "Windows is taking the PC down; no iPad was streaming" + ); + return; + } + PowerDown::Ended(end) => end, }; info!( power = event.name(), @@ -131,25 +158,115 @@ fn going_down(shared: &SharedControl, event: PowerEvent, wait: Duration) { ); // A PC that sleeps comes back with this process and its log writers; // one that shuts down or signs out may end it any moment now. - match event { - PowerEvent::Sleep => crate::diagnostics::log_session_end(&end, shared), - PowerEvent::Shutdown | PowerEvent::Logoff => { - crate::diagnostics::log_session_end_now(&end, shared) - } - } - if !sending { + let write = match event { + PowerEvent::Sleep => Write::Log, + PowerEvent::Shutdown | PowerEvent::Logoff => Write::Disk, + }; + finish( + shared, + end, + write, + told, + before, + started, + wait, + event.name(), + ); +} + +/// Restart Manager closes the app for an installer while the PC stays up: +/// the stream ends as on a quit, with the goodbye every iPad reads (connect +/// again, when an update hands over to its new copy), and the summary on +/// disk, since the process ends right after. +fn closing_app(shared: &SharedControl) { + let started = Instant::now(); + let before = shared.power.count(); + let end = crate::diagnostics::end_stream(shared, crate::diagnostics::quit_stop(shared)); + let told = shared.session.lock().goodbye_pending(); + let Some(end) = end else { + info!("Windows is closing the app for an installer; no iPad was streaming"); return; - } - while shared.power.count() == before && started.elapsed() < wait { - std::thread::sleep(Duration::from_millis(5)); - } - let sent = shared.power.count() != before; + }; info!( - power = event.name(), - sent, - waited_ms = started.elapsed().as_millis() as u64, - "Goodbye before the PC goes down" + told, + "Windows is closing the app for an installer — ended the iPad's session" ); + finish( + shared, + end, + Write::Disk, + told, + before, + started, + END_SESSION_WAIT, + "close_app", + ); +} + +/// Where an ended session's summary goes. +enum Write { + /// Through the log, which outlives a sleep. + Log, + /// Straight to disk, on a thread of its own, waited for until + /// [`SUMMARY_WAIT`] after the message came. + Disk, +} + +/// Writes `end`'s summary and waits for the goodbye the transport was told +/// to send, each within its bound from `started`. +#[allow(clippy::too_many_arguments)] +fn finish( + shared: &SharedControl, + end: SessionEnd, + write: Write, + told: bool, + before: u64, + started: Instant, + wait: Duration, + why: &'static str, +) { + // Only a running transport can send it. + let sending = told && shared.transport_beat.read().is_some(); + if sending { + shared.power.wake.notify_one(); + } + let written = match write { + Write::Log => { + crate::diagnostics::log_session_end(&end, shared); + None + } + Write::Disk => { + let (done, written) = std::sync::mpsc::channel(); + let writer = shared.clone(); + let spawned = std::thread::Builder::new() + .name("summary-now".into()) + .spawn(move || { + crate::diagnostics::log_session_end_now(&end, &writer); + let _ = done.send(()); + }); + spawned.ok().map(|_| written) + } + }; + if sending { + while shared.power.count() == before && started.elapsed() < wait { + std::thread::sleep(Duration::from_millis(5)); + } + info!( + power = why, + sent = shared.power.count() != before, + waited_ms = started.elapsed().as_millis() as u64, + "Goodbye before the PC goes down" + ); + } + if let Some(written) = written { + let left = SUMMARY_WAIT.saturating_sub(started.elapsed()); + if written.recv_timeout(left).is_err() { + warn!( + power = why, + "The session summary did not reach the disk in time" + ); + } + } } fn back_up(shared: &SharedControl, why: &'static str) { @@ -185,7 +302,7 @@ mod windows_impl { }; use windows::Win32::UI::WindowsAndMessaging::{ CreateWindowExW, DefWindowProcW, DispatchMessageW, GetMessageW, RegisterClassW, - TranslateMessage, DEVICE_NOTIFY_WINDOW_HANDLE, ENDSESSION_LOGOFF, MSG, + TranslateMessage, DEVICE_NOTIFY_WINDOW_HANDLE, ENDSESSION_CLOSEAPP, ENDSESSION_LOGOFF, MSG, PBT_APMRESUMEAUTOMATIC, PBT_APMRESUMESUSPEND, PBT_APMSUSPEND, WINDOW_EX_STYLE, WINDOW_STYLE, WM_ENDSESSION, WM_POWERBROADCAST, WM_QUERYENDSESSION, WM_WTSSESSION_CHANGE, WNDCLASSW, WTS_SESSION_LOGOFF, @@ -276,6 +393,7 @@ mod windows_impl { WM_ENDSESSION => Message::EndSession { ending: wparam != 0, logoff: lparam as u32 & ENDSESSION_LOGOFF != 0, + close_app: lparam as u32 & ENDSESSION_CLOSEAPP != 0, }, WM_WTSSESSION_CHANGE if wparam as u32 == WTS_SESSION_LOGOFF => Message::SessionLogoff, _ => return None, @@ -329,6 +447,18 @@ mod windows_impl { Some(Message::EndSession { ending: true, logoff: false, + close_app: false, + }), + ), + // Restart Manager closing the app for an installer. + ( + WM_ENDSESSION, + 1, + ENDSESSION_CLOSEAPP as isize, + Some(Message::EndSession { + ending: true, + logoff: false, + close_app: true, }), ), ( @@ -338,6 +468,7 @@ mod windows_impl { Some(Message::EndSession { ending: true, logoff: true, + close_app: false, }), ), ( @@ -347,6 +478,7 @@ mod windows_impl { Some(Message::EndSession { ending: false, logoff: false, + close_app: false, }), ), ( diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index 902d1d10..293749c6 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -245,6 +245,28 @@ impl PowerEvent { } } +/// What [`Session::power_down`] did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PowerDown { + /// The session ended; its summary is the caller's to write. + Ended(SessionEnd), + /// Nobody was streaming. + NoSession, + /// The PC was already going down: Windows sends some of this news twice + /// (the end of the session, then the session change, for a sign-out; + /// the broadcast and the registration, for sleep). Nothing changed. + Repeat, +} + +impl PowerDown { + pub fn ended(self) -> Option { + match self { + Self::Ended(end) => Some(end), + Self::NoSession | Self::Repeat => None, + } + } +} + /// How far a session got and how it ended, for its support summary. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SessionEnd { @@ -730,18 +752,35 @@ impl Session { /// [`Self::power_up`] or [`POWER_DOWN_HOLD`], HELLO2s go unanswered, as /// they would a moment later from a PC that is down. The stream keeps /// running, so the PC streams again when it wakes. - pub fn power_down(&mut self, now: Instant, event: PowerEvent) -> Option { + pub fn power_down(&mut self, now: Instant, event: PowerEvent) -> PowerDown { + if self.going_down(now) { + return PowerDown::Repeat; + } self.powering_down = Some((event, now)); - let understands = - self.active.as_ref()?.accepted_hello.feature_caps & FEATURE_POWER_BYE != 0; - if understands { + let Some(session) = self.active.as_ref() else { + return PowerDown::NoSession; + }; + let end = if session.accepted_hello.feature_caps & FEATURE_POWER_BYE != 0 { let end = self.end_saying(event.bye(), EndReason::Shutdown, now); self.power_bye = true; end } else { self.lost_unsaid = true; self.end(EndReason::Shutdown, now) - } + }; + end.map_or(PowerDown::NoSession, PowerDown::Ended) + } + + /// Windows said the PC is going down, and hasn't said since that it is + /// back, within [`POWER_DOWN_HOLD`]. + fn going_down(&self, now: Instant) -> bool { + self.powering_down + .is_some_and(|(_, since)| now.saturating_duration_since(since) < POWER_DOWN_HOLD) + } + + /// A goodbye waits for the transport. + pub fn goodbye_pending(&self) -> bool { + self.goodbye.is_some() } /// A goodbye [`Self::power_down`] left waits for the transport. @@ -765,10 +804,7 @@ impl Session { /// quits). The first and last are turned away with Busy before anything /// else is looked at; a retry once the stream runs again works. fn closed_to_clients(&self, now: Instant) -> Option { - if self - .powering_down - .is_some_and(|(_, since)| now.saturating_duration_since(since) < POWER_DOWN_HOLD) - { + if self.going_down(now) { Some(Closed::Silent("Client unanswered while the PC goes down")) } else if self.updating { Some(Closed::Busy("Client turned away while an update starts")) @@ -4246,7 +4282,6 @@ mod tests { ); } - /// The goodbye's destination, session id and reason. fn power_hello(nonce: u32, feature_caps: u16) -> ControlMessage { let ControlMessage::Hello2(mut hello) = hello(nonce, 50000) else { unreachable!() @@ -4271,7 +4306,10 @@ mod tests { let mut session = Session::new(500); session.handle_control(peer, 0, power_hello(1, FEATURE_POWER_BYE), &TestConfig, now); let id = session.session_id().unwrap(); - let end = session.power_down(now, event).expect("the session ended"); + let end = session + .power_down(now, event) + .ended() + .expect("the session ended"); assert_eq!((end.client_nonce, end.reason), (1, EndReason::Shutdown)); assert!(!session.is_active()); assert!(session.power_goodbye_pending()); @@ -4298,7 +4336,7 @@ mod tests { &TestConfig, now, ); - let end = session.power_down(now, PowerEvent::Sleep).unwrap(); + let end = session.power_down(now, PowerEvent::Sleep).ended().unwrap(); assert_eq!(end.reason, EndReason::Shutdown); assert!(!session.power_goodbye_pending()); let next = session.tick(&TestConfig, false, now); @@ -4318,7 +4356,7 @@ mod tests { let mut session = Session::new(500); assert_eq!( session.power_down(now, PowerEvent::Sleep), - None, + PowerDown::NoSession, "nobody streamed" ); let unanswered = @@ -4343,6 +4381,50 @@ mod tests { assert_eq!(admitted.new_target, Some(peer), "the hold ran out"); } + /// Windows tells some of it twice: the end of the session, then the + /// session change, for a sign-out. The repeat changes nothing: the hold + /// keeps its start, and nothing is ended or told again. + #[test] + fn news_of_the_pc_going_down_counts_once() { + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + let mut session = Session::new(500); + session.handle_control(peer, 0, power_hello(1, FEATURE_POWER_BYE), &TestConfig, now); + assert!(session + .power_down(now, PowerEvent::Logoff) + .ended() + .is_some()); + let again = now + Duration::from_secs(20); + assert_eq!( + session.power_down(again, PowerEvent::Logoff), + PowerDown::Repeat + ); + assert_eq!( + session.power_down(again, PowerEvent::Sleep), + PowerDown::Repeat + ); + assert!(session.take_goodbye().is_some(), "told once"); + assert_eq!(session.take_goodbye(), None); + // The hold still runs from the first news. + let admitted = session.handle_control( + peer, + 0, + power_hello(2, FEATURE_POWER_BYE), + &TestConfig, + now + POWER_DOWN_HOLD, + ); + assert_eq!(admitted.new_target, Some(peer)); + // After waking, the next news counts again. + session.power_up(); + assert_eq!( + session + .power_down(now + POWER_DOWN_HOLD, PowerEvent::Sleep) + .ended() + .map(|end| end.client_nonce), + Some(2) + ); + } + /// A goodbye no transport sent before the PC slept is for a session /// that is over by the time it wakes: dropped. Any other goodbye stays. #[test] @@ -4351,7 +4433,7 @@ mod tests { let peer = addr([10, 0, 0, 5], 50000); let mut session = Session::new(500); session.handle_control(peer, 0, power_hello(1, FEATURE_POWER_BYE), &TestConfig, now); - session.power_down(now, PowerEvent::Sleep).unwrap(); + session.power_down(now, PowerEvent::Sleep).ended().unwrap(); session.power_up(); assert_eq!(session.take_goodbye(), None); @@ -4374,7 +4456,7 @@ mod tests { for caps in [FEATURE_POWER_BYE, 0] { let mut session = Session::new(500); session.handle_control(peer, 0, power_hello(1, caps), &TestConfig, now); - session.power_down(now, PowerEvent::Sleep).unwrap(); + session.power_down(now, PowerEvent::Sleep).ended().unwrap(); session.tick(&TestConfig, false, now); let woke = now + Duration::from_secs(3 * 3600); session.power_up(); @@ -4387,6 +4469,7 @@ mod tests { } } + /// The goodbye's destination, session id and reason. fn goodbye_of(actions: &Actions) -> (PeerId, u32, ByeReason) { let (to, bytes) = actions.goodbye.clone().expect("a goodbye"); let (header, message) = eternal_wire::v2::control::parse_control(&bytes).unwrap(); diff --git a/host/tests/power_goodbye.rs b/host/tests/power_goodbye.rs index 6229b36f..f095a86f 100644 --- a/host/tests/power_goodbye.rs +++ b/host/tests/power_goodbye.rs @@ -9,7 +9,7 @@ use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; use eternal_host::control::SharedControl; -use eternal_host::power::{self, Message, END_SESSION_WAIT, SLEEP_WAIT}; +use eternal_host::power::{self, Message, SLEEP_WAIT, SUMMARY_WAIT}; use eternal_host::transport; use eternal_wire::v2::control::{ encode_control, parse_control, ByeReason, ControlMessage, Hello2, HelloStatus, CAP_DECODE_H264, @@ -248,6 +248,7 @@ fn shutting_down_or_signing_out_says_so_and_saves_the_summary() { Message::EndSession { ending: true, logoff: false, + close_app: false, }, ), ( @@ -255,6 +256,7 @@ fn shutting_down_or_signing_out_says_so_and_saves_the_summary() { Message::EndSession { ending: true, logoff: true, + close_app: false, }, ), (13, Message::SessionLogoff), @@ -266,7 +268,8 @@ fn shutting_down_or_signing_out_says_so_and_saves_the_summary() { assert_eq!( rig.tell(Message::EndSession { ending: false, - logoff: false + logoff: false, + close_app: false, }) .0, 0 @@ -280,19 +283,21 @@ fn shutting_down_or_signing_out_says_so_and_saves_the_summary() { let (answer, took) = rig.tell(message); assert_eq!(answer, 0, "{message:?}"); - assert!( - took < END_SESSION_WAIT + Duration::from_millis(100), - "{took:?}" - ); - let log = session_log(); + assert!(took < SUMMARY_WAIT + Duration::from_millis(100), "{took:?}"); + // A sign-out's session change follows its end of session: the + // repeat ends nothing, tells nothing and writes nothing again. + let (_, repeat_took) = rig.tell(Message::SessionLogoff); + assert!(repeat_took < Duration::from_millis(100), "{repeat_took:?}"); let summary = format!(" nonce={nonce:016x} "); - assert!( - log.lines() - .any(|line| line.contains("session_summary v=1 role=pc") + let summaries = session_log() + .lines() + .filter(|line| { + line.contains("session_summary v=1 role=pc") && line.contains(&summary) - && line.contains(" end=shutdown ")), - "{message:?}: no summary in the log" - ); + && line.contains(" end=shutdown ") + }) + .count(); + assert_eq!(summaries, 1, "{message:?}: one summary in the log"); assert_eq!( rig.goodbyes(Duration::from_millis(500)), [ByeReason::HostPoweringOff; 3], @@ -301,6 +306,47 @@ fn shutting_down_or_signing_out_says_so_and_saves_the_summary() { } } +/// Restart Manager closes the app for an installer, as Setup does on an +/// update, while the PC stays up: the stream ends as on a quit, with the +/// goodbye every iPad reads, not as the PC powering off. An update that +/// hands over to its new copy says connect again. +#[test] +fn an_installer_closing_the_app_ends_the_stream_as_a_quit() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + for (nonce, handoff, bye) in [ + (21, false, ByeReason::UserDisconnect), + (22, true, ByeReason::Error), + ] { + let mut rig = Rig::start(FEATURE_POWER_BYE); + assert_eq!(rig.connect(nonce), Some(HelloStatus::Ok)); + if handoff { + rig.shared + .handoff + .claim(eternal_host::handoff::Handoff::Update) + .unwrap(); + } + let (answer, took) = rig.tell(Message::EndSession { + ending: true, + logoff: false, + close_app: true, + }); + assert_eq!(answer, 0); + assert!(took < SUMMARY_WAIT + Duration::from_millis(100), "{took:?}"); + assert_eq!(rig.goodbyes(Duration::from_millis(500)), [bye; 3]); + assert!( + !rig.shared.running.load(Ordering::SeqCst), + "the stream stopped" + ); + let summary = format!(" nonce={nonce:016x} "); + assert!( + session_log() + .lines() + .any(|line| line.contains("session_summary v=1 role=pc") && line.contains(&summary)), + "no summary in the log" + ); + } +} + /// The real window, given the message Windows sends before sleep. Windows /// only: it checks the window gets Windows' messages and answers them. #[cfg(windows)] diff --git a/ios/EternalMonitor/App/ConnectionProblem.swift b/ios/EternalMonitor/App/ConnectionProblem.swift index dfeaaebd..6e484ff7 100644 --- a/ios/EternalMonitor/App/ConnectionProblem.swift +++ b/ios/EternalMonitor/App/ConnectionProblem.swift @@ -192,7 +192,7 @@ enum ConnectionProblem: Equatable { case .pcAsleep: return "Wake the PC, then tap Try again." case .pcPoweringOff: - return "Tap Try again once the PC is back on and EternalMonitor is open on it." + return "Tap Try again once the PC is on, someone has signed in, and EternalMonitor is open." case .pcUpdating(.reconnecting): return "This iPad reconnects when it's back. If it doesn't, check that EternalMonitor opened again on the PC, then try again." case .pcUpdating(.manual): diff --git a/ios/EternalMonitorTests/ConnectionProblemTests.swift b/ios/EternalMonitorTests/ConnectionProblemTests.swift index 3c604a3d..865f84bd 100644 --- a/ios/EternalMonitorTests/ConnectionProblemTests.swift +++ b/ios/EternalMonitorTests/ConnectionProblemTests.swift @@ -301,7 +301,7 @@ final class ConnectionProblemTests: XCTestCase { .hostShuttingDown: (.pcUpdating(.reconnecting), "This iPad reconnects when it's back. If it doesn't, check that EternalMonitor opened again on the PC, then try again."), .hostSleeping: (.pcAsleep, "Wake the PC, then tap Try again."), - .hostPoweringOff: (.pcPoweringOff, "Tap Try again once the PC is back on and EternalMonitor is open on it."), + .hostPoweringOff: (.pcPoweringOff, "Tap Try again once the PC is on, someone has signed in, and EternalMonitor is open."), ] for reason in reasons { let problem = ConnectionProblem.pcGoodbye(reason) From 96cb943c66cc0cc1410e9f7421cd076a09bec13c Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 09:01:39 -0500 Subject: [PATCH 196/226] Run the link's silent-peer test on a paused clock too It waited out the real 2 s handshake and then gave the dropped link 1 s of real time to close its end, the other link test with a 1 s deadline. On a paused clock the handshake runs out in virtual time and a link that stops closing its end still fails, now at once. --- host/src/transport/link.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/host/src/transport/link.rs b/host/src/transport/link.rs index 0f909745..c182f565 100644 --- a/host/src/transport/link.rs +++ b/host/src/transport/link.rs @@ -1031,7 +1031,7 @@ mod tests { .is_err()); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn silent_listener_peer_times_out_and_dropping_the_link_closes_io() { let (server, _silent) = tokio::io::duplex(32); let failed = From e7dde1a19648226a3e71ecc35fd4d344bd7a186b Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 09:04:29 -0500 Subject: [PATCH 197/226] Look at the chosen network first when a stream starts A stream's first look at its chosen network was the first tick of a 10 s interval, which could lose to the 250 ms liveness tick. A stream stopped as it started then ended without looking, and never asked to restart on the network's new address, so the test that checks such a request is turned down failed now and then under load. The stream now looks once as soon as it may, before its loop, and then every 10 s as before. The supervisor still restarts it only while it runs. --- host/src/transport/mod.rs | 48 +++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index d53977c0..b2b9d1df 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -308,6 +308,35 @@ impl ConfigSource for SharedConfigSource<'_> { } } +/// How often a stream looks at its chosen network while nobody streams. +const NETWORK_CHECK: Duration = Duration::from_secs(10); + +/// With Listen on set to one network: while nobody streams, a stream bound +/// to an address that network no longer gives restarts to listen on the +/// one it gives now. The supervisor does that only while this stream still +/// runs, never once the user stopped it. +fn look_at_network( + shared: &SharedControl, + listen_on: &crate::network::ListenOn, + bind_ip: std::net::Ipv4Addr, + supervisor: &StreamSupervisor, +) { + if *listen_on == crate::network::ListenOn::All + || shared.client_connected() + || crate::network::listen_ip_now(listen_on) == bind_ip + { + return; + } + info!( + listen_on = listen_on.label(), + "The chosen network changed: restarting the stream to listen on it" + ); + shared.stop(); + if let Err(error) = supervisor.restart() { + warn!(error = %error, "Failed to request a pipeline restart for the chosen network"); + } +} + /// The supervisor a stream answers to, and the pipeline generation the /// stream belongs to. struct StreamSupervisor { @@ -463,8 +492,13 @@ pub async fn start_sender( let mut liveness_tick = tokio::time::interval(Duration::from_millis(250)); liveness_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); // A chosen network can connect, or get a new address, after the stream - // starts. It is looked at again while nobody streams. - let mut network_check = tokio::time::interval(Duration::from_secs(10)); + // bound. It is looked at now, before anything else, and again every + // 10 s while nobody streams. The first look used to be the interval's + // first tick, which could lose to the liveness tick: a stream stopped + // meanwhile then ended without looking. + look_at_network(&shared, &listen_on, bind_ip, &supervisor); + let mut network_check = + tokio::time::interval_at(tokio::time::Instant::now() + NETWORK_CHECK, NETWORK_CHECK); network_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); let mut pointer_budget = Deadline::new("pointer-budget")?; @@ -500,15 +534,9 @@ pub async fn start_sender( let actions = shared.session.lock().tick(&config, true, now()); act(actions, &mut state, &links, &shared, &config, &supervisor).await; } - _ = network_check.tick(), if listen_on != crate::network::ListenOn::All => { + _ = network_check.tick() => { beat.mark(Phase::Tick); - if !shared.client_connected() && crate::network::listen_ip_now(&listen_on) != bind_ip { - info!(listen_on = listen_on.label(), "The chosen network changed: restarting the stream to listen on it"); - shared.stop(); - if let Err(error) = supervisor.restart() { - warn!(error = %error, "Failed to request a pipeline restart for the chosen network"); - } - } + look_at_network(&shared, &listen_on, bind_ip, &supervisor); } _ = liveness_tick.tick() => { beat.mark(Phase::Tick); From ac866f6944d59cac42e6e1c2218cbb083f05007e Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 09:19:55 -0500 Subject: [PATCH 198/226] Mirror at once on Windows 10 and say Second screen needs Windows 11 The display driver Setup installs needs Windows 11's display driver framework. On Windows 10 the driver starts, but Windows never adds its screen, so every Second screen session waited and then fell back to the mirror. Now, on a build below 22000, a Second screen connection mirrors at once without turning the driver on, ahead of any failure an earlier version saved. Try again and choosing Second screen again don't turn it on either. The log line names the existing `no_driver` fallback, with `reason=windows_10`. The Stream page says "Second screen needs Windows 11, so the iPad shows your main screen instead." with no Try again, the note under the mode choice and the Settings description say the same, and for a few seconds after each connection the heartbeat carries a new notice value for the iPad. Windows 11 and an unknown build are unchanged. ETERNAL_SECOND_SCREEN_ON_WINDOWS_10=1 tries Second screen on Windows 10 anyway, for testing. The build check is one function, so a driver that works on Windows 10 can lift it. --- host/src/capture/second_screen.rs | 146 +++++++++++++++++++++++ host/src/capture/second_screen/memory.rs | 110 ++++++++++++++++- host/src/gui/page_settings.rs | 13 +- host/src/gui/page_stream.rs | 28 ++++- host/src/gui/tests.rs | 74 +++++++++++- host/src/main.rs | 2 + host/src/transport/mod.rs | 52 +++++++- proto/src/v2/control.rs | 12 +- 8 files changed, 425 insertions(+), 12 deletions(-) diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 5c5a5600..6e533550 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -103,6 +103,33 @@ pub(crate) const EXTEND_TIMEOUT: Duration = Duration::from_secs(30); #[cfg(any(windows, test, debug_assertions))] pub(crate) const BLOCKED_EXTEND_RETRY: Duration = Duration::from_secs(120); +/// Why Second screen mirrors on this PC without trying: Windows is older +/// than the display driver needs ([`driver_supports_windows`]). Not saved: +/// each app run reads Windows again. +pub(crate) const WINDOWS_10: &str = "windows_10"; + +/// The first Windows 11 build. The display driver Setup installs is built +/// for Windows 11's display driver framework: on Windows 10 it starts, but +/// Windows never adds its screen. +#[cfg(any(windows, test))] +const FIRST_WINDOWS_11_BUILD: u32 = 22000; + +/// Set to `1` to try Second screen on Windows 10 anyway, for testing. +pub const WINDOWS_10_OVERRIDE: &str = "ETERNAL_SECOND_SCREEN_ON_WINDOWS_10"; + +/// How long the iPad says that Second screen needs Windows 11, from the +/// connection that mirrors for it. The mirror works, so the notice doesn't +/// stay over it. +pub(crate) const WINDOWS_10_IPAD_NOTICE: Duration = Duration::from_secs(10); + +/// Whether the bundled display driver can show a second screen on Windows +/// `build`, or a tester asked to try anyway (`forced`). An unknown build is +/// tried, as before. A driver that works on Windows 10 lifts the limit here. +#[cfg(any(windows, test))] +pub(crate) fn driver_supports_windows(build: Option, forced: bool) -> bool { + forced || build.is_none_or(|build| build >= FIRST_WINDOWS_11_BUILD) +} + /// How long quitting waits for the worker to exit after the last Enable. const QUIT_JOIN: Duration = Duration::from_secs(5); @@ -212,6 +239,12 @@ pub(crate) struct State { /// The graphics cards and the screens' wiring, as last read off the /// transport loop: see [`facts`]. facts: facts::Cache, + /// Windows is too old for the display driver: Second screen mirrors + /// without turning it on ([`note_windows`]). + windows_unsupported: bool, + /// When a connection last began mirroring for that, for the iPad's + /// notice ([`windows_notice`]). + windows_held_at: Option, /// Whether the user's desktop is away, as an end-to-end test's fake /// display says in place of Windows: see [`desktop_away`]. #[cfg(debug_assertions)] @@ -243,6 +276,7 @@ impl State { memory: std::mem::take(&mut self.memory), mirror_refused: self.mirror_refused.take(), facts: std::mem::take(&mut self.facts), + windows_unsupported: self.windows_unsupported, #[cfg(debug_assertions)] fake_desktop_away: self.fake_desktop_away.take(), ..State::default() @@ -254,6 +288,14 @@ impl State { } self.retry_if_back(status, connected, now); match *status { + // Before any saved failure: trying again can't help. + VddStatus::WaitingForClient if connected && self.windows_unsupported => { + *status = VddStatus::Failed; + self.failure = Some(WINDOWS_10); + self.windows_held_at = Some(now); + log_windows_hold(); + Step::Nothing + } VddStatus::WaitingForClient if connected => { let held = self.memory.hold(now); // A blocked mirror is no reason to take the panel again. @@ -354,6 +396,7 @@ impl State { memory: std::mem::take(&mut self.memory), mirror_refused: self.mirror_refused.take(), facts: std::mem::take(&mut self.facts), + windows_unsupported: self.windows_unsupported, #[cfg(debug_assertions)] fake_desktop_away: self.fake_desktop_away.take(), ..State::default() @@ -542,6 +585,78 @@ fn log_hold(hold: Hold) { ); } +/// The one log line for a connection that mirrors because Windows is too +/// old for the display driver. The summary names it `no_driver`: there is +/// no display driver for this Windows. +fn log_windows_hold() { + tracing::info!( + second_screen_fallback = "no_driver", + reason = WINDOWS_10, + "Second screen needs Windows 11 — mirroring the primary display without turning the \ + display driver on" + ); +} + +/// Records whether this Windows can run the display driver (see +/// [`driver_supports_windows`]), at startup. +#[cfg(any(windows, test))] +pub(crate) fn note_windows(shared: &SharedControl, build: Option, forced: bool) { + let supported = driver_supports_windows(build, forced); + if !driver_supports_windows(build, false) { + if forced { + tracing::info!( + "Windows is older than Windows 11, but {WINDOWS_10_OVERRIDE} asks to try Second \ + screen anyway" + ); + } else { + tracing::info!( + "Windows is older than Windows 11 — Second screen mirrors the primary display \ + without turning the display driver on" + ); + } + } + shared.second_screen.lock().windows_unsupported = !supported; +} + +/// [`note_windows`] for this PC: its build, and the tester's override. +#[cfg(windows)] +pub fn check_windows(shared: &SharedControl) { + let forced = std::env::var(WINDOWS_10_OVERRIDE).is_ok_and(|value| value.trim() == "1"); + note_windows(shared, windows_build(), forced); +} + +/// This PC's Windows build, as the registry names it. Windows 11 keeps +/// "Windows 10" as its product name, so only the build tells them apart. +#[cfg(windows)] +fn windows_build() -> Option { + crate::diagnostics::registry_string( + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion", + "CurrentBuildNumber", + ) + .trim() + .parse() + .ok() +} + +/// Whether Second screen mirrors on this PC because Windows is too old for +/// the display driver, for the Stream page and Settings. +pub(crate) fn windows_unsupported(shared: &SharedControl) -> bool { + shared.second_screen.lock().windows_unsupported +} + +/// Whether the heartbeat tells the iPad now that Second screen needs +/// Windows 11: for [`WINDOWS_10_IPAD_NOTICE`] after a connection began +/// mirroring for it. +pub(crate) fn windows_notice(shared: &SharedControl, now: Instant) -> bool { + let status = *shared.vdd_status.lock(); + let state = shared.second_screen.lock(); + status == VddStatus::Failed + && state.failure == Some(WINDOWS_10) + && state + .windows_held_at + .is_some_and(|at| now.saturating_duration_since(at) < WINDOWS_10_IPAD_NOTICE) +} + /// Read the failure an earlier app run saved, for this PC as it is now. #[cfg(windows)] pub fn load_memory(shared: &SharedControl) { @@ -579,6 +694,13 @@ pub(crate) fn note_connected(shared: &SharedControl, now: Instant) { let blocked = mirror_blocked_now(shared, |status, _| status == VddStatus::Failed); let mut status = shared.vdd_status.lock(); let mut state = shared.second_screen.lock(); + // Nothing to try on this Windows, even when the mirror shows nothing. + if *status == VddStatus::Failed && state.failure == Some(WINDOWS_10) { + state.windows_held_at = Some(now); + drop((state, status)); + log_windows_hold(); + return; + } // The mirror would show nothing: the second screen is the only picture. // Not after a lost panel: the next try takes it again. if *status == VddStatus::Failed && blocked && state.failure != Some(PANEL_LOST) { @@ -1939,6 +2061,30 @@ mod tests { use std::cell::{Cell, RefCell}; use std::sync::atomic::Ordering; + /// Every Windows has a build number to read. + #[cfg(windows)] + #[test] + fn this_windows_build_reads_from_the_registry() { + let build = windows_build().expect("a build number"); + assert!(build >= 10240, "{build}"); + } + + /// Native, on a Windows 11 PC: the startup check reads the real build, + /// and a Second screen connection starts the second screen as before. + #[cfg(windows)] + #[test] + #[ignore = "needs Windows 11"] + fn this_windows_11_pc_tries_the_second_screen() { + let shared = second_screen_selected(); + check_windows(&shared); + assert!(!windows_unsupported(&shared), "{:?}", windows_build()); + connect_viewer(&shared); + let now = Instant::now(); + note_connected(&shared, now); + assert_eq!(next_step(&shared, now), Step::Attach); + assert!(!windows_notice(&shared, now)); + } + /// The iPad's connect timeout in 0.3.1 and 0.3.2: it disconnects when no /// frame has decoded this long after the handshake. const IPAD_TIMEOUT: Duration = Duration::from_secs(10); diff --git a/host/src/capture/second_screen/memory.rs b/host/src/capture/second_screen/memory.rs index ce9f682e..72cf6da1 100644 --- a/host/src/capture/second_screen/memory.rs +++ b/host/src/capture/second_screen/memory.rs @@ -958,9 +958,10 @@ fn overwrite_file(path: &Path, data: &[u8]) -> std::io::Result<()> { #[cfg(test)] mod tests { use super::super::{ - attach, next_step, note_attached, note_connected, note_image, notice_failure, - remembered_failure, retries_later, set_facts, start_proving, Driver, Failure, Step, - TurnedOn, BLOCKED_EXTEND_RETRY, PANEL_LOST_BEFORE, PROVE_TIMEOUT, + attach, driver_supports_windows, next_step, note_attached, note_connected, note_image, + note_windows, notice_failure, remembered_failure, retries_later, set_facts, start_proving, + windows_notice, Driver, Failure, Step, TurnedOn, BLOCKED_EXTEND_RETRY, PANEL_LOST_BEFORE, + PROVE_TIMEOUT, WINDOWS_10, WINDOWS_10_IPAD_NOTICE, }; use super::*; use crate::capture::tests::{connect_viewer, hybrid_laptop, refuse_moved_mirror, NVIDIA}; @@ -2568,6 +2569,109 @@ mod tests { ))); } + /// Windows 10: a Second screen connection mirrors at once, with no + /// Enable, and says why: the log names `no_driver`, and the iPad's + /// notice lasts a few seconds from each connection. Neither a failure + /// an earlier version saved, Try again, nor choosing Second screen again + /// turns the driver on. + #[test] + fn on_windows_10_second_screen_mirrors_without_turning_the_driver_on() { + let path = temp_file("windows-10"); + let (mut memory, start) = app_run(&path, Duration::ZERO); + memory.note_failure("attach_timeout", start); + let shared = second_screen_with(memory); + note_windows(&shared, Some(19045), false); + connect_viewer(&shared); + note_connected(&shared, start); + let (log, step) = logs(|| next_step(&shared, start)); + assert_eq!(step, Step::Nothing, "no Enable"); + assert!( + log.contains("second_screen_fallback") + && log.contains("no_driver") + && log.contains(WINDOWS_10) + && log.contains("Second screen needs Windows 11"), + "{log}" + ); + assert!(!log.contains("remembered"), "{log}"); + assert!(shared.vdd_worker.queue.lock().commands.is_empty()); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + assert_eq!(remembered_failure(&shared), Some(WINDOWS_10)); + assert_eq!(notice_failure(&shared), Some(WINDOWS_10)); + let generation = + crate::supervisor::CURRENT_GENERATION.load(std::sync::atomic::Ordering::SeqCst); + assert_eq!( + crate::capture::reconcile_virtual_display( + &CaptureTarget::VirtualExtended, + &shared, + generation + ), + Some(CaptureTarget::PrimaryAuto), + "the mirror" + ); + // The iPad's notice, for a while from the connection. + assert!(windows_notice(&shared, start)); + assert!(windows_notice( + &shared, + start + WINDOWS_10_IPAD_NOTICE - secs(1) + )); + assert!(!windows_notice(&shared, start + WINDOWS_10_IPAD_NOTICE)); + + // The next connection says it again, without the saved failure. + let later = start + Duration::from_secs(60); + let (log, step) = logs(|| { + reconnect(&shared, later); + next_step(&shared, later) + }); + assert_eq!(step, Step::Nothing); + assert!( + log.contains("no_driver") && !log.contains("remembered"), + "{log}" + ); + assert!(windows_notice(&shared, later)); + + shared.retry_second_screen(); + assert_eq!(next_step(&shared, later), Step::Nothing, "Try again"); + *shared.capture_target.lock() = CaptureTarget::PrimaryAuto; + assert_eq!(next_step(&shared, later), Step::Nothing); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + *shared.vdd_status.lock() = VddStatus::WaitingForClient; + assert_eq!(next_step(&shared, later), Step::Nothing, "chosen again"); + assert_eq!(remembered_failure(&shared), Some(WINDOWS_10)); + assert!(shared.vdd_worker.queue.lock().commands.is_empty()); + } + + /// Windows 11, an unknown build and the tester's override try the + /// second screen as before, with no Windows 10 notice. + #[test] + fn windows_11_and_the_windows_10_override_try_the_second_screen() { + assert!(!driver_supports_windows(Some(19045), false)); + assert!(!driver_supports_windows(Some(21999), false)); + assert!(driver_supports_windows(Some(22000), false)); + assert!(driver_supports_windows(Some(26200), false)); + assert!(driver_supports_windows(None, false)); + assert!(driver_supports_windows(Some(19045), true)); + for (build, forced) in [ + (Some(26200), false), + (Some(22000), false), + (None, false), + (Some(19045), true), + ] { + let path = temp_file("windows-11"); + let (memory, start) = app_run(&path, Duration::ZERO); + let shared = second_screen_with(memory); + note_windows(&shared, build, forced); + connect_viewer(&shared); + note_connected(&shared, start); + assert_eq!( + next_step(&shared, start), + Step::Attach, + "{build:?} {forced}" + ); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Attaching); + assert!(!windows_notice(&shared, start)); + } + } + /// The manifest names this PC: Second screen sessions start in the /// mirror with the usual notice, and Try again works and is kept for /// this PC's cards, through driver and Windows updates. diff --git a/host/src/gui/page_settings.rs b/host/src/gui/page_settings.rs index 6b47d083..dacc6164 100644 --- a/host/src/gui/page_settings.rs +++ b/host/src/gui/page_settings.rs @@ -24,10 +24,11 @@ impl AnalyzerApp { widgets::list_card().show(ui, |ui| { ui.set_width(ui.available_width()); let mut refresh = false; + let windows_10 = crate::capture::second_screen::windows_unsupported(&self.control.shared); widgets::setting_row( ui, tr("How the iPad shows your PC"), - "Mirror my screen shows your main screen. Second screen adds the iPad as an extra screen while it is connected. You can also mirror one particular monitor.", + display_description(windows_10), 300.0, |ui| { refresh = widgets::icon_button(ui, super::icons::Icon::Refresh, tr("Refresh displays")).clicked(); @@ -328,6 +329,16 @@ pub(super) fn env_or(env: Option<&str>, description: &str) -> String { /// What "Match the iPad's resolution" does, and when a change applies: the /// driver reads the modes when the second screen starts, and restarting /// the stream keeps the second screen that is already there. +/// What the display choice does, and on Windows 10 that Second screen +/// can't run there. +pub(super) fn display_description(windows_10: bool) -> &'static str { + if windows_10 { + "Mirror my screen shows your main screen. You can also mirror one particular monitor. Second screen needs Windows 11, so on this PC the iPad shows your main screen instead." + } else { + "Mirror my screen shows your main screen. Second screen adds the iPad as an extra screen while it is connected. You can also mirror one particular monitor." + } +} + pub(super) fn vdd_match_description(extending: bool) -> &'static str { if extending { "Makes the second screen exactly as sharp and smooth as the iPad's display. A change applies the next time the second screen starts, when the iPad connects again." diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index e9d4170d..264f6669 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -574,12 +574,14 @@ impl AnalyzerApp { } }); ui.add_space(2.0); + let windows_10 = crate::capture::second_screen::windows_unsupported(&self.control.shared); ui.add( egui::Label::new(muted(screen_mode_text( ¤t, output.as_deref(), vdd, running, + windows_10, ))) .wrap(), ); @@ -872,6 +874,10 @@ pub(super) fn software_fallback_body( "The picture may lag and the PC may run hot. Update your graphics driver from NVIDIA, AMD or Intel, then click Restart.".to_string() } } +/// Why Second screen mirrors on Windows 10: the notice, and the note beside +/// the choice. +pub(super) const WINDOWS_10_NOTE: &str = + "Second screen needs Windows 11, so the iPad shows your main screen instead."; pub(super) const SECOND_SCREEN_FAILED_TITLE: &str = "The second screen couldn't start, so your main screen is mirrored"; const INSTALLER_URL: &str = "https://eternalmonitor.dev/download.html"; @@ -950,6 +956,14 @@ pub(super) fn second_screen_notice( }) }; match (status, health, problem) { + // Nothing was tried, and trying again can't help. + (Failed | TaskFailed, _, _) if failure == Some(crate::capture::second_screen::WINDOWS_10) => { + notice( + &format!("{WINDOWS_10_NOTE} Mirror my screen works as usual."), + false, + false, + ) + } // Windows left the device as not extended too, but Extend can't help. (Failed, _, _) if failure == Some("panel_lost") => notice( "Your laptop's built-in screen turned off when the second screen started, and Windows only turns it back on after a restart. Until then your desktop is on the iPad: save your work there, then restart the PC. On this PC the second screen can turn the built-in screen off again, so the iPad shows your main screen until you click Try again or choose Second screen again.", @@ -1195,18 +1209,23 @@ pub(super) fn capture_block_notice( } /// What the Stream page says under the mirror / second screen choice. +/// `windows_10`: Second screen can't run on this Windows +/// ([`WINDOWS_10_NOTE`]). pub(super) fn screen_mode_text( capture: &str, output: Option<&str>, vdd: VddStatus, running: bool, + windows_10: bool, ) -> String { let restart = if running { " Switching restarts the stream, and the iPad reconnects by itself." } else { "" }; - let what = if capture == CAPTURE_VIRTUAL_SENTINEL { + let what = if capture == CAPTURE_VIRTUAL_SENTINEL && windows_10 { + WINDOWS_10_NOTE.to_string() + } else if capture == CAPTURE_VIRTUAL_SENTINEL { match vdd { VddStatus::Active => "The iPad is an extra screen. Drag windows past the edge of your main screen to move them onto it.", VddStatus::Failed | VddStatus::TaskFailed => { @@ -1222,7 +1241,12 @@ pub(super) fn screen_mode_text( } else { "The iPad shows the same picture as your main screen.".to_string() }; - what + restart + let windows = if windows_10 && capture != CAPTURE_VIRTUAL_SENTINEL { + " Second screen needs Windows 11." + } else { + "" + }; + what + windows + restart } /// The home network's address, without the port (the iPad has its own Port diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index ab0c7b49..6819657f 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -332,12 +332,12 @@ fn screen_choice_says_what_the_ipad_shows_and_that_it_reconnects() { use super::page_stream::screen_mode_text; use crate::control::VddStatus; - let mirror = screen_mode_text("", None, VddStatus::Inactive, false); + let mirror = screen_mode_text("", None, VddStatus::Inactive, false, false); assert_eq!( mirror, "The iPad shows the same picture as your main screen." ); - let running = screen_mode_text("", None, VddStatus::Inactive, true); + let running = screen_mode_text("", None, VddStatus::Inactive, true, false); assert!(running.contains("reconnects by itself"), "{running}"); let monitor = screen_mode_text( @@ -345,6 +345,7 @@ fn screen_choice_says_what_the_ipad_shows_and_that_it_reconnects() { Some("DISPLAY2"), VddStatus::Inactive, false, + false, ); assert!( monitor.contains("DISPLAY2") && monitor.contains("Settings"), @@ -356,12 +357,19 @@ fn screen_choice_says_what_the_ipad_shows_and_that_it_reconnects() { None, VddStatus::WaitingForClient, true, + false, ); assert!(waiting.contains("as soon as it connects"), "{waiting}"); - let active = screen_mode_text(CAPTURE_VIRTUAL_SENTINEL, None, VddStatus::Active, true); + let active = screen_mode_text( + CAPTURE_VIRTUAL_SENTINEL, + None, + VddStatus::Active, + true, + false, + ); assert!(active.contains("Drag windows"), "{active}"); for failed in [VddStatus::Failed, VddStatus::TaskFailed] { - let text = screen_mode_text(CAPTURE_VIRTUAL_SENTINEL, None, failed, true); + let text = screen_mode_text(CAPTURE_VIRTUAL_SENTINEL, None, failed, true, false); assert!(text.contains("mirrored"), "{text}"); } // No driver or protocol words on the main screen. @@ -926,6 +934,64 @@ fn a_lost_built_in_screen_from_before_the_restart_only_says_it_is_paused() { } } +/// On Windows 10 Second screen isn't tried: the notice, the note beside the +/// choice and Settings say it needs Windows 11, with no Try again, and the +/// note goes from Windows 11's text only. +#[test] +fn windows_10_says_second_screen_needs_windows_11() { + use super::page_settings::display_description; + use super::page_stream::{add_later_retry, screen_mode_text, WINDOWS_10_NOTE}; + use crate::control::VddStatus; + use crate::vdd::VddTaskHealth; + let windows_10 = Some(crate::capture::second_screen::WINDOWS_10); + for status in [VddStatus::Failed, VddStatus::TaskFailed] { + for health in [VddTaskHealth::Ok, VddTaskHealth::TasksMissing] { + let mut notice = second_screen_notice(status, health, None, windows_10).unwrap(); + assert!(notice.body.starts_with(WINDOWS_10_NOTE), "{}", notice.body); + assert!(!notice.retry && !notice.installer); + // A failure saved by an earlier version doesn't offer a retry. + let before = notice.body.clone(); + add_later_retry(&mut notice, true); + assert_eq!(notice.body, before); + } + } + assert!(WINDOWS_10_NOTE.contains("needs Windows 11")); + + for status in [ + VddStatus::WaitingForClient, + VddStatus::Failed, + VddStatus::Active, + ] { + let second = screen_mode_text(CAPTURE_VIRTUAL_SENTINEL, None, status, false, true); + assert_eq!(second, WINDOWS_10_NOTE); + } + let mirror = screen_mode_text("", None, VddStatus::Inactive, true, true); + assert_eq!( + mirror, + "The iPad shows the same picture as your main screen. Second screen needs Windows 11. \ + Switching restarts the stream, and the iPad reconnects by itself." + ); + assert!(display_description(true).contains("Second screen needs Windows 11")); + + // Windows 11 reads as before. + assert!(!display_description(false).contains("Windows")); + for (capture, status) in [ + ("", VddStatus::Inactive), + (CAPTURE_VIRTUAL_SENTINEL, VddStatus::WaitingForClient), + (CAPTURE_VIRTUAL_SENTINEL, VddStatus::Failed), + ] { + assert!(!screen_mode_text(capture, None, status, true, false).contains("Windows")); + } + let attach = second_screen_notice( + VddStatus::Failed, + VddTaskHealth::Ok, + None, + Some("attach_timeout"), + ) + .unwrap(); + assert!(!attach.body.contains("Windows 11") && attach.retry); +} + /// A second screen failure remembered for this PC is tried again later by /// itself; the notice says so beside Try again. #[test] diff --git a/host/src/main.rs b/host/src/main.rs index fb76194a..584d70ba 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -210,6 +210,8 @@ fn main() -> Result<(), Box> { eternal_host::remote_config::load(); #[cfg(windows)] eternal_host::capture::second_screen::load_memory(&shared); + #[cfg(windows)] + eternal_host::capture::second_screen::check_windows(&shared); eternal_host::remote_config::log_launch(); shared .hevc_enabled diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index d53977c0..c8ba688e 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -274,7 +274,13 @@ impl ConfigSource for SharedConfigSource<'_> { } fn notice(&self) -> u8 { - crate::capture::recovery::heartbeat_notice(*self.shared.capture_block.lock()) + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11}; + let notice = crate::capture::recovery::heartbeat_notice(*self.shared.capture_block.lock()); + if notice == NOTICE_NONE && second_screen::windows_notice(self.shared, Instant::now()) { + NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11 + } else { + notice + } } fn host_caps(&self) -> u16 { @@ -2106,6 +2112,50 @@ mod tests { assert_eq!(notices(), [NOTICE_NONE]); } + /// On Windows 10 a Second screen session mirrors, and its heartbeats + /// tell the iPad why. Windows 11 starts the second screen with no + /// such notice. + #[test] + fn heartbeats_say_second_screen_needs_windows_11_on_windows_10() { + use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11}; + for (build, step, notice) in [ + ( + 19045, + second_screen::Step::Nothing, + NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11, + ), + (26200, second_screen::Step::Attach, NOTICE_NONE), + ] { + let shared = SharedControl::new(9876, 15_000_000); + shared.pairing.lock().required = false; + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + *shared.vdd_status.lock() = VddStatus::WaitingForClient; + second_screen::note_windows(&shared, Some(build), false); + let now = Instant::now(); + repairable_session(&shared, "127.0.0.1:50000".parse().unwrap(), now); + let config = SharedConfigSource { + shared: &shared, + stream_epoch: 1, + pen_available: false, + listen: SocketAddr::from(([127, 0, 0, 1], 0)), + }; + let notices = || { + let actions = shared.session.lock().tick(&config, true, now); + actions + .replies + .iter() + .filter_map(|(_, bytes)| match parse_control(bytes) { + Ok((_, ControlMessage::Heartbeat(heartbeat))) => Some(heartbeat.notice), + _ => None, + }) + .collect::>() + }; + assert_eq!(notices(), [NOTICE_NONE], "{build}"); + assert_eq!(second_screen::next_step(&shared, now), step, "{build}"); + assert_eq!(notices(), [notice], "{build}"); + } + } + /// A NACK of a session that ended for liveness, while another frame is /// paced, is answered with the goodbye, as any of its late packets is. #[tokio::test] diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index bcd7351b..1dcc81f6 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -286,6 +286,11 @@ pub const NOTICE_PC_LOCKED: u8 = 1; /// No display is attached to the PC's desktop (a closed lid, screens off or /// unplugged), so there is nothing to capture until one comes back. pub const NOTICE_NO_DISPLAY: u8 = 2; +/// Second screen needs Windows 11, so the iPad shows the PC's main screen +/// instead. Sent for a few seconds after the session starts mirroring for +/// it, over a picture that works. (3 to 5 are kept for the graphics card and +/// restart notices.) +pub const NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11: u8 = 6; /// Host → client, every heartbeat interval while a session is active. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1612,7 +1617,12 @@ mod tests { // A notice is one byte more. Such a body, cut back to the old length // as an older iPad reads it, is the plain heartbeat; an unknown value // reaches the iPad as it is, for it to ignore. - for notice in [NOTICE_PC_LOCKED, NOTICE_NO_DISPLAY, 0xC8] { + for notice in [ + NOTICE_PC_LOCKED, + NOTICE_NO_DISPLAY, + NOTICE_SECOND_SCREEN_NEEDS_WINDOWS_11, + 0xC8, + ] { let datagram = encode_control(7, 1, &heartbeat(notice)); assert_eq!(datagram.len(), plain.len() + 1); assert_eq!(datagram[..plain.len()][8..], plain[8..]); From a616664de74303617a8279849359b3a736b4b7a1 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 09:21:25 -0500 Subject: [PATCH 199/226] Ask the PC's listings until one proves the pairing A PC that closed without a goodbye stays listed on Bonjour, unchanged, after its app opens again, so no appearance ever marked its return. The end-to-end run met this: the iPad's search began while Bonjour still held the stopped host's listing, and no removal followed. While the card is up, each listing of the PC that hasn't had its try is asked to prove the saved pairing: at once when it turns up or changes, then every five seconds. Nothing that needs the pairing goes out before the proof, and the card doesn't change. A listing that proves it gets the one try; only a new or changed listing gets another. There is no first-second baseline any more. A PC gives the pairing back only when it is new, so the session of an automatic try must give the saved PC's name, when this iPad knows it, and no other pairing; the proof just before is what shows it holds the saved one. --- .../App/ConnectionManager.swift | 203 +++++++++--------- ios/EternalMonitor/App/PCReturn.swift | 83 +++---- ios/EternalMonitorTests/PCReturnTests.swift | 172 ++++++++------- scripts/e2e_pc_returns.sh | 4 +- 4 files changed, 231 insertions(+), 231 deletions(-) diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 96835f61..dbea18e0 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -255,9 +255,11 @@ final class ConnectionManager: ObservableObject { /// next no-video step (see `NoVideoFallback`). var noVideoStepSeconds = 1.5 /// How long after the PC that didn't answer shows up on the network - /// the one automatic try waits, so its stream can start and a few - /// Bonjour updates count as one. + /// it is asked, so a few Bonjour updates count as one. var pcReturnDebounceSeconds = 1.0 + /// How often a listing of the PC that hasn't proved the pairing yet + /// is asked again while the card is up. + var pcReturnProbeSeconds = 5.0 /// How long the "didn't answer" card watches for the PC. var pcReturnLimitSeconds: TimeInterval = 180 } @@ -313,18 +315,26 @@ final class ConnectionManager: ObservableObject { /// Watches Bonjour for the PC that didn't answer. Tests pass a fake. var makePCBrowser: () -> any PCBrowser = { BonjourPCBrowser() } /// The PC watched for, its search (stopped in the background), the - /// automatic try waiting out its debounce, and the time limit. + /// task asking its listings, and the time limit. private var pcReturn: PCReturn? private var pcReturnBrowser: (any PCBrowser)? private var pcReturnPaused = false private var pcReturnTry: Task? private var pcReturnLimit: Task? - /// The automatic try checking that the PC is the same one. - private var pcReturnCheck: UUID? - /// The pairing an automatic try's session must give back, for the next - /// attempt and the current one. - private var pendingPairing: Data? - private var attemptPairing: Data? + /// The PC an automatic try's session must be, for the next attempt and + /// the current one. The PC proved the pairing just before; a PC gives + /// the pairing back only when it is new, so the session must give the + /// saved PC's name, when this iPad knows it, and no other pairing. + private struct ExpectedPC { + let pairing: Data + let name: String? + + func accepts(_ info: ControlChannel.SessionInfo) -> Bool { + (!PairingStore.valid(info.authToken) || info.authToken == pairing) && (name == nil || info.hostName == name) + } + } + private var pendingPC: ExpectedPC? + private var attemptPC: ExpectedPC? /// The automatic try is starting: the attempt it starts doesn't end the watch. private var startingPCReturnTry = false @@ -574,8 +584,8 @@ final class ConnectionManager: ObservableObject { pendingMove = nil attemptRedirect = isUSB ? nil : pendingRedirect pendingRedirect = nil - attemptPairing = isUSB ? nil : pendingPairing - pendingPairing = nil + attemptPC = isUSB ? nil : pendingPC + pendingPC = nil noVideoFallback = nil noVideoStep = nil sessionMediaSize = nil @@ -613,8 +623,8 @@ final class ConnectionManager: ObservableObject { sessionWantsAudio = wantsAudio audioStats = AudioStats() debugState = ConnectionDebugState(host: normalizedHost, port: port) - // Watched for from the start, so the search knows what Bonjour - // already listed before this attempt's HELLO2s stop. + // Watched for from the start: a PC listed by the time the card goes + // up is asked then. if !isUSB, attemptMove == nil, attemptRedirect == nil, reconnectCycleID == nil { watchForPC(host: normalizedHost, port: port) } @@ -834,9 +844,9 @@ final class ConnectionManager: ObservableObject { let acceptedAt = ProcessInfo.processInfo.systemUptime Task { @MainActor in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } - // An automatic try streams only from the PC that gives the - // saved pairing back, whatever answered at that address. - if let pairing = self.attemptPairing, info.authToken != pairing { + // An automatic try streams only from the PC it expects, + // whatever answered at that address. + if let expected = self.attemptPC, !expected.accepts(info) { self.record(.warning, "connection", "pc_return result=other_pc") self.disconnect(.otherPC) self.problem = .noReply(usb: false) @@ -901,7 +911,7 @@ final class ConnectionManager: ObservableObject { guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } // Nor did the PC an automatic try reached: no pairing prompt, // and the saved pairing stays. - if self.attemptPairing != nil, self.attemptMove == nil, status == .unauthorized || status == .rateLimited { + if self.attemptPC != nil, self.attemptMove == nil, status == .unauthorized || status == .rateLimited { self.record(.warning, "connection", "pc_return result=other_pc") self.disconnect(.otherPC) self.problem = .noReply(usb: false) @@ -1563,7 +1573,7 @@ final class ConnectionManager: ObservableObject { guard savedPairing(host: host, port: port) != nil else { return } let name = RecentConnectionStore.shared.entry(host: host, port: port)?.name ?? pairingStore.pcName(address: PairingStore.address(host: host, port: port)) - pcReturn = PCReturn(name: name, host: host, port: port, at: now()) + pcReturn = PCReturn(name: name, host: host, port: port) let browser = makePCBrowser() browser.onHosts = { [weak self] hosts in self?.pcReturnHeard(hosts) } pcReturnBrowser = browser @@ -1571,105 +1581,105 @@ final class ConnectionManager: ObservableObject { if isForeground { browser.start() } } - /// The "didn't answer" card is up for the PC watched for: it connects - /// once the PC shows up, until the time limit. + /// The "didn't answer" card is up for the PC watched for: its listings + /// are asked until one proves the pairing, until the time limit. private func armPCReturn() { guard pcReturn != nil else { return } - guard pcReturn?.armedAt == nil else { return considerPCReturn() } - pcReturn?.arm(at: now()) - watchingForPC = true - record(.info, "connection", "pc_return result=watching") - let limit = timing.pcReturnLimitSeconds - pcReturnLimit = Task { @MainActor [weak self] in - try? await Task.sleep(for: .seconds(limit)) - guard !Task.isCancelled, let self, self.pcReturn != nil else { return } - self.record(.info, "connection", "pc_return result=expired") - self.endPCReturn() + if pcReturn?.armedAt == nil { + pcReturn?.arm(at: now()) + watchingForPC = true + record(.info, "connection", "pc_return result=watching") + let limit = timing.pcReturnLimitSeconds + pcReturnLimit = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(limit)) + guard !Task.isCancelled, let self, self.pcReturn != nil else { return } + self.record(.info, "connection", "pc_return result=expired") + self.endPCReturn() + } } - considerPCReturn() + askPCReturn(after: timing.pcReturnDebounceSeconds) } private func pcReturnHeard(_ hosts: [DiscoveredHost]) { guard pcReturn != nil else { return } - pcReturn?.heard(hosts, at: now()) - considerPCReturn() + if pcReturn?.heard(hosts) == true { askPCReturn(after: timing.pcReturnDebounceSeconds) } } - /// The card is up, in the foreground, with no try under way. + /// The card is up, in the foreground, within the time limit. private var pcReturnMayTry: Bool { - guard let watch = pcReturn else { return false } - return isForeground && state == .disconnected && showsNoReplyOverWiFi && pcReturnCheck == nil + guard let watch = pcReturn, watch.armedAt != nil else { return false } + return isForeground && state == .disconnected && showsNoReplyOverWiFi && !watch.expired(at: now(), limit: timing.pcReturnLimitSeconds) } - /// The PC showed up while the card is up, in the foreground: one try, - /// after the debounce. Updates meanwhile count as the same appearance. - private func considerPCReturn() { - guard let watch = pcReturn, watch.armedAt != nil, watch.appeared != nil else { return } + /// Asks the PC's listings after `delay`, so a few Bonjour updates count + /// as one, then every few seconds, until one proves the pairing and has + /// its try or none is left to ask. A newer call starts over. + private func askPCReturn(after delay: Double) { + guard let watch = pcReturn, watch.armedAt != nil else { return } guard !watch.expired(at: now(), limit: timing.pcReturnLimitSeconds) else { record(.info, "connection", "pc_return result=expired") return endPCReturn() } - guard pcReturnMayTry, pcReturnTry == nil else { return } - let wait = timing.pcReturnDebounceSeconds + pcReturnTry?.cancel() pcReturnTry = Task { @MainActor [weak self] in - try? await Task.sleep(for: .seconds(wait)) - guard !Task.isCancelled, let self else { return } - self.pcReturnTry = nil - guard self.pcReturnMayTry, let found = self.pcReturn?.takeAppearance() else { return } - self.tryPCAgain(at: found) + var wait = delay + while true { + try? await Task.sleep(for: .seconds(wait)) + guard !Task.isCancelled, let self, self.pcReturnMayTry, let pc = self.pcReturn?.candidate else { return } + let proved = await self.provePCReturn(pc) + guard !Task.isCancelled, self.pcReturnMayTry else { return } + if proved { + self.pcReturnTry = nil + self.pcReturn?.didTry(pc) + return self.tryPCAgain(at: pc) + } + wait = self.timing.pcReturnProbeSeconds + } } } - /// One try at where Bonjour has the PC now. First the PC there shows it - /// holds the saved pairing, as a moved PC does, so nothing goes to - /// another PC that took the address; an address other than the one - /// tried must also be on this iPad's own network. Then the attempt - /// carries the pairing, and only a session that gives it back counts. - /// If the PC still doesn't answer, the card comes back and waits again. + /// Whether the PC at `pc`'s address shows it holds the saved pairing, + /// as a moved PC does; nothing with the pairing goes to it before. An + /// address other than the one tried must be on this iPad's own network + /// first: one that isn't never gets a try. + private func provePCReturn(_ pc: DiscoveredHost) async -> Bool { + guard let watch = pcReturn, let host = pc.address, + let token = savedPairing(host: watch.host, port: watch.port) else { return false } + if host != watch.host { + let route = homeAddress(host) ? await localRoute(host, pc.port) : nil + guard let route, MovedPC.isLocal(host, over: route) else { + if !Task.isCancelled, pcReturn != nil { + pcReturn?.didTry(pc) + record(.info, "connection", "pc_return result=not_local") + } + return false + } + } + return await MovedPC.confirm(host: host, port: pc.port, token: token, link: makeUDPLink(pc.port)) + } + + /// The one try at where Bonjour has the PC now, which proved the + /// pairing. The attempt carries the pairing, and only a session from + /// the PC it expects counts (see `ExpectedPC`). A PC at a new address or port moves the saved + /// entry once it shows a picture. If the PC still doesn't answer, the + /// card comes back and waits again. private func tryPCAgain(at found: DiscoveredHost) { guard let watch = pcReturn, let host = found.address, let token = savedPairing(host: watch.host, port: watch.port) else { return } - let port = found.port - let moved = host != watch.host || port != watch.port record(.info, "connection", "pc_return result=tried") - let check = UUID() - pcReturnCheck = check - // The card may go, or another attempt start, while the PC is asked. - let current = { [weak self] () -> Bool in - guard let self, self.pcReturnCheck == check, self.pcReturn != nil else { return false } - return self.isForeground && self.state == .disconnected && self.showsNoReplyOverWiFi - } - Task { @MainActor [weak self] in - guard let self else { return } - let done = { (result: String) in - if self.pcReturnCheck == check { self.pcReturnCheck = nil } - self.record(.info, "connection", "pc_return result=\(result)") - } - if host != watch.host { - guard self.homeAddress(host) else { return done("not_local") } - let route = await self.localRoute(host, port) - guard current() else { return } - guard let route, MovedPC.isLocal(host, over: route) else { return done("not_local") } - } - let same = await MovedPC.confirm(host: host, port: port, token: token, link: self.makeUDPLink(port)) - guard current() else { return } - guard same else { return done("not_confirmed") } - self.pcReturnCheck = nil - self.startingPCReturnTry = true - defer { self.startingPCReturnTry = false } - self.problem = nil - self.connectionError = nil - // A PC at a new address or port moves the saved entry once it shows a picture. - if moved, let name = watch.name { - self.pendingMove = Move(from: (watch.host, watch.port), name: name, token: token) - } - self.pendingPairing = token - self.explicitPairingToken = token - self.connect(target: .udp(host: host, port: port)) - self.pendingMove = nil - self.pendingPairing = nil + startingPCReturnTry = true + defer { startingPCReturnTry = false } + problem = nil + connectionError = nil + if host != watch.host || found.port != watch.port, let name = watch.name { + pendingMove = Move(from: (watch.host, watch.port), name: name, token: token) } + pendingPC = ExpectedPC(pairing: token, name: watch.name) + explicitPairingToken = token + connect(target: .udp(host: host, port: found.port)) + pendingMove = nil + pendingPC = nil } /// Dismissed, or replaced by another card: the watch ends. An attempt @@ -1681,30 +1691,27 @@ final class ConnectionManager: ObservableObject { endPCReturn() } - /// The app left the foreground: the search stops and no try starts. + /// The app left the foreground: the search stops and nothing is asked. private func pausePCReturn() { pcReturnTry?.cancel() pcReturnTry = nil - pcReturnCheck = nil guard let browser = pcReturnBrowser, !pcReturnPaused else { return } pcReturnPaused = true browser.stop() } - /// Back in the foreground: the search starts again, and a PC that - /// showed up meanwhile gets its try. + /// Back in the foreground: the search starts again, and the PC's + /// listings are asked again. private func resumePCReturn() { guard let browser = pcReturnBrowser, pcReturnPaused else { return } pcReturnPaused = false - pcReturn?.restarted(at: now()) browser.start() - considerPCReturn() + askPCReturn(after: timing.pcReturnDebounceSeconds) } private func endPCReturn() { pcReturnTry?.cancel() pcReturnTry = nil - pcReturnCheck = nil pcReturnLimit?.cancel() pcReturnLimit = nil pcReturnBrowser?.onHosts = nil diff --git a/ios/EternalMonitor/App/PCReturn.swift b/ios/EternalMonitor/App/PCReturn.swift index 5d190b88..ea63fce7 100644 --- a/ios/EternalMonitor/App/PCReturn.swift +++ b/ios/EternalMonitor/App/PCReturn.swift @@ -39,37 +39,29 @@ final class BonjourPCBrowser: PCBrowser { /// The PC that didn't answer, watched for on Bonjour from the start of the /// attempt. Its HELLO2s go out for two seconds only, so a PC app that opens -/// after that never hears this iPad: the PC's service turning up, or -/// changing, is the sign to try once more. What Bonjour already listed in -/// the search's first second is no sign; the PC was there and didn't answer. +/// after that never hears this iPad. While the card is up, each listing of +/// the PC is asked to prove it holds the saved pairing: at once when it +/// turns up or changes, then every few seconds, because a PC that closed +/// without saying goodbye stays listed, unchanged, after it opens again. +/// A listing that proves it gets one try; only a new or changed one gets +/// another. struct PCReturn { - /// The first second of a search lists what Bonjour already knew. It ends - /// before the attempt's HELLO2s do. - static let settleSeconds = 1.0 - /// The PC's name, when this iPad knows it: its Bonjour service is the /// PC, wherever it is. The service at the address tried is the PC too. let name: String? let host: String let port: UInt16 - /// Until when the latest search lists what Bonjour already knew. - private var settlesAt: Date - /// A search ran before this one: what it listed is what was known. - private var resumed = false - /// The PC's listings, by name, address, port and version. - private var listed = Set() - /// The latest list while settling, which replaces `listed` after it. - private var settled: Set? - /// The listing that turned up last and has had no try yet. - private(set) var appeared: DiscoveredHost? + /// The PC's listings now, by name, address, port and version. + private(set) var listed: [String: DiscoveredHost] = [:] + /// Listings that had their try, while they stay listed. + private var tried = Set() /// When the card went up and the watch's time limit started. private(set) var armedAt: Date? - init(name: String?, host: String, port: UInt16, at now: Date) { + init(name: String?, host: String, port: UInt16) { self.name = name.flatMap { $0.isEmpty ? nil : $0 } self.host = host self.port = port - settlesAt = now.addingTimeInterval(Self.settleSeconds) } var service: String? { name.map(MovedPC.serviceName) } @@ -79,35 +71,30 @@ struct PCReturn { return (service != nil && pc.name == service) || (address == host && pc.port == port) } - /// Bonjour's list now. A listing of the PC that wasn't there before is - /// an appearance; a new address, port or version is one too. - mutating func heard(_ hosts: [DiscoveredHost], at now: Date) { + private static func key(_ pc: DiscoveredHost) -> String { + "\(pc.name)|\(pc.address ?? "")|\(pc.port)|\(pc.version ?? "")" + } + + /// Bonjour's list now. True when a listing of the PC turned up, or + /// changed address, port or version: it is asked at once. A listing + /// that leaves and comes back may have a try again. + mutating func heard(_ hosts: [DiscoveredHost]) -> Bool { var current: [String: DiscoveredHost] = [:] - for pc in hosts where matches(pc) { - current["\(pc.name)|\(pc.address ?? "")|\(pc.port)|\(pc.version ?? "")"] = pc - } - let keys = Set(current.keys) - let new = { (known: Set) in keys.subtracting(known).sorted().compactMap { current[$0] }.first } - if now < settlesAt { - // A search that starts again lists only what changed meanwhile as new. - if resumed, let pc = new(listed) { appeared = pc } - listed.formUnion(keys) - settled = keys - return - } - if let settled { - listed = settled - self.settled = nil - } - if let pc = new(listed) { appeared = pc } - // A listing that left and comes back is a new appearance. - listed = keys + for pc in hosts where matches(pc) { current[Self.key(pc)] = pc } + let new = Set(current.keys).subtracting(listed.keys) + listed = current + tried.formIntersection(current.keys) + return !new.isEmpty + } + + /// The listing to ask next: one that hasn't had its try. + var candidate: DiscoveredHost? { + listed.keys.sorted().first { !tried.contains($0) }.flatMap { listed[$0] } } - /// The search starts again, as when the app comes back to the foreground. - mutating func restarted(at now: Date) { - settlesAt = now.addingTimeInterval(Self.settleSeconds) - resumed = true + /// `pc` had its try, or can't have one. + mutating func didTry(_ pc: DiscoveredHost) { + tried.insert(Self.key(pc)) } /// The card is up: the time limit starts, once. @@ -118,10 +105,4 @@ struct PCReturn { func expired(at now: Date, limit: TimeInterval) -> Bool { armedAt.map { now.timeIntervalSince($0) >= limit } ?? false } - - /// Takes the appearance for its one try: where the PC is now. - mutating func takeAppearance() -> DiscoveredHost? { - defer { appeared = nil } - return appeared - } } diff --git a/ios/EternalMonitorTests/PCReturnTests.swift b/ios/EternalMonitorTests/PCReturnTests.swift index f8f6e5e2..b81d005a 100644 --- a/ios/EternalMonitorTests/PCReturnTests.swift +++ b/ios/EternalMonitorTests/PCReturnTests.swift @@ -37,77 +37,52 @@ final class PCReturnTests: XCTestCase { DiscoveredHost(name: name, address: address, port: port, version: version) } - // MARK: - What counts as the PC showing up + // MARK: - Which listings are asked - func testOnlyAListingThatWasNotThereBeforeCounts() { - var watch = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876, at: start) + func testEachListingOfThePCGetsOneTry() { + var watch = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876) let listed = pc(port: 9876) - // What Bonjour already knew when the search began. - watch.heard([listed], at: start.addingTimeInterval(0.2)) - watch.heard([listed], at: start.addingTimeInterval(PCReturn.settleSeconds + 1)) - XCTAssertNil(watch.appeared, "the PC was listed and didn't answer") + XCTAssertTrue(watch.heard([listed]), "asked at once") + // Not proved yet: asked again, every few seconds. + XCTAssertFalse(watch.heard([listed])) + XCTAssertEqual(watch.candidate, listed) + watch.didTry(listed) + XCTAssertNil(watch.candidate, "one try per listing") + XCTAssertFalse(watch.heard([listed])) + XCTAssertNil(watch.candidate) // It leaves (the app closed) and comes back. - watch.heard([], at: start.addingTimeInterval(5)) - XCTAssertNil(watch.appeared) - watch.heard([listed], at: start.addingTimeInterval(9)) - XCTAssertEqual(watch.takeAppearance(), listed) - XCTAssertNil(watch.takeAppearance(), "one try per appearance") - // The same listing again is no new appearance; a change is, and the - // try goes where the PC is now. - watch.heard([listed], at: start.addingTimeInterval(10)) - XCTAssertNil(watch.appeared) - watch.heard([pc(port: 9877)], at: start.addingTimeInterval(11)) - XCTAssertEqual(watch.takeAppearance()?.port, 9877, "a new port") - watch.heard([pc(port: 9877, version: "0.3.6")], at: start.addingTimeInterval(12)) - XCTAssertEqual(watch.takeAppearance()?.version, "0.3.6", "a new version") - } - - func testAPCMissingAtTheStartCountsAsSoonAsItShowsUp() { - var watch = PCReturn(name: nil, host: "192.0.2.20", port: 9876, at: start) - watch.heard([], at: start.addingTimeInterval(0.1)) - watch.heard([pc("EternalMonitor (Test PC)", "192.0.2.20", port: 9876)], - at: start.addingTimeInterval(PCReturn.settleSeconds + 0.5)) - XCTAssertNotNil(watch.appeared) + XCTAssertFalse(watch.heard([])) + XCTAssertTrue(watch.heard([listed])) + XCTAssertEqual(watch.candidate, listed) + watch.didTry(listed) + // A change is a new listing, and the try goes where the PC is now. + XCTAssertTrue(watch.heard([pc(port: 9877)])) + XCTAssertEqual(watch.candidate?.port, 9877, "a new port") + watch.didTry(pc(port: 9877)) + XCTAssertTrue(watch.heard([pc(port: 9877, version: "0.3.6")])) + XCTAssertEqual(watch.candidate?.version, "0.3.6", "a new version") } func testOnlyThisPCCounts() { - var named = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876, at: start) - var unnamed = PCReturn(name: nil, host: "192.0.2.20", port: 9876, at: start) - let later = start.addingTimeInterval(5) + var named = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876) + var unnamed = PCReturn(name: nil, host: "192.0.2.20", port: 9876) let others = [pc("EternalMonitor (Other PC)", "192.0.2.30", port: 9876), pc("EternalMonitor (Test PC 2)", "192.0.2.31", port: 9876), pc("EternalMonitor (Other PC)", "192.0.2.20", port: 9900), pc("EternalMonitor (Test PC)", nil, port: 9876)] - named.heard(others, at: later) - unnamed.heard(others, at: later) - XCTAssertNil(named.appeared) - XCTAssertNil(unnamed.appeared) + XCTAssertFalse(named.heard(others)) + XCTAssertFalse(unnamed.heard(others)) + XCTAssertNil(named.candidate) + XCTAssertNil(unnamed.candidate) // By name at another address, or by the address tried. - named.heard(others + [pc("EternalMonitor (Test PC)", "192.0.2.40", port: 9876)], at: later) - XCTAssertEqual(named.appeared?.address, "192.0.2.40") - unnamed.heard(others + [pc("EternalMonitor (Any)", "192.0.2.20", port: 9876)], at: later) - XCTAssertEqual(unnamed.appeared?.name, "EternalMonitor (Any)") - } - - /// Back from the background, the new search lists what changed while - /// the app was away as new, and what was there before as known. - func testASearchStartedAgainKeepsWhatItKnew() { - var watch = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876, at: start) - watch.heard([pc(port: 9876)], at: start.addingTimeInterval(0.1)) - watch.restarted(at: start.addingTimeInterval(30)) - watch.heard([], at: start.addingTimeInterval(30.1)) - watch.heard([pc(port: 9876)], at: start.addingTimeInterval(30.2)) - XCTAssertNil(watch.appeared, "still the listing that didn't answer") - - var away = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876, at: start) - away.heard([], at: start.addingTimeInterval(0.1)) - away.restarted(at: start.addingTimeInterval(30)) - away.heard([pc(port: 9876)], at: start.addingTimeInterval(30.2)) - XCTAssertNotNil(away.appeared, "it showed up while the app was away") + XCTAssertTrue(named.heard(others + [pc("EternalMonitor (Test PC)", "192.0.2.40", port: 9876)])) + XCTAssertEqual(named.candidate?.address, "192.0.2.40") + XCTAssertTrue(unnamed.heard(others + [pc("EternalMonitor (Any)", "192.0.2.20", port: 9876)])) + XCTAssertEqual(unnamed.candidate?.name, "EternalMonitor (Any)") } func testTheTimeLimitStartsWhenTheCardGoesUp() { - var watch = PCReturn(name: nil, host: "192.0.2.20", port: 9876, at: start) + var watch = PCReturn(name: nil, host: "192.0.2.20", port: 9876) XCTAssertFalse(watch.expired(at: start.addingTimeInterval(1_000), limit: 180), "not armed yet") watch.arm(at: start.addingTimeInterval(10)) watch.arm(at: start.addingTimeInterval(100)) @@ -164,7 +139,8 @@ final class PCReturnTests: XCTestCase { let pairings = PairingStore(storage: MemoryPairingStorage()) try pairings.save(token: Self.token, hostName: name, address: PairingStore.address(host: "127.0.0.1", port: host.port)) let manager = ConnectionManager(timing: .init(connectTimeoutSeconds: 4, reconnectGraceSeconds: 0.1, - reconnectBackoffSeconds: { _ in 0.05 }, pcReturnDebounceSeconds: 0.1), + reconnectBackoffSeconds: { _ in 0.05 }, pcReturnDebounceSeconds: 0.1, + pcReturnProbeSeconds: 0.5), pairingStore: pairings) manager.homeAddress = { _ in true } manager.localRoute = { _, _ in MovedPC.Route(interface: "en0", direct: true, address: "127.0.0.1", netmask: "255.0.0.0") } @@ -185,7 +161,6 @@ final class PCReturnTests: XCTestCase { setup.manager.connect(host: "127.0.0.1", port: setup.host.port) XCTAssertTrue(setup.browser.running, "the search runs from the attempt's start") // Bonjour has nothing for the PC: its app isn't open. - setup.clock = start.addingTimeInterval(PCReturn.settleSeconds + 1) try await until("didn't answer", seconds: 15) { setup.manager.problem == .noReply(usb: false) } XCTAssertTrue(setup.manager.watchingForPC) XCTAssertTrue(setup.browser.running, "the search runs while the card is up") @@ -204,9 +179,12 @@ final class PCReturnTests: XCTestCase { try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) } - /// The PC proves the pairing, then accepts and streams. + /// The PC proves the pairing, then accepts and streams. As a real PC + /// does, it gives the pairing back only when it is new: not here. private func stream(_ setup: Setup, from pc: FakeUDPHost) async throws { - let accepted = await Task.detached { pc.answerIdentify(token: Self.token) && pc.acceptSession(authToken: Self.token) }.value + let accepted = await Task.detached { + pc.answerIdentify(token: Self.token) && pc.acceptSession(authToken: Data(repeating: 0, count: 16)) + }.value XCTAssertTrue(accepted, "no automatic attempt reached the PC") XCTAssertFalse(pc.helloBeforeIdentify, "nothing with the pairing before the PC proved it holds it") try await until("handshake", seconds: 15) { setup.manager.hostInfo != nil } @@ -231,12 +209,11 @@ final class PCReturnTests: XCTestCase { } /// The PC opened just after the attempt's HELLO2s stopped, before the - /// card: its listing is a return, not what Bonjour already knew. + /// card: its listing is asked once the card is up. func testAPCOpenedJustAfterTheHellosIsNotMissed() async throws { let setup = try paired() defer { setup.manager.cancel() } setup.manager.connect(host: "127.0.0.1", port: setup.host.port) - setup.clock = start.addingTimeInterval(PCReturn.settleSeconds + 0.5) setup.browser.list([setup.listing]) try await until("didn't answer", seconds: 15) { setup.manager.problem == .noReply(usb: false) } _ = await Task.detached { [host = setup.host] in host.helloAttempts(within: 0.2) }.value @@ -244,6 +221,25 @@ final class PCReturnTests: XCTestCase { XCTAssertEqual(setup.tries, 1) } + /// The PC closed without a goodbye, so Bonjour still lists it, the + /// same as before, when its app opens again. The listing is asked + /// until the PC answers, with nothing sent that needs the pairing. + func testAPCStillListedFromBeforeIsAskedUntilItAnswers() async throws { + let setup = try paired() + defer { setup.manager.cancel() } + setup.browser.current = [setup.listing] + setup.manager.connect(host: "127.0.0.1", port: setup.host.port) + try await until("didn't answer", seconds: 15) { setup.manager.problem == .noReply(usb: false) } + _ = await Task.detached { [host = setup.host] in host.helloAttempts(within: 0.05) }.value + let asked = await Task.detached { [host = setup.host] in host.received(within: 6) }.value + XCTAssertGreaterThan(Set(asked).count, 0) + XCTAssertFalse(asked.contains(.hello2), "\(asked)") + XCTAssertGreaterThan(asked.filter { $0 == .identify }.count, 8, "asked more than once") + XCTAssertEqual(setup.tries, 0) + try await stream(setup, from: setup.host) + XCTAssertEqual(setup.tries, 1) + } + /// Bonjour has the saved PC at a new port: the try goes there, after the /// PC there proves the pairing, and the saved entry moves with it. func testAPCBackAtANewPortIsTriedThere() async throws { @@ -259,30 +255,37 @@ final class PCReturnTests: XCTestCase { } /// Another PC took the saved address, with pairing off: it can't prove - /// the pairing, so nothing is sent to it. One that got past that and - /// accepts without giving the pairing back is dropped before a picture. + /// the pairing, so nothing is sent to it. A session that gets past that + /// from a PC with another name, or that gives another pairing back, is + /// dropped before a picture, and the saved pairing stays. func testADifferentPCAtTheSavedAddressGetsNoAutomaticSession() async throws { - let setup = try await noReply(named: false) + let setup = try await noReply() defer { setup.manager.cancel() } let other = DiscoveredHost(name: "EternalMonitor (Other PC)", address: "127.0.0.1", port: setup.host.port) setup.browser.list([other]) let types = await Task.detached { [host = setup.host] in host.received(within: 4) }.value XCTAssertTrue(types.contains(.identify), "\(types)") XCTAssertFalse(types.contains(.hello2), "\(types)") - try await until("not confirmed", seconds: 5) { setup.lines.contains("pc_return result=not_confirmed") } + XCTAssertEqual(setup.tries, 0) XCTAssertEqual(setup.attempts, 1) XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) - setup.browser.list([]) - setup.browser.list([other]) - let accepted = await Task.detached { [host = setup.host] in - host.answerIdentify(token: Self.token) && host.acceptSession(hostName: "Other PC", authToken: Data(repeating: 0, count: 16)) - }.value - XCTAssertTrue(accepted) - try await until("dropped", seconds: 15) { setup.lines.contains("pc_return result=other_pc") } - XCTAssertEqual(setup.manager.state, .disconnected) - XCTAssertNil(setup.manager.hostInfo) - XCTAssertEqual(setup.manager.problem, .noReply(usb: false)) + for (round, (name, pairing)) in [("Other PC", Data(repeating: 0, count: 16)), ("Test PC", Data(1...16))].enumerated() { + if round > 0 { + setup.browser.list([]) + setup.browser.list([other]) + } + let accepted = await Task.detached { [host = setup.host] in + host.answerIdentify(token: Self.token) && host.acceptSession(hostName: name, authToken: pairing) + }.value + XCTAssertTrue(accepted, name) + try await until("dropped: \(name)", seconds: 15) { + setup.lines.filter { $0 == "pc_return result=other_pc" }.count == round + 1 + } + XCTAssertEqual(setup.manager.state, .disconnected, name) + XCTAssertNil(setup.manager.hostInfo, name) + XCTAssertEqual(setup.manager.problem, .noReply(usb: false), name) + } XCTAssertEqual(try setup.pairings.token(address: PairingStore.address(host: "127.0.0.1", port: setup.host.port)), Self.token) XCTAssertFalse(setup.lines.contains { $0.hasPrefix("Connected to") }) } @@ -302,13 +305,17 @@ final class PCReturnTests: XCTestCase { setup.manager.state == .disconnected && setup.manager.problem == .noReply(usb: false) } XCTAssertTrue(setup.manager.watchingForPC, "still waiting for the PC") + // The same listing isn't asked again. setup.browser.list([listing]) - try await settle() + let after = await Task.detached { [host = setup.host] in host.received(within: 2) }.value + XCTAssertFalse(after.contains(.identify), "\(after)") XCTAssertEqual(setup.tries, 1) XCTAssertEqual(setup.attempts, 2) // It leaves and comes back: one more try. setup.browser.list([]) setup.browser.list([listing]) + let again = await Task.detached { [host = setup.host] in host.answerIdentify(token: Self.token) }.value + XCTAssertTrue(again) try await until("tried again", seconds: 5) { setup.tries == 2 } } @@ -319,7 +326,8 @@ final class PCReturnTests: XCTestCase { XCTAssertFalse(setup.manager.watchingForPC) XCTAssertFalse(setup.browser.running) setup.browser.list([setup.listing]) - try await settle() + let asked = await Task.detached { [host = setup.host] in host.received(within: 1.5) }.value + XCTAssertEqual(asked, []) XCTAssertEqual(setup.tries, 0) } @@ -343,10 +351,12 @@ final class PCReturnTests: XCTestCase { setup.manager.handleAppBackgrounded() XCTAssertFalse(setup.browser.running, "no search in the background") setup.browser.current = [setup.listing] - try await settle() - XCTAssertEqual(setup.tries, 0) + let asked = await Task.detached { [host = setup.host] in host.received(within: 1.5) }.value + XCTAssertEqual(asked, [], "nothing asked in the background") setup.manager.handleAppForegrounded() XCTAssertTrue(setup.browser.running) + let proved = await Task.detached { [host = setup.host] in host.answerIdentify(token: Self.token) }.value + XCTAssertTrue(proved) try await until("tried on return", seconds: 5) { setup.tries == 1 } } diff --git a/scripts/e2e_pc_returns.sh b/scripts/e2e_pc_returns.sh index be3106bf..16be95fa 100755 --- a/scripts/e2e_pc_returns.sh +++ b/scripts/e2e_pc_returns.sh @@ -4,7 +4,9 @@ # The host is then stopped and the app opened again: it connects to the saved # PC, which doesn't answer, and shows "didn't answer". The host then starts # again and advertises with Bonjour, and the app should connect and stream by -# itself, with no tap. +# itself, with no tap. The app opens a second after the host stopped, while +# Bonjour may still list the PC unchanged: the listing is asked until the PC +# proves its pairing, so a return that changes nothing on Bonjour still counts. # # On a shared Mac this is an iOS e2e run: hold the machine's e2e lock and use # your own simulator (EM_SIM_UDID). The PC's port is picked free when the run From 8117d4a9d192cc3f0a9de74a2598973c6ca53555 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 09:19:55 -0500 Subject: [PATCH 200/226] Show on the iPad that Second screen needs Windows 11 When the PC's heartbeat says Second screen needs Windows 11, the iPad shows "Second screen needs Windows 11. Showing your PC's screen instead." over the mirrored picture until the heartbeats stop saying it. The string is in the catalog, waiting for translation. --- .../App/ConnectionManager.swift | 10 ++++++++ ios/EternalMonitor/App/DisplayView.swift | 25 +++++++++++++++++++ ios/EternalMonitor/App/UIPreview.swift | 8 ++++-- ios/EternalMonitor/Network/WireProtocol.swift | 3 +++ .../Resources/Localizable.xcstrings | 12 +++++++++ .../LocalizationTests.swift | 4 ++- .../StreamRecoveryTests.swift | 19 ++++++++++++++ 7 files changed, 78 insertions(+), 3 deletions(-) diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 74ef73b7..c55b5a0e 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -122,6 +122,9 @@ final class ConnectionManager: ObservableObject { /// The PC has no display turned on, so there is nothing to capture and /// the picture is held. Its heartbeats say so until one comes back. @Published var pcNoDisplay = false + /// Second screen needs Windows 11, so the PC mirrors its main screen. + /// Its heartbeats say so for a few seconds after the session starts. + @Published var secondScreenNeedsWindows11 = false /// Nonzero while an automatic reconnect cycle is running. @Published private(set) var reconnectAttempt = 0 @@ -562,6 +565,7 @@ final class ConnectionManager: ObservableObject { signalLost = false pcLocked = false pcNoDisplay = false + secondScreenNeedsWindows11 = false degradedSinceUs = nil prevCounters = FrameAssembler.Counters() diagnostics.removeAll() @@ -983,6 +987,11 @@ final class ConnectionManager: ObservableObject { self.pcNoDisplay = noDisplay self.record(.info, "ctrl", noDisplay ? "PC has no active display: holding the picture" : "PC display back") } + let windows10 = heartbeat.notice == Heartbeat.noticeSecondScreenNeedsWindows11 + if windows10 != self.secondScreenNeedsWindows11 { + self.secondScreenNeedsWindows11 = windows10 + if windows10 { self.record(.info, "ctrl", "PC mirrors: Second screen needs Windows 11") } + } } } channel.onStreamConfig = { [weak self, weak assembler] config in @@ -1774,6 +1783,7 @@ final class ConnectionManager: ObservableObject { pictureRestoring = false pcLocked = false pcNoDisplay = false + secondScreenNeedsWindows11 = false degradedSinceUs = nil prevCounters = FrameAssembler.Counters() hostInfo = nil diff --git a/ios/EternalMonitor/App/DisplayView.swift b/ios/EternalMonitor/App/DisplayView.swift index bb557073..e929135a 100644 --- a/ios/EternalMonitor/App/DisplayView.swift +++ b/ios/EternalMonitor/App/DisplayView.swift @@ -63,6 +63,10 @@ struct DisplayView: View { restoringPill .transition(.opacity) .allowsHitTesting(false) + } else if connectionManager.secondScreenNeedsWindows11 { + needsWindows11Pill + .transition(.opacity) + .allowsHitTesting(false) } VStack { @@ -123,6 +127,7 @@ struct DisplayView: View { .animation(.easeInOut(duration: 0.25), value: connectionManager.pictureRestoring) .animation(.easeInOut(duration: 0.25), value: connectionManager.pcLocked) .animation(.easeInOut(duration: 0.25), value: connectionManager.pcNoDisplay) + .animation(.easeInOut(duration: 0.25), value: connectionManager.secondScreenNeedsWindows11) .animation(reduceMotion ? nil : .easeInOut(duration: 0.35), value: appeared) .onAppear { appeared = true @@ -460,6 +465,26 @@ struct DisplayView: View { .accessibilityIdentifier("display.pcNoDisplay") } + /// Over the mirrored picture for a few seconds, when the PC chose Second + /// screen but its Windows can't run it. + private var needsWindows11Pill: some View { + HStack(spacing: 10) { + Image(systemName: "rectangle.on.rectangle") + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(Theme.text) + .accessibilityHidden(true) + Text(L("Second screen needs Windows 11. Showing your PC's screen instead.")) + .font(.app(15, .semibold, relativeTo: .subheadline)) + .foregroundStyle(Theme.text) + } + .padding(.horizontal, 18) + .frame(minHeight: 44) + .background(glass(Capsule())) + .accessibilityElement(children: .ignore) + .accessibilityLabel(L("Second screen needs Windows 11. Showing your PC's screen instead.")) + .accessibilityIdentifier("display.needsWindows11") + } + /// The PC is still connected but a lost frame left the picture waiting /// for a fresh one; usually gone in well under a second. private var restoringPill: some View { diff --git a/ios/EternalMonitor/App/UIPreview.swift b/ios/EternalMonitor/App/UIPreview.swift index 5766d5d0..a6af085f 100644 --- a/ios/EternalMonitor/App/UIPreview.swift +++ b/ios/EternalMonitor/App/UIPreview.swift @@ -4,7 +4,7 @@ import SwiftUI /// Debug builds only: put the UI in a state that normally needs a live PC, /// for screenshots and design review. Never opens a connection. /// -/// `EM_UI_PREVIEW=display|quality|restoring|connecting|error|error-no-picture|error-pairing-code|error-router|error-updating|error-local-network|error-wifi-off|mismatch|pairing|settings|settings-about|feedback|recent|old-pc|searchFailed|searchBlocked|setup|setup-blocked`. +/// `EM_UI_PREVIEW=display|quality|restoring|needs-windows-11|connecting|error|error-no-picture|error-pairing-code|error-router|error-updating|error-local-network|error-wifi-off|mismatch|pairing|settings|settings-about|feedback|recent|old-pc|searchFailed|searchBlocked|setup|setup-blocked`. /// `setup` opens first-run setup whatever the saved state (launch with /// `-didSeeOnboarding NO`); `setup-blocked` also fails its search as if /// Local Network were off. @@ -12,7 +12,7 @@ import SwiftUI /// Launch with `-allowUSB NO` so the USB listener stays off as well. enum UIPreview { static let mode = ProcessInfo.processInfo.environment["EM_UI_PREVIEW"] ?? "" - static var showsBackdrop: Bool { mode == "display" || mode == "quality" || mode == "restoring" || mode == "locked" || mode == "no-display" } + static var showsBackdrop: Bool { mode == "display" || mode == "quality" || mode == "restoring" || mode == "locked" || mode == "no-display" || mode == "needs-windows-11" } static var opensQuality: Bool { mode == "quality" } static var opensSettings: Bool { mode == "settings" || mode == "settings-about" } /// Scrolls Settings to Support and About. @@ -54,6 +54,10 @@ enum UIPreview { manager.transportMode = "WiFi" manager.state = .connected manager.pcNoDisplay = true + case "needs-windows-11": + manager.transportMode = "WiFi" + manager.state = .connected + manager.secondScreenNeedsWindows11 = true case "connecting": manager.state = .connecting case "error": diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index 4b87b593..8b748ed7 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -358,6 +358,9 @@ struct Heartbeat: Equatable { static let noticePCLocked: UInt8 = 1 /// No display is on at the PC (a closed lid, screens off or unplugged). static let noticeNoDisplay: UInt8 = 2 + /// Second screen needs Windows 11, so the PC shows its main screen + /// instead. Sent for a few seconds, over a picture that works. + static let noticeSecondScreenNeedsWindows11: UInt8 = 6 /// The video frames the PC sent this session, from a PC that offered /// `HelloAck.hostCapMediaSize`; nil from others. Appended after /// `notice`, which is then always written. diff --git a/ios/EternalMonitor/Resources/Localizable.xcstrings b/ios/EternalMonitor/Resources/Localizable.xcstrings index 7c85e091..a012f927 100644 --- a/ios/EternalMonitor/Resources/Localizable.xcstrings +++ b/ios/EternalMonitor/Resources/Localizable.xcstrings @@ -6835,6 +6835,18 @@ } } }, + "Second screen needs Windows 11. Showing your PC's screen instead." : { + "comment" : "Over the mirrored picture for a few seconds, when the PC chose Second screen but runs Windows 10, which Second screen doesn't support", + "extractionState" : "manual", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "Second screen needs Windows 11. Showing your PC's screen instead." + } + } + } + }, "Second screen or mirror" : { "extractionState" : "manual", "localizations" : { diff --git a/ios/EternalMonitorTests/LocalizationTests.swift b/ios/EternalMonitorTests/LocalizationTests.swift index 0db3ce8d..81063edf 100644 --- a/ios/EternalMonitorTests/LocalizationTests.swift +++ b/ios/EternalMonitorTests/LocalizationTests.swift @@ -22,7 +22,9 @@ final class LocalizationTests: XCTestCase { /// English strings whose translations come in a separate pass. Until /// then every language shows them in English. - private let awaitingTranslation: Set = [] + private let awaitingTranslation: Set = [ + "Second screen needs Windows 11. Showing your PC's screen instead.", + ] /// Every language has exactly English's strings, none empty, each with /// the same placeholders. diff --git a/ios/EternalMonitorTests/StreamRecoveryTests.swift b/ios/EternalMonitorTests/StreamRecoveryTests.swift index c0317ecc..1d2467a5 100644 --- a/ios/EternalMonitorTests/StreamRecoveryTests.swift +++ b/ios/EternalMonitorTests/StreamRecoveryTests.swift @@ -133,6 +133,25 @@ final class StreamRecoveryTests: XCTestCase { XCTAssertFalse(manager.pcNoDisplay, "the next session starts without it") } + /// The PC chose Second screen, but its Windows can't run it, so it + /// mirrors. Its heartbeats say why for a few seconds, and the iPad says + /// so over the working picture until they stop. + func testAPCThatNeedsWindows11ForSecondScreenSaysSoWhileItMirrors() async throws { + let host = FakeUDPHost() + let manager = try await streaming(host) + XCTAssertFalse(manager.secondScreenNeedsWindows11) + host.sendHeartbeat(notice: Heartbeat.noticeSecondScreenNeedsWindows11) + try await until("the PC says Second screen needs Windows 11") { manager.secondScreenNeedsWindows11 } + XCTAssertEqual(manager.state, .connected) + XCTAssertFalse(manager.pcLocked || manager.pcNoDisplay) + host.sendHeartbeat() + try await until("the notice ends") { !manager.secondScreenNeedsWindows11 } + host.sendHeartbeat(notice: Heartbeat.noticeSecondScreenNeedsWindows11) + try await until("said again") { manager.secondScreenNeedsWindows11 } + manager.cancel() + XCTAssertFalse(manager.secondScreenNeedsWindows11, "the next session starts without it") + } + func testContiguousNumberingRevealsALostReopenKeyframe() async throws { // The PC reopened its encoder and its new keyframe (seq 2) was lost; // the new encoder's first P-frame continues the codec's count. A PC From 3ca530dd36a3c113fda6cf78cb4c66f6e0ebd1b2 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 09:29:27 -0500 Subject: [PATCH 201/226] Bring Second screen back after a lock Windows reports late A secure screen (the lock screen, a UAC prompt, Ctrl+Alt+Del) can take the desktop before Windows says so. A second screen that lost the desktop and showed no picture was retried once the desktop came back only if it read as away at the moment of the failure. When Windows said so seconds later, the app run stayed on the mirror until Try again. - A no-picture failure that lost the desktop, with no driver call outstanding and no display that keeps dropping out, now watches for 30 s: Windows saying the desktop is away, or capture reporting a secure screen, tries the second screen again once the desktop is back, as for a failure while it is away. - A failure whose loss came while the desktop was away, and that is back by the failure, is tried again at once. - Once per lock: a retried second screen that fails again needs a new lock to be tried again. Try again, another mode or a new failure ends the wait. Driver failures and failures with no lock change nothing. --- host/src/capture/second_screen.rs | 251 +++++++++++++++++++++++++++++- 1 file changed, 247 insertions(+), 4 deletions(-) diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 5c5a5600..106a8655 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -150,6 +150,9 @@ struct Proving { /// A time-limited driver call of this generation is outstanding (see /// [`driver_call`]): a failure now may be the driver's, not the PC's. call_pending: bool, + /// Windows lost the desktop during this proof while it was away (a + /// secure screen, the displays off), even if it is back by the failure. + away_seen: bool, } /// A second screen that loses the user's desktop more often than this in one @@ -168,6 +171,11 @@ pub(crate) type ProofOutput = (crate::gpu_preference::AdapterId, String); /// back. const DESKTOP_CHECK: Duration = Duration::from_secs(1); +/// How long after a second screen fails, having lost the desktop, Windows +/// may still say a secure screen took it or the displays went off, and be +/// taken as the cause. Windows can say so many seconds after the loss. +const LATE_AWAY_WINDOW: Duration = Duration::from_secs(30); + #[derive(Debug, Default)] pub(crate) struct State { /// The display attached; the stream has not moved onto it yet. @@ -209,6 +217,11 @@ pub(crate) struct State { /// [`desktop_away`]): once it is back, a connected viewer gets the /// second screen again. When to look at the desktop next. retry_when_back: Option, + /// The last failure lost the desktop with the user's desktop there: until + /// then, Windows saying it was away after all (see [`LATE_AWAY_WINDOW`]) + /// tries the second screen again once it is back. When to ask next. + late_away_until: Option, + late_away_check: Option, /// The graphics cards and the screens' wiring, as last read off the /// transport loop: see [`facts`]. facts: facts::Cache, @@ -221,6 +234,10 @@ pub(crate) struct State { impl State { /// `blocked`: [`mirror_blocked_now`], read for this decision when /// [`State::decides_on_mirror`] says it needs it. + /// `secure`: capture says a secure screen holds the desktop now + /// ([`CaptureBlock::SecureDesktop`]). + /// + /// [`CaptureBlock::SecureDesktop`]: crate::capture::recovery::CaptureBlock::SecureDesktop pub(crate) fn step( &mut self, selected: bool, @@ -228,6 +245,7 @@ impl State { connected: bool, now: Instant, blocked: bool, + secure: bool, ) -> Step { self.memory.retry_clear(now); if !selected { @@ -252,7 +270,8 @@ impl State { if let Some(reason) = self.fail_unproven(status, now, blocked) { return Step::Fallback(reason); } - self.retry_if_back(status, connected, now); + self.watch_late_away(*status, now, secure); + self.retry_if_back(status, connected, now, secure); match *status { VddStatus::WaitingForClient if connected => { let held = self.memory.hold(now); @@ -361,20 +380,93 @@ impl State { *status = VddStatus::Failed; let passing = proving.interrupted || proving.call_pending || proving.losses > LOSS_LIMIT; self.remember(reason, now, passing, blocked); + self.after_desktop_loss(reason, proving, now, blocked); Some(reason) } + /// A second screen that showed no picture after Windows took its + /// desktop away: the lock screen, a UAC prompt or Ctrl+Alt+Del can take + /// the input before Windows says so, so the desktop may still read as + /// the user's at the failure. When the loss came while the desktop was + /// away, try again once it is back, as for a failure while it is away. + /// Otherwise watch for Windows to say so within [`LATE_AWAY_WINDOW`]. + /// Neither for a driver call that didn't return, a display that keeps + /// dropping out, or a failure [`State::remember`] already set a retry + /// for. + fn after_desktop_loss( + &mut self, + reason: &'static str, + proving: Proving, + now: Instant, + blocked: bool, + ) { + let desktop_loss = reason == "no_frames" + && proving.interrupted + && !proving.call_pending + && proving.losses <= LOSS_LIMIT; + if !desktop_loss || blocked || self.retry_when_back.is_some() { + return; + } + if proving.away_seen { + tracing::info!( + reason, + "The second screen lost the desktop while Windows had it away — trying the \ + second screen again now that the desktop is back" + ); + self.retry_when_back = Some(now); + } else { + self.late_away_until = Some(now + LATE_AWAY_WINDOW); + self.late_away_check = None; + } + } + + /// While a failure that lost the desktop waits (see + /// [`State::after_desktop_loss`]): a secure screen or the displays off + /// now means that was the cause, and the second screen is tried again + /// once the desktop is back. Once per failure, and only while it is still + /// the failure: Try again, another mode or a retry ends the wait. + fn watch_late_away(&mut self, status: VddStatus, now: Instant, secure: bool) { + let Some(until) = self.late_away_until else { + return; + }; + if status != VddStatus::Failed || now >= until { + self.late_away_until = None; + self.late_away_check = None; + return; + } + if !secure && self.late_away_check.is_some_and(|at| now < at) { + return; + } + self.late_away_check = Some(now + DESKTOP_CHECK); + if secure || self.desktop_away() { + self.late_away_until = None; + self.late_away_check = None; + tracing::info!( + reason = self.failure.unwrap_or("no_frames"), + "Windows took the desktop away just after the second screen failed — trying \ + the second screen again once the desktop is back" + ); + self.retry_when_back = Some(now + DESKTOP_CHECK); + } + } + /// After a failure while the desktop was away, try the second screen /// again for a connected viewer once the desktop is back. Without this /// the app run stayed on the mirror after the PC was unlocked or its /// displays woke, until the user clicked Try again. Windows is asked at /// most once a second: this runs on the transport loop. - fn retry_if_back(&mut self, status: &mut VddStatus, connected: bool, now: Instant) { + fn retry_if_back( + &mut self, + status: &mut VddStatus, + connected: bool, + now: Instant, + secure: bool, + ) { let due = self.retry_when_back.is_some_and(|at| now >= at); if !due || !connected || *status != VddStatus::Failed { return; } - if self.desktop_away() { + if secure || self.desktop_away() { self.retry_when_back = Some(now + DESKTOP_CHECK); return; } @@ -491,11 +583,13 @@ pub(crate) fn next_step(shared: &SharedControl, now: Instant) -> Step { let blocked = mirror_blocked_now(shared, |status, state| { state.decides_on_mirror(selected, status, connected, now) }); + let secure = + *shared.capture_block.lock() == Some(crate::capture::recovery::CaptureBlock::SecureDesktop); let mut status = shared.vdd_status.lock(); shared .second_screen .lock() - .step(selected, &mut status, connected, now, blocked) + .step(selected, &mut status, connected, now, blocked, secure) } /// A new capture generation ends a second screen that already failed to @@ -916,6 +1010,7 @@ pub(crate) fn start_proving_on( losses: 0, persistent: false, call_pending: false, + away_seen: false, }) } } @@ -1004,6 +1099,7 @@ fn note_interruption_while(shared: &SharedControl, generation: u64, away: bool) if let Some(proving) = shared.second_screen.lock().proving.as_mut() { if proving.generation == generation { proving.interrupted = true; + proving.away_seen |= away; if !away { proving.losses += 1; proving.capture_failed |= proving.losses > LOSS_LIMIT; @@ -2632,6 +2728,153 @@ mod tests { assert!(shared.second_screen.lock().retry_when_back.is_none()); } + /// A second screen proving itself, with a viewer and a fake desktop, + /// whose capture then lost the desktop (`away` at that moment) and showed + /// no picture. Returns the shared state, the fake desktop and when it + /// fell back. + fn fell_back_after_a_desktop_loss( + away_at_loss: bool, + away_at_failure: bool, + ) -> ( + SharedControl, + std::sync::Arc, + Instant, + ) { + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + let shared = second_screen_selected(); + let away = Arc::new(AtomicBool::new(false)); + shared.second_screen.lock().fake_desktop_away = Some(away.clone()); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + let start = Instant::now(); + assert_eq!(next_step(&shared, start), Step::Switch); + start_proving(&shared, 7, start); + note_interruption_while(&shared, 7, away_at_loss); + away.store(away_at_failure, Ordering::Relaxed); + let failed = start + PROVE_TIMEOUT; + assert_eq!(next_step(&shared, failed), Step::Fallback("no_frames")); + (shared, away, failed) + } + + /// Steps every quarter second from `from` for `span`, expecting nothing. + fn nothing_for(shared: &SharedControl, from: Instant, span: Duration) -> Instant { + let mut now = from; + while now < from + span { + now += secs(0.25); + assert_eq!(next_step(shared, now), Step::Nothing, "{:?}", now - from); + } + now + } + + /// Windows said a secure screen had the desktop 13 s after the second + /// screen failed, having lost it: on 0.3.4 the app run stayed on the + /// mirror until Try again. It is tried again once the desktop is back. + #[test] + fn a_lock_windows_reports_after_the_fallback_brings_the_second_screen_back() { + let (shared, away, failed) = fell_back_after_a_desktop_loss(false, false); + let locked = nothing_for(&shared, failed, secs(13.0)); + away.store(true, Ordering::Relaxed); + // Locked: the mirror stays, and the driver is left alone. + let unlocked = nothing_for(&shared, locked, secs(20.0)); + assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); + away.store(false, Ordering::Relaxed); + assert_eq!(next_step(&shared, unlocked + DESKTOP_CHECK), Step::Attach); + assert_eq!(remembered_failure(&shared), None); + let state = shared.second_screen.lock(); + assert!(state.retry_when_back.is_none() && state.late_away_until.is_none()); + } + + /// Capture saying a secure screen holds the desktop counts as well, and + /// the second screen waits until capture has the desktop again. + #[test] + fn a_secure_screen_capture_reports_after_the_fallback_brings_the_second_screen_back() { + use crate::capture::recovery::CaptureBlock; + let (shared, _away, failed) = fell_back_after_a_desktop_loss(false, false); + let blocked = nothing_for(&shared, failed, secs(13.0)); + *shared.capture_block.lock() = Some(CaptureBlock::SecureDesktop); + let resumed = nothing_for(&shared, blocked, secs(10.0)); + *shared.capture_block.lock() = None; + assert_eq!(next_step(&shared, resumed + DESKTOP_CHECK), Step::Attach); + } + + /// A loss while the desktop was away, back by the failure (Windows said + /// so before the fallback): tried again at once. + #[test] + fn a_loss_while_the_desktop_was_away_is_tried_again_once_it_is_back() { + let (shared, _away, failed) = fell_back_after_a_desktop_loss(true, false); + assert_eq!(next_step(&shared, failed + secs(0.25)), Step::Attach); + } + + /// Away at the failure itself: #207's retry, unchanged. + #[test] + fn a_failure_while_the_desktop_is_away_still_waits_for_it() { + let (shared, away, failed) = fell_back_after_a_desktop_loss(false, true); + let back = nothing_for(&shared, failed, secs(20.0)); + away.store(false, Ordering::Relaxed); + assert_eq!(next_step(&shared, back + DESKTOP_CHECK), Step::Attach); + } + + /// No secure screen, or one only after the window: the mirror stays + /// until Try again, as before. + #[test] + fn a_desktop_loss_with_no_lock_is_not_tried_again() { + let (shared, away, failed) = fell_back_after_a_desktop_loss(false, false); + let later = nothing_for(&shared, failed, LATE_AWAY_WINDOW + secs(1.0)); + away.store(true, Ordering::Relaxed); + let back = nothing_for(&shared, later, secs(5.0)); + away.store(false, Ordering::Relaxed); + nothing_for(&shared, back, secs(30.0)); + assert_eq!(remembered_failure(&shared), Some("no_frames")); + } + + /// One retry per lock: a retried second screen that fails the same way + /// with no new lock stays on the mirror. + #[test] + fn a_late_lock_brings_the_second_screen_back_once() { + let (shared, away, failed) = fell_back_after_a_desktop_loss(false, false); + away.store(true, Ordering::Relaxed); + let locked = nothing_for(&shared, failed, secs(2.0)); + away.store(false, Ordering::Relaxed); + let retry = locked + DESKTOP_CHECK; + assert_eq!(next_step(&shared, retry), Step::Attach); + note_attached(&shared, "virtual".into()); + assert_eq!(next_step(&shared, retry), Step::Switch); + start_proving(&shared, 8, retry); + note_interruption_while(&shared, 8, false); + let failed_again = retry + PROVE_TIMEOUT; + assert_eq!( + next_step(&shared, failed_again), + Step::Fallback("no_frames") + ); + nothing_for(&shared, failed_again, LATE_AWAY_WINDOW + secs(5.0)); + } + + /// A driver call that didn't return, or a display that keeps dropping + /// out, is not a desktop transition: a lock afterwards changes nothing. + #[test] + fn a_driver_failure_is_not_tried_again_after_a_lock() { + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + let shared = second_screen_selected(); + let away = Arc::new(AtomicBool::new(false)); + shared.second_screen.lock().fake_desktop_away = Some(away.clone()); + connect_viewer(&shared); + note_attached(&shared, "virtual".into()); + let start = Instant::now(); + assert_eq!(next_step(&shared, start), Step::Switch); + start_proving(&shared, 7, start); + note_interruption_while(&shared, 7, false); + let _call = driver_call(&shared, 7); + let failed = start + PROVE_TIMEOUT; + assert_eq!(next_step(&shared, failed), Step::Fallback("no_frames")); + away.store(true, Ordering::Relaxed); + let back = nothing_for(&shared, failed, secs(5.0)); + away.store(false, Ordering::Relaxed); + nothing_for(&shared, back, secs(10.0)); + assert!(shared.second_screen.lock().late_away_until.is_none()); + } + /// A proof that lost the desktop while the user was there is excused /// too, but tried again only at Try again: retrying at once could turn /// a display that keeps dropping out on and off forever. From 85e3b578da23f8cc00b275f7287b956495b81f58 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 05:05:13 -0500 Subject: [PATCH 202/226] Run each AMF encoder test alone, from a clean launch The encode stage's state is process-wide: the encoders that failed to open this launch, PIPELINE_STATS, the lane FFmpeg's hardware opens share, and an AMD driver's limit on concurrent sessions. The AMF probes ran outside the stage lock, beside another test's AMF session, so one failed open there could send every later AMF stage at that size to libx264. Each test now holds one guard from its first line, through its probes and stages; the guard waits for the lane to clear and forgets earlier failures. A stage that started on libx264 only because its hardware open was still going when the first picture was due runs once more. The stage now sends frame 0, then the rest once it is out. --- host/tests/amf_encoder.rs | 215 ++++++++++++++++++++++++++++++-------- 1 file changed, 169 insertions(+), 46 deletions(-) diff --git a/host/tests/amf_encoder.rs b/host/tests/amf_encoder.rs index cdd2cf9e..2f4135ac 100644 --- a/host/tests/amf_encoder.rs +++ b/host/tests/amf_encoder.rs @@ -12,21 +12,80 @@ //! desktop mode under a running stream. use std::sync::atomic::Ordering; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; use eternal_host::capture::synthetic::{decode_frame_counter_from_luma, render_synthetic_frame}; use eternal_host::capture::{frame_buffer, frame_slot, RawFrame}; use eternal_host::control::SharedControl; -use eternal_host::encoder::{run_encode_stage, NALUnit, NAL_CHANNEL_CAPACITY}; +use eternal_host::encoder::{ + forget_launch_failures, run_encode_stage, NALUnit, NAL_CHANNEL_CAPACITY, +}; use eternal_host::gpu::GpuInfo; use eternal_host::stats::PIPELINE_STATS; use eternal_wire::h264::{contains_nal_type, nal_type, parse_annex_b_units}; use ffmpeg_next::codec::Id; -/// The stage and PIPELINE_STATS are process-global, so stages run one at a time. +/// The encode stage's state is process-wide: PIPELINE_STATS, the encoders +/// that failed to open this launch, and the lane FFmpeg's hardware opens +/// share. So is an AMD driver's limit on concurrent AMF sessions. Each test +/// holds [`Serial`] from its first line, through its AMF probes and stages. static STAGE_LOCK: Mutex<()> = Mutex::new(()); +/// One test's turn at the encoders. A stage and an AMF probe take it by +/// reference, so neither runs beside another test's, and PIPELINE_STATS is +/// read only for the stage that just ran. +struct Serial(#[allow(dead_code)] MutexGuard<'static, ()>); + +/// Waits for this test's turn, then starts it from a clean launch: no +/// earlier test's encoder open is still running, and no earlier failure +/// keeps an encoder out of this test's stages. +fn serial() -> Serial { + let guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + wait_for_clear_lane(); + forget_launch_failures(); + Serial(guard) +} + +/// An open that went on in the background (see `encoder::Opening`) holds +/// the encoders' lane until it returns, and a stage's open in that lane is +/// refused meanwhile. +fn lane_busy() -> bool { + eternal_host::stall::abandoned(eternal_host::stall::Call::EncoderOpen) > 0 +} + +fn wait_for_clear_lane() { + // Past the 15 s limit of the slowest open in the lane. + let deadline = Instant::now() + Duration::from_secs(30); + while lane_busy() { + assert!( + Instant::now() < deadline, + "an earlier encoder open is still running" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + +/// A stage whose hardware open outlasted the first picture's wait starts on +/// libx264 while the open goes on in the background, as it should when a +/// driver is slow. On a busy test PC that says nothing about what a test +/// checks, so the stage runs once more. +fn started_on_the_fallback(_: &Serial) -> bool { + PIPELINE_STATS.lock().using_software_fallback && lane_busy() +} + +const STAGE_TRIES: u32 = 2; + +/// Before a stage runs again: its open in the background is released, and +/// what it remembered is forgotten, as for a new test. +fn before_another_try(encoder: &str) { + eprintln!( + "{encoder} was still opening when the first picture was due; running the stage again" + ); + wait_for_clear_lane(); + forget_launch_failures(); +} + const FRAME_INTERVAL: Duration = Duration::from_millis(20); const FRAME_COUNT: u64 = 90; /// Mid-GOP. The encoder's own IDRs land on frames 0, 30 and 60. @@ -71,11 +130,11 @@ fn amf_expectation() -> Option { /// Whether an AMF encoder opens on this machine. Cases that need a working /// AMF skip when it does not, except under `ETERNAL_AMF_TEST=hardware`. -fn amf_available(name: &str) -> bool { +fn amf_available(serial: &Serial, name: &str) -> bool { let Some(expectation) = amf_expectation() else { return false; }; - let opens = amf_opens(name); + let opens = amf_opens(serial, name); if expectation == AmfExpectation::Hardware { assert!( opens, @@ -88,7 +147,7 @@ fn amf_available(name: &str) -> bool { opens } -fn amf_opens(name: &str) -> bool { +fn amf_opens(_: &Serial, name: &str) -> bool { let Some(codec) = ffmpeg_next::encoder::find_by_name(name) else { return false; }; @@ -115,10 +174,29 @@ struct StageRun { /// `FRAME_INTERVAL`, sized by `size_of`, and raises a keyframe request just /// before frame `keyframe_at`. Panics if the stage fails or panics. fn run_stage( + serial: &Serial, encoder: &str, size_of: impl Fn(u64) -> (u32, u32), keyframe_at: Option, ) -> StageRun { + for _ in 1..STAGE_TRIES { + match run_stage_once(serial, encoder, &size_of, keyframe_at) { + Some(run) => return run, + None => before_another_try(encoder), + } + } + run_stage_once(serial, encoder, &size_of, keyframe_at) + .unwrap_or_else(|| panic!("{encoder} was still opening when the first picture was due")) +} + +/// [`run_stage`] once. None if the stage started on the fallback while its +/// encoder was still opening (see [`started_on_the_fallback`]). +fn run_stage_once( + serial: &Serial, + encoder: &str, + size_of: impl Fn(u64) -> (u32, u32), + keyframe_at: Option, +) -> Option { ffmpeg_next::init().unwrap(); let (producer, consumer) = frame_slot(); let (tx, mut rx) = tokio::sync::mpsc::channel(NAL_CHANNEL_CAPACITY); @@ -135,18 +213,19 @@ fn run_stage( run_encode_stage(consumer, tx, stage_shared, gpu, generation).map_err(|e| e.to_string()) }); // The output channel holds only a few frames, so drain it as we go. + let (first_tx, first) = std::sync::mpsc::channel(); let collector = std::thread::spawn(move || { let mut packets = Vec::new(); while let Some(nal) = rx.blocking_recv() { + if packets.is_empty() { + let _ = first_tx.send(()); + } packets.push(nal); } packets }); - for number in 0..FRAME_COUNT { - if keyframe_at == Some(number) { - shared.force_next_idr.store(true, Ordering::SeqCst); - } + let publish = |number: u64| { let (width, height) = size_of(number); let mut data = frame_buffer((width * height * 4) as usize); render_synthetic_frame(&mut data, width, height, number); @@ -157,6 +236,23 @@ fn run_stage( width, height, }); + }; + // The stage opens its encoder on the first frame. The rest follow once + // it is out, however long the open took. + publish(0); + if first.recv_timeout(Duration::from_secs(20)).is_ok() && started_on_the_fallback(serial) { + producer.close(); + let _ = stage.join(); + let _ = collector.join(); + return None; + } + std::thread::sleep(FRAME_INTERVAL); + + for number in 1..FRAME_COUNT { + if keyframe_at == Some(number) { + shared.force_next_idr.store(true, Ordering::SeqCst); + } + publish(number); std::thread::sleep(FRAME_INTERVAL); } producer.close(); @@ -169,11 +265,11 @@ fn run_stage( let packets = collector.join().unwrap(); assert!(!packets.is_empty(), "{encoder} produced no packets"); let stats = PIPELINE_STATS.lock(); - StageRun { + Some(StageRun { packets, software_fallback: stats.using_software_fallback, codec_name: stats.codec_name.clone(), - } + }) } /// Decodes `packets` with a fresh, strict software decoder and returns each @@ -322,15 +418,15 @@ fn assert_follows_mode_changes(run: &StageRun, codec: Id) { #[test] fn software_encoder_follows_capture_mode_changes() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("libx264", mode_changes, None); + let serial = serial(); + let run = run_stage(&serial, "libx264", mode_changes, None); assert_follows_mode_changes(&run, Id::H264); } #[test] fn software_encoder_honours_keyframe_requests() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("libx264", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); + let serial = serial(); + let run = run_stage(&serial, "libx264", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); assert!( !run.software_fallback, "explicit CPU selection is not a fallback" @@ -344,8 +440,8 @@ fn software_encoder_honours_keyframe_requests() { #[test] #[ignore = "requires the NVIDIA/AMD reference PC without an Intel GPU"] fn unavailable_quick_sync_uses_another_gpu_and_decodes() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("h264_qsv", |_| LARGE, None); + let serial = serial(); + let run = run_stage(&serial, "h264_qsv", |_| LARGE, None); assert!(!run.software_fallback, "fell straight from QSV to CPU"); assert_eq!(run.codec_name, "H.264 (NVENC)"); assert_h264_keyframe(&run.packets[0], "the first packet"); @@ -371,11 +467,11 @@ fn unavailable_quick_sync_uses_another_gpu_and_decodes() { #[test] #[ignore = "native: an NVIDIA card, with this exe on another card"] fn nvenc_encodes_while_the_process_runs_on_another_card() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let serial = serial(); let facts = eternal_host::gpu_preference::facts().expect("graphics card facts"); let app_gpu = facts.running_on().expect("a hardware adapter").name.clone(); let started = Instant::now(); - let run = run_stage("h264_nvenc", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); + let run = run_stage(&serial, "h264_nvenc", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); assert!(!run.software_fallback, "NVENC fell back to the CPU"); assert!(run.codec_name.contains("NVENC"), "{}", run.codec_name); assert_keyframe_on_request(&run, Id::H264); @@ -388,8 +484,8 @@ fn nvenc_encodes_while_the_process_runs_on_another_card() { #[test] fn strict_decoder_rejects_broken_access_units() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("libx264", |_| SMALL, None); + let serial = serial(); + let run = run_stage(&serial, "libx264", |_| SMALL, None); let predicted = run .packets .iter() @@ -413,11 +509,11 @@ fn strict_decoder_rejects_broken_access_units() { #[test] fn amf_honours_keyframe_requests() { + let serial = serial(); let Some(expectation) = amf_expectation() else { return; }; - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("h264_amf", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); + let run = run_stage(&serial, "h264_amf", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); match expectation { AmfExpectation::Hardware => assert!( !run.software_fallback, @@ -460,33 +556,33 @@ fn amf_honours_keyframe_requests() { #[test] fn amf_follows_capture_mode_changes() { - if !amf_available("h264_amf") { + let serial = serial(); + if !amf_available(&serial, "h264_amf") { return; } - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("h264_amf", mode_changes, None); + let run = run_stage(&serial, "h264_amf", mode_changes, None); assert!(!run.software_fallback, "AMF fell back to libx264"); assert_follows_mode_changes(&run, Id::H264); } #[test] fn amf_hevc_follows_capture_mode_changes() { - if !amf_available("hevc_amf") { + let serial = serial(); + if !amf_available(&serial, "hevc_amf") { return; } - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("hevc_amf", mode_changes, None); + let run = run_stage(&serial, "hevc_amf", mode_changes, None); assert!(!run.software_fallback, "AMF fell back to libx264"); assert_follows_mode_changes(&run, Id::HEVC); } #[test] fn amf_hevc_honours_keyframe_requests() { - if !amf_available("hevc_amf") { + let serial = serial(); + if !amf_available(&serial, "hevc_amf") { return; } - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let run = run_stage("hevc_amf", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); + let run = run_stage(&serial, "hevc_amf", |_| LARGE, Some(KEYFRAME_REQUEST_AT)); assert_hevc_keyframe(&run.packets[0], "the first packet"); assert_keyframe_on_request(&run, Id::HEVC); } @@ -508,7 +604,22 @@ const WARM_UP_QUIET: Duration = Duration::from_millis(300); /// then requests a keyframe with one resent picture and nothing after it, as /// for a reconnecting viewer. Returns how long that picture's keyframe took /// to leave the stage, or `None` if it did not within [`STILL_WINDOW`]. -fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { +fn keyframe_on_a_still_desktop(serial: &Serial, encoder: &str) -> (Option, bool) { + for _ in 1..STAGE_TRIES { + match keyframe_on_a_still_desktop_once(serial, encoder) { + Some(result) => return result, + None => before_another_try(encoder), + } + } + keyframe_on_a_still_desktop_once(serial, encoder) + .unwrap_or_else(|| panic!("{encoder} was still opening when the first picture was due")) +} + +/// [`keyframe_on_a_still_desktop`] once, or None as [`run_stage_once`]. +fn keyframe_on_a_still_desktop_once( + serial: &Serial, + encoder: &str, +) -> Option<(Option, bool)> { ffmpeg_next::init().unwrap(); let (producer, consumer) = frame_slot(); let (tx, mut rx) = tokio::sync::mpsc::channel(NAL_CHANNEL_CAPACITY); @@ -555,7 +666,15 @@ fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { let mut seen = Vec::new(); loop { match packets.recv_timeout(WARM_UP_QUIET) { - Ok(packet) => seen.push(packet), + Ok(packet) => { + if seen.is_empty() && started_on_the_fallback(serial) { + producer.close(); + let _ = stage.join(); + let _ = collector.join(); + return None; + } + seen.push(packet); + } Err(std::sync::mpsc::RecvTimeoutError::Timeout) if !seen.is_empty() => break, Err(std::sync::mpsc::RecvTimeoutError::Timeout) => assert!( Instant::now() < warm_up_until, @@ -608,11 +727,11 @@ fn keyframe_on_a_still_desktop(encoder: &str) -> (Option, bool) { )) .collect::>() ); - (keyframe, software_fallback) + Some((keyframe, software_fallback)) } -fn assert_still_keyframe(encoder: &str) { - let (keyframe, software_fallback) = keyframe_on_a_still_desktop(encoder); +fn assert_still_keyframe(serial: &Serial, encoder: &str) { + let (keyframe, software_fallback) = keyframe_on_a_still_desktop(serial, encoder); assert!( encoder == "libx264" || !software_fallback, "{encoder} fell back to libx264" @@ -625,18 +744,18 @@ fn assert_still_keyframe(encoder: &str) { #[test] fn software_encoder_sends_a_requested_keyframe_on_a_still_desktop() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - assert_still_keyframe("libx264"); + let serial = serial(); + assert_still_keyframe(&serial, "libx264"); } #[test] fn amf_sends_a_requested_keyframe_on_a_still_desktop() { + let serial = serial(); for name in ["h264_amf", "hevc_amf"] { - if !amf_available(name) { + if !amf_available(&serial, name) { continue; } - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - assert_still_keyframe(name); + assert_still_keyframe(&serial, name); } } @@ -644,20 +763,24 @@ fn amf_sends_a_requested_keyframe_on_a_still_desktop() { #[test] #[ignore = "native: an NVIDIA card"] fn nvenc_sends_a_requested_keyframe_on_a_still_desktop() { - let _guard = STAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let serial = serial(); for name in ["h264_nvenc", "hevc_nvenc"] { - assert_still_keyframe(name); + assert_still_keyframe(&serial, name); } } #[test] fn amf_fails_cleanly_without_an_amd_driver() { + let serial = serial(); if amf_expectation() != Some(AmfExpectation::Fallback) { return; } // Without an AMD driver, both AMF encoders must fail to open rather than // hang or crash. The HEVC one is chosen at runtime for HEVC clients. for name in ["h264_amf", "hevc_amf"] { - assert!(!amf_opens(name), "{name} opened without an AMD driver"); + assert!( + !amf_opens(&serial, name), + "{name} opened without an AMD driver" + ); } } From 3b7ea7cac4d9f98b65d979c474e6731199752304 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 09:41:31 -0500 Subject: [PATCH 203/226] Offer a one-click fix for a Windows Firewall block on EternalMonitor The Stream page said to run Setup again when a Block rule for this exe, the kind Windows adds when its prompt is dismissed, kept iPads out. - The notice for that block now has Fix it. It starts this exe as administrator, with one Windows prompt, and that copy runs the firewall lines Setup runs, in the same order: remove the installer's rules and every inbound rule for this exe, then add the installer's Allow rule. Each line is fixed text with only this exe's own path in it, run with the system's netsh.exe and no shell. - The firewall is read again as soon as it ran, so the notice follows. A No to the Windows prompt changes nothing. A fix that didn't work says so in English under the notice. - A network that blocks all incoming connections gets no button: only the switch in Windows Security turns that off, as the notice says. - Fix it is English in the catalog, awaiting translation. --- host/src/firewall.rs | 48 +++- host/src/firewall_fix.rs | 456 ++++++++++++++++++++++++++++++++++ host/src/gui/page_stream.rs | 44 +++- host/src/gui/tests.rs | 11 +- host/src/i18n/strings/en.json | 1 + host/src/i18n/tests.rs | 10 +- host/src/lib.rs | 1 + host/src/main.rs | 4 + 8 files changed, 567 insertions(+), 8 deletions(-) create mode 100644 host/src/firewall_fix.rs diff --git a/host/src/firewall.rs b/host/src/firewall.rs index 60921c28..e2544b77 100644 --- a/host/src/firewall.rs +++ b/host/src/firewall.rs @@ -536,6 +536,32 @@ pub fn judge( static LATEST: Mutex> = Mutex::new(None); +/// Set to read the policy again now rather than at the next [`RECHECK`]. +static RECHECK_NOW: (Mutex, std::sync::Condvar) = + (Mutex::new(false), std::sync::Condvar::new()); + +/// Reads the policy again at once (after "Fix it" ran), on the thread +/// [`watch`] started. +pub fn recheck_now() { + let (wanted, wake) = &RECHECK_NOW; + if let Ok(mut wanted) = wanted.lock() { + *wanted = true; + wake.notify_all(); + } +} + +/// Waits up to `timeout`, or until [`recheck_now`]. +#[cfg(any(windows, test))] +fn wait_for_recheck(timeout: Duration) { + let (wanted, wake) = &RECHECK_NOW; + let Ok(guard) = wanted.lock() else { + return; + }; + if let Ok((mut guard, _)) = wake.wait_timeout_while(guard, timeout, |wanted| !*wanted) { + *guard = false; + } +} + /// The last reading, if the last read worked. `None` off Windows. pub fn latest() -> Option { LATEST.lock().ok()?.clone() @@ -592,7 +618,7 @@ pub fn watch(port: u16) { }); let read = windows_policy::read(&exe, port, lan).map_err(|e| e.to_string()); observe(read, &mut failed); - std::thread::sleep(RECHECK); + wait_for_recheck(RECHECK); } }); } @@ -1459,6 +1485,26 @@ mod tests { ); } + /// After "Fix it", the policy is read again at once, not at the next + /// minute's check. + #[test] + fn a_recheck_wakes_the_watch_at_once() { + let started = std::time::Instant::now(); + let waiting = std::thread::spawn(|| wait_for_recheck(Duration::from_secs(30))); + std::thread::sleep(Duration::from_millis(50)); + recheck_now(); + waiting.join().unwrap(); + assert!( + started.elapsed() < Duration::from_secs(10), + "{:?}", + started.elapsed() + ); + // Used up: the next wait waits its time. + let started = std::time::Instant::now(); + wait_for_recheck(Duration::from_millis(100)); + assert!(started.elapsed() >= Duration::from_millis(100)); + } + #[test] fn the_log_line_names_no_path_or_rule() { let scoped = Rule { diff --git a/host/src/firewall_fix.rs b/host/src/firewall_fix.rs new file mode 100644 index 00000000..9dda3690 --- /dev/null +++ b/host/src/firewall_fix.rs @@ -0,0 +1,456 @@ +//! The Stream page's "Fix it" for a Windows Firewall block on this exe. +//! +//! The block Windows adds when its "allow access" prompt is dismissed beats +//! the installer's Allow rule, and the only fix the PC offered was to run +//! Setup again. "Fix it" runs the same netsh lines Setup runs for the +//! firewall, in the same order, with one Windows prompt: this exe starts a +//! copy of itself as administrator with [`FIX_ARG`], and that copy runs +//! them and exits. Each line is a fixed string with only this exe's own +//! path in it, run with the system's netsh.exe and no shell, so nothing +//! anyone types reaches an elevated command line. Only rules for this exe +//! and the installer's own rule names are touched. +//! +//! A user who answers No to the Windows prompt changes nothing: the notice +//! stays as it was. After the copy ran, the firewall is read again at once. + +use std::path::Path; + +/// The argument the elevated copy runs with. +pub const FIX_ARG: &str = "--fix-firewall"; + +/// The netsh lines of the installer's `[Run]` section, as Setup runs them, +/// for this exe at `exe`. `{exe}` is the only thing filled in. +const SETUP_LINES: [&str; 4] = [ + r#"advfirewall firewall delete rule name="EternalMonitor Host UDP""#, + r#"advfirewall firewall delete rule name="EternalMonitor Host TCP""#, + r#"advfirewall firewall delete rule name=all dir=in program="{exe}""#, + r#"advfirewall firewall add rule name="EternalMonitor Host UDP" dir=in action=allow program="{exe}" protocol=UDP profile=domain,private,public enable=yes"#, +]; + +/// The netsh parameter lines the fix runs, in order, for this exe at `exe`. +/// None for a path that can't go inside quotes as it is. +pub fn command_lines(exe: &Path) -> Option> { + let exe = exe.to_str()?; + if exe.is_empty() || exe.contains('"') || exe.chars().any(char::is_control) { + return None; + } + Some( + SETUP_LINES + .iter() + .map(|line| line.replace("{exe}", exe)) + .collect(), + ) +} + +/// How starting the elevated copy went. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Elevated { + /// It ran and exited with this code. + Exited(u32), + /// The user answered No to the Windows prompt. + Declined, + /// Windows couldn't start it, or it didn't finish in time. + Failed(String), +} + +/// Starts this exe again as administrator with `arg`, and waits for it. +/// The real one asks Windows; tests fake it. +pub trait Elevate { + fn run_elevated(&self, exe: &Path, arg: &str) -> Elevated; +} + +/// What the Stream page shows after "Fix it". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Outcome { + /// The lines ran: the firewall was read again, and the notice follows it. + Ran, + /// No to the Windows prompt: nothing changed. + Declined, + /// The fix didn't run or didn't finish. English, as other errors are. + Failed(&'static str), +} + +/// Runs the fix: the elevated copy, then a fresh read of the firewall +/// (`recheck`), whatever it did, so the notice shows what is true now. +pub fn fix(elevate: &impl Elevate, exe: &Path, recheck: impl FnOnce()) -> Outcome { + if command_lines(exe).is_none() { + return Outcome::Failed(FAILED); + } + match elevate.run_elevated(exe, FIX_ARG) { + Elevated::Declined => Outcome::Declined, + Elevated::Exited(0) => { + recheck(); + Outcome::Ran + } + Elevated::Exited(_) | Elevated::Failed(_) => { + recheck(); + Outcome::Failed(FAILED) + } + } +} + +/// Where the Stream page's "Fix it" is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Progress { + #[default] + Idle, + /// The Windows prompt or the netsh lines are running. + Running, + Done(Outcome), +} + +static PROGRESS: std::sync::Mutex = std::sync::Mutex::new(Progress::Idle); + +pub fn progress() -> Progress { + PROGRESS.lock().map_or(Progress::Idle, |progress| *progress) +} + +/// Runs `fix` off the window's thread, once at a time: the Windows prompt +/// waits for the user, and netsh for Windows. +pub fn start(fix: impl FnOnce() -> Outcome + Send + 'static) { + { + let Ok(mut progress) = PROGRESS.lock() else { + return; + }; + if *progress == Progress::Running { + return; + } + *progress = Progress::Running; + } + let spawned = std::thread::Builder::new() + .name("firewall-fix".into()) + .spawn(move || { + let outcome = fix(); + tracing::info!(?outcome, "Firewall fix finished"); + if let Ok(mut progress) = PROGRESS.lock() { + *progress = Progress::Done(outcome); + } + }); + if spawned.is_err() { + if let Ok(mut progress) = PROGRESS.lock() { + *progress = Progress::Done(Outcome::Failed(FAILED)); + } + } +} + +/// The error under the notice when the fix didn't work. +pub const FAILED: &str = + "Windows didn't apply the fix. Run the EternalMonitor installer again instead."; + +/// The elevated copy's work: runs each line with `netsh` and returns the +/// exit code. Removing a rule that isn't there fails, as it does for Setup, +/// and doesn't count; only the Allow rule must be added. +pub fn run_lines(lines: &[String], mut netsh: impl FnMut(&str) -> std::io::Result) -> i32 { + let mut code = 0; + for line in lines { + let added = line.starts_with("advfirewall firewall add rule "); + match netsh(line) { + Ok(0) => {} + Ok(_) | Err(_) if !added => {} + Ok(failed) => code = failed.max(1), + Err(_) => code = 2, + } + } + code +} + +/// If this process is the elevated copy ([`FIX_ARG`]), runs the fix and +/// exits with its code. Called first thing in `main`. +pub fn run_if_requested() { + if std::env::args().nth(1).as_deref() != Some(FIX_ARG) { + return; + } + let code = std::env::current_exe() + .ok() + .and_then(|exe| command_lines(&exe)) + .map_or(3, |lines| run_lines(&lines, windows_netsh::run)); + std::process::exit(code); +} + +/// The fix as the Stream page runs it, on Windows. +#[cfg(windows)] +pub fn fix_now() -> Outcome { + let Ok(exe) = std::env::current_exe() else { + return Outcome::Failed(FAILED); + }; + fix(&windows_elevate::Shell, &exe, crate::firewall::recheck_now) +} + +/// Off Windows there is no firewall notice, so no fix. +#[cfg(not(windows))] +pub fn fix_now() -> Outcome { + Outcome::Failed(FAILED) +} + +#[cfg(windows)] +mod windows_netsh { + use std::os::windows::process::CommandExt; + + /// Runs the system's netsh.exe with `line` exactly as written, no shell. + pub fn run(line: &str) -> std::io::Result { + let mut system = [0u16; 260]; + let len = unsafe { + windows::Win32::System::SystemInformation::GetSystemDirectoryW(Some(&mut system)) + }; + if len == 0 || len as usize >= system.len() { + return Err(std::io::Error::other("no system folder")); + } + let netsh = std::path::PathBuf::from(String::from_utf16_lossy(&system[..len as usize])) + .join("netsh.exe"); + let status = std::process::Command::new(netsh) + .raw_arg(line) + .creation_flags(0x0800_0000) // CREATE_NO_WINDOW + .status()?; + Ok(status.code().unwrap_or(1)) + } +} + +#[cfg(not(windows))] +mod windows_netsh { + pub fn run(_line: &str) -> std::io::Result { + Err(std::io::Error::other("Windows only")) + } +} + +#[cfg(windows)] +mod windows_elevate { + use std::os::windows::ffi::OsStrExt; + use std::path::Path; + + use windows::core::PCWSTR; + use windows::Win32::Foundation::{CloseHandle, ERROR_CANCELLED, WAIT_OBJECT_0}; + use windows::Win32::System::Threading::{GetExitCodeProcess, WaitForSingleObject}; + use windows::Win32::UI::Shell::{ShellExecuteExW, SEE_MASK_NOCLOSEPROCESS, SHELLEXECUTEINFOW}; + use windows::Win32::UI::WindowsAndMessaging::SW_HIDE; + + use super::{Elevate, Elevated}; + + /// How long the copy may take: four netsh calls. + const WAIT_MS: u32 = 60_000; + + pub struct Shell; + + fn wide(text: &std::ffi::OsStr) -> Vec { + text.encode_wide().chain(Some(0)).collect() + } + + impl Elevate for Shell { + fn run_elevated(&self, exe: &Path, arg: &str) -> Elevated { + let verb = wide("runas".as_ref()); + let file = wide(exe.as_os_str()); + let params = wide(arg.as_ref()); + let mut info = SHELLEXECUTEINFOW { + cbSize: std::mem::size_of::() as u32, + fMask: SEE_MASK_NOCLOSEPROCESS, + lpVerb: PCWSTR(verb.as_ptr()), + lpFile: PCWSTR(file.as_ptr()), + lpParameters: PCWSTR(params.as_ptr()), + nShow: SW_HIDE.0, + ..Default::default() + }; + if let Err(error) = unsafe { ShellExecuteExW(&mut info) } { + return if error.code() == ERROR_CANCELLED.to_hresult() { + Elevated::Declined + } else { + Elevated::Failed(error.to_string()) + }; + } + if info.hProcess.is_invalid() { + return Elevated::Failed("no process".into()); + } + let elevated = unsafe { + if WaitForSingleObject(info.hProcess, WAIT_MS) != WAIT_OBJECT_0 { + Elevated::Failed("did not finish in time".into()) + } else { + let mut code = 0u32; + match GetExitCodeProcess(info.hProcess, &mut code) { + Ok(()) => Elevated::Exited(code), + Err(error) => Elevated::Failed(error.to_string()), + } + } + }; + unsafe { + let _ = CloseHandle(info.hProcess); + } + elevated + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::{Cell, RefCell}; + + const EXE: &str = r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"; + + /// The installer's [Run] netsh Parameters, as Setup passes them to + /// netsh, for the installed exe at `exe`. + fn setup_lines(exe: &str) -> Vec { + let iss = include_str!("../../installer/EternalMonitor.iss"); + let run = iss + .split("\n[Run]") + .nth(1) + .and_then(|section| section.split("\n[").next()) + .expect("EternalMonitor.iss has a [Run] section"); + run.lines() + .filter(|line| !line.trim_start().starts_with(';') && line.contains(r"\netsh.exe")) + .map(|line| { + line.split("Parameters: \"") + .nth(1) + .and_then(|rest| rest.split("\"; ").next()) + .expect("a netsh line has Parameters") + .replace("\"\"", "\"") + .replace(r"{app}\EternalMonitor-host.exe", exe) + }) + .collect() + } + + /// The fix runs exactly what Setup runs, in the same order. + #[test] + fn the_fix_runs_the_lines_setup_runs() { + let lines = command_lines(Path::new(EXE)).unwrap(); + assert_eq!(lines, setup_lines(EXE)); + assert_eq!(lines.len(), 4); + // Only this exe's rules and the installer's own names. + for line in &lines { + assert!( + line.contains(r#"name="EternalMonitor Host "#) + || line.contains(&format!(r#"program="{EXE}""#)), + "{line}" + ); + } + // A copy elsewhere fixes its own rules. + let other = r"D:\Apps\EM\EternalMonitor-host.exe"; + assert!( + command_lines(Path::new(other)).unwrap()[2].ends_with(&format!(r#"program="{other}""#)) + ); + } + + /// A path that would break out of its quotes never reaches netsh. + #[test] + fn a_path_that_cant_be_quoted_is_refused() { + for bad in [r#"C:\a"b\x.exe"#, "C:\\a\nb\\x.exe", ""] { + assert_eq!(command_lines(Path::new(bad)), None, "{bad:?}"); + } + } + + struct Fake { + answer: Elevated, + asked: RefCell>, + } + + impl Elevate for Fake { + fn run_elevated(&self, exe: &Path, arg: &str) -> Elevated { + self.asked + .borrow_mut() + .push((exe.display().to_string(), arg.to_string())); + self.answer.clone() + } + } + + fn fake(answer: Elevated) -> Fake { + Fake { + answer, + asked: RefCell::new(Vec::new()), + } + } + + /// The elevated copy is this exe with the fixed argument, and the + /// firewall is read again once it ran. + #[test] + fn the_fix_starts_this_exe_elevated_then_reads_the_firewall_again() { + let elevate = fake(Elevated::Exited(0)); + let rechecked = Cell::new(false); + assert_eq!( + fix(&elevate, Path::new(EXE), || rechecked.set(true)), + Outcome::Ran + ); + assert!(rechecked.get()); + assert_eq!( + *elevate.asked.borrow(), + [(EXE.to_string(), FIX_ARG.to_string())] + ); + } + + /// No to the Windows prompt: nothing ran, nothing is read again, and no + /// error shows. + #[test] + fn declining_the_windows_prompt_changes_nothing() { + let elevate = fake(Elevated::Declined); + let rechecked = Cell::new(false); + assert_eq!( + fix(&elevate, Path::new(EXE), || rechecked.set(true)), + Outcome::Declined + ); + assert!(!rechecked.get()); + } + + /// A copy that failed, or that Windows couldn't start: the error, and + /// the firewall read again in case some lines ran. + #[test] + fn a_fix_that_fails_says_so() { + for answer in [Elevated::Exited(1), Elevated::Failed("x".into())] { + let rechecked = Cell::new(false); + assert_eq!( + fix(&fake(answer.clone()), Path::new(EXE), || rechecked + .set(true)), + Outcome::Failed(FAILED), + "{answer:?}" + ); + assert!(rechecked.get(), "{answer:?}"); + } + let elevate = fake(Elevated::Exited(0)); + assert_eq!(fix(&elevate, Path::new(""), || {}), Outcome::Failed(FAILED)); + assert!( + elevate.asked.borrow().is_empty(), + "nothing elevated for a bad path" + ); + } + + /// The copy runs every line in order. A rule to remove that isn't there + /// doesn't count; the Allow rule that isn't added does. + #[test] + fn the_elevated_copy_runs_every_line_and_fails_only_on_the_allow_rule() { + let lines = command_lines(Path::new(EXE)).unwrap(); + let ran = RefCell::new(Vec::new()); + let code = run_lines(&lines, |line| { + ran.borrow_mut().push(line.to_string()); + Ok(if line.contains("delete") { 1 } else { 0 }) + }); + assert_eq!(code, 0); + assert_eq!(*ran.borrow(), lines); + assert_ne!( + run_lines(&lines, |line| Ok(i32::from(line.contains(" add ")))), + 0 + ); + assert_ne!( + run_lines(&lines, |line| if line.contains(" add ") { + Err(std::io::Error::other("x")) + } else { + Ok(0) + }), + 0 + ); + } + + /// One fix at a time, and the window hears how it went. + #[test] + fn the_page_sees_the_fix_run_and_finish() { + let (go, wait) = std::sync::mpsc::channel::<()>(); + start(move || { + let _ = wait.recv(); + Outcome::Declined + }); + assert_eq!(progress(), Progress::Running); + // A second click while it runs starts nothing. + start(|| panic!("a second fix started")); + go.send(()).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while progress() == Progress::Running { + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert_eq!(progress(), Progress::Done(Outcome::Declined)); + } +} diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index e9d4170d..a0185565 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -56,10 +56,22 @@ impl AnalyzerApp { /// Update, restart and fault notices, most urgent first. fn notices(&mut self, ui: &mut Ui, snap: &StatsSnapshot, client: &ClientSnapshot) { self.port_notice(ui); - if let Some((title, body)) = - firewall_notice(crate::firewall::latest().as_ref(), client.peer.is_some()) - { - widgets::banner(ui, WARNING, Icon::Warning, &title, &body, 0.0, |_| {}); + let firewall = crate::firewall::latest(); + if let Some((title, mut body)) = firewall_notice(firewall.as_ref(), client.peer.is_some()) { + let progress = crate::firewall_fix::progress(); + if let crate::firewall_fix::Progress::Done(crate::firewall_fix::Outcome::Failed( + error, + )) = progress + { + body = format!("{body} {error}"); + } + let fixable = firewall_fix_offered(firewall.as_ref()); + let width = if fixable { 110.0 } else { 0.0 }; + widgets::banner(ui, WARNING, Icon::Warning, &title, &body, width, |ui| { + if fixable { + firewall_fix_action(ui, progress); + } + }); ui.add_space(GAP - 10.0); } if let Some((title, body)) = usb_cable_notice( @@ -1397,6 +1409,30 @@ pub(super) fn usb_cable_notice( }) } +/// "Fix it" on the firewall notice: for this exe's own Block rule, which +/// Setup's firewall lines remove, not for "Blocks all incoming +/// connections", which only the user's switch in Windows Security turns off. +pub(super) fn firewall_fix_offered(report: Option<&crate::firewall::Report>) -> bool { + matches!( + report.and_then(crate::firewall::Report::certain_block), + Some(crate::firewall::CertainBlock::Rule { .. }) + ) +} + +/// Runs the fix; it asks Windows for administrator rights once. +fn firewall_fix_action(ui: &mut Ui, progress: crate::firewall_fix::Progress) { + let running = progress == crate::firewall_fix::Progress::Running; + let clicked = ui + .add_enabled_ui(!running, |ui| { + widgets::button(ui, Tone::Primary, None, tr("Fix it")) + }) + .inner + .clicked(); + if clicked { + crate::firewall_fix::start(crate::firewall_fix::fix_now); + } +} + /// What the Stream page says while Windows Firewall keeps iPads out: only /// for a block certain on the network an iPad would use that Setup or one /// switch removes (see [`crate::firewall::Report::certain_block`]), and diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index ab0c7b49..e9a196ff 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -1,6 +1,6 @@ use super::page_stream::{ - audio_summary, capture_block_notice, firewall_notice, second_screen_notice, - software_fallback_body, usb_cable_notice, usb_summary, + audio_summary, capture_block_notice, firewall_fix_offered, firewall_notice, + second_screen_notice, software_fallback_body, usb_cable_notice, usb_summary, }; use super::*; @@ -26,6 +26,8 @@ fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { ); let (title, body) = firewall_notice(Some(&blocked), false).unwrap(); assert_eq!(title, "Windows Firewall is blocking EternalMonitor"); + // This exe's own Block rule: Setup's firewall lines remove it. + assert!(firewall_fix_offered(Some(&blocked))); assert!( body.contains("Run the EternalMonitor installer again"), "{body}" @@ -39,6 +41,9 @@ fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { let shields = report(&[profile(PUBLIC, true)], std::slice::from_ref(&allow), true); let (title, body) = firewall_notice(Some(&shields), false).unwrap(); assert_eq!(title, "Windows Firewall blocks every incoming connection"); + // Only the user's switch turns that off: no button. + assert!(!firewall_fix_offered(Some(&shields))); + assert!(!firewall_fix_offered(None)); assert!(body.contains("choose the public network"), "{body}"); let both = report( &[profile(PRIVATE, false), profile(PUBLIC, true)], @@ -51,6 +56,8 @@ fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { body.contains("installer again") && body.contains("choose the public network"), "{body}" ); + // The rule part has the button; the switch keeps its instructions. + assert!(firewall_fix_offered(Some(&both))); // A rule for every program: Setup can't remove it, so it is only // logged, alone or beside this exe's own block. diff --git a/host/src/i18n/strings/en.json b/host/src/i18n/strings/en.json index 4e691899..d434a5fb 100644 --- a/host/src/i18n/strings/en.json +++ b/host/src/i18n/strings/en.json @@ -28,6 +28,7 @@ "EternalMonitor runs in the background so your iPad can find this PC.": "EternalMonitor runs in the background so your iPad can find this PC.", "Excellent": "Excellent", "Fair": "Fair", + "Fix it": "Fix it", "Forget all iPads": "Forget all iPads", "Frame rate": "Frame rate", "General": "General", diff --git a/host/src/i18n/tests.rs b/host/src/i18n/tests.rs index 94a3db96..6751f2ee 100644 --- a/host/src/i18n/tests.rs +++ b/host/src/i18n/tests.rs @@ -22,17 +22,25 @@ fn placeholders(text: &str) -> Vec<&str> { found } +/// English keys whose translations come in a separate pass. Until then +/// every language shows them in English. A language that has one already +/// is checked like any other key. +const AWAITING_TRANSLATION: &[&str] = &["Fix it"]; + #[test] fn every_language_has_exactly_the_english_keys_and_placeholders() { let english = strings(Lang::En); for (key, value) in &english { assert_eq!(key, value, "en.json maps each key to itself"); } + for key in AWAITING_TRANSLATION { + assert!(english.contains_key(*key), "{key:?} is not an English key"); + } for lang in Lang::ALL { let translated = strings(lang); let missing: Vec<_> = english .keys() - .filter(|k| !translated.contains_key(*k)) + .filter(|k| !translated.contains_key(*k) && !AWAITING_TRANSLATION.contains(&k.as_str())) .collect(); let extra: Vec<_> = translated .keys() diff --git a/host/src/lib.rs b/host/src/lib.rs index 1e91fcbf..c2d4300e 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -13,6 +13,7 @@ pub mod display_guard; pub mod encoder; pub mod feedback; pub mod firewall; +pub mod firewall_fix; pub mod gpu_preference; pub mod gui; pub mod handoff; diff --git a/host/src/main.rs b/host/src/main.rs index fb76194a..f9ec2ad9 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -14,6 +14,10 @@ use tracing_subscriber::prelude::*; use tracing_subscriber::EnvFilter; fn main() -> Result<(), Box> { + // The Stream page's "Fix it" for a Windows Firewall block starts this + // exe as administrator to run Setup's firewall lines, and nothing else: + // no settings, log, window or single-instance check. + eternal_host::firewall_fix::run_if_requested(); // Before anything below creates the settings folder or a file in it: a // folder that held nothing means a fresh install, which opens first-run // setup. Upgrades never see it. From 14d3a28d70eb6259a3155bd095eda773e6b96a7e Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 09:46:26 -0500 Subject: [PATCH 204/226] Ask every listing of the PC in turn Each round asked only the first listing that hadn't had its try, and one that never proved the pairing stayed first. Another PC at the saved address that sorts first kept the PC's own listing at its new address from ever being asked. Each round now asks every listing that hasn't had its try, in turn, then waits the usual five seconds. A listing that doesn't answer stays for the next round; one that proves the pairing gets the one try. --- .../App/ConnectionManager.swift | 24 +++++---- ios/EternalMonitor/App/PCReturn.swift | 19 +++++-- ios/EternalMonitorTests/PCReturnTests.swift | 51 +++++++++++++++---- 3 files changed, 69 insertions(+), 25 deletions(-) diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index dbea18e0..7dc39439 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -1612,9 +1612,10 @@ final class ConnectionManager: ObservableObject { && !watch.expired(at: now(), limit: timing.pcReturnLimitSeconds) } - /// Asks the PC's listings after `delay`, so a few Bonjour updates count - /// as one, then every few seconds, until one proves the pairing and has - /// its try or none is left to ask. A newer call starts over. + /// Asks each of the PC's listings in turn after `delay`, so a few + /// Bonjour updates count as one, then again every few seconds, until one + /// proves the pairing and has its try or none is left to ask. A newer + /// call starts over. private func askPCReturn(after delay: Double) { guard let watch = pcReturn, watch.armedAt != nil else { return } guard !watch.expired(at: now(), limit: timing.pcReturnLimitSeconds) else { @@ -1626,13 +1627,16 @@ final class ConnectionManager: ObservableObject { var wait = delay while true { try? await Task.sleep(for: .seconds(wait)) - guard !Task.isCancelled, let self, self.pcReturnMayTry, let pc = self.pcReturn?.candidate else { return } - let proved = await self.provePCReturn(pc) - guard !Task.isCancelled, self.pcReturnMayTry else { return } - if proved { - self.pcReturnTry = nil - self.pcReturn?.didTry(pc) - return self.tryPCAgain(at: pc) + guard !Task.isCancelled, let self, self.pcReturnMayTry, + let round = self.pcReturn?.candidates, !round.isEmpty else { return } + for pc in round where self.pcReturn?.awaits(pc) == true { + let proved = await self.provePCReturn(pc) + guard !Task.isCancelled, self.pcReturnMayTry else { return } + if proved { + self.pcReturnTry = nil + self.pcReturn?.didTry(pc) + return self.tryPCAgain(at: pc) + } } wait = self.timing.pcReturnProbeSeconds } diff --git a/ios/EternalMonitor/App/PCReturn.swift b/ios/EternalMonitor/App/PCReturn.swift index ea63fce7..8cd80776 100644 --- a/ios/EternalMonitor/App/PCReturn.swift +++ b/ios/EternalMonitor/App/PCReturn.swift @@ -43,8 +43,10 @@ final class BonjourPCBrowser: PCBrowser { /// the PC is asked to prove it holds the saved pairing: at once when it /// turns up or changes, then every few seconds, because a PC that closed /// without saying goodbye stays listed, unchanged, after it opens again. -/// A listing that proves it gets one try; only a new or changed one gets -/// another. +/// Every such listing is asked in turn, so one that never answers, such as +/// another PC at the saved address, can't keep the PC from being asked +/// where it is now. A listing that proves it gets one try; only a new or +/// changed one gets another. struct PCReturn { /// The PC's name, when this iPad knows it: its Bonjour service is the /// PC, wherever it is. The service at the address tried is the PC too. @@ -87,9 +89,16 @@ struct PCReturn { return !new.isEmpty } - /// The listing to ask next: one that hasn't had its try. - var candidate: DiscoveredHost? { - listed.keys.sorted().first { !tried.contains($0) }.flatMap { listed[$0] } + /// The listings to ask, each in its turn: those that haven't had their + /// try. One that doesn't answer stays among them. + var candidates: [DiscoveredHost] { + listed.keys.sorted().filter { !tried.contains($0) }.compactMap { listed[$0] } + } + + /// Whether `pc` is still listed and hasn't had its try. + func awaits(_ pc: DiscoveredHost) -> Bool { + let key = Self.key(pc) + return listed[key] != nil && !tried.contains(key) } /// `pc` had its try, or can't have one. diff --git a/ios/EternalMonitorTests/PCReturnTests.swift b/ios/EternalMonitorTests/PCReturnTests.swift index b81d005a..a7ac524c 100644 --- a/ios/EternalMonitorTests/PCReturnTests.swift +++ b/ios/EternalMonitorTests/PCReturnTests.swift @@ -45,22 +45,37 @@ final class PCReturnTests: XCTestCase { XCTAssertTrue(watch.heard([listed]), "asked at once") // Not proved yet: asked again, every few seconds. XCTAssertFalse(watch.heard([listed])) - XCTAssertEqual(watch.candidate, listed) + XCTAssertEqual(watch.candidates.first, listed) watch.didTry(listed) - XCTAssertNil(watch.candidate, "one try per listing") + XCTAssertNil(watch.candidates.first, "one try per listing") XCTAssertFalse(watch.heard([listed])) - XCTAssertNil(watch.candidate) + XCTAssertNil(watch.candidates.first) // It leaves (the app closed) and comes back. XCTAssertFalse(watch.heard([])) XCTAssertTrue(watch.heard([listed])) - XCTAssertEqual(watch.candidate, listed) + XCTAssertEqual(watch.candidates.first, listed) watch.didTry(listed) // A change is a new listing, and the try goes where the PC is now. XCTAssertTrue(watch.heard([pc(port: 9877)])) - XCTAssertEqual(watch.candidate?.port, 9877, "a new port") + XCTAssertEqual(watch.candidates.first?.port, 9877, "a new port") watch.didTry(pc(port: 9877)) XCTAssertTrue(watch.heard([pc(port: 9877, version: "0.3.6")])) - XCTAssertEqual(watch.candidate?.version, "0.3.6", "a new version") + XCTAssertEqual(watch.candidates.first?.version, "0.3.6", "a new version") + } + + /// A listing that never answers stays to be asked, beside the others. + func testEveryUntriedListingIsAsked() { + var watch = PCReturn(name: "Test PC", host: "192.0.2.20", port: 9876) + let stale = pc("EternalMonitor (Another PC)", "192.0.2.20", port: 9876) + let moved = pc("EternalMonitor (Test PC)", "192.0.2.40", port: 9876) + XCTAssertTrue(watch.heard([stale, moved])) + XCTAssertEqual(watch.candidates, [stale, moved]) + XCTAssertTrue(watch.awaits(moved)) + watch.didTry(moved) + XCTAssertEqual(watch.candidates, [stale]) + XCTAssertFalse(watch.awaits(moved)) + XCTAssertFalse(watch.heard([stale])) + XCTAssertFalse(watch.awaits(moved), "no longer listed") } func testOnlyThisPCCounts() { @@ -72,13 +87,13 @@ final class PCReturnTests: XCTestCase { pc("EternalMonitor (Test PC)", nil, port: 9876)] XCTAssertFalse(named.heard(others)) XCTAssertFalse(unnamed.heard(others)) - XCTAssertNil(named.candidate) - XCTAssertNil(unnamed.candidate) + XCTAssertNil(named.candidates.first) + XCTAssertNil(unnamed.candidates.first) // By name at another address, or by the address tried. XCTAssertTrue(named.heard(others + [pc("EternalMonitor (Test PC)", "192.0.2.40", port: 9876)])) - XCTAssertEqual(named.candidate?.address, "192.0.2.40") + XCTAssertEqual(named.candidates.first?.address, "192.0.2.40") XCTAssertTrue(unnamed.heard(others + [pc("EternalMonitor (Any)", "192.0.2.20", port: 9876)])) - XCTAssertEqual(unnamed.candidate?.name, "EternalMonitor (Any)") + XCTAssertEqual(unnamed.candidates.first?.name, "EternalMonitor (Any)") } func testTheTimeLimitStartsWhenTheCardGoesUp() { @@ -254,6 +269,22 @@ final class PCReturnTests: XCTestCase { XCTAssertEqual(try setup.pairings.token(address: PairingStore.address(host: "127.0.0.1", port: moved.port)), Self.token) } + /// Another PC sits at the saved address and never answers; Bonjour has + /// the saved PC at a new port too, and that listing sorts after it. Both + /// are asked in turn, so the PC streams from its new port. + func testASilentListingDoesNotKeepThePCFromBeingAsked() async throws { + let setup = try await noReply() + defer { setup.manager.cancel() } + let moved = FakeUDPHost() + let stale = DiscoveredHost(name: "EternalMonitor (Another PC)", address: "127.0.0.1", port: setup.host.port) + let pc = DiscoveredHost(name: "EternalMonitor (Test PC)", address: "127.0.0.1", port: moved.port) + setup.browser.list([stale, pc]) + try await stream(setup, from: moved) + let atStale = await Task.detached { [host = setup.host] in host.received(within: 0.3) }.value + XCTAssertFalse(atStale.contains(.hello2), "\(atStale)") + XCTAssertEqual(setup.tries, 1) + } + /// Another PC took the saved address, with pairing off: it can't prove /// the pairing, so nothing is sent to it. A session that gets past that /// from a PC with another name, or that gives another pairing back, is From e40494a76a787c2805d5753e98116946d97d013f Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 10:04:28 -0500 Subject: [PATCH 205/226] Quit when an installer closes the app, and keep the power window Review round 2. - After Restart Manager's end of session with ENDSESSION_CLOSEAPP (also with ENDSESSION_CRITICAL), the stream ends as before and the app then takes its quit path, the one closing the window takes, on a thread of its own, so the window answers at once. - A WM_CLOSE to the hidden window takes the same quit path, once, and never destroys the window: until the process ends it still hears sleep and shutdown, which it could not open again. - The window test sends the end of session for an installer and then WM_CLOSE to the test's own window: the app quits once, and the window stays and still answers. --- host/src/main.rs | 17 ++++-- host/src/power.rs | 112 +++++++++++++++++++++++++++++++++--- host/tests/power_goodbye.rs | 59 +++++++++++++++++-- 3 files changed, 171 insertions(+), 17 deletions(-) diff --git a/host/src/main.rs b/host/src/main.rs index c3735b1d..27b56e28 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -241,10 +241,6 @@ fn main() -> Result<(), Box> { shared: shared.clone(), supervisor_tx: supervisor_tx.clone(), }; - // Sleep, shutdown and sign-out end the iPad's session with a goodbye. - #[cfg(windows)] - eternal_host::power::watch(shared.clone()); - /// Set by the termination-signal handler; the headless loop polls it. static SHUTDOWN_REQUESTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); @@ -341,6 +337,19 @@ fn main() -> Result<(), Box> { }, )))); + // Sleep, shutdown and sign-out end the iPad's session with a goodbye. An + // installer closing the app takes the quit path, as closing the window does. + #[cfg(windows)] + eternal_host::power::watch(gui_control.shared.clone(), { + let quit_once = std::sync::Arc::clone(&quit_once); + Box::new(move || { + let quit = quit_once.lock().take(); + if let Some(quit) = quit { + quit(None); + } + }) + }); + // Headless mode for automation (the iOS E2E harness runs the host without // a window): stream until SIGTERM/SIGINT, then fall through to the same // clean teardown as a GUI quit — mDNS goodbye, VDD disable, bounded diff --git a/host/src/power.rs b/host/src/power.rs index 57e02251..f337af86 100644 --- a/host/src/power.rs +++ b/host/src/power.rs @@ -72,6 +72,25 @@ pub enum Message { }, /// WM_WTSSESSION_CHANGE WTS_SESSION_LOGOFF. SessionLogoff, + /// WM_CLOSE: someone asks the app to close, as Restart Manager does + /// after the end of session when the app still runs. + Close, +} + +/// Whether the app quits once `message` is answered, the way it does when +/// its window is closed: Restart Manager closing it for an installer, or a +/// WM_CLOSE. The hidden window stays meanwhile, so sleep and shutdown are +/// still heard should the quit not end the process. +pub fn quits(message: Message) -> bool { + matches!( + message, + Message::Close + | Message::EndSession { + ending: true, + logoff: false, + close_app: true, + } + ) } /// Longest the end-of-session message waits for the session summary to @@ -122,6 +141,8 @@ pub fn handle(shared: &SharedControl, message: Message) -> isize { going_down(shared, PowerEvent::Logoff, END_SESSION_WAIT); 0 } + // The quit ends the stream ([`quits`]). + Message::Close => 0, } } @@ -275,10 +296,12 @@ fn back_up(shared: &SharedControl, why: &'static str) { } /// Starts listening for Windows' sleep, shutdown and sign-out messages, on a -/// hidden window of its own. Later calls do nothing. +/// hidden window of its own. `quit` is the app's quit path, taken on a +/// thread of its own after a message that [`quits`]; it runs once. Later +/// calls do nothing. #[cfg(windows)] -pub fn watch(shared: SharedControl) { - windows_impl::watch(shared); +pub fn watch(shared: SharedControl, quit: Box) { + windows_impl::watch(shared, quit); } /// The hidden window's handle, once it exists, for tests that send it @@ -291,7 +314,7 @@ pub fn window() -> Option { #[cfg(windows)] mod windows_impl { use std::sync::atomic::{AtomicIsize, Ordering}; - use std::sync::OnceLock; + use std::sync::{Mutex, OnceLock}; use windows::core::w; use windows::Win32::Foundation::{HANDLE, HINSTANCE, HWND, LPARAM, LRESULT, WPARAM}; @@ -304,8 +327,8 @@ mod windows_impl { CreateWindowExW, DefWindowProcW, DispatchMessageW, GetMessageW, RegisterClassW, TranslateMessage, DEVICE_NOTIFY_WINDOW_HANDLE, ENDSESSION_CLOSEAPP, ENDSESSION_LOGOFF, MSG, PBT_APMRESUMEAUTOMATIC, PBT_APMRESUMESUSPEND, PBT_APMSUSPEND, WINDOW_EX_STYLE, - WINDOW_STYLE, WM_ENDSESSION, WM_POWERBROADCAST, WM_QUERYENDSESSION, WM_WTSSESSION_CHANGE, - WNDCLASSW, WTS_SESSION_LOGOFF, + WINDOW_STYLE, WM_CLOSE, WM_ENDSESSION, WM_POWERBROADCAST, WM_QUERYENDSESSION, + WM_WTSSESSION_CHANGE, WNDCLASSW, WTS_SESSION_LOGOFF, }; use super::Message; @@ -313,11 +336,15 @@ mod windows_impl { static SHARED: OnceLock = OnceLock::new(); static WINDOW: AtomicIsize = AtomicIsize::new(0); + /// The app's quit path, until a message takes it. + #[allow(clippy::type_complexity)] + static QUIT: Mutex>> = Mutex::new(None); - pub(super) fn watch(shared: SharedControl) { + pub(super) fn watch(shared: SharedControl, quit: Box) { if SHARED.set(shared).is_err() { return; } + *QUIT.lock().unwrap_or_else(|e| e.into_inner()) = Some(quit); let started = std::thread::Builder::new() .name("power-watch".into()) .spawn(|| { @@ -396,6 +423,7 @@ mod windows_impl { close_app: lparam as u32 & ENDSESSION_CLOSEAPP != 0, }, WM_WTSSESSION_CHANGE if wparam as u32 == WTS_SESSION_LOGOFF => Message::SessionLogoff, + WM_CLOSE => Message::Close, _ => return None, }) } @@ -407,14 +435,38 @@ mod windows_impl { lparam: LPARAM, ) -> LRESULT { match (message(kind, wparam.0, lparam.0), SHARED.get()) { - (Some(message), Some(shared)) => LRESULT(super::handle(shared, message)), + (Some(message), Some(shared)) => { + let answer = super::handle(shared, message); + if super::quits(message) { + quit(message); + } + LRESULT(answer) + } _ => DefWindowProcW(hwnd, kind, wparam, lparam), } } + /// Takes the app's quit path on a thread of its own, so the window + /// answers Windows at once; the first time only. The window is never + /// destroyed here: until the process ends, it still hears Windows. + fn quit(message: Message) { + let Some(quit) = QUIT.lock().unwrap_or_else(|e| e.into_inner()).take() else { + tracing::debug!(?message, "Windows asked again for the app to close"); + return; + }; + tracing::info!(?message, "Windows asked the app to close — quitting"); + if let Err(error) = std::thread::Builder::new() + .name("power-quit".into()) + .spawn(quit) + { + tracing::warn!(%error, "Could not start quitting for Windows"); + } + } + #[cfg(test)] mod tests { use super::*; + use windows::Win32::UI::WindowsAndMessaging::ENDSESSION_CRITICAL; #[test] fn window_messages_map_to_what_windows_means() { @@ -450,7 +502,8 @@ mod windows_impl { close_app: false, }), ), - // Restart Manager closing the app for an installer. + // Restart Manager closing the app for an installer, also + // when it must. ( WM_ENDSESSION, 1, @@ -461,6 +514,17 @@ mod windows_impl { close_app: true, }), ), + ( + WM_ENDSESSION, + 1, + (ENDSESSION_CLOSEAPP | ENDSESSION_CRITICAL) as i32 as isize, + Some(Message::EndSession { + ending: true, + logoff: false, + close_app: true, + }), + ), + (WM_CLOSE, 0, 0, Some(Message::Close)), ( WM_ENDSESSION, 1, @@ -500,3 +564,33 @@ mod windows_impl { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Only an installer closing the app, or a WM_CLOSE, quits it; the PC + /// going down, or a sign-out, leaves that to Windows. + #[test] + fn closing_the_app_quits_it_and_the_pc_going_down_does_not() { + let end = |ending, logoff, close_app| Message::EndSession { + ending, + logoff, + close_app, + }; + assert!(quits(Message::Close)); + assert!(quits(end(true, false, true))); + for message in [ + end(true, false, false), + end(true, true, false), + end(true, true, true), + end(false, false, true), + Message::Suspend, + Message::Resume, + Message::QueryEndSession, + Message::SessionLogoff, + ] { + assert!(!quits(message), "{message:?}"); + } + } +} diff --git a/host/tests/power_goodbye.rs b/host/tests/power_goodbye.rs index f095a86f..8e3eb5e4 100644 --- a/host/tests/power_goodbye.rs +++ b/host/tests/power_goodbye.rs @@ -347,19 +347,27 @@ fn an_installer_closing_the_app_ends_the_stream_as_a_quit() { } } -/// The real window, given the message Windows sends before sleep. Windows -/// only: it checks the window gets Windows' messages and answers them. +/// The real window, given the message Windows sends before sleep, then +/// those Restart Manager sends to close the app for an installer. Windows +/// only: it checks the window gets Windows' messages and answers them, and +/// that closing the app takes the app's quit path once while the window +/// stays. #[cfg(windows)] #[test] fn the_hidden_window_turns_windows_messages_into_goodbyes() { use windows::Win32::Foundation::{HWND, LPARAM, WPARAM}; use windows::Win32::UI::WindowsAndMessaging::{ - SendMessageW, PBT_APMRESUMEAUTOMATIC, PBT_APMSUSPEND, WM_POWERBROADCAST, + IsWindow, SendMessageW, ENDSESSION_CLOSEAPP, PBT_APMRESUMEAUTOMATIC, PBT_APMSUSPEND, + WM_CLOSE, WM_ENDSESSION, WM_POWERBROADCAST, }; let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let rig = Rig::start(FEATURE_POWER_BYE); - power::watch(rig.shared.clone()); + let (quit_tx, quits) = std::sync::mpsc::channel(); + power::watch( + rig.shared.clone(), + Box::new(move || quit_tx.send(()).unwrap()), + ); let started = Instant::now(); let hwnd = loop { if let Some(hwnd) = power::window() { @@ -394,4 +402,47 @@ fn the_hidden_window_turns_windows_messages_into_goodbyes() { }; assert_eq!(answer.0, 1); assert_eq!(rig.ask(2, Duration::from_secs(10)), Some(HelloStatus::Ok)); + + // Restart Manager closes the app: the quit goodbye, then the quit path. + let answer = unsafe { + SendMessageW( + hwnd, + WM_ENDSESSION, + WPARAM(1), + LPARAM(ENDSESSION_CLOSEAPP as isize), + ) + }; + assert_eq!(answer.0, 0); + assert_eq!( + rig.goodbyes(Duration::from_millis(500)), + [ByeReason::UserDisconnect; 3] + ); + assert_eq!( + quits.recv_timeout(Duration::from_secs(5)), + Ok(()), + "the app quits" + ); + // Its WM_CLOSE that follows neither quits again nor destroys the window, + // which still hears Windows. + let answer = unsafe { SendMessageW(hwnd, WM_CLOSE, WPARAM(0), LPARAM(0)) }; + assert_eq!(answer.0, 0); + assert!(unsafe { IsWindow(hwnd) }.as_bool(), "the window stays"); + assert!(quits.recv_timeout(Duration::from_millis(500)).is_err()); + let answer = unsafe { + SendMessageW( + hwnd, + WM_POWERBROADCAST, + WPARAM(PBT_APMSUSPEND as usize), + LPARAM(0), + ) + }; + assert_eq!(answer.0, 1, "still answers Windows"); + unsafe { + SendMessageW( + hwnd, + WM_POWERBROADCAST, + WPARAM(PBT_APMRESUMEAUTOMATIC as usize), + LPARAM(0), + ) + }; } From 887d65d0965822980948992daca68d169e52a9a6 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 10:15:33 -0500 Subject: [PATCH 206/226] Clear the renderer marker when Windows closes the app Review round 3. When Windows closes the app for an installer, or sends the hidden window WM_CLOSE, while the window is still starting, the quit left this launch's renderer marker behind, and the next launch counted it as a failed start and avoided that backend. Before taking the quit path, the power watcher now clears the marker without counting, as a quit from the graphics driver notice does (gui::quitting_from_outside), and a probe still in the driver can't save it again. --- host/src/gui/mod.rs | 9 +++++++++ host/src/gui/renderer.rs | 22 +++++++++++++++++++++- host/src/main.rs | 1 + 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/host/src/gui/mod.rs b/host/src/gui/mod.rs index 79c9e882..670017eb 100644 --- a/host/src/gui/mod.rs +++ b/host/src/gui/mod.rs @@ -1787,6 +1787,15 @@ pub fn shown_address(ip: std::net::Ipv4Addr, listen_port: u16) -> String { } } +/// The app quits from outside its window, as when Windows closes it for an +/// installer, perhaps before the window is confirmed drawing. As for a quit +/// from the graphics driver notice, this launch's renderer marker goes +/// without counting, so the next launch avoids no backend, and nothing still +/// in the driver saves it again. +pub fn quitting_from_outside() { + renderer::abandon(); +} + /// Launch the GUI window. Blocks the calling thread. /// Runs the window until it closes. `quit_while_stuck` is the app's quit /// path, for a Cancel in the graphics driver notice while the window's diff --git a/host/src/gui/renderer.rs b/host/src/gui/renderer.rs index 64a3e3d5..b2403584 100644 --- a/host/src/gui/renderer.rs +++ b/host/src/gui/renderer.rs @@ -782,7 +782,6 @@ pub(super) fn when_drawing(hook: OnDrawing) { /// The user quit before the window drew. A deliberate quit is no failed /// start: the marker goes without counting, and no worker still in the /// driver writes it again. -#[cfg_attr(not(windows), allow(dead_code))] pub(super) fn abandon() { if let (Some(directory), Ok(mut launch)) = (STATE_DIR.get(), LAUNCH.lock()) { launch.abandon(directory, ProcessId::current()); @@ -2171,6 +2170,27 @@ mod tests { let _ = fs::remove_dir_all(&directory); } + /// The app quits from outside its window (Windows closing it for an + /// installer) before the window is confirmed drawing, through this + /// launch's own marker: none is left for the next launch to count, and a + /// probe still in the driver doesn't write it again. The only test of + /// the launch-wide marker; it puts it back as it found it. + #[test] + fn a_quit_from_outside_the_window_before_it_draws_leaves_no_marker() { + let directory = STATE_DIR.get_or_init(|| temp_dir("quit-outside")).clone(); + let owner = ProcessId::current(); + assert!(start(marker("10de:2c02 32.0", "dx12"))); + assert!(marker_path(&directory, owner).exists()); + crate::gui::quitting_from_outside(); + assert!(!marker_path(&directory, owner).exists()); + mark_thread(Some(WINDOW)); + assert!(!start(marker("10de:2c02 32.0", "vulkan"))); + assert!(!marker_path(&directory, owner).exists()); + assert!(sweep(&directory, ProcessId::current()).is_empty()); + *LAUNCH.lock().unwrap() = Launch::new(); + let _ = fs::remove_dir_all(&directory); + } + #[test] fn a_launch_that_dies_listing_the_graphics_cards_skips_the_list_next_time() { let directory = temp_dir("listing"); diff --git a/host/src/main.rs b/host/src/main.rs index 27b56e28..8415cda5 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -343,6 +343,7 @@ fn main() -> Result<(), Box> { eternal_host::power::watch(gui_control.shared.clone(), { let quit_once = std::sync::Arc::clone(&quit_once); Box::new(move || { + gui::quitting_from_outside(); let quit = quit_once.lock().take(); if let Some(quit) = quit { quit(None); From 55bf14de921eec12044cbac5f193af680a053f6c Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 10:25:34 -0500 Subject: [PATCH 207/226] Log the Windows 10 hold as its own line, not as a Second screen fallback Holding Second screen on Windows 10 is expected, so it no longer logs a `second_screen_fallback` field, which diagnostics count as a failed start. Each held connection logs "Second screen held: needs Windows 11" instead. The session summary still names it `no_driver`, recorded directly. A start that fails without a driver logs the field as before. --- host/src/capture/second_screen.rs | 14 +++--- host/src/capture/second_screen/memory.rs | 16 ++++--- host/src/diagnostics.rs | 54 +++++++++++++++++++++++- 3 files changed, 70 insertions(+), 14 deletions(-) diff --git a/host/src/capture/second_screen.rs b/host/src/capture/second_screen.rs index 6e533550..55fa4110 100644 --- a/host/src/capture/second_screen.rs +++ b/host/src/capture/second_screen.rs @@ -586,14 +586,16 @@ fn log_hold(hold: Hold) { } /// The one log line for a connection that mirrors because Windows is too -/// old for the display driver. The summary names it `no_driver`: there is -/// no display driver for this Windows. -fn log_windows_hold() { +/// old for the display driver. It is expected, not a failure, so it has no +/// `second_screen_fallback` field: the daily check counts those. The +/// session summary still names it `no_driver`: there is no display driver +/// for this Windows. +pub(crate) fn log_windows_hold() { + crate::diagnostics::note_fallback("no_driver"); tracing::info!( - second_screen_fallback = "no_driver", reason = WINDOWS_10, - "Second screen needs Windows 11 — mirroring the primary display without turning the \ - display driver on" + "Second screen held: needs Windows 11 — mirroring the primary display without turning \ + the display driver on" ); } diff --git a/host/src/capture/second_screen/memory.rs b/host/src/capture/second_screen/memory.rs index 72cf6da1..675f0596 100644 --- a/host/src/capture/second_screen/memory.rs +++ b/host/src/capture/second_screen/memory.rs @@ -2570,7 +2570,7 @@ mod tests { } /// Windows 10: a Second screen connection mirrors at once, with no - /// Enable, and says why: the log names `no_driver`, and the iPad's + /// Enable, and says why: its own log line, and the iPad's /// notice lasts a few seconds from each connection. Neither a failure /// an earlier version saved, Try again, nor choosing Second screen again /// turns the driver on. @@ -2586,13 +2586,13 @@ mod tests { let (log, step) = logs(|| next_step(&shared, start)); assert_eq!(step, Step::Nothing, "no Enable"); assert!( - log.contains("second_screen_fallback") - && log.contains("no_driver") - && log.contains(WINDOWS_10) - && log.contains("Second screen needs Windows 11"), + log.contains("Second screen held: needs Windows 11") && log.contains(WINDOWS_10), "{log}" ); - assert!(!log.contains("remembered"), "{log}"); + assert!( + !log.contains("second_screen_fallback") && !log.contains("remembered"), + "an expected hold is no fallback for the daily check: {log}" + ); assert!(shared.vdd_worker.queue.lock().commands.is_empty()); assert_eq!(*shared.vdd_status.lock(), VddStatus::Failed); assert_eq!(remembered_failure(&shared), Some(WINDOWS_10)); @@ -2624,7 +2624,9 @@ mod tests { }); assert_eq!(step, Step::Nothing); assert!( - log.contains("no_driver") && !log.contains("remembered"), + log.contains("Second screen held: needs Windows 11") + && !log.contains("second_screen_fallback") + && !log.contains("remembered"), "{log}" ); assert!(windows_notice(&shared, later)); diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index ffe78b58..b1653a49 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -262,7 +262,11 @@ pub fn note_session_start(session_id: u32, shared: &SharedControl) { }; } -fn note_fallback(reason: &str) { +/// Notes `reason` for the live session's summary, as a +/// `second_screen_fallback` log field does. Also called directly for a +/// Second screen that is held by design rather than failed, so the field +/// (which the daily check counts as a failure) isn't logged for it. +pub(crate) fn note_fallback(reason: &str) { let (session_id, mode) = { let slot = SESSION_FALLBACK.lock(); match (slot.session_id, &slot.mode) { @@ -1062,6 +1066,54 @@ pub(crate) mod tests { }); } + /// The Windows 10 Second screen hold names `no_driver` in the session + /// summary without logging a `second_screen_fallback` field, which the + /// daily check counts as a failure. A failed start without a driver + /// still logs the field. + #[test] + fn a_windows_10_hold_is_summarised_as_no_driver_without_a_fallback_line() { + use tracing_subscriber::prelude::*; + struct Fields(std::sync::Arc>>); + impl tracing_subscriber::Layer for Fields { + fn on_event( + &self, + event: &tracing::Event<'_>, + _: tracing_subscriber::layer::Context<'_, S>, + ) { + let names = event + .metadata() + .fields() + .iter() + .map(|f| f.name().to_string()); + self.0.lock().extend(names); + } + } + let _slot = FALLBACK_SLOT.lock(); + let shared = SharedControl::new(9876, 15_000_000); + *shared.capture_target.lock() = CaptureTarget::VirtualExtended; + let fields = std::sync::Arc::new(parking_lot::Mutex::new(Vec::::new())); + let subscriber = tracing_subscriber::registry() + .with(FallbackWatch) + .with(Fields(std::sync::Arc::clone(&fields))); + crate::logging::prepare_log_capture(); + let has_field = || { + fields + .lock() + .iter() + .any(|name| name == "second_screen_fallback") + }; + tracing::subscriber::with_default(subscriber, || { + note_session_start(1, &shared); + crate::capture::second_screen::log_windows_hold(); + assert!(!has_field(), "{:?}", fields.lock()); + assert_eq!(take_fallback(1), Some("no_driver")); + note_session_start(2, &shared); + tracing::info!(second_screen_fallback = "no_driver", "no driver here"); + assert!(has_field()); + assert_eq!(take_fallback(2), Some("no_driver")); + }); + } + /// A fallback counts for the live session in the mode it has when the /// fallback is logged. Session 1 starts the virtual display and leaves, /// and its start fails late: neither the gap before session 2 nor a From 4deecfd824527aa0303d0561be69cdc1b0d87ab9 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 10:37:18 -0500 Subject: [PATCH 208/226] Install the virtual display driver on Windows 11 only Second screen needs Windows 11: on Windows 10 the bundled driver starts, but Windows never adds its screen, and the app mirrors there without turning it on. On a fresh install on Windows 10 (a build below 22000), Setup no longer installs the driver or registers its tasks. A driver that is already installed, by an earlier version or another app, stays as it is: its tasks are kept so it stays off, and Setup neither repairs nor checks it, so Setup doesn't change devices or fail over a driver Windows 10 doesn't use. Once the PC runs Windows 11, the next Setup run installs the driver. Windows 11 is unchanged. Test-InstallerWindows10.ps1 checks these rules in the script and rejects broken copies of it. CI runs it with the other installer tests. --- .github/workflows/ci.yml | 3 + installer/EternalMonitor.iss | 48 +++++++++++-- installer/tests/Test-InstallerWindows10.ps1 | 75 +++++++++++++++++++++ 3 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 installer/tests/Test-InstallerWindows10.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 691a0493..633ac21f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,9 @@ jobs: - name: Verify Setup hands its language to the app shell: powershell run: ./installer/tests/Test-InstallerLanguages.ps1 + - name: Verify Setup leaves the virtual display driver off Windows 10 + shell: powershell + run: ./installer/tests/Test-InstallerWindows10.ps1 rust-macos: name: Rust (macOS, full workspace + E2E) diff --git a/installer/EternalMonitor.iss b/installer/EternalMonitor.iss index d97ca563..e66679f4 100644 --- a/installer/EternalMonitor.iss +++ b/installer/EternalMonitor.iss @@ -216,12 +216,14 @@ Filename: "{sys}\netsh.exe"; Parameters: "advfirewall firewall add rule name=""E ; screen at all. The check after them puts the saved screens back only then. Both always exit 0. Filename: "powershell.exe"; Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\scripts\display-guard.ps1"" -Action save -Path ""{tmp}\display-layout.json"""; Flags: runhidden waituntilterminated logoutput ; The vendor setup prompts to uninstall an existing installation even with silent -; switches. Preserve an existing driver and install only when it is absent. +; switches. Preserve an existing driver and install only when it is absent, and only on +; Windows 11: Second screen needs it (see WindowsRunsVdd). Filename: "{app}\driver\vdd-setup-x64.exe"; Parameters: "/VERYSILENT /SUPPRESSMSGBOXES /NORESTART"; StatusMsg: "{cm:DriverStatus}"; Flags: waituntilterminated; Check: NeedsVddInstall; AfterInstall: RecordVddOwnership ; Register the enable/disable scheduled tasks and leave the virtual display OFF by default — ; EternalMonitor turns it on only while streaming to it, so there's no phantom monitor. -; A preserved driver whose device was removed or broke is repaired first. -Filename: "powershell.exe"; Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\scripts\vdd-tasks-setup.ps1"""; StatusMsg: "{cm:DisplayStatus}"; Flags: runhidden waituntilterminated; Check: VddInstallSucceeded; BeforeInstall: RepairVddDriver; AfterInstall: VerifyVddDriver +; A preserved driver whose device was removed or broke is repaired first. On Windows 10 a +; driver an earlier version installed keeps its tasks, which keep it off. +Filename: "powershell.exe"; Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\scripts\vdd-tasks-setup.ps1"""; StatusMsg: "{cm:DisplayStatus}"; Flags: runhidden waituntilterminated; Check: VddTasksWanted; BeforeInstall: RepairVddDriver; AfterInstall: VerifyVddDriver ; After the Disable above, and also when a failed driver step skipped it. Filename: "powershell.exe"; Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\scripts\display-guard.ps1"" -Action check -Path ""{tmp}\display-layout.json"""; Flags: runhidden waituntilterminated logoutput #endif @@ -306,19 +308,50 @@ begin Result := Result and (Directory <> '') and (Version <> ''); end; +const + FirstWindows11Build = 22000; + +{ The bundled driver needs Windows 11's display driver framework. On Windows 10 + it installs and starts, but Windows never adds its screen, and the app shows + the mirror there without turning it on (second_screen::driver_supports_windows + in the host). } +function WindowsRunsVdd: Boolean; +var + Windows: TWindowsVersion; +begin + GetWindowsVersionEx(Windows); + Result := Windows.Build >= FirstWindows11Build; +end; + function NeedsVddInstall: Boolean; var Directory, Version: String; begin Result := not ReadVddRegistration(Directory, Version); if not Result then - Log('Preserving existing Virtual Display Driver package ' + Version); + Log('Preserving existing Virtual Display Driver package ' + Version) + else if not WindowsRunsVdd then begin + Log('Not installing the Virtual Display Driver: Second screen needs Windows 11'); + Result := False; + end; +end; + +{ The driver's tasks: wherever a driver is installed, so one an earlier version + installed on Windows 10 stays off. None on Windows 10 without one. } +function VddTasksWanted: Boolean; +var + Directory, Version: String; +begin + Result := VddInstallSucceeded and (WindowsRunsVdd or ReadVddRegistration(Directory, Version)); end; procedure VerifyVddDriver; var ExitCode: Integer; begin + { Windows 10 doesn't use the driver: a kept one isn't checked. } + if not WindowsRunsVdd then + exit; { [Run] ignores a child process's exit code. Check the actual PnP binding explicitly, including upgrades that preserve an existing registration. } if not Exec(ExpandConstant('{sys}\WindowsPowerShell\v1.0\powershell.exe'), @@ -336,7 +369,12 @@ var begin { Setup keeps an existing driver instead of running its setup again. When Setup installed it and no device works, put one back (exit codes in the - script). A driver another app installed is left alone. } + script). A driver another app installed is left alone. Windows 10 doesn't + use the driver: a kept one is left as it is, with no device changes. } + if not WindowsRunsVdd then begin + Log('Leaving the Virtual Display Driver as it is: Second screen needs Windows 11'); + exit; + end; if not Exec(ExpandConstant('{sys}\WindowsPowerShell\v1.0\powershell.exe'), '-NoProfile -ExecutionPolicy Bypass -File "' + ExpandConstant('{app}\scripts\vdd-driver-repair.ps1') + '"', '', SW_HIDE, ewWaitUntilTerminated, ExitCode) then diff --git a/installer/tests/Test-InstallerWindows10.ps1 b/installer/tests/Test-InstallerWindows10.ps1 new file mode 100644 index 00000000..165004d7 --- /dev/null +++ b/installer/tests/Test-InstallerWindows10.ps1 @@ -0,0 +1,75 @@ +# Static checks for Setup on Windows 10, where Second screen needs Windows 11: a fresh install +# skips the Virtual Display Driver and its tasks, and an upgrade keeps a driver an earlier +# version installed, with its tasks, without repairing or verifying it. Windows 11 is +# unchanged. Nothing is compiled or installed. The checks also run against broken copies of +# the script, which they must reject. +param([string]$IssPath) +$ErrorActionPreference = 'Stop' +$source = if ($IssPath) { $IssPath } else { Join-Path $PSScriptRoot '..\EternalMonitor.iss' } +$iss = Get-Content -Raw -Encoding UTF8 $source + +# One [Code] routine, from its header to its closing "end;". +function Get-Routine([string]$Text, [string]$Header) { + [regex]::Match($Text, '(?s)\n' + [regex]::Escape($Header) + '.*?\nend;').Value +} + +function Get-Failures([string]$Text) { + $failures = @() + if ($Text -notmatch '(?m)^\s*FirstWindows11Build = 22000;') { + $failures += 'The first Windows 11 build is not 22000' + } + $runs = Get-Routine $Text 'function WindowsRunsVdd: Boolean;' + if ($runs -notmatch 'GetWindowsVersionEx\((\w+)\);' -or + $runs -notmatch "Result := $($Matches[1])\.Build >= FirstWindows11Build;") { + $failures += 'WindowsRunsVdd does not compare the Windows build with the first Windows 11 build' + } + + # Fresh install: a registered driver is kept on every Windows; with none, only Windows 11 + # installs one. + $needs = Get-Routine $Text 'function NeedsVddInstall: Boolean;' + if ($needs -notmatch '(?s)Result := not ReadVddRegistration\(Directory, Version\);.*else if not WindowsRunsVdd then begin.*Result := False;') { + $failures += 'NeedsVddInstall installs the driver on Windows 10' + } + if ($Text -notmatch '(?m)^Filename: "\{app\}\\driver\\vdd-setup-x64\.exe";.*Check: NeedsVddInstall;') { + $failures += 'The vendor setup does not run under NeedsVddInstall' + } + + # The tasks: wherever a driver is installed, and on Windows 11. + $wanted = Get-Routine $Text 'function VddTasksWanted: Boolean;' + if ($wanted -notmatch 'Result := VddInstallSucceeded and \(WindowsRunsVdd or ReadVddRegistration\(Directory, Version\)\);') { + $failures += 'VddTasksWanted does not keep the tasks for an installed driver only' + } + $tasks = $Text -split "\r?\n" | Where-Object { $_ -match '^Filename: "powershell\.exe";.*vdd-tasks-setup\.ps1' } + if (@($tasks).Count -ne 1 -or $tasks -notmatch 'Check: VddTasksWanted;') { + $failures += 'The driver tasks are set up without VddTasksWanted' + } + + # A driver kept on Windows 10 is neither repaired nor checked: no device changes, and no + # failed Setup over a driver Windows 10 doesn't use. + foreach ($routine in 'procedure RepairVddDriver;', 'procedure VerifyVddDriver;') { + $body = Get-Routine $Text $routine + $first = [regex]::Match($body, '(?s)\nbegin\s*(?:\{[^}]*\}\s*)*(?if not WindowsRunsVdd then(?: begin)?\s*(?:Log\([^;]*\);\s*)?exit;)') + if (!$first.Success) { $failures += "$($routine.Split(' ')[1].TrimEnd(';')) runs on Windows 10" } + } + $failures +} + +$failures = @(Get-Failures $iss) +if ($failures) { throw "Windows 10 checks failed:`n " + ($failures -join "`n ") } + +# Each broken copy must fail. +$broken = [ordered]@{ + 'Windows 10 build' = $iss.Replace('FirstWindows11Build = 22000;', 'FirstWindows11Build = 19041;') + 'build not read' = $iss.Replace('Result := Windows.Build >= FirstWindows11Build;', 'Result := True;') + 'fresh install on Windows 10' = $iss.Replace(" else if not WindowsRunsVdd then begin`r`n", " else if False then begin`r`n").Replace(" else if not WindowsRunsVdd then begin`n", " else if False then begin`n") + 'vendor setup unchecked' = $iss.Replace('Check: NeedsVddInstall;', 'Check: VddInstallSucceeded;') + 'tasks without a driver' = $iss.Replace('(WindowsRunsVdd or ReadVddRegistration(Directory, Version))', '(True or ReadVddRegistration(Directory, Version))') + 'tasks unchecked' = $iss.Replace('Check: VddTasksWanted;', 'Check: VddInstallSucceeded;') + 'repair on Windows 10' = [regex]::Replace($iss, '(?s)(procedure RepairVddDriver;.*?\nbegin.*?)if not WindowsRunsVdd then begin', '$1if False then begin') + 'verify on Windows 10' = [regex]::Replace($iss, '(?s)(procedure VerifyVddDriver;.*?\nbegin.*?)if not WindowsRunsVdd then', '$1if False then') +} +foreach ($case in $broken.Keys) { + if ($broken[$case] -ceq $iss) { throw "The '$case' copy is the real script; update this test" } + if (!@(Get-Failures $broken[$case])) { throw "The checks pass a broken script: $case" } +} +Write-Output "PASS: Windows 10 skips a fresh driver install and its tasks, keeps an installed driver unrepaired and unchecked; $($broken.Count) broken copies rejected" From e29b29fbd56a5db455c7c13b5d10f8c4084f8f8b Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 10:37:18 -0500 Subject: [PATCH 209/226] Run no Disable task when Windows confirms there is no display device At startup and quit the host reads the virtual display device before its Disable task. When Windows lists every device and none is the driver's, as on Windows 10 where Setup installs no driver, there is nothing to turn off: the task isn't run, so a PC without the driver no longer logs a missing task at every launch. A device list that can't be read still runs the task, as before. --- host/src/vdd.rs | 67 ++++++++++++++++++++++++++++++++---------- host/src/vdd_device.rs | 38 ++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 15 deletions(-) diff --git a/host/src/vdd.rs b/host/src/vdd.rs index 140bc949..6aca9a52 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -600,23 +600,20 @@ pub fn disable_in_background() { #[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. const DEVICE_READ_LIMIT: Duration = Duration::from_secs(2); -/// Whether Configuration Manager reads the device disabled, said in the log. +/// Whether Configuration Manager reads the device disabled, or confirms +/// there is none, said in the log. #[cfg(windows)] fn device_off() -> bool { - let off = off_within(crate::vdd_device::query, DEVICE_READ_LIMIT); - if off { - info!("Virtual display already off — not running the Disable task"); - } - off + off_within(crate::vdd_device::query_present, DEVICE_READ_LIMIT) } -/// Whether `query`, run on its own thread, reads the device disabled within -/// `limit`. A read that doesn't answer in time, or whose thread can't start, -/// counts as not disabled, so the Disable task runs under its own limit; a -/// late answer is dropped. +/// Whether `query`, run on its own thread, reads the device disabled, or +/// confirms there is no device (`None`), within `limit`. A read that doesn't +/// answer in time, or whose thread can't start, counts as not disabled, so +/// the Disable task runs under its own limit; a late answer is dropped. #[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. fn off_within( - query: impl FnOnce() -> crate::vdd_device::DeviceState + Send + 'static, + query: impl FnOnce() -> Option + Send + 'static, limit: Duration, ) -> bool { let (answer, answered) = std::sync::mpsc::channel(); @@ -626,7 +623,17 @@ fn off_within( let _ = answer.send(query()); }); match spawned.ok().map(|_| answered.recv_timeout(limit)) { - Some(Ok(device)) => reports_disabled(device), + Some(Ok(None)) => { + tracing::info!("No virtual display device — not running the Disable task"); + true + } + Some(Ok(Some(device))) => { + let off = reports_disabled(device); + if off { + tracing::info!("Virtual display already off — not running the Disable task"); + } + off + } _ => { tracing::warn!( limit_ms = limit.as_millis() as u64, @@ -892,7 +899,7 @@ mod tests { let ran = Cell::new(false); let hung = move || { let _ = held.recv(); - DeviceState::Problem(22) + Some(DeviceState::Problem(22)) }; tasks.disable_unless_off( || {}, @@ -921,15 +928,45 @@ mod tests { let started = Instant::now(); assert!(tasks.disable_unless_off( || {}, - || off_within(|| DeviceState::Problem(22), DEVICE_READ_LIMIT), + || off_within(|| Some(DeviceState::Problem(22)), DEVICE_READ_LIMIT), |_, _| panic!("ran the Disable task for a display that reads disabled") )); assert!(started.elapsed() < DEVICE_READ_LIMIT); for device in [DeviceState::Ready, DeviceState::Missing] { - assert!(!off_within(move || device, DEVICE_READ_LIMIT), "{device:?}"); + assert!( + !off_within(move || Some(device), DEVICE_READ_LIMIT), + "{device:?}" + ); } } + /// Windows 10, where Setup installs no driver: Windows confirms there is + /// no device, so the Disable at startup and quit runs no task, and no + /// missing task is logged at every launch. A device list that couldn't + /// be read (`Missing`) still runs it. + #[test] + fn a_confirmed_absent_device_runs_no_disable_task() { + use crate::vdd_device::DeviceState; + use std::cell::Cell; + let tasks = DriverTasks::new(); + assert!(tasks.disable_unless_off( + || {}, + || off_within(|| None, DEVICE_READ_LIMIT), + |_, _| panic!("ran the Disable task with no device to turn off") + )); + let ran = Cell::new(false); + tasks.disable_unless_off( + || {}, + || off_within(|| Some(DeviceState::Missing), DEVICE_READ_LIMIT), + |_, _| { + ran.set(true); + TaskOutcome::Missing + }, + ); + assert!(ran.get(), "an unread device list skipped the Disable task"); + assert_eq!(tasks.health(), VddTaskHealth::TasksMissing); + } + #[test] fn a_hung_task_is_not_waited_out_again_until_the_cooldown_ends() { let mut gate = TaskGate::new(); diff --git a/host/src/vdd_device.rs b/host/src/vdd_device.rs index d7fd9787..5889d449 100644 --- a/host/src/vdd_device.rs +++ b/host/src/vdd_device.rs @@ -137,6 +137,26 @@ pub fn from_task_stderr(stderr: &str) -> Option { DeviceState::from_task_result(digits.parse().ok()?) } +/// As [`query`], but `None` when Windows lists the present root devices, +/// reads each of them, and none is the driver's: there is no device to turn +/// off, as on Windows 10, where Setup installs none. [`query`] reads that as +/// `Missing`, as it does a list it couldn't get. +#[cfg(windows)] +pub fn query_present() -> Option { + match root_devnodes() { + Some(devnodes) if confirmed_absent(&devnodes) => None, + _ => Some(query()), + } +} + +/// Whether every present root device was read and none is the driver's. +#[cfg(any(windows, test))] +fn confirmed_absent(devnodes: &[Devnode]) -> bool { + devnodes + .iter() + .all(|devnode| matches!(devnode, Devnode::Other)) +} + /// Read the driver's device from Configuration Manager. #[cfg(windows)] pub fn query() -> DeviceState { @@ -447,6 +467,24 @@ pub fn query() -> DeviceState { mod tests { use super::*; + /// Only a list where every root device was read, and none is the + /// driver's, says there is no device to turn off. + #[test] + fn only_a_fully_read_list_without_the_driver_is_an_absent_device() { + let driver = Devnode::Driver { + devinst: 7, + status: Some((0, 22)), + }; + assert!(confirmed_absent(&[])); + assert!(confirmed_absent(&[Devnode::Other, Devnode::Other])); + assert!(!confirmed_absent(&[Devnode::Other, driver])); + assert!(!confirmed_absent(&[Devnode::Other, Devnode::Unread])); + assert!(!confirmed_absent(&[Devnode::Driver { + devinst: 7, + status: None + }])); + } + /// Review r6: a root device whose location or hardware IDs couldn't be /// read may be the driver's, so the driver's state is unreadable, not /// off, and an Enable after it keeps no record. One whose hardware IDs From 707b25a4e8c741756fd3d79ac89bea6e34d2ebe6 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 10:43:37 -0500 Subject: [PATCH 210/226] Fix the firewall block without ever removing a rule that works The fix ran Setup's deletions before adding the Allow rule. A failed add then left the exe with fewer rules than before; failed deletions counted as success; and a fix that went part of the way could hide its own error with the notice it was shown in. - The elevated copy now changes the firewall through its COM interface. It adds the installer's Allow rule first, unless one that works is there, so a failed add changes nothing. It then removes only this exe's own Block rules that reach the stream port with no other scope, the rules the Stream page's verdict counts. Removing a Block rule never stops anything that worked; Allow rules, scoped rules and other programs' rules stay. - A Block rule that another rule shares a name with is left, since rules are removed by name. - Any failed step fails the fix, and the rules are read again at the end: the fix fails unless the Allow rule is there and none of those Block rules is left. - The elevated copy gets the stream's port, a number, besides its fixed argument, to judge reach as the verdict does. - A failed fix shows its error in its own banner, with Try again and Dismiss, whatever the firewall reads now. Try again and Dismiss are English in the catalog, awaiting translation. --- host/src/firewall.rs | 38 +- host/src/firewall_fix.rs | 793 ++++++++++++++++++++++++++-------- host/src/gui/page_stream.rs | 55 ++- host/src/gui/tests.rs | 23 +- host/src/i18n/strings/en.json | 2 + host/src/i18n/tests.rs | 2 +- 6 files changed, 697 insertions(+), 216 deletions(-) diff --git a/host/src/firewall.rs b/host/src/firewall.rs index e2544b77..b1bb686f 100644 --- a/host/src/firewall.rs +++ b/host/src/firewall.rs @@ -260,6 +260,10 @@ enum Whose { Other, } +pub(crate) fn ours(rule: &Rule, exe: &Path, lookup: impl Fn(&str) -> Option) -> bool { + whose(rule, exe, lookup) == Whose::Ours +} + fn whose(rule: &Rule, exe: &Path, lookup: impl Fn(&str) -> Option) -> Whose { match rule.program.as_deref() { None => Whose::Unknown, @@ -627,7 +631,7 @@ pub fn watch(port: u16) { } #[cfg(windows)] -mod windows_policy { +pub(crate) mod windows_policy { use std::path::Path; use windows::core::{IUnknown, Interface, VARIANT}; @@ -649,10 +653,10 @@ mod windows_policy { /// VT_DISPATCH: what the rule enumerator hands out. const VT_DISPATCH: u16 = 9; - struct Apartment; + pub(crate) struct Apartment; impl Apartment { - fn new() -> windows::core::Result { + pub(crate) fn new() -> windows::core::Result { unsafe { CoInitializeEx(None, COINIT_MULTITHREADED).ok()? }; Ok(Self) } @@ -749,13 +753,27 @@ mod windows_policy { /// Every inbound rule, each with what could be read of it. fn inbound_rules(policy: &INetFwPolicy2) -> windows::core::Result> { - let all = unsafe { policy.Rules()? }; - let each: IEnumVARIANT = unsafe { all._NewEnum()? }.cast()?; let mut rules = Vec::new(); + for_each_rule(policy, |rule| { + if let Some(rule) = rule_from(rule).map_err(|e| windows::core::Error::new(E_FAIL, e))? { + rules.push(rule); + } + Ok(()) + })?; + Ok(rules) + } + + /// Hands every rule of the policy, in any direction, to `each`. + pub(crate) fn for_each_rule( + policy: &INetFwPolicy2, + mut each: impl FnMut(&INetFwRule) -> windows::core::Result<()>, + ) -> windows::core::Result<()> { + let all = unsafe { policy.Rules()? }; + let rules: IEnumVARIANT = unsafe { all._NewEnum()? }.cast()?; loop { let mut item = [VARIANT::default()]; let mut fetched = 0; - unsafe { each.Next(&mut item, &mut fetched).ok()? }; + unsafe { rules.Next(&mut item, &mut fetched).ok()? }; if fetched == 0 { break; } @@ -776,13 +794,9 @@ mod windows_policy { let Ok(rule) = unknown.cast::() else { continue; }; - if let Some(rule) = - rule_from(&rule).map_err(|e| windows::core::Error::new(E_FAIL, e))? - { - rules.push(rule); - } + each(&rule)?; } - Ok(rules) + Ok(()) } } diff --git a/host/src/firewall_fix.rs b/host/src/firewall_fix.rs index 9dda3690..e2763685 100644 --- a/host/src/firewall_fix.rs +++ b/host/src/firewall_fix.rs @@ -2,44 +2,140 @@ //! //! The block Windows adds when its "allow access" prompt is dismissed beats //! the installer's Allow rule, and the only fix the PC offered was to run -//! Setup again. "Fix it" runs the same netsh lines Setup runs for the -//! firewall, in the same order, with one Windows prompt: this exe starts a -//! copy of itself as administrator with [`FIX_ARG`], and that copy runs -//! them and exits. Each line is a fixed string with only this exe's own -//! path in it, run with the system's netsh.exe and no shell, so nothing -//! anyone types reaches an elevated command line. Only rules for this exe -//! and the installer's own rule names are touched. +//! Setup again. "Fix it" starts this exe as administrator, with one Windows +//! prompt, with [`FIX_ARG`] and the stream's port. That copy changes the +//! firewall through its COM interface, never a shell, and exits. //! -//! A user who answers No to the Windows prompt changes nothing: the notice -//! stays as it was. After the copy ran, the firewall is read again at once. +//! It can't leave the PC worse off than before. Setup deletes every rule +//! for the exe, then adds its Allow rule; deletions with no add after them +//! could lose rules that worked. The fix is narrower ([`fix_rules`]): +//! +//! 1. It adds the installer's Allow rule, unless one that works is there. +//! It does this first, so a failure here changes nothing. +//! 2. It removes only this exe's own Block rules that reach the stream +//! port with no other scope: the rules the Stream page's verdict counts. +//! Removing a Block rule never stops anything that worked. Allow rules, +//! scoped rules and other programs' rules stay as they are. +//! 3. It reads the rules again and fails unless the Allow rule is there and +//! none of those Block rules is left. +//! +//! The exe's path comes from the elevated process itself, and the port is +//! only ever a number. A user who answers No to the Windows prompt changes +//! nothing: the notice stays as it was. After the copy ran, the firewall +//! is read again at once. use std::path::Path; -/// The argument the elevated copy runs with. +use crate::firewall::{reach, Reach, Rule, DOMAIN, PRIVATE, PUBLIC}; + +/// The argument the elevated copy runs with, before the stream's port. pub const FIX_ARG: &str = "--fix-firewall"; -/// The netsh lines of the installer's `[Run]` section, as Setup runs them, -/// for this exe at `exe`. `{exe}` is the only thing filled in. -const SETUP_LINES: [&str; 4] = [ - r#"advfirewall firewall delete rule name="EternalMonitor Host UDP""#, - r#"advfirewall firewall delete rule name="EternalMonitor Host TCP""#, - r#"advfirewall firewall delete rule name=all dir=in program="{exe}""#, - r#"advfirewall firewall add rule name="EternalMonitor Host UDP" dir=in action=allow program="{exe}" protocol=UDP profile=domain,private,public enable=yes"#, -]; - -/// The netsh parameter lines the fix runs, in order, for this exe at `exe`. -/// None for a path that can't go inside quotes as it is. -pub fn command_lines(exe: &Path) -> Option> { - let exe = exe.to_str()?; - if exe.is_empty() || exe.contains('"') || exe.chars().any(char::is_control) { - return None; - } - Some( - SETUP_LINES - .iter() - .map(|line| line.replace("{exe}", exe)) - .collect(), - ) +/// The installer's Allow rule, as its `[Run]` section adds it: this name, +/// inbound, UDP, every profile, enabled. +pub const ALLOW_NAME: &str = "EternalMonitor Host UDP"; +pub const ALLOW_PROFILES: u32 = DOMAIN | PRIVATE | PUBLIC; + +/// A firewall rule with its name. `rule` is None for an outbound rule. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Named { + pub name: String, + pub rule: Option, +} + +/// The firewall, as the elevated copy changes it. The real one is the +/// Windows Firewall COM interface; tests fake it. +pub trait RuleStore { + /// Every rule, in any direction. + fn rules(&mut self) -> Result, String>; + /// Adds the installer's Allow rule for `exe`. + fn add_allow(&mut self, exe: &str) -> Result<(), String>; + /// Removes the rule named `name`. + fn remove(&mut self, name: &str) -> Result<(), String>; +} + +/// Why the fix failed, as the elevated copy's exit code. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Failure { + /// The copy got no usable path or port. + Arguments = 3, + /// The rules couldn't be read. + Read = 10, + /// The Allow rule couldn't be added. Nothing was changed. + Add = 11, + /// A Block rule couldn't be removed. + Remove = 12, + /// A Block rule shares its name with another rule, so removing it by + /// name could remove that one too. It was left. + SharedName = 13, + /// Read again, a Block rule is still there, or the Allow rule isn't. + StillBlocked = 14, +} + +impl Failure { + pub fn code(self) -> u32 { + self as u32 + } +} + +fn ours(rule: &Rule, exe: &Path) -> bool { + crate::firewall::ours(rule, exe, |name| std::env::var(name).ok()) +} + +/// This exe's own Block rules that reach every datagram to `port`: what +/// the verdict counts as a block, and all the fix removes. +fn is_block(rule: &Rule, exe: &Path, port: u16) -> bool { + !rule.allow && rule.enabled && rule.udp && ours(rule, exe) && reach(rule, port) == Reach::All +} + +/// An Allow rule that lets every datagram to `port` through on every +/// profile, as the installer's does. +fn is_working_allow(rule: &Rule, exe: &Path, port: u16) -> bool { + rule.allow + && rule.enabled + && rule.udp + && rule.profiles & ALLOW_PROFILES == ALLOW_PROFILES + && ours(rule, exe) + && reach(rule, port) == Reach::All +} + +fn inbound(rules: &[Named]) -> impl Iterator { + rules + .iter() + .filter_map(|named| named.rule.as_ref().map(|rule| (named.name.as_str(), rule))) +} + +/// The fix, on `store`, for this exe at `exe` and the stream's `port`. +pub fn fix_rules(store: &mut impl RuleStore, exe: &Path, port: u16) -> Result<(), Failure> { + let program = exe + .to_str() + .filter(|path| !path.is_empty()) + .ok_or(Failure::Arguments)?; + let rules = store.rules().map_err(|_| Failure::Read)?; + if !inbound(&rules).any(|(_, rule)| is_working_allow(rule, exe, port)) { + store.add_allow(program).map_err(|_| Failure::Add)?; + } + let mut shared = false; + for (name, rule) in inbound(&rules) { + if !is_block(rule, exe, port) { + continue; + } + if rules.iter().filter(|other| other.name == name).count() > 1 { + shared = true; + continue; + } + store.remove(name).map_err(|_| Failure::Remove)?; + } + let after = store.rules().map_err(|_| Failure::Read)?; + let blocked = inbound(&after).any(|(_, rule)| is_block(rule, exe, port)); + let allowed = inbound(&after).any(|(_, rule)| is_working_allow(rule, exe, port)); + if blocked && shared { + Err(Failure::SharedName) + } else if blocked || !allowed { + Err(Failure::StillBlocked) + } else { + Ok(()) + } } /// How starting the elevated copy went. @@ -53,48 +149,60 @@ pub enum Elevated { Failed(String), } -/// Starts this exe again as administrator with `arg`, and waits for it. +/// Starts this exe again as administrator with `args`, and waits for it. /// The real one asks Windows; tests fake it. pub trait Elevate { - fn run_elevated(&self, exe: &Path, arg: &str) -> Elevated; + fn run_elevated(&self, exe: &Path, args: &str) -> Elevated; } /// What the Stream page shows after "Fix it". #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Outcome { - /// The lines ran: the firewall was read again, and the notice follows it. - Ran, + /// The firewall now has the Allow rule and none of the Block rules; it + /// was read again, and the notice follows it. + Fixed, /// No to the Windows prompt: nothing changed. Declined, /// The fix didn't run or didn't finish. English, as other errors are. Failed(&'static str), } +/// The arguments the elevated copy gets: [`FIX_ARG`] and the port. +pub fn arguments(port: u16) -> String { + format!("{FIX_ARG} {port}") +} + /// Runs the fix: the elevated copy, then a fresh read of the firewall -/// (`recheck`), whatever it did, so the notice shows what is true now. -pub fn fix(elevate: &impl Elevate, exe: &Path, recheck: impl FnOnce()) -> Outcome { - if command_lines(exe).is_none() { +/// (`recheck`), unless the user said No, so the notice shows what is true. +pub fn fix(elevate: &impl Elevate, exe: &Path, port: u16, recheck: impl FnOnce()) -> Outcome { + if exe.to_str().is_none_or(str::is_empty) { return Outcome::Failed(FAILED); } - match elevate.run_elevated(exe, FIX_ARG) { - Elevated::Declined => Outcome::Declined, - Elevated::Exited(0) => { - recheck(); - Outcome::Ran + let outcome = match elevate.run_elevated(exe, &arguments(port)) { + Elevated::Declined => return Outcome::Declined, + Elevated::Exited(0) => Outcome::Fixed, + Elevated::Exited(code) if code == Failure::Add.code() => Outcome::Failed(NOT_CHANGED), + Elevated::Exited(code) if code == Failure::SharedName.code() => { + Outcome::Failed(SHARED_NAME) } - Elevated::Exited(_) | Elevated::Failed(_) => { - recheck(); - Outcome::Failed(FAILED) - } - } + Elevated::Exited(_) | Elevated::Failed(_) => Outcome::Failed(FAILED), + }; + recheck(); + outcome } +/// The errors the Stream page shows when the fix didn't work. +pub const FAILED: &str = + "Windows didn't apply the whole fix. Run the EternalMonitor installer again."; +pub const NOT_CHANGED: &str = "Windows didn't let EternalMonitor add its firewall rule, so nothing was changed. Run the EternalMonitor installer again."; +pub const SHARED_NAME: &str = "A firewall rule blocking EternalMonitor shares its name with another rule, so it was left. Remove it in Windows Defender Firewall, or run the EternalMonitor installer again."; + /// Where the Stream page's "Fix it" is. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum Progress { #[default] Idle, - /// The Windows prompt or the netsh lines are running. + /// The Windows prompt or the fix is running. Running, Done(Outcome), } @@ -105,8 +213,17 @@ pub fn progress() -> Progress { PROGRESS.lock().map_or(Progress::Idle, |progress| *progress) } +/// Forgets how the last fix went: the user dismissed its message. +pub fn dismiss() { + if let Ok(mut progress) = PROGRESS.lock() { + if matches!(*progress, Progress::Done(_)) { + *progress = Progress::Idle; + } + } +} + /// Runs `fix` off the window's thread, once at a time: the Windows prompt -/// waits for the user, and netsh for Windows. +/// waits for the user, and the fix for Windows. pub fn start(fix: impl FnOnce() -> Outcome + Send + 'static) { { let Ok(mut progress) = PROGRESS.lock() else { @@ -133,82 +250,140 @@ pub fn start(fix: impl FnOnce() -> Outcome + Send + 'static) { } } -/// The error under the notice when the fix didn't work. -pub const FAILED: &str = - "Windows didn't apply the fix. Run the EternalMonitor installer again instead."; - -/// The elevated copy's work: runs each line with `netsh` and returns the -/// exit code. Removing a rule that isn't there fails, as it does for Setup, -/// and doesn't count; only the Allow rule must be added. -pub fn run_lines(lines: &[String], mut netsh: impl FnMut(&str) -> std::io::Result) -> i32 { - let mut code = 0; - for line in lines { - let added = line.starts_with("advfirewall firewall add rule "); - match netsh(line) { - Ok(0) => {} - Ok(_) | Err(_) if !added => {} - Ok(failed) => code = failed.max(1), - Err(_) => code = 2, - } +/// The stream's port the last fix was for, for Try again. +static LAST_PORT: std::sync::atomic::AtomicU16 = std::sync::atomic::AtomicU16::new(0); + +/// Starts the fix for the stream on `port` ([`fix_now`]). +pub fn start_for(port: u16) { + LAST_PORT.store(port, std::sync::atomic::Ordering::Relaxed); + start(move || fix_now(port)); +} + +/// The last fix's port, if one was started. +pub fn last_port() -> Option { + Some(LAST_PORT.load(std::sync::atomic::Ordering::Relaxed)).filter(|port| *port != 0) +} + +/// The elevated copy's arguments after [`FIX_ARG`]: the port, a number. +pub fn parse_port(args: &[String]) -> Option { + match args { + [port] => port.parse().ok().filter(|port| *port != 0), + _ => None, } - code } /// If this process is the elevated copy ([`FIX_ARG`]), runs the fix and /// exits with its code. Called first thing in `main`. pub fn run_if_requested() { - if std::env::args().nth(1).as_deref() != Some(FIX_ARG) { + let args: Vec = std::env::args().skip(1).collect(); + if args.first().map(String::as_str) != Some(FIX_ARG) { return; } - let code = std::env::current_exe() - .ok() - .and_then(|exe| command_lines(&exe)) - .map_or(3, |lines| run_lines(&lines, windows_netsh::run)); - std::process::exit(code); + let code = match (std::env::current_exe(), parse_port(&args[1..])) { + (Ok(exe), Some(port)) => windows_firewall::fix(&exe, port), + _ => Failure::Arguments.code(), + }; + std::process::exit(code as i32); } /// The fix as the Stream page runs it, on Windows. #[cfg(windows)] -pub fn fix_now() -> Outcome { +pub fn fix_now(port: u16) -> Outcome { let Ok(exe) = std::env::current_exe() else { return Outcome::Failed(FAILED); }; - fix(&windows_elevate::Shell, &exe, crate::firewall::recheck_now) + fix( + &windows_elevate::Shell, + &exe, + port, + crate::firewall::recheck_now, + ) } /// Off Windows there is no firewall notice, so no fix. #[cfg(not(windows))] -pub fn fix_now() -> Outcome { +pub fn fix_now(_port: u16) -> Outcome { Outcome::Failed(FAILED) } #[cfg(windows)] -mod windows_netsh { - use std::os::windows::process::CommandExt; - - /// Runs the system's netsh.exe with `line` exactly as written, no shell. - pub fn run(line: &str) -> std::io::Result { - let mut system = [0u16; 260]; - let len = unsafe { - windows::Win32::System::SystemInformation::GetSystemDirectoryW(Some(&mut system)) - }; - if len == 0 || len as usize >= system.len() { - return Err(std::io::Error::other("no system folder")); +mod windows_firewall { + use std::path::Path; + + use windows::core::BSTR; + use windows::Win32::Foundation::{E_FAIL, VARIANT_TRUE}; + use windows::Win32::NetworkManagement::WindowsFirewall::{ + INetFwPolicy2, INetFwRule, NetFwPolicy2, NetFwRule, NET_FW_ACTION_ALLOW, NET_FW_RULE_DIR_IN, + }; + use windows::Win32::System::Com::{CoCreateInstance, CLSCTX_INPROC_SERVER}; + + use super::{fix_rules, Failure, Named, RuleStore, ALLOW_NAME, ALLOW_PROFILES}; + use crate::firewall::rule_from; + use crate::firewall::windows_policy::{for_each_rule, Apartment}; + + /// UDP, as `NET_FW_IP_PROTOCOL`. + const UDP: i32 = 17; + + struct Com(INetFwPolicy2); + + impl RuleStore for Com { + fn rules(&mut self) -> Result, String> { + let mut rules = Vec::new(); + for_each_rule(&self.0, |rule| { + let name = unsafe { rule.Name()? }.to_string(); + let rule = rule_from(rule).map_err(|e| windows::core::Error::new(E_FAIL, e))?; + rules.push(Named { name, rule }); + Ok(()) + }) + .map_err(|e| e.to_string())?; + Ok(rules) + } + + fn add_allow(&mut self, exe: &str) -> Result<(), String> { + let add = || -> windows::core::Result<()> { + let rule: INetFwRule = + unsafe { CoCreateInstance(&NetFwRule, None, CLSCTX_INPROC_SERVER)? }; + unsafe { + rule.SetName(&BSTR::from(ALLOW_NAME))?; + rule.SetApplicationName(&BSTR::from(exe))?; + rule.SetProtocol(UDP)?; + rule.SetDirection(NET_FW_RULE_DIR_IN)?; + rule.SetAction(NET_FW_ACTION_ALLOW)?; + rule.SetProfiles(ALLOW_PROFILES as i32)?; + rule.SetEnabled(VARIANT_TRUE)?; + self.0.Rules()?.Add(&rule) + } + }; + add().map_err(|e| e.to_string()) + } + + fn remove(&mut self, name: &str) -> Result<(), String> { + unsafe { + self.0 + .Rules() + .and_then(|rules| rules.Remove(&BSTR::from(name))) + } + .map_err(|e| e.to_string()) } - let netsh = std::path::PathBuf::from(String::from_utf16_lossy(&system[..len as usize])) - .join("netsh.exe"); - let status = std::process::Command::new(netsh) - .raw_arg(line) - .creation_flags(0x0800_0000) // CREATE_NO_WINDOW - .status()?; - Ok(status.code().unwrap_or(1)) + } + + /// The elevated copy's work: the exit code. + pub fn fix(exe: &Path, port: u16) -> u32 { + let run = || -> Result<(), Failure> { + let _apartment = Apartment::new().map_err(|_| Failure::Read)?; + let policy: INetFwPolicy2 = + unsafe { CoCreateInstance(&NetFwPolicy2, None, CLSCTX_INPROC_SERVER) } + .map_err(|_| Failure::Read)?; + fix_rules(&mut Com(policy), exe, port) + }; + run().map_or_else(Failure::code, |()| 0) } } #[cfg(not(windows))] -mod windows_netsh { - pub fn run(_line: &str) -> std::io::Result { - Err(std::io::Error::other("Windows only")) +mod windows_firewall { + pub fn fix(_exe: &std::path::Path, _port: u16) -> u32 { + super::Failure::Read.code() } } @@ -225,7 +400,7 @@ mod windows_elevate { use super::{Elevate, Elevated}; - /// How long the copy may take: four netsh calls. + /// How long the copy may take. const WAIT_MS: u32 = 60_000; pub struct Shell; @@ -235,10 +410,10 @@ mod windows_elevate { } impl Elevate for Shell { - fn run_elevated(&self, exe: &Path, arg: &str) -> Elevated { + fn run_elevated(&self, exe: &Path, args: &str) -> Elevated { let verb = wide("runas".as_ref()); let file = wide(exe.as_os_str()); - let params = wide(arg.as_ref()); + let params = wide(args.as_ref()); let mut info = SHELLEXECUTEINFOW { cbSize: std::mem::size_of::() as u32, fMask: SEE_MASK_NOCLOSEPROCESS, @@ -283,93 +458,347 @@ mod tests { use std::cell::{Cell, RefCell}; const EXE: &str = r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"; + const PORT: u16 = 9876; + + fn exe() -> &'static Path { + Path::new(EXE) + } + + /// What Windows adds when its prompt is dismissed. + fn prompt_block(profiles: u32) -> Rule { + Rule::unscoped(&EXE.to_lowercase(), false, profiles) + } + + fn installer_allow() -> Rule { + Rule::unscoped(EXE, true, ALLOW_PROFILES) + } + + fn named(name: &str, rule: Rule) -> Named { + Named { + name: name.into(), + rule: Some(rule), + } + } + + /// A firewall in memory. Removes go by name, as INetFwRules::Remove does. + #[derive(Default)] + struct Fake { + rules: Vec, + add_fails: bool, + remove_fails: bool, + read_fails: bool, + calls: Vec, + } + + impl RuleStore for Fake { + fn rules(&mut self) -> Result, String> { + if self.read_fails { + return Err("read".into()); + } + Ok(self.rules.clone()) + } + + fn add_allow(&mut self, exe: &str) -> Result<(), String> { + self.calls.push(format!("add {exe}")); + if self.add_fails { + return Err("add".into()); + } + self.rules + .push(named(ALLOW_NAME, Rule::unscoped(exe, true, ALLOW_PROFILES))); + Ok(()) + } + + fn remove(&mut self, name: &str) -> Result<(), String> { + self.calls.push(format!("remove {name}")); + if self.remove_fails { + return Err("remove".into()); + } + self.rules.retain(|rule| rule.name != name); + Ok(()) + } + } + + const PROMPT_UDP: &str = "UDP Query User{0-1-2-3-4}eternalmonitor-host.exe"; + + fn dismissed_prompt() -> Vec { + vec![ + named(ALLOW_NAME, installer_allow()), + named(PROMPT_UDP, prompt_block(PRIVATE | PUBLIC)), + named( + "TCP Query User{0-1-2-3-4}eternalmonitor-host.exe", + Rule { + udp: false, + ..prompt_block(PRIVATE) + }, + ), + named( + "Other", + Rule::unscoped(r"C:\Tools\Other.exe", false, PUBLIC), + ), + Named { + name: "Outbound".into(), + rule: None, + }, + ] + } + + /// Windows' Block rule for the exe goes; its Allow rule, the TCP block + /// that doesn't reach the stream, and other programs' rules stay. + #[test] + fn the_fix_removes_only_this_exes_block_and_keeps_the_rest() { + let mut store = Fake { + rules: dismissed_prompt(), + ..Fake::default() + }; + assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + let mut expected = dismissed_prompt(); + expected.remove(1); + assert_eq!(store.rules, expected); + assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")]); + } - /// The installer's [Run] netsh Parameters, as Setup passes them to - /// netsh, for the installed exe at `exe`. - fn setup_lines(exe: &str) -> Vec { + /// No Allow rule at all (deleted, or never added): it comes first, as + /// the installer adds it, then the block goes. + #[test] + fn a_missing_allow_rule_is_added_before_anything_is_removed() { + let mut rules = dismissed_prompt(); + rules.remove(0); + let mut store = Fake { + rules, + ..Fake::default() + }; + assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + assert_eq!( + store.calls, + [format!("add {EXE}"), format!("remove {PROMPT_UDP}")] + ); + assert!(store.rules.contains(&named(ALLOW_NAME, installer_allow()))); + } + + /// The fix's Allow rule is the one the installer's [Run] section adds. + #[test] + fn the_allow_rule_is_the_installers() { let iss = include_str!("../../installer/EternalMonitor.iss"); let run = iss .split("\n[Run]") .nth(1) .and_then(|section| section.split("\n[").next()) - .expect("EternalMonitor.iss has a [Run] section"); - run.lines() - .filter(|line| !line.trim_start().starts_with(';') && line.contains(r"\netsh.exe")) - .map(|line| { - line.split("Parameters: \"") - .nth(1) - .and_then(|rest| rest.split("\"; ").next()) - .expect("a netsh line has Parameters") - .replace("\"\"", "\"") - .replace(r"{app}\EternalMonitor-host.exe", exe) - }) - .collect() + .unwrap(); + let add = run + .lines() + .find(|line| line.contains("advfirewall firewall add rule")) + .expect("the installer adds its Allow rule"); + assert!(add.contains(&format!(r#"name=""{ALLOW_NAME}"""#)), "{add}"); + assert!( + add.contains(r#"program=""{app}\EternalMonitor-host.exe"""#), + "{add}" + ); + for part in [ + "dir=in", + "action=allow", + "protocol=UDP", + "profile=domain,private,public", + "enable=yes", + ] { + assert!(add.contains(part), "{part} in {add}"); + } } - /// The fix runs exactly what Setup runs, in the same order. + /// Never worse than before: when the Allow rule can't be added, + /// nothing is removed, and the rules are exactly as they were. #[test] - fn the_fix_runs_the_lines_setup_runs() { - let lines = command_lines(Path::new(EXE)).unwrap(); - assert_eq!(lines, setup_lines(EXE)); - assert_eq!(lines.len(), 4); - // Only this exe's rules and the installer's own names. - for line in &lines { - assert!( - line.contains(r#"name="EternalMonitor Host "#) - || line.contains(&format!(r#"program="{EXE}""#)), - "{line}" - ); + fn a_failed_add_leaves_the_rules_exactly_as_they_were() { + let mut rules = dismissed_prompt(); + rules.remove(0); + let before = rules.clone(); + let mut store = Fake { + rules, + add_fails: true, + ..Fake::default() + }; + assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::Add)); + assert_eq!(store.rules, before); + assert_eq!(store.calls, [format!("add {EXE}")]); + } + + /// A Block rule that can't be removed is a failure, not a success, and + /// nothing that let traffic through went. + #[test] + fn a_failed_remove_is_a_failure() { + let mut store = Fake { + rules: dismissed_prompt(), + remove_fails: true, + ..Fake::default() + }; + assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::Remove)); + assert_eq!(store.rules, dismissed_prompt()); + } + + /// A remove that reports success but leaves the block, or an add that + /// leaves no Allow rule, is caught when the rules are read again. + #[test] + fn a_fix_that_did_not_take_is_a_failure() { + struct Stubborn(Fake); + impl RuleStore for Stubborn { + fn rules(&mut self) -> Result, String> { + self.0.rules() + } + fn add_allow(&mut self, exe: &str) -> Result<(), String> { + self.0.add_allow(exe) + } + fn remove(&mut self, _name: &str) -> Result<(), String> { + Ok(()) + } } - // A copy elsewhere fixes its own rules. - let other = r"D:\Apps\EM\EternalMonitor-host.exe"; - assert!( - command_lines(Path::new(other)).unwrap()[2].ends_with(&format!(r#"program="{other}""#)) + let mut store = Stubborn(Fake { + rules: dismissed_prompt(), + ..Fake::default() + }); + assert_eq!( + fix_rules(&mut store, exe(), PORT), + Err(Failure::StillBlocked) + ); + + struct NoAdd(Fake); + impl RuleStore for NoAdd { + fn rules(&mut self) -> Result, String> { + self.0.rules() + } + fn add_allow(&mut self, _exe: &str) -> Result<(), String> { + Ok(()) + } + fn remove(&mut self, name: &str) -> Result<(), String> { + self.0.remove(name) + } + } + let mut rules = dismissed_prompt(); + rules.remove(0); + let mut store = NoAdd(Fake { + rules, + ..Fake::default() + }); + assert_eq!( + fix_rules(&mut store, exe(), PORT), + Err(Failure::StillBlocked) ); } - /// A path that would break out of its quotes never reaches netsh. + /// A block whose name another rule has too is left: removing it by name + /// could remove a rule that works. + #[test] + fn a_block_sharing_its_name_is_left() { + let mut rules = dismissed_prompt(); + rules.push(named( + PROMPT_UDP, + Rule::unscoped(r"C:\Tools\Other.exe", true, PUBLIC), + )); + let before = rules.clone(); + let mut store = Fake { + rules, + ..Fake::default() + }; + assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::SharedName)); + assert_eq!(store.rules, before); + } + + /// A scoped Block rule, a disabled one, or one for another port is the + /// user's own: left, as the verdict leaves it. #[test] - fn a_path_that_cant_be_quoted_is_refused() { - for bad in [r#"C:\a"b\x.exe"#, "C:\\a\nb\\x.exe", ""] { - assert_eq!(command_lines(Path::new(bad)), None, "{bad:?}"); + fn blocks_the_verdict_does_not_count_are_left() { + let rules = vec![ + named(ALLOW_NAME, installer_allow()), + named( + "Scoped", + Rule { + remote_addresses: Some("10.0.0.0/8".into()), + ..prompt_block(PUBLIC) + }, + ), + named( + "Off", + Rule { + enabled: false, + ..prompt_block(PUBLIC) + }, + ), + named( + "Port", + Rule { + local_ports: Some("80".into()), + ..prompt_block(PUBLIC) + }, + ), + ]; + let mut store = Fake { + rules: rules.clone(), + ..Fake::default() + }; + assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + assert_eq!(store.rules, rules); + assert!(store.calls.is_empty()); + } + + #[test] + fn rules_that_cant_be_read_are_a_failure() { + let mut store = Fake { + read_fails: true, + ..Fake::default() + }; + assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::Read)); + assert!(store.calls.is_empty()); + assert_eq!( + fix_rules(&mut Fake::default(), Path::new(""), PORT), + Err(Failure::Arguments) + ); + } + + /// The elevated copy gets the fixed argument and a port, nothing else. + #[test] + fn the_elevated_copy_takes_only_a_port() { + assert_eq!(arguments(PORT), "--fix-firewall 9876"); + assert_eq!(parse_port(&["9876".into()]), Some(9876)); + let bad: [&[&str]; 5] = [&[], &["0"], &["98765"], &["9876 x"], &["1", "2"]]; + for args in bad { + let args: Vec = args.iter().map(|arg| arg.to_string()).collect(); + assert_eq!(parse_port(&args), None, "{args:?}"); } } - struct Fake { + struct FakeElevate { answer: Elevated, asked: RefCell>, } - impl Elevate for Fake { - fn run_elevated(&self, exe: &Path, arg: &str) -> Elevated { + impl Elevate for FakeElevate { + fn run_elevated(&self, exe: &Path, args: &str) -> Elevated { self.asked .borrow_mut() - .push((exe.display().to_string(), arg.to_string())); + .push((exe.display().to_string(), args.to_string())); self.answer.clone() } } - fn fake(answer: Elevated) -> Fake { - Fake { + fn elevate(answer: Elevated) -> FakeElevate { + FakeElevate { answer, asked: RefCell::new(Vec::new()), } } - /// The elevated copy is this exe with the fixed argument, and the + /// The elevated copy is this exe with the fixed arguments, and the /// firewall is read again once it ran. #[test] fn the_fix_starts_this_exe_elevated_then_reads_the_firewall_again() { - let elevate = fake(Elevated::Exited(0)); + let elevate = elevate(Elevated::Exited(0)); let rechecked = Cell::new(false); assert_eq!( - fix(&elevate, Path::new(EXE), || rechecked.set(true)), - Outcome::Ran + fix(&elevate, exe(), PORT, || rechecked.set(true)), + Outcome::Fixed ); assert!(rechecked.get()); assert_eq!( *elevate.asked.borrow(), - [(EXE.to_string(), FIX_ARG.to_string())] + [(EXE.to_string(), "--fix-firewall 9876".to_string())] ); } @@ -377,80 +806,64 @@ mod tests { /// error shows. #[test] fn declining_the_windows_prompt_changes_nothing() { - let elevate = fake(Elevated::Declined); let rechecked = Cell::new(false); assert_eq!( - fix(&elevate, Path::new(EXE), || rechecked.set(true)), + fix(&elevate(Elevated::Declined), exe(), PORT, || rechecked + .set(true)), Outcome::Declined ); assert!(!rechecked.get()); } - /// A copy that failed, or that Windows couldn't start: the error, and - /// the firewall read again in case some lines ran. + /// Each failure says what happened, and the firewall is read again in + /// case some of it ran. #[test] fn a_fix_that_fails_says_so() { - for answer in [Elevated::Exited(1), Elevated::Failed("x".into())] { + for (answer, said) in [ + (Elevated::Exited(Failure::Add.code()), NOT_CHANGED), + (Elevated::Exited(Failure::SharedName.code()), SHARED_NAME), + (Elevated::Exited(Failure::Remove.code()), FAILED), + (Elevated::Exited(Failure::StillBlocked.code()), FAILED), + (Elevated::Exited(1), FAILED), + (Elevated::Failed("x".into()), FAILED), + ] { let rechecked = Cell::new(false); assert_eq!( - fix(&fake(answer.clone()), Path::new(EXE), || rechecked + fix(&elevate(answer.clone()), exe(), PORT, || rechecked .set(true)), - Outcome::Failed(FAILED), + Outcome::Failed(said), "{answer:?}" ); assert!(rechecked.get(), "{answer:?}"); } - let elevate = fake(Elevated::Exited(0)); - assert_eq!(fix(&elevate, Path::new(""), || {}), Outcome::Failed(FAILED)); - assert!( - elevate.asked.borrow().is_empty(), - "nothing elevated for a bad path" - ); - } - - /// The copy runs every line in order. A rule to remove that isn't there - /// doesn't count; the Allow rule that isn't added does. - #[test] - fn the_elevated_copy_runs_every_line_and_fails_only_on_the_allow_rule() { - let lines = command_lines(Path::new(EXE)).unwrap(); - let ran = RefCell::new(Vec::new()); - let code = run_lines(&lines, |line| { - ran.borrow_mut().push(line.to_string()); - Ok(if line.contains("delete") { 1 } else { 0 }) - }); - assert_eq!(code, 0); - assert_eq!(*ran.borrow(), lines); - assert_ne!( - run_lines(&lines, |line| Ok(i32::from(line.contains(" add ")))), - 0 - ); - assert_ne!( - run_lines(&lines, |line| if line.contains(" add ") { - Err(std::io::Error::other("x")) - } else { - Ok(0) - }), - 0 + let nobody = elevate(Elevated::Exited(0)); + assert_eq!( + fix(&nobody, Path::new(""), PORT, || {}), + Outcome::Failed(FAILED) ); + assert!(nobody.asked.borrow().is_empty()); } - /// One fix at a time, and the window hears how it went. + /// One fix at a time; the window hears how it went until dismissed. #[test] fn the_page_sees_the_fix_run_and_finish() { let (go, wait) = std::sync::mpsc::channel::<()>(); start(move || { let _ = wait.recv(); - Outcome::Declined + Outcome::Failed(FAILED) }); assert_eq!(progress(), Progress::Running); - // A second click while it runs starts nothing. start(|| panic!("a second fix started")); + dismiss(); + assert_eq!(progress(), Progress::Running, "a running fix stays"); go.send(()).unwrap(); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); while progress() == Progress::Running { assert!(std::time::Instant::now() < deadline); std::thread::sleep(std::time::Duration::from_millis(5)); } - assert_eq!(progress(), Progress::Done(Outcome::Declined)); + assert_eq!(progress(), Progress::Done(Outcome::Failed(FAILED))); + dismiss(); + assert_eq!(progress(), Progress::Idle); } } diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index a0185565..57c696f6 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -57,19 +57,36 @@ impl AnalyzerApp { fn notices(&mut self, ui: &mut Ui, snap: &StatsSnapshot, client: &ClientSnapshot) { self.port_notice(ui); let firewall = crate::firewall::latest(); - if let Some((title, mut body)) = firewall_notice(firewall.as_ref(), client.peer.is_some()) { - let progress = crate::firewall_fix::progress(); - if let crate::firewall_fix::Progress::Done(crate::firewall_fix::Outcome::Failed( + let progress = crate::firewall_fix::progress(); + // Its own banner, whatever the firewall reads now: a fix that went + // part of the way can leave no notice to show the failure in. + if let Some(error) = firewall_fix_failure(progress) { + widgets::banner( + ui, + WARNING, + Icon::Warning, + FIREWALL_FIX_FAILED_TITLE, error, - )) = progress - { - body = format!("{body} {error}"); - } + 220.0, + |ui| { + if let Some(port) = crate::firewall_fix::last_port() { + if widgets::button(ui, Tone::Primary, None, tr("Try again")).clicked() { + crate::firewall_fix::start_for(port); + } + } + if widgets::button(ui, Tone::Secondary, None, tr("Dismiss")).clicked() { + crate::firewall_fix::dismiss(); + } + }, + ); + ui.add_space(GAP - 10.0); + } + if let Some((title, body)) = firewall_notice(firewall.as_ref(), client.peer.is_some()) { let fixable = firewall_fix_offered(firewall.as_ref()); let width = if fixable { 110.0 } else { 0.0 }; widgets::banner(ui, WARNING, Icon::Warning, &title, &body, width, |ui| { - if fixable { - firewall_fix_action(ui, progress); + if let (true, Some(report)) = (fixable, firewall.as_ref()) { + firewall_fix_action(ui, progress, report.port); } }); ui.add_space(GAP - 10.0); @@ -1420,7 +1437,7 @@ pub(super) fn firewall_fix_offered(report: Option<&crate::firewall::Report>) -> } /// Runs the fix; it asks Windows for administrator rights once. -fn firewall_fix_action(ui: &mut Ui, progress: crate::firewall_fix::Progress) { +fn firewall_fix_action(ui: &mut Ui, progress: crate::firewall_fix::Progress, port: u16) { let running = progress == crate::firewall_fix::Progress::Running; let clicked = ui .add_enabled_ui(!running, |ui| { @@ -1429,7 +1446,23 @@ fn firewall_fix_action(ui: &mut Ui, progress: crate::firewall_fix::Progress) { .inner .clicked(); if clicked { - crate::firewall_fix::start(crate::firewall_fix::fix_now); + crate::firewall_fix::start_for(port); + } +} + +/// The title of the banner for a firewall fix that didn't work. +pub(super) const FIREWALL_FIX_FAILED_TITLE: &str = "EternalMonitor couldn't fix Windows Firewall"; + +/// The error to show for the last fix, until dismissed: a failure only. +/// A fix that worked, or a No to the Windows prompt, shows nothing. +pub(super) fn firewall_fix_failure( + progress: crate::firewall_fix::Progress, +) -> Option<&'static str> { + match progress { + crate::firewall_fix::Progress::Done(crate::firewall_fix::Outcome::Failed(error)) => { + Some(error) + } + _ => None, } } diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index e9a196ff..71859fe7 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -1,9 +1,28 @@ use super::page_stream::{ - audio_summary, capture_block_notice, firewall_fix_offered, firewall_notice, - second_screen_notice, software_fallback_body, usb_cable_notice, usb_summary, + audio_summary, capture_block_notice, firewall_fix_failure, firewall_fix_offered, + firewall_notice, second_screen_notice, software_fallback_body, usb_cable_notice, usb_summary, }; use super::*; +/// A fix that failed says so in its own banner, whatever the firewall +/// reads now. One that worked, or a No to the Windows prompt, says nothing. +#[test] +fn a_failed_firewall_fix_shows_its_error_on_its_own() { + use crate::firewall_fix::{Outcome, Progress, FAILED}; + assert_eq!( + firewall_fix_failure(Progress::Done(Outcome::Failed(FAILED))), + Some(FAILED) + ); + for quiet in [ + Progress::Idle, + Progress::Running, + Progress::Done(Outcome::Fixed), + Progress::Done(Outcome::Declined), + ] { + assert_eq!(firewall_fix_failure(quiet), None, "{quiet:?}"); + } +} + #[test] fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { use crate::firewall::{judge, Profile, Rule, PRIVATE, PUBLIC}; diff --git a/host/src/i18n/strings/en.json b/host/src/i18n/strings/en.json index d434a5fb..9c2b59dd 100644 --- a/host/src/i18n/strings/en.json +++ b/host/src/i18n/strings/en.json @@ -23,6 +23,7 @@ "Copied": "Copied", "Copy address": "Copy address", "Delay": "Delay", + "Dismiss": "Dismiss", "Display": "Display", "Done": "Done", "EternalMonitor runs in the background so your iPad can find this PC.": "EternalMonitor runs in the background so your iPad can find this PC.", @@ -90,6 +91,7 @@ "The iPad is an extra screen next to your main one.": "The iPad is an extra screen next to your main one.", "The iPad shows the same picture as your main screen.": "The iPad shows the same picture as your main screen.", "Then tap this PC, or tap Scan QR code and point the camera at this code.": "Then tap this PC, or tap Scan QR code and point the camera at this code.", + "Try again": "Try again", "USB connection": "USB connection", "USB needs Apple's free Apple Devices app.": "USB needs Apple's free Apple Devices app.", "Unavailable": "Unavailable", diff --git a/host/src/i18n/tests.rs b/host/src/i18n/tests.rs index 6751f2ee..892301d7 100644 --- a/host/src/i18n/tests.rs +++ b/host/src/i18n/tests.rs @@ -25,7 +25,7 @@ fn placeholders(text: &str) -> Vec<&str> { /// English keys whose translations come in a separate pass. Until then /// every language shows them in English. A language that has one already /// is checked like any other key. -const AWAITING_TRANSLATION: &[&str] = &["Fix it"]; +const AWAITING_TRANSLATION: &[&str] = &["Dismiss", "Fix it", "Try again"]; #[test] fn every_language_has_exactly_the_english_keys_and_placeholders() { From 146101a23b1c4860bfcd2ed5609272c142783ff8 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 10:59:59 -0500 Subject: [PATCH 211/226] Log what started each held picture on the iPad A held picture asks for keyframes, rebuilds the decoder at 2 s and reconnects at 5 s, but nothing said which check started the hold. On a still, lossless stream (no drops, no NACKs) the field showed hold after hold with no way to tell why. Each hold now logs one info line when it starts, beside the hold's own lines, which keep their wording: "Picture hold started reason=... " with the check and its numbers, the wire frame number when a picture started it, and the time since the last decoded picture. The reasons are frame_seq_gap, stream_restarted, frame_num_jump and poc_step from the reference-continuity check, decode_error at submission or output, decoder_session_failed, and decoder_session_replaced. Nothing else changes: the same pictures are held, and the same requests go out. --- .../Decode/ReferenceContinuity.swift | 70 ++++++-- ios/EternalMonitor/Decode/VideoDecoder.swift | 66 +++++-- .../VideoDecoderTests.swift | 166 ++++++++++++++++++ 3 files changed, 281 insertions(+), 21 deletions(-) diff --git a/ios/EternalMonitor/Decode/ReferenceContinuity.swift b/ios/EternalMonitor/Decode/ReferenceContinuity.swift index d37425ae..c64ceaf0 100644 --- a/ios/EternalMonitor/Decode/ReferenceContinuity.swift +++ b/ios/EternalMonitor/Decode/ReferenceContinuity.swift @@ -34,6 +34,23 @@ struct ReferenceContinuity { /// Set from the session's host caps: a forward gap in `frame_seq` is a loss. var frameSeqContiguous = false + /// Why a picture was turned down, for the log line of the hold it starts. + enum Loss: Equatable { + /// `frame_seq` skipped from `previous` to `seq` on a PC that numbers + /// contiguously. + case frameSeqGap(previous: UInt32, seq: UInt32) + /// Another epoch, or a lower `frame_seq` in this one, and this picture + /// is not its keyframe. + case streamRestarted(previousEpoch: UInt32, epoch: UInt32, previousSeq: UInt32, seq: UInt32) + /// H.264: `frame_num` is not the one after the latest reference picture. + case frameNumJump(expected: UInt32, got: UInt32) + /// HEVC: the POC moved by other than the learned step. + case pocStep(expected: UInt32, got: UInt32) + } + + /// Why the latest picture turned down was, until another is. + private(set) var lastLoss: Loss? + struct H264Parameters: Equatable { var log2MaxFrameNum: Int var separateColourPlane: Bool @@ -91,17 +108,21 @@ struct ReferenceContinuity { /// frame number, `epoch` its stream epoch; a lower `seq` or another epoch /// means the host restarted its stream. mutating func admitH264(firstSlice nal: Data, isKeyframe: Bool, seq: UInt32, epoch: UInt32) -> Bool { - let (restarted, gap) = noteStream(seq: seq, epoch: epoch) + let stream = noteStream(seq: seq, epoch: epoch) guard let params = h264, let frameNum = Self.h264FrameNum(nal, params) else { previousReferenceFrameNum = nil - return isKeyframe || !(restarted || gap) + return isKeyframe || admitStream(stream, seq: seq, epoch: epoch) } let isReference = (nal[nal.startIndex] >> 5) & 0x03 != 0 if !isKeyframe { - if restarted || gap { return false } + if !admitStream(stream, seq: seq, epoch: epoch) { return false } if let previous = previousReferenceFrameNum, !params.gapsAllowed { let maxFrameNum = UInt32(1) << params.log2MaxFrameNum - guard frameNum == (previous + 1) % maxFrameNum else { return false } + let expected = (previous + 1) % maxFrameNum + guard frameNum == expected else { + lastLoss = .frameNumJump(expected: expected, got: frameNum) + return false + } } } if isReference { previousReferenceFrameNum = frameNum } @@ -110,18 +131,19 @@ struct ReferenceContinuity { mutating func admitHEVC(firstSlice nal: Data, isKeyframe: Bool, seq: UInt32, epoch: UInt32) -> Bool { let contiguous = previousSeq.map { seq == $0 &+ 1 } ?? false - let (restarted, gap) = noteStream(seq: seq, epoch: epoch) + let stream = noteStream(seq: seq, epoch: epoch) guard let params = hevc, let poc = Self.hevcPocLsb(nal, params) else { previousPoc = nil - return isKeyframe || !(restarted || gap) + return isKeyframe || admitStream(stream, seq: seq, epoch: epoch) } if !isKeyframe { - if restarted || gap { return false } + if !admitStream(stream, seq: seq, epoch: epoch) { return false } if let previous = previousPoc, !pocUnreliable { let delta = (poc &- previous) & ((UInt32(1) << params.log2MaxPocLsb) - 1) if contiguous { if pocStep == nil { pocStep = delta } else if delta != pocStep { pocUnreliable = true } } else if let step = pocStep, delta != step { + lastLoss = .pocStep(expected: step, got: delta) return false } } @@ -130,21 +152,47 @@ struct ReferenceContinuity { return true } + /// What [`noteStream`] found, with the frame number and epoch before. + private struct StreamStep { + var restarted = false + var gap = false + var previousSeq: UInt32 = 0 + var previousEpoch: UInt32 = 0 + } + /// Records the wire frame number and epoch. `restarted`: another epoch, /// or a lower number in this one, so a new encoder whose counts, POC step /// and reliability start over. `gap`: a number skipped on a PC that /// numbers contiguously. - private mutating func noteStream(seq: UInt32, epoch: UInt32) -> (restarted: Bool, gap: Bool) { + private mutating func noteStream(seq: UInt32, epoch: UInt32) -> StreamStep { defer { previousSeq = seq; previousEpoch = epoch } - guard let previous = previousSeq else { return (false, false) } + guard let previous = previousSeq else { return StreamStep() } + var step = StreamStep(previousSeq: previous, previousEpoch: previousEpoch ?? epoch) guard previousEpoch == epoch, seq >= previous else { previousReferenceFrameNum = nil previousPoc = nil pocStep = nil pocUnreliable = false - return (true, false) + step.restarted = true + return step + } + step.gap = frameSeqContiguous && seq > previous &+ 1 + return step + } + + /// Whether a picture that isn't a keyframe may follow `step`; when not, + /// [`lastLoss`] says why. + private mutating func admitStream(_ step: StreamStep, seq: UInt32, epoch: UInt32) -> Bool { + if step.restarted { + lastLoss = .streamRestarted(previousEpoch: step.previousEpoch, epoch: epoch, + previousSeq: step.previousSeq, seq: seq) + return false + } + if step.gap { + lastLoss = .frameSeqGap(previous: step.previousSeq, seq: seq) + return false } - return (false, frameSeqContiguous && seq > previous &+ 1) + return true } // MARK: - Parsing diff --git a/ios/EternalMonitor/Decode/VideoDecoder.swift b/ios/EternalMonitor/Decode/VideoDecoder.swift index 9d81e4a8..48a202b9 100644 --- a/ios/EternalMonitor/Decode/VideoDecoder.swift +++ b/ios/EternalMonitor/Decode/VideoDecoder.swift @@ -84,6 +84,42 @@ final class VideoDecoder { var reportedStall = false } private let hold = OSAllocatedUnfairLock(initialState: nil) + /// When the decoder last put out a picture. + private let lastDecodedUs = OSAllocatedUnfairLock(initialState: nil) + + /// What started a hold, for the one line it logs. + private enum HoldCause { + /// The reference-continuity check turned a picture down. + case continuity(ReferenceContinuity.Loss?) + /// VideoToolbox refused a picture, at submission or in its output. + case decodeError(stage: String, status: OSStatus, keyframe: Bool) + /// A decoder session for new parameter sets didn't open. + case sessionFailed(OSStatus) + /// A new decoder session opened mid-stream, for new parameter sets, + /// and waits for its keyframe. + case sessionReplaced + + var fields: String { + switch self { + case .continuity(.frameSeqGap(let previous, _)?): + return "reason=frame_seq_gap previous_seq=\(previous)" + case .continuity(.streamRestarted(let previousEpoch, let epoch, let previousSeq, _)?): + return "reason=stream_restarted previous_epoch=\(previousEpoch) epoch=\(epoch) previous_seq=\(previousSeq)" + case .continuity(.frameNumJump(let expected, let got)?): + return "reason=frame_num_jump expected_frame_num=\(expected) frame_num=\(got)" + case .continuity(.pocStep(let expected, let got)?): + return "reason=poc_step expected_step=\(expected) step=\(got)" + case .continuity(nil): + return "reason=continuity" + case .decodeError(let stage, let status, let keyframe): + return "reason=decode_error stage=\(stage) status=\(status) keyframe=\(keyframe)" + case .sessionFailed(let status): + return "reason=decoder_session_failed status=\(status)" + case .sessionReplaced: + return "reason=decoder_session_replaced" + } + } + } private var recoveryTimer: DispatchSourceTimer? private let nowUs: () -> UInt64 /// Whether each picture follows a lost reference picture. @@ -174,14 +210,22 @@ final class VideoDecoder { decodeQueue.async { [weak self] in self?.continuity.frameSeqContiguous = contiguous } } - private func beginWaitingForKeyframe() { + /// `seq`: the wire frame number of the picture that started it, when one did. + private func beginWaitingForKeyframe(_ cause: HoldCause, seq: UInt32? = nil) { guard !waitingForSyncSample else { return } waitingForSyncSample = true let now = nowUs() // A keyframe that failed to decode continues the same hold. - hold.withLock { current in - if current == nil { current = Hold(start: now) } else { current?.resumeGeneration = nil } + let started = hold.withLock { current -> Bool in + if current == nil { current = Hold(start: now); return true } + current?.resumeGeneration = nil + return false } + guard started else { return } + // Once per hold: which check started it, whatever the hold then does. + let frame = seq.map { " frame_seq=\($0)" } ?? "" + let since = lastDecodedUs.withLock { $0 }.map { String((now &- $0) / 1000) } ?? "-" + onEvent?("Picture hold started \(cause.fields)\(frame) since_decoded_ms=\(since)") } /// One timer tick, queued behind the frames already given to `decode`. @@ -630,7 +674,7 @@ final class VideoDecoder { // only those) try again instead of matching it and being ignored. // Until then the last picture stays, with keyframe requests. self.formatDescription = nil - beginWaitingForKeyframe() + beginWaitingForKeyframe(.sessionFailed(status)) failedSessionAttempts += 1 if failedSessionAttempts == 1 { reportFailure("Failed to create VideoToolbox session status=\(status); retrying on the next keyframe") @@ -652,7 +696,7 @@ final class VideoDecoder { let hardwareKind = hardwareStatus == noErr ? (hardware ? "hardware" : "software") : "unknown" decompressionSession = session recoveryGeneration &+= 1 - beginWaitingForKeyframe() + beginWaitingForKeyframe(.sessionReplaced) if notReadyDrops > 0 { onEvent?("Dropped \(notReadyDrops) more access units because decoder was not ready") } @@ -697,7 +741,7 @@ final class VideoDecoder { if !admitted(nalUnits, isSyncSample: isSyncSample, seq: seq, epoch: epoch) { // A reference picture before this one never arrived (lost whole, // or dropped unfinished). Hold and ask for a keyframe now. - beginWaitingForKeyframe() + beginWaitingForKeyframe(.continuity(continuity.lastLoss), seq: seq) requestKeyframe(at: nowUs()) return } @@ -823,12 +867,13 @@ final class VideoDecoder { guard status == noErr else { self.decodeQueue.async { self.recoverFromDecodeFailure( - status, generation: generation, source: "Decoder callback", onSyncSample: isSyncSample + status, generation: generation, source: "Decoder callback", onSyncSample: isSyncSample, seq: seq ) } return } guard let pixelBuffer = imageBuffer else { return } + self.lastDecodedUs.withLock { $0 = self.nowUs() } let ended = self.hold.withLock { current -> UInt64? in guard let held = current, held.resumeGeneration == generation else { return nil } current = nil @@ -848,7 +893,7 @@ final class VideoDecoder { if status != noErr || infoFlags.contains(.frameDropped) { finish() } if status != noErr { - recoverFromDecodeFailure(status, generation: generation, source: "VTDecodeFrame", onSyncSample: isSyncSample) + recoverFromDecodeFailure(status, generation: generation, source: "VTDecodeFrame", onSyncSample: isSyncSample, seq: seq) } } @@ -872,9 +917,10 @@ final class VideoDecoder { /// Hold dependent frames until a replacement sync sample, and coalesce errors /// already in flight so one damaged frame cannot cause a request storm. private func recoverFromDecodeFailure(_ status: OSStatus, generation: UInt64, source: String, - onSyncSample: Bool) { + onSyncSample: Bool, seq: UInt32) { guard !isShutdown, generation == recoveryGeneration, !waitingForSyncSample else { return } - beginWaitingForKeyframe() + let stage = source == "VTDecodeFrame" ? "submit" : "output" + beginWaitingForKeyframe(.decodeError(stage: stage, status: status, keyframe: onSyncSample), seq: seq) reportFailure("\(source) status=\(status) codec=\(codec) depth=\(decodeDepth) generation=\(generation) sync=\(onSyncSample); waiting for a keyframe") if status == kVTInvalidSessionErr || onSyncSample { // The session died underneath us (typical after app backgrounding), diff --git a/ios/EternalMonitorTests/VideoDecoderTests.swift b/ios/EternalMonitorTests/VideoDecoderTests.swift index 3e604ee3..f9340207 100644 --- a/ios/EternalMonitorTests/VideoDecoderTests.swift +++ b/ios/EternalMonitorTests/VideoDecoderTests.swift @@ -701,3 +701,169 @@ final class VideoDecoderTests: XCTestCase { XCTAssertNil(decoder.keyframeWaitUs) } } + +// MARK: - Why a hold started + +/// Each hold logs one line saying which check started it, beside the hold's +/// own lines, which keep their wording. In the field a still, lossless stream +/// (no drops, no NACKs) was held and reconnected over and over, and nothing +/// said what started the hold. +extension VideoDecoderTests { + private func holdLines(_ events: [String]) -> [String] { + events.filter { $0.hasPrefix("Picture hold started ") } + } + + /// The lines without their time since the last picture, which the real + /// clock and VideoToolbox's asynchronous output decide (see + /// `testTheLineSaysHowLongAgoThePictureWasDecoded`). + private func holdReasons(_ events: [String]) -> [String] { + holdLines(events).map { line in + line.range(of: " since_decoded_ms=").map { String(line[..<$0.lowerBound]) } ?? line + } + } + + func testAFrameNumberJumpOrPOCStepStartsOneHoldThatSaysSo() throws { + // seq 3 lost whole on a PC that doesn't number contiguously: only the + // codec's own count shows it. + let (h264, _) = try run("h264", [(1, 0), (2, 1), (4, 3), (5, 3), (6, 4), (7, 5)]) + XCTAssertEqual(holdReasons(h264.events), + ["Picture hold started reason=frame_num_jump expected_frame_num=2 frame_num=3 frame_seq=4"]) + let (hevc, _) = try run("h265", [(1, 0), (2, 1), (4, 3), (5, 3), (6, 4), (7, 5)]) + let line = try XCTUnwrap(holdLines(hevc.events).first) + XCTAssertEqual(holdLines(hevc.events).count, 1, "\(hevc.events)") + XCTAssertTrue(line.hasPrefix("Picture hold started reason=poc_step expected_step="), line) + XCTAssertTrue(line.contains(" frame_seq=4 since_decoded_ms="), line) + } + + func testAFrameNumberGapOrStreamRestartStartsOneHoldThatSaysSo() throws { + let numbered = VideoDecoder() + numbered.setFrameSeqContiguous(true) + let (gap, _) = try run("h264", [(1, 0), (3, 5), (4, 5), (5, 4), (6, 5)], decoder: numbered) + XCTAssertEqual(holdReasons(gap.events), + ["Picture hold started reason=frame_seq_gap previous_seq=1 frame_seq=3"]) + let (restart, _) = try run("h264", [(7, 0), (8, 1), (1, 1), (2, 2), (3, 4), (4, 5)]) + XCTAssertEqual(holdReasons(restart.events), + ["Picture hold started reason=stream_restarted previous_epoch=0 epoch=0 previous_seq=8 frame_seq=1"]) + let (epoch, _) = try run("h264", [(1, 0, 1), (2, 5, 2), (3, 4, 2), (4, 5, 2)]) + XCTAssertEqual(holdReasons(epoch.events), + ["Picture hold started reason=stream_restarted previous_epoch=1 epoch=2 previous_seq=1 frame_seq=2"]) + } + + func testEachHoldSaysWhyOnceAndALosslessStreamNever() throws { + // Two holds, each with frames held behind it: one line each. + let (twice, _) = try run("h264", [(1, 0), (2, 1), (4, 3), (5, 3), (6, 4), (7, 5), (9, 7), (10, 7), (11, 0)]) + XCTAssertEqual(holdLines(twice.events).count, 2, "\(twice.events)") + XCTAssertTrue(holdLines(twice.events).allSatisfy { $0.contains("reason=frame_num_jump") }) + let (clean, _) = try run("h264", (0..<8).map { (UInt32($0 + 1), $0) }) + XCTAssertEqual(holdLines(clean.events), []) + // The first session waits for its keyframe without a hold. + let (start, _) = try run("h264", [(1, 1), (2, 0), (3, 1)]) + XCTAssertEqual(holdLines(start.events), []) + } + + func testTheLineSaysHowLongAgoThePictureWasDecoded() throws { + let units = try twoGOPs("h264") + let clock = OSAllocatedUnfairLock(initialState: 5_000_000) + let decoder = VideoDecoder(nowUs: { clock.withLock { $0 } }, runsRecoveryTimer: false) + defer { decoder.shutdown() } + let events = OSAllocatedUnfairLock(initialState: [String]()) + decoder.onEvent = { message in events.withLock { $0.append(message) } } + let decoded = OSAllocatedUnfairLock(initialState: 0) + decoder.onFrameDecoded = { _, _, _ in decoded.withLock { $0 += 1 } } + decoder.decode(packet: packet(seq: 1, data: units[0], keyframe: true)) + decoder.decode(packet: packet(seq: 2, data: units[1], keyframe: false)) + let both = expectation(for: NSPredicate { _, _ in decoded.withLock { $0 } == 2 }, evaluatedWith: nil) + wait(for: [both], timeout: 10) + clock.withLock { $0 = 5_750_000 } + decoder.decode(packet: packet(seq: 4, data: units[3], keyframe: false)) + let held = expectation(for: NSPredicate { _, _ in decoder.keyframeWaitUs != nil }, evaluatedWith: nil) + wait(for: [held], timeout: 10) + XCTAssertEqual(holdLines(events.withLock { $0 }), + ["Picture hold started reason=frame_num_jump expected_frame_num=2 frame_num=3 frame_seq=4 since_decoded_ms=750"]) + } + + func testADecodeErrorStartsOneHoldAndAFailedKeyframeContinuesIt() throws { + let data = try fixtureData() + let decoder = VideoDecoder() + defer { decoder.shutdown() } + let events = OSAllocatedUnfairLock(initialState: [String]()) + decoder.onEvent = { message in events.withLock { $0.append(message) } } + let first = expectation(description: "first picture") + let recovered = expectation(description: "recovered") + decoder.onFrameDecoded = { _, timestamp, _ in + if timestamp == 16_667 { first.fulfill() } + if timestamp == 20 * 16_667 { recovered.fulfill() } + } + let failures = OSAllocatedUnfairLock(initialState: 0) + decoder.onFailure = { message in + if message.hasPrefix("Decoder callback") || message.hasPrefix("VTDecodeFrame") { failures.withLock { $0 += 1 } } + } + func failed(_ count: Int) { + let met = expectation(for: NSPredicate { _, _ in failures.withLock { $0 } == count }, evaluatedWith: nil) + wait(for: [met], timeout: 10) + } + decoder.decode(packet: packet(seq: 1, data: data, keyframe: true)) + wait(for: [first], timeout: 10) + // VideoToolbox reports the error asynchronously: each step waits for it. + let corrupt = Data([0, 0, 0, 1, 0x41, 0x80]) + decoder.decode(packet: packet(seq: 2, data: corrupt, keyframe: false)) + failed(1) + for seq in 3...5 { decoder.decode(packet: packet(seq: UInt32(seq), data: corrupt, keyframe: false)) } + // A keyframe VideoToolbox rejects continues the same hold. + decoder.decode(packet: packet(seq: 6, data: Data([0, 0, 0, 1, 0x65, 0x80]), keyframe: true)) + failed(2) + decoder.decode(packet: packet(seq: 20, data: data, keyframe: true)) + wait(for: [recovered], timeout: 10) + let lines = holdLines(events.withLock { $0 }) + XCTAssertEqual(lines.count, 1, "\(lines)") + let line = try XCTUnwrap(lines.first) + XCTAssertTrue(line.hasPrefix("Picture hold started reason=decode_error stage="), line) + XCTAssertTrue(line.contains(" keyframe=false frame_seq=2 since_decoded_ms="), line) + } + + func testANewDecoderSessionMidStreamStartsOneHoldThatSaysWhetherItOpened() throws { + let first = try gopFixture() + let bigger = try fixture("idr_96x64", "h264") + // It opened: the new size's keyframe is shown at once. + let replaced = VideoDecoder() + defer { replaced.shutdown() } + let replacedEvents = OSAllocatedUnfairLock(initialState: [String]()) + replaced.onEvent = { message in replacedEvents.withLock { $0.append(message) } } + let shownBigger = expectation(description: "new size shown") + replaced.onFrameDecoded = { buffer, _, _ in if CVPixelBufferGetWidth(buffer) == 96 { shownBigger.fulfill() } } + replaced.decode(packet: packet(seq: 1, data: first.idr, keyframe: true)) + replaced.decode(packet: packet(seq: 2, data: bigger, keyframe: true)) + wait(for: [shownBigger], timeout: 10) + let lines = holdLines(replacedEvents.withLock { $0 }) + XCTAssertEqual(lines.count, 1, "\(lines)") + XCTAssertTrue(lines.first?.hasPrefix("Picture hold started reason=decoder_session_replaced since_decoded_ms=") == true, + "\(lines)") + // It didn't open; the next keyframe's session does, in the same hold. + let attempts = OSAllocatedUnfairLock(initialState: 0) + let failing = VideoDecoder(makeSession: { format, specification, attributes in + let attempt = attempts.withLock { value -> Int in value += 1; return value } + if attempt == 2 { return (kVTVideoDecoderMalfunctionErr, nil) } + var session: VTDecompressionSession? + let status = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: format, + decoderSpecification: specification, imageBufferAttributes: attributes, + outputCallback: nil, decompressionSessionOut: &session) + return (status, session) + }) + defer { failing.shutdown() } + let failingEvents = OSAllocatedUnfairLock(initialState: [String]()) + failing.onEvent = { message in failingEvents.withLock { $0.append(message) } } + let retried = expectation(description: "picture after the retry") + failing.onFrameDecoded = { buffer, timestamp, _ in + if timestamp == 9 * 16_667, CVPixelBufferGetWidth(buffer) == 96 { retried.fulfill() } + } + failing.decode(packet: packet(seq: 1, data: first.idr, keyframe: true)) + failing.decode(packet: packet(seq: 2, data: bigger, keyframe: true)) + failing.decode(packet: packet(seq: 3, data: first.p, keyframe: false)) + failing.decode(packet: packet(seq: 9, data: bigger, keyframe: true)) + wait(for: [retried], timeout: 10) + let failed = try XCTUnwrap(holdLines(failingEvents.withLock { $0 }).first) + XCTAssertEqual(holdLines(failingEvents.withLock { $0 }).count, 1) + XCTAssertTrue(failed.hasPrefix("Picture hold started reason=decoder_session_failed status=\(kVTVideoDecoderMalfunctionErr) since_decoded_ms="), + failed) + } +} From 16a6bff241f26773e2083c63724bd5297a3fbd03 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 10:51:08 -0500 Subject: [PATCH 212/226] Check every capture notice comes before the restarting one The precedence test covered a locked PC only. Since the graphics card notices joined the capture's own, each of them, and no display, must also come before "restarting", which returns once the block clears. --- host/src/transport/session.rs | 43 +++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index fe34cff1..0f0aef3d 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -5913,13 +5913,17 @@ mod tests { } /// The capture's own notice says more than "restarting": a stream that - /// comes back into a locked PC, or one with no display, says that, and + /// comes back into a locked PC, one with no display, or one whose + /// graphics card Windows won't capture from, says that, and /// "restarting" shows only when nothing else does. #[test] fn a_capture_notice_comes_before_the_restarting_one() { - use eternal_wire::v2::control::{NOTICE_NONE, NOTICE_PC_LOCKED, NOTICE_RESTARTING}; - struct Locked; - impl ConfigSource for Locked { + use eternal_wire::v2::control::{ + NOTICE_NONE, NOTICE_NO_DISPLAY, NOTICE_PC_LOCKED, NOTICE_RESTARTING, + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, NOTICE_UNSUPPORTED_GPU, + }; + struct Blocked(u8); + impl ConfigSource for Blocked { fn stream_config(&self) -> StreamConfig { TestConfig.stream_config() } @@ -5927,7 +5931,7 @@ mod tests { TestConfig.host_name() } fn notice(&self) -> u8 { - NOTICE_PC_LOCKED + self.0 } } let now = Instant::now(); @@ -5945,19 +5949,24 @@ mod tests { notices(&session.restart_tick(&RestartConfig, at)), [NOTICE_RESTARTING] ); - // The next stream runs into a locked PC, before its first frame. + // The next stream runs into a blocked capture, before its first frame. session.set_not_streaming(None); let at = now + Duration::from_secs(2); - session.handle_control_authed(peer, report(10), &Locked, at); - assert_eq!( - notices(&session.tick(&Locked, true, at)), - [NOTICE_PC_LOCKED] - ); - // Unlocked, still no frame: restarting again. - assert_eq!( - notices(&session.tick(&TestConfig, true, at)), - [NOTICE_RESTARTING] - ); - assert_ne!(NOTICE_PC_LOCKED, NOTICE_NONE); + for notice in [ + NOTICE_PC_LOCKED, + NOTICE_NO_DISPLAY, + NOTICE_UNSUPPORTED_GPU, + NOTICE_SECOND_SCREEN_UNSUPPORTED_GPU, + ] { + assert_ne!(notice, NOTICE_NONE); + session.handle_control_authed(peer, report(10), &Blocked(notice), at); + assert_eq!(notices(&session.tick(&Blocked(notice), true, at)), [notice]); + // Unblocked, still no frame: restarting again. + assert_eq!( + notices(&session.tick(&TestConfig, true, at)), + [NOTICE_RESTARTING], + "after notice {notice}" + ); + } } } From f9a5fe77c9e5a9acd968d816eb98eed1e454a95d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 11:05:14 -0500 Subject: [PATCH 213/226] Pass the GPU refusal to the summary written before the PC goes down The candidate's session summary now says whether Windows refused the screen from the graphics card. The summary written straight to disk before a shutdown, sign-out or installer close still called it the old way. Both summaries now come from one helper. --- host/src/diagnostics.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index 109a1172..04f928f6 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -343,16 +343,19 @@ impl tracing_subscriber::Layer for FallbackWatch { /// Logs the summary of a session, or of a HELLO2 the host turned down. The /// upload sends it as category `summary` (`crate::telemetry`). pub fn log_session_end(end: &SessionEnd, shared: &SharedControl) { + info!("{}", summary_line(end, shared)); +} + +/// The summary of `end`, with what the PC knows at the end, for +/// [`log_session_end`] and [`log_session_end_now`] alike. +fn summary_line(end: &SessionEnd, shared: &SharedControl) -> String { let second_screen = *shared.capture_target.lock() == CaptureTarget::VirtualExtended; // `mode=` is the mode at the end; the fallback was counted in Second // screen mode, perhaps before a switch to Mirror. let fallback = end.session_id.and_then(take_fallback); let refused = *shared.capture_block.lock() == Some(crate::capture::recovery::CaptureBlock::UnsupportedOnThisGpu); - info!( - "{}", - session_summary(end, second_screen, fallback, refused, host_facts()) - ); + session_summary(end, second_screen, fallback, refused, host_facts()) } /// The app is quitting: the summary of the stream it stops. Its iPad is @@ -395,10 +398,7 @@ pub fn end_stream(shared: &SharedControl, why: StreamStop) -> Option /// the session log and the upload outbox on disk, so it survives the /// process ending a moment later, and the next launch uploads it. pub fn log_session_end_now(end: &SessionEnd, shared: &SharedControl) { - let second_screen = *shared.capture_target.lock() == CaptureTarget::VirtualExtended; - let fallback = end.session_id.and_then(take_fallback); - let line = - crate::startup::info_line(&session_summary(end, second_screen, fallback, host_facts())); + let line = crate::startup::info_line(&summary_line(end, shared)); let written = crate::logging::append_now(&line).and(crate::telemetry::queue_now(&line, "info")); if let Err(error) = written { tracing::warn!(%error, "Could not save the session summary before the PC went down"); From 5cee55787f67a30cc9928b345f78894026e4d69b Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 11:05:31 -0500 Subject: [PATCH 214/226] Start a mirror from the screen's picture when duplication gives none Desktop Duplication delivers an image only once the desktop presents one. A display that went to sleep, or a desktop that changes nothing, presents none, and capture skips a new duplication's blank first frame. The encoder then opened only at the 750 ms keepalive, which sent the black start frame: a viewer that connected at launch got a black mirror after about a second, and kept it until the desktop changed. When a mirror's duplication has given no image 100 ms after it was created (or created again, or resized), capture now reads the output's picture once through GDI, which still returns what the desktop shows, and publishes it as the start picture. The second screen is unchanged: its proof is an image from its own duplication. The GDI picture has no pointer; the next image from the duplication replaces it. --- host/src/capture/dxgi.rs | 113 ++++++++++++++++++++++++++++++++++++++- host/src/capture/mod.rs | 83 ++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 1 deletion(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index f72ea660..0a9572c8 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -21,7 +21,7 @@ use super::{ idle_heartbeat, keepalive_allowed, keepalive_due, pointer_apart, pointer_place, read_back_image, reconcile_virtual_display, recovery_poll, redraw_cached, resolve_target, Acquired, FrameSlot, FrameUse, IdleGate, OutputInfo, Picture, PointerPlace, RawFrame, Redraw, - ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, PICTURE_HOLD, + StartPicture, ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, PICTURE_HOLD, }; use crate::control::{CaptureTarget, SharedControl}; use crate::gpu_preference::Outcome; @@ -307,6 +307,7 @@ pub fn run_capture_loop( let mut resumed = false; let mut idle = IdleGate::new(); let mut picture = Picture::default(); + let mut start_picture = StartPicture::new(Instant::now()); // ACCESS_LOST from a call after the acquire (ReleaseFrame, the staging // Map). It is handled as if the next acquire had returned it: before, // it ended the capture loop and restarted the pipeline. @@ -438,6 +439,33 @@ pub fn run_capture_loop( } } + if start_picture.due(picture.real(), second_screen, Instant::now()) { + if let Some(pixels) = read_screen_gdi(&output1, (tex_width, tex_height)) { + info!( + device = %chosen.device_name, + "Desktop duplication gave no image yet — starting from the screen's picture" + ); + spare = None; + last_frame = Arc::new(pixels); + last_frame_has_pointer = false; + picture.cached(); + let now = Instant::now(); + if idle.on_frame(client_connected(&shared), now) == FrameUse::Publish { + *frame_number += 1; + PIPELINE_STATS.lock().record_capture(tex_width, tex_height); + heartbeat(&shared.hb_capture_frame_ms); + slot.publish(RawFrame { + frame_number: *frame_number, + timestamp: now, + data: Arc::clone(&last_frame), + width: tex_width, + height: tex_height, + }); + last_send = now; + } + } + } + let frame_start = Instant::now(); let frame_budget = frame_budget_for(shared.effective_fps()); // Half the frame budget, clamped: at 60 fps this is the classic 8 ms; @@ -849,6 +877,7 @@ pub fn run_capture_loop( Ok(d) => { duplication = Some(d); picture.recreated(Instant::now()); + start_picture.duplicated(Instant::now()); super::second_screen::note_recovery( &shared, generation, @@ -912,6 +941,7 @@ pub fn run_capture_loop( spare = None; last_frame = Arc::new(frame_buffer(frame_bytes)); picture.resized(); + start_picture.duplicated(Instant::now()); staging_texture = unsafe { let mut tex = None; device.CreateTexture2D( @@ -1197,6 +1227,87 @@ fn publish_capture_geometry(output: &IDXGIOutput1, shared: &SharedControl) { } } +/// The picture `output` shows, read through GDI, as `width`x`height` BGRA +/// rows in a [`frame_buffer`]: for a duplication that gives no image (see +/// [`StartPicture`]). The pointer is not in it. None if the output's desktop +/// rect isn't that size (a rotated display) or GDI can't read it (no +/// interactive desktop). +fn read_screen_gdi(output: &IDXGIOutput1, (width, height): (u32, u32)) -> Option> { + use windows::Win32::Graphics::Gdi::{ + BitBlt, CreateCompatibleDC, CreateDIBSection, DeleteDC, DeleteObject, GdiFlush, GetDC, + ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, DIB_RGB_COLORS, SRCCOPY, + }; + let rect = unsafe { output.GetDesc() }.ok()?.DesktopCoordinates; + if (rect.right - rect.left, rect.bottom - rect.top) != (width as i32, height as i32) { + return None; + } + let row_bytes = width as usize * 4; + let info = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: std::mem::size_of::() as u32, + biWidth: width as i32, + // Negative: top-down rows, as capture's. + biHeight: -(height as i32), + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0, + ..Default::default() + }, + ..Default::default() + }; + unsafe { + let screen = GetDC(None); + if screen.is_invalid() { + return None; + } + let memory = CreateCompatibleDC(screen); + let mut bits = std::ptr::null_mut(); + let pixels = match CreateDIBSection(memory, &info, DIB_RGB_COLORS, &mut bits, None, 0) { + Ok(bitmap) if !bits.is_null() => { + let previous = SelectObject(memory, bitmap); + let copied = BitBlt( + memory, + 0, + 0, + width as i32, + height as i32, + screen, + rect.left, + rect.top, + SRCCOPY, + ) + .is_ok(); + let _ = GdiFlush(); + let pixels = copied.then(|| { + let mut buf = frame_buffer(row_bytes * height as usize); + std::ptr::copy_nonoverlapping( + bits as *const u8, + buf.as_mut_ptr(), + row_bytes * height as usize, + ); + // GDI leaves the alpha byte 0; Desktop Duplication's is opaque. + let (pixels, _) = buf[..row_bytes * height as usize].as_chunks_mut::<4>(); + for pixel in pixels { + pixel[3] = 255; + } + buf + }); + SelectObject(memory, previous); + let _ = DeleteObject(bitmap); + pixels + } + Ok(bitmap) => { + let _ = DeleteObject(bitmap); + None + } + Err(_) => None, + }; + let _ = DeleteDC(memory); + ReleaseDC(None, screen); + pixels + } +} + /// Reads the staging texture's image into `buf`, row by row (its pitch may /// differ from `row_bytes`). fn copy_staging( diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index b4e430a9..d5e742be 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -1103,6 +1103,54 @@ impl Picture { } } +/// How long a mirror's new duplication may go without an image before its +/// start picture is read through GDI instead (see [`StartPicture`]). +#[cfg(any(windows, test))] +pub(crate) const START_PICTURE_GRACE: Duration = Duration::from_millis(100); + +/// A mirror's start picture when its duplication gives none. Desktop +/// Duplication delivers an image only once the desktop presents one. A +/// display that went to sleep, or a desktop that changes nothing, presents +/// none: the encoder then opened only at the 750 ms keepalive, which sent the +/// black start frame, and the viewer saw black until the desktop changed. +/// GDI still reads what the desktop shows. +#[cfg(any(windows, test))] +#[derive(Debug)] +pub(crate) struct StartPicture { + since: Instant, + tried: bool, +} + +#[cfg(any(windows, test))] +impl StartPicture { + pub fn new(now: Instant) -> Self { + Self { + since: now, + tried: false, + } + } + + /// A duplication was created again, or its picture size changed. + pub fn duplicated(&mut self, now: Instant) { + *self = Self::new(now); + } + + /// Whether to read the start picture through GDI now: once per + /// duplication, [`START_PICTURE_GRACE`] after it was created, while the + /// cache holds no real picture (`real`, see [`Picture::real`]). Never on + /// the second screen, whose image from the duplication is its proof. + pub fn due(&mut self, real: bool, second_screen: bool, now: Instant) -> bool { + if real || second_screen || self.tried { + return false; + } + if now.saturating_duration_since(self.since) < START_PICTURE_GRACE { + return false; + } + self.tried = true; + true + } +} + /// Whether an idle keepalive may resend the last frame. Before the second /// screen has delivered an image, that frame is the black placeholder: /// sending it would replace the iPad's mirror with black, so the switch @@ -1403,6 +1451,41 @@ mod tests { assert!(kept.is_none()); } + #[test] + fn a_mirror_without_an_image_reads_its_start_picture_once_after_the_grace() { + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let mut gdi = super::StartPicture::new(start); + assert!( + !gdi.due(false, false, at(50)), + "the duplication may still give one" + ); + assert!(gdi.due(false, false, at(100))); + assert!(!gdi.due(false, false, at(200)), "once per duplication"); + // A new duplication (after access was lost, or a size change) whose + // cache is black tries again. + gdi.duplicated(at(1000)); + assert!(!gdi.due(false, false, at(1050))); + assert!(gdi.due(false, false, at(1100))); + } + + #[test] + fn a_real_picture_or_a_second_screen_never_reads_the_start_picture() { + let start = Instant::now(); + let later = start + Duration::from_secs(5); + let mut gdi = super::StartPicture::new(start); + assert!( + !gdi.due(true, false, later), + "the duplication gave an image" + ); + assert!( + !gdi.due(false, true, later), + "a second screen proves itself" + ); + // Neither used up the one try. + assert!(gdi.due(false, false, later)); + } + #[test] fn pointer_only_frames_are_not_read_back_before_the_first_image() { assert!(!frame_has_image(0, false), "blank first frame"); From 361b1edd82a9f68a0e021330b88ad764bcfd9acf Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 11:05:32 -0500 Subject: [PATCH 215/226] End the idle capture nap when a viewer connects With nobody watching a still desktop, the capture loop slept 250 ms at a time. A viewer that connected during a nap waited out the rest of it before capture resent the picture for its first keyframe: up to 250 ms on the first session after the PC sat idle. The nap now runs in 50 ms slices and ends as soon as a client connects or capture stops. --- host/src/capture/dxgi.rs | 24 +++++++++++------ host/src/capture/mod.rs | 57 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 0a9572c8..34a14952 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -18,7 +18,7 @@ use super::recovery::{ }; use super::{ acquired, client_connected, composite_pointer, frame_budget_for, frame_buffer, heartbeat, - idle_heartbeat, keepalive_allowed, keepalive_due, pointer_apart, pointer_place, + idle_heartbeat, keepalive_allowed, keepalive_due, nap_unless, pointer_apart, pointer_place, read_back_image, reconcile_virtual_display, recovery_poll, redraw_cached, resolve_target, Acquired, FrameSlot, FrameUse, IdleGate, OutputInfo, Picture, PointerPlace, RawFrame, Redraw, StartPicture, ACQUIRE_TIMEOUT_MS, FPS_WINDOW, FRAME_PADDING, PICTURE_HOLD, @@ -750,13 +750,21 @@ pub fn run_capture_loop( && last_send.elapsed() >= Duration::from_secs(2) { // Idle throttle: nobody is watching and the desktop is - // static — stop spinning at capture rate. 4 Hz polling - // still notices a client promptly (registration also - // restarts the pipeline for virtual-display targets). - // The 250 ms naps stay far under the supervisor's 3 s - // loop-heartbeat wedge threshold. - idle_heartbeat(&shared, crate::clock::host_now_us() / 1000); - std::thread::sleep(Duration::from_millis(250)); + // static — stop spinning at capture rate. 4 Hz polling, + // whose nap ends within 50 ms of a client connecting + // (registration also restarts the pipeline for + // virtual-display targets). The 250 ms naps stay far + // under the supervisor's 3 s loop-heartbeat wedge + // threshold. + nap_unless( + Duration::from_millis(250), + || { + client_connected(&shared) + || !super::capture_is_current(&shared, generation) + }, + || idle_heartbeat(&shared, crate::clock::host_now_us() / 1000), + std::thread::sleep, + ); } continue; } diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index d5e742be..beb5ab87 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -1103,6 +1103,27 @@ impl Picture { } } +/// Naps up to `total` in slices of at most 50 ms, calling `heartbeat` before +/// each, and stops as soon as `wake` says to: a viewer that connected during +/// the idle throttle's 250 ms nap waited out the rest of it before capture +/// resent the picture. +#[cfg(any(windows, test))] +pub(crate) fn nap_unless( + total: Duration, + mut wake: impl FnMut() -> bool, + mut heartbeat: impl FnMut(), + mut sleep: impl FnMut(Duration), +) { + const SLICE: Duration = Duration::from_millis(50); + let mut left = total; + while !left.is_zero() && !wake() { + heartbeat(); + let nap = left.min(SLICE); + sleep(nap); + left -= nap; + } +} + /// How long a mirror's new duplication may go without an image before its /// start picture is read through GDI instead (see [`StartPicture`]). #[cfg(any(windows, test))] @@ -1451,6 +1472,42 @@ mod tests { assert!(kept.is_none()); } + #[test] + fn the_idle_nap_ends_when_a_viewer_connects() { + let slept = std::cell::RefCell::new(Vec::new()); + let checks = std::cell::Cell::new(0); + super::nap_unless( + Duration::from_millis(250), + || { + checks.set(checks.get() + 1); + checks.get() > 2 + }, + || {}, + |nap| slept.borrow_mut().push(nap), + ); + let total: Duration = slept.borrow().iter().sum(); + assert!( + total <= Duration::from_millis(100), + "napped {total:?} after the viewer connected" + ); + assert!(slept + .borrow() + .iter() + .all(|nap| *nap <= Duration::from_millis(50))); + // Nobody connects: the whole nap. + let slept = std::cell::RefCell::new(Vec::new()); + super::nap_unless( + Duration::from_millis(250), + || false, + || {}, + |nap| slept.borrow_mut().push(nap), + ); + assert_eq!( + slept.borrow().iter().sum::(), + Duration::from_millis(250) + ); + } + #[test] fn a_mirror_without_an_image_reads_its_start_picture_once_after_the_grace() { let start = Instant::now(); From b83bfb88cc68c0a97e10e5e276b9c6cc3bb30f82 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 11:57:51 -0500 Subject: [PATCH 216/226] Judge the firewall fix as the Stream page does, and remove only the rules it counts The fix now decides from the same verdict the Stream page shows, on the networks in use with Windows Firewall on. It adds the installer's Allow rule only when no Allow rule already lets the stream in there, so a narrower rule the user made stays the only one. It removes only this exe's own Block rules on those networks, leaving blocks for other networks alone, and reads everything back through the verdict before it reports success. Each Block rule is renamed to a fresh name and removed by that name, so no other rule sharing its name can go with it. The firewall objects are released before COM is shut down. The Windows prompt belongs to the window that asked for it, and the page waits for the fix however long it runs, so Try again never starts a second copy. --- host/src/firewall.rs | 14 +- host/src/firewall_fix.rs | 587 ++++++++++++++++++++++++------------ host/src/gui/page_stream.rs | 4 +- host/src/gui/tests.rs | 2 +- host/src/main.rs | 4 +- 5 files changed, 408 insertions(+), 203 deletions(-) diff --git a/host/src/firewall.rs b/host/src/firewall.rs index b1bb686f..13192118 100644 --- a/host/src/firewall.rs +++ b/host/src/firewall.rs @@ -672,6 +672,15 @@ pub(crate) mod windows_policy { let _apartment = Apartment::new()?; let policy: INetFwPolicy2 = unsafe { CoCreateInstance(&NetFwPolicy2, None, CLSCTX_INPROC_SERVER)? }; + let profiles = current_profiles(&policy)?; + let rules = inbound_rules(&policy)?; + Ok(judge(&profiles, &rules, exe, port, lan, |name| { + std::env::var(name).ok() + })) + } + + /// The profiles of the networks in use now, as the verdict sees them. + pub(crate) fn current_profiles(policy: &INetFwPolicy2) -> windows::core::Result> { let current = unsafe { policy.CurrentProfileTypes()? } as u32; let mut profiles = Vec::new(); for bit in [DOMAIN, PRIVATE, PUBLIC] { @@ -685,10 +694,7 @@ pub(crate) mod windows_policy { block_all_inbound: unsafe { policy.get_BlockAllInboundTraffic(profile)? }.as_bool(), }); } - let rules = inbound_rules(&policy)?; - Ok(judge(&profiles, &rules, exe, port, lan, |name| { - std::env::var(name).ok() - })) + Ok(profiles) } impl RuleSource for INetFwRule { diff --git a/host/src/firewall_fix.rs b/host/src/firewall_fix.rs index e2763685..442f016c 100644 --- a/host/src/firewall_fix.rs +++ b/host/src/firewall_fix.rs @@ -8,16 +8,22 @@ //! //! It can't leave the PC worse off than before. Setup deletes every rule //! for the exe, then adds its Allow rule; deletions with no add after them -//! could lose rules that worked. The fix is narrower ([`fix_rules`]): +//! could lose rules that worked. The fix is narrower ([`fix_rules`]), and +//! judges every step as the Stream page's verdict does +//! ([`crate::firewall::judge`]), on the networks in use now: //! -//! 1. It adds the installer's Allow rule, unless one that works is there. -//! It does this first, so a failure here changes nothing. -//! 2. It removes only this exe's own Block rules that reach the stream -//! port with no other scope: the rules the Stream page's verdict counts. -//! Removing a Block rule never stops anything that worked. Allow rules, -//! scoped rules and other programs' rules stay as they are. -//! 3. It reads the rules again and fails unless the Allow rule is there and -//! none of those Block rules is left. +//! 1. Unless the exe would be allowed on every network in use once its +//! blocks are gone, it adds the installer's Allow rule, first, so a +//! failure here changes nothing. An Allow rule the user made, however +//! narrow, that already covers those networks means no add. +//! 2. It removes only this exe's own enabled Block rules that reach the +//! stream port with no other scope, on a network in use: the blocks the +//! verdict counts. Each is renamed to a fresh name, then removed by that +//! name, so no other rule can go with it. Removing a Block rule never +//! stops anything that worked. Allow rules, scoped rules, blocks for +//! networks not in use and other programs' rules stay as they are. +//! 3. It reads everything again and fails unless the verdict now finds the +//! exe allowed, and not blocked by its own rule, on every network in use. //! //! The exe's path comes from the elevated process itself, and the port is //! only ever a number. A user who answers No to the Windows prompt changes @@ -26,7 +32,7 @@ use std::path::Path; -use crate::firewall::{reach, Reach, Rule, DOMAIN, PRIVATE, PUBLIC}; +use crate::firewall::{judge, reach, Profile, Reach, Rule, Verdict, DOMAIN, PRIVATE, PUBLIC}; /// The argument the elevated copy runs with, before the stream's port. pub const FIX_ARG: &str = "--fix-firewall"; @@ -36,22 +42,26 @@ pub const FIX_ARG: &str = "--fix-firewall"; pub const ALLOW_NAME: &str = "EternalMonitor Host UDP"; pub const ALLOW_PROFILES: u32 = DOMAIN | PRIVATE | PUBLIC; -/// A firewall rule with its name. `rule` is None for an outbound rule. +/// A firewall rule as the last read found it. `id` names it for +/// [`RuleStore::remove`] until the next read. `rule` is None for an +/// outbound rule. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct Named { - pub name: String, +pub struct Found { + pub id: usize, pub rule: Option, } /// The firewall, as the elevated copy changes it. The real one is the /// Windows Firewall COM interface; tests fake it. pub trait RuleStore { + /// The profiles of the networks in use now. + fn profiles(&mut self) -> Result, String>; /// Every rule, in any direction. - fn rules(&mut self) -> Result, String>; + fn rules(&mut self) -> Result, String>; /// Adds the installer's Allow rule for `exe`. fn add_allow(&mut self, exe: &str) -> Result<(), String>; - /// Removes the rule named `name`. - fn remove(&mut self, name: &str) -> Result<(), String>; + /// Removes the rule the last read found as `id`, and no other. + fn remove(&mut self, id: usize) -> Result<(), String>; } /// Why the fix failed, as the elevated copy's exit code. @@ -59,16 +69,14 @@ pub trait RuleStore { pub enum Failure { /// The copy got no usable path or port. Arguments = 3, - /// The rules couldn't be read. + /// The rules or the networks in use couldn't be read. Read = 10, /// The Allow rule couldn't be added. Nothing was changed. Add = 11, /// A Block rule couldn't be removed. Remove = 12, - /// A Block rule shares its name with another rule, so removing it by - /// name could remove that one too. It was left. - SharedName = 13, - /// Read again, a Block rule is still there, or the Allow rule isn't. + /// Read again, the verdict still finds the exe blocked by its own rule, + /// or not allowed, on a network in use. StillBlocked = 14, } @@ -78,31 +86,50 @@ impl Failure { } } -fn ours(rule: &Rule, exe: &Path) -> bool { - crate::firewall::ours(rule, exe, |name| std::env::var(name).ok()) +fn env(name: &str) -> Option { + std::env::var(name).ok() } -/// This exe's own Block rules that reach every datagram to `port`: what -/// the verdict counts as a block, and all the fix removes. -fn is_block(rule: &Rule, exe: &Path, port: u16) -> bool { - !rule.allow && rule.enabled && rule.udp && ours(rule, exe) && reach(rule, port) == Reach::All +/// The bits of the profiles in use with Windows Firewall on: where the +/// verdict counts rules. +fn in_use(profiles: &[Profile]) -> u32 { + profiles + .iter() + .filter(|profile| profile.on) + .map(|profile| profile.bit) + .sum() } -/// An Allow rule that lets every datagram to `port` through on every -/// profile, as the installer's does. -fn is_working_allow(rule: &Rule, exe: &Path, port: u16) -> bool { - rule.allow +/// This exe's own Block rules the verdict counts: enabled, reaching every +/// datagram to `port`, on a network in use. All the fix removes. +fn is_target(rule: &Rule, exe: &Path, port: u16, in_use: u32) -> bool { + !rule.allow && rule.enabled && rule.udp - && rule.profiles & ALLOW_PROFILES == ALLOW_PROFILES - && ours(rule, exe) + && rule.profiles & in_use != 0 + && crate::firewall::ours(rule, exe, env) && reach(rule, port) == Reach::All } -fn inbound(rules: &[Named]) -> impl Iterator { - rules +/// Whether the verdict on `rules` finds the exe allowed, and not blocked by +/// its own rule, on every network in use. A network that blocks all +/// incoming connections or has the firewall off is the user's switch, not +/// a rule the fix could change. +fn allowed_everywhere(profiles: &[Profile], rules: &[&Rule], exe: &Path, port: u16) -> bool { + let rules: Vec = rules.iter().map(|rule| (*rule).clone()).collect(); + let report = judge(profiles, &rules, exe, port, true, env); + report.profiles.iter().all(|(_, verdict)| { + matches!( + verdict, + Verdict::Allowed | Verdict::BlocksAll | Verdict::Off + ) + }) +} + +fn inbound(found: &[Found]) -> impl Iterator { + found .iter() - .filter_map(|named| named.rule.as_ref().map(|rule| (named.name.as_str(), rule))) + .filter_map(|found| found.rule.as_ref().map(|rule| (found.id, rule))) } /// The fix, on `store`, for this exe at `exe` and the stream's `port`. @@ -111,33 +138,50 @@ pub fn fix_rules(store: &mut impl RuleStore, exe: &Path, port: u16) -> Result<() .to_str() .filter(|path| !path.is_empty()) .ok_or(Failure::Arguments)?; - let rules = store.rules().map_err(|_| Failure::Read)?; - if !inbound(&rules).any(|(_, rule)| is_working_allow(rule, exe, port)) { + let profiles = store.profiles().map_err(|_| Failure::Read)?; + let found = store.rules().map_err(|_| Failure::Read)?; + let bits = in_use(&profiles); + let targets: Vec = inbound(&found) + .filter(|(_, rule)| is_target(rule, exe, port, bits)) + .map(|(id, _)| id) + .collect(); + let without_targets: Vec<&Rule> = inbound(&found) + .filter(|(id, _)| !targets.contains(id)) + .map(|(_, rule)| rule) + .collect(); + if !allowed_everywhere(&profiles, &without_targets, exe, port) { store.add_allow(program).map_err(|_| Failure::Add)?; } - let mut shared = false; - for (name, rule) in inbound(&rules) { - if !is_block(rule, exe, port) { - continue; - } - if rules.iter().filter(|other| other.name == name).count() > 1 { - shared = true; - continue; - } - store.remove(name).map_err(|_| Failure::Remove)?; + for id in targets { + store.remove(id).map_err(|_| Failure::Remove)?; } + let profiles = store.profiles().map_err(|_| Failure::Read)?; let after = store.rules().map_err(|_| Failure::Read)?; - let blocked = inbound(&after).any(|(_, rule)| is_block(rule, exe, port)); - let allowed = inbound(&after).any(|(_, rule)| is_working_allow(rule, exe, port)); - if blocked && shared { - Err(Failure::SharedName) - } else if blocked || !allowed { - Err(Failure::StillBlocked) - } else { + let after: Vec<&Rule> = inbound(&after).map(|(_, rule)| rule).collect(); + if allowed_everywhere(&profiles, &after, exe, port) { Ok(()) + } else { + Err(Failure::StillBlocked) } } +/// Runs `fix` on a store opened inside an apartment, and drops the store +/// before the apartment: COM objects released after `CoUninitialize` may +/// call into a library Windows has already unloaded. +#[cfg(any(windows, test))] +pub(crate) fn in_apartment( + apartment: impl FnOnce() -> Result, + store: impl FnOnce() -> Result, + fix: impl FnOnce(&mut S) -> Result, +) -> Result { + let apartment = apartment()?; + let mut store = store()?; + let result = fix(&mut store); + drop(store); + drop(apartment); + result +} + /// How starting the elevated copy went. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Elevated { @@ -145,12 +189,13 @@ pub enum Elevated { Exited(u32), /// The user answered No to the Windows prompt. Declined, - /// Windows couldn't start it, or it didn't finish in time. + /// Windows couldn't start it. Failed(String), } -/// Starts this exe again as administrator with `args`, and waits for it. -/// The real one asks Windows; tests fake it. +/// Starts this exe again as administrator with `args`, and waits for it to +/// exit, however long it takes: a second copy never starts meanwhile. The +/// real one asks Windows; tests fake it. pub trait Elevate { fn run_elevated(&self, exe: &Path, args: &str) -> Elevated; } @@ -158,8 +203,8 @@ pub trait Elevate { /// What the Stream page shows after "Fix it". #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Outcome { - /// The firewall now has the Allow rule and none of the Block rules; it - /// was read again, and the notice follows it. + /// Read again, the verdict finds the exe allowed on every network in + /// use; the Stream page reads it again too, and the notice follows. Fixed, /// No to the Windows prompt: nothing changed. Declined, @@ -182,9 +227,6 @@ pub fn fix(elevate: &impl Elevate, exe: &Path, port: u16, recheck: impl FnOnce() Elevated::Declined => return Outcome::Declined, Elevated::Exited(0) => Outcome::Fixed, Elevated::Exited(code) if code == Failure::Add.code() => Outcome::Failed(NOT_CHANGED), - Elevated::Exited(code) if code == Failure::SharedName.code() => { - Outcome::Failed(SHARED_NAME) - } Elevated::Exited(_) | Elevated::Failed(_) => Outcome::Failed(FAILED), }; recheck(); @@ -195,7 +237,6 @@ pub fn fix(elevate: &impl Elevate, exe: &Path, port: u16, recheck: impl FnOnce() pub const FAILED: &str = "Windows didn't apply the whole fix. Run the EternalMonitor installer again."; pub const NOT_CHANGED: &str = "Windows didn't let EternalMonitor add its firewall rule, so nothing was changed. Run the EternalMonitor installer again."; -pub const SHARED_NAME: &str = "A firewall rule blocking EternalMonitor shares its name with another rule, so it was left. Remove it in Windows Defender Firewall, or run the EternalMonitor installer again."; /// Where the Stream page's "Fix it" is. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -315,28 +356,41 @@ mod windows_firewall { use windows::Win32::NetworkManagement::WindowsFirewall::{ INetFwPolicy2, INetFwRule, NetFwPolicy2, NetFwRule, NET_FW_ACTION_ALLOW, NET_FW_RULE_DIR_IN, }; - use windows::Win32::System::Com::{CoCreateInstance, CLSCTX_INPROC_SERVER}; + use windows::Win32::System::Com::{CoCreateGuid, CoCreateInstance, CLSCTX_INPROC_SERVER}; - use super::{fix_rules, Failure, Named, RuleStore, ALLOW_NAME, ALLOW_PROFILES}; - use crate::firewall::rule_from; - use crate::firewall::windows_policy::{for_each_rule, Apartment}; + use super::{fix_rules, in_apartment, Failure, Found, RuleStore, ALLOW_NAME, ALLOW_PROFILES}; + use crate::firewall::windows_policy::{current_profiles, for_each_rule, Apartment}; + use crate::firewall::{rule_from, Profile}; /// UDP, as `NET_FW_IP_PROTOCOL`. const UDP: i32 = 17; - struct Com(INetFwPolicy2); + /// The policy, and the rules its last read found, by id. + struct Com { + policy: INetFwPolicy2, + found: Vec, + } impl RuleStore for Com { - fn rules(&mut self) -> Result, String> { + fn profiles(&mut self) -> Result, String> { + current_profiles(&self.policy).map_err(|e| e.to_string()) + } + + fn rules(&mut self) -> Result, String> { + let mut found = Vec::new(); let mut rules = Vec::new(); - for_each_rule(&self.0, |rule| { - let name = unsafe { rule.Name()? }.to_string(); - let rule = rule_from(rule).map_err(|e| windows::core::Error::new(E_FAIL, e))?; - rules.push(Named { name, rule }); + for_each_rule(&self.policy, |rule| { + let read = rule_from(rule).map_err(|e| windows::core::Error::new(E_FAIL, e))?; + found.push(Found { + id: rules.len(), + rule: read, + }); + rules.push(rule.clone()); Ok(()) }) .map_err(|e| e.to_string())?; - Ok(rules) + self.found = rules; + Ok(found) } fn add_allow(&mut self, exe: &str) -> Result<(), String> { @@ -351,32 +405,46 @@ mod windows_firewall { rule.SetAction(NET_FW_ACTION_ALLOW)?; rule.SetProfiles(ALLOW_PROFILES as i32)?; rule.SetEnabled(VARIANT_TRUE)?; - self.0.Rules()?.Add(&rule) + self.policy.Rules()?.Add(&rule) } }; add().map_err(|e| e.to_string()) } - fn remove(&mut self, name: &str) -> Result<(), String> { - unsafe { - self.0 - .Rules() - .and_then(|rules| rules.Remove(&BSTR::from(name))) - } - .map_err(|e| e.to_string()) + /// Renames the rule to a fresh name first, so removing by name takes + /// it alone: not a rule that shares its name, whatever the case, nor + /// one added since the read. A failed remove leaves a renamed Block + /// rule, no worse than before. + fn remove(&mut self, id: usize) -> Result<(), String> { + let rule = self.found.get(id).ok_or("no such rule")?; + let remove = || -> windows::core::Result<()> { + let guid = unsafe { CoCreateGuid()? }; + let name = BSTR::from(format!("EternalMonitor fix {guid:?}")); + unsafe { + rule.SetName(&name)?; + self.policy.Rules()?.Remove(&name) + } + }; + remove().map_err(|e| e.to_string()) } } /// The elevated copy's work: the exit code. pub fn fix(exe: &Path, port: u16) -> u32 { - let run = || -> Result<(), Failure> { - let _apartment = Apartment::new().map_err(|_| Failure::Read)?; - let policy: INetFwPolicy2 = - unsafe { CoCreateInstance(&NetFwPolicy2, None, CLSCTX_INPROC_SERVER) } - .map_err(|_| Failure::Read)?; - fix_rules(&mut Com(policy), exe, port) - }; - run().map_or_else(Failure::code, |()| 0) + in_apartment( + || Apartment::new().map_err(|_| Failure::Read), + || { + let policy: INetFwPolicy2 = + unsafe { CoCreateInstance(&NetFwPolicy2, None, CLSCTX_INPROC_SERVER) } + .map_err(|_| Failure::Read)?; + Ok(Com { + policy, + found: Vec::new(), + }) + }, + |store| fix_rules(store, exe, port), + ) + .map_or_else(Failure::code, |()| 0) } } @@ -393,14 +461,17 @@ mod windows_elevate { use std::path::Path; use windows::core::PCWSTR; - use windows::Win32::Foundation::{CloseHandle, ERROR_CANCELLED, WAIT_OBJECT_0}; + use windows::Win32::Foundation::{CloseHandle, ERROR_CANCELLED, WAIT_OBJECT_0, WAIT_TIMEOUT}; + use windows::Win32::System::Com::{ + CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED, COINIT_DISABLE_OLE1DDE, + }; use windows::Win32::System::Threading::{GetExitCodeProcess, WaitForSingleObject}; use windows::Win32::UI::Shell::{ShellExecuteExW, SEE_MASK_NOCLOSEPROCESS, SHELLEXECUTEINFOW}; - use windows::Win32::UI::WindowsAndMessaging::SW_HIDE; + use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, SW_HIDE}; use super::{Elevate, Elevated}; - /// How long the copy may take. + /// How long to wait between looks at a copy that hasn't finished. const WAIT_MS: u32 = 60_000; pub struct Shell; @@ -409,14 +480,31 @@ mod windows_elevate { text.encode_wide().chain(Some(0)).collect() } + /// The single-threaded apartment ShellExecuteEx expects, for this call. + struct Sta(bool); + + impl Drop for Sta { + fn drop(&mut self) { + if self.0 { + unsafe { CoUninitialize() }; + } + } + } + impl Elevate for Shell { fn run_elevated(&self, exe: &Path, args: &str) -> Elevated { + let _sta = Sta(unsafe { + CoInitializeEx(None, COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE).is_ok() + }); let verb = wide("runas".as_ref()); let file = wide(exe.as_os_str()); let params = wide(args.as_ref()); let mut info = SHELLEXECUTEINFOW { cbSize: std::mem::size_of::() as u32, fMask: SEE_MASK_NOCLOSEPROCESS, + // The window just clicked: the Windows prompt belongs to it, + // not to whatever is behind. + hwnd: unsafe { GetForegroundWindow() }, lpVerb: PCWSTR(verb.as_ptr()), lpFile: PCWSTR(file.as_ptr()), lpParameters: PCWSTR(params.as_ptr()), @@ -433,15 +521,21 @@ mod windows_elevate { if info.hProcess.is_invalid() { return Elevated::Failed("no process".into()); } - let elevated = unsafe { - if WaitForSingleObject(info.hProcess, WAIT_MS) != WAIT_OBJECT_0 { - Elevated::Failed("did not finish in time".into()) - } else { - let mut code = 0u32; - match GetExitCodeProcess(info.hProcess, &mut code) { - Ok(()) => Elevated::Exited(code), - Err(error) => Elevated::Failed(error.to_string()), + // Until it exits, however long: the page stays on "running", so + // Try again can't start a second copy. + let elevated = loop { + match unsafe { WaitForSingleObject(info.hProcess, WAIT_MS) } { + WAIT_OBJECT_0 => { + let mut code = 0u32; + break match unsafe { GetExitCodeProcess(info.hProcess, &mut code) } { + Ok(()) => Elevated::Exited(code), + Err(error) => Elevated::Failed(error.to_string()), + }; + } + WAIT_TIMEOUT => { + tracing::warn!("The firewall fix is taking a while; still waiting for it"); } + _ => break Elevated::Failed("lost track of the fix".into()), } }; unsafe { @@ -464,6 +558,14 @@ mod tests { Path::new(EXE) } + fn on(bit: u32) -> Profile { + Profile { + bit, + on: true, + block_all_inbound: false, + } + } + /// What Windows adds when its prompt is dismissed. fn prompt_block(profiles: u32) -> Rule { Rule::unscoped(&EXE.to_lowercase(), false, profiles) @@ -473,29 +575,58 @@ mod tests { Rule::unscoped(EXE, true, ALLOW_PROFILES) } - fn named(name: &str, rule: Rule) -> Named { - Named { - name: name.into(), - rule: Some(rule), - } - } - - /// A firewall in memory. Removes go by name, as INetFwRules::Remove does. - #[derive(Default)] + /// A firewall in memory. Removes take the rule they name and no other. struct Fake { - rules: Vec, + profiles: Vec, + rules: Vec<(String, Option)>, add_fails: bool, remove_fails: bool, read_fails: bool, calls: Vec, } + impl Fake { + fn new(profiles: &[Profile], rules: Vec<(&str, Rule)>) -> Self { + Self { + profiles: profiles.to_vec(), + rules: rules + .into_iter() + .map(|(name, rule)| (name.to_string(), Some(rule))) + .chain([("Outbound".to_string(), None)]) + .collect(), + add_fails: false, + remove_fails: false, + read_fails: false, + calls: Vec::new(), + } + } + + fn names(&self) -> Vec<&str> { + self.rules.iter().map(|(name, _)| name.as_str()).collect() + } + } + impl RuleStore for Fake { - fn rules(&mut self) -> Result, String> { + fn profiles(&mut self) -> Result, String> { if self.read_fails { return Err("read".into()); } - Ok(self.rules.clone()) + Ok(self.profiles.clone()) + } + + fn rules(&mut self) -> Result, String> { + if self.read_fails { + return Err("read".into()); + } + Ok(self + .rules + .iter() + .enumerate() + .map(|(id, (_, rule))| Found { + id, + rule: rule.clone(), + }) + .collect()) } fn add_allow(&mut self, exe: &str) -> Result<(), String> { @@ -503,57 +634,96 @@ mod tests { if self.add_fails { return Err("add".into()); } - self.rules - .push(named(ALLOW_NAME, Rule::unscoped(exe, true, ALLOW_PROFILES))); + self.rules.push(( + ALLOW_NAME.into(), + Some(Rule::unscoped(exe, true, ALLOW_PROFILES)), + )); Ok(()) } - fn remove(&mut self, name: &str) -> Result<(), String> { - self.calls.push(format!("remove {name}")); + fn remove(&mut self, id: usize) -> Result<(), String> { + self.calls.push(format!("remove {}", self.rules[id].0)); if self.remove_fails { return Err("remove".into()); } - self.rules.retain(|rule| rule.name != name); + self.rules.remove(id); Ok(()) } } const PROMPT_UDP: &str = "UDP Query User{0-1-2-3-4}eternalmonitor-host.exe"; - fn dismissed_prompt() -> Vec { + fn dismissed_prompt() -> Vec<(&'static str, Rule)> { vec![ - named(ALLOW_NAME, installer_allow()), - named(PROMPT_UDP, prompt_block(PRIVATE | PUBLIC)), - named( + (ALLOW_NAME, installer_allow()), + (PROMPT_UDP, prompt_block(PRIVATE | PUBLIC)), + ( "TCP Query User{0-1-2-3-4}eternalmonitor-host.exe", Rule { udp: false, ..prompt_block(PRIVATE) }, ), - named( + ( "Other", Rule::unscoped(r"C:\Tools\Other.exe", false, PUBLIC), ), - Named { - name: "Outbound".into(), - rule: None, - }, ] } /// Windows' Block rule for the exe goes; its Allow rule, the TCP block - /// that doesn't reach the stream, and other programs' rules stay. + /// that doesn't reach the stream, other programs' rules and outbound + /// rules stay. #[test] fn the_fix_removes_only_this_exes_block_and_keeps_the_rest() { - let mut store = Fake { - rules: dismissed_prompt(), - ..Fake::default() - }; + let mut store = Fake::new(&[on(PUBLIC)], dismissed_prompt()); + assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + assert_eq!( + store.names(), + [ + ALLOW_NAME, + "TCP Query User{0-1-2-3-4}eternalmonitor-host.exe", + "Other", + "Outbound" + ] + ); + assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")]); + } + + /// The user's own block for a network not in use was never counted, + /// and stays. + #[test] + fn a_block_on_a_network_not_in_use_is_left() { + let mut rules = dismissed_prompt(); + rules.push(("Block EternalMonitor on Public", prompt_block(PUBLIC))); + let mut store = Fake::new(&[on(PRIVATE)], rules); + assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + assert!(store.names().contains(&"Block EternalMonitor on Public")); + assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")]); + } + + /// A network with Windows Firewall off, or one that blocks all incoming + /// connections, is the user's switch: a block there is left, and the + /// fix still works on the other networks. + #[test] + fn the_users_switches_are_left_as_they_are() { + let mut rules = dismissed_prompt(); + rules.push(("Block EternalMonitor on Domain", prompt_block(DOMAIN))); + let switches = [ + Profile { + bit: DOMAIN, + on: false, + block_all_inbound: false, + }, + Profile { + bit: PRIVATE, + on: true, + block_all_inbound: true, + }, + ]; + let mut store = Fake::new(&[switches[0], switches[1], on(PUBLIC)], rules); assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); - let mut expected = dismissed_prompt(); - expected.remove(1); - assert_eq!(store.rules, expected); + assert!(store.names().contains(&"Block EternalMonitor on Domain")); assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")]); } @@ -563,16 +733,32 @@ mod tests { fn a_missing_allow_rule_is_added_before_anything_is_removed() { let mut rules = dismissed_prompt(); rules.remove(0); - let mut store = Fake { - rules, - ..Fake::default() - }; + let mut store = Fake::new(&[on(PUBLIC)], rules); assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); assert_eq!( store.calls, [format!("add {EXE}"), format!("remove {PROMPT_UDP}")] ); - assert!(store.rules.contains(&named(ALLOW_NAME, installer_allow()))); + } + + /// The user's own Allow rule that already lets the stream in on the + /// networks in use, however narrow, means no add: the fix doesn't + /// widen what the user chose. + #[test] + fn the_users_own_allow_rule_means_no_add() { + let private_only = Rule::unscoped(EXE, true, PRIVATE); + let lan_only = Rule { + remote_addresses: Some("LocalSubnet".into()), + ..Rule::unscoped(EXE, true, PRIVATE) + }; + for allow in [private_only, lan_only] { + let mut store = Fake::new( + &[on(PRIVATE)], + vec![("Mine", allow.clone()), (PROMPT_UDP, prompt_block(PRIVATE))], + ); + assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(()), "{allow:?}"); + assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")], "{allow:?}"); + } } /// The fix's Allow rule is the one the installer's [Run] section adds. @@ -610,12 +796,9 @@ mod tests { fn a_failed_add_leaves_the_rules_exactly_as_they_were() { let mut rules = dismissed_prompt(); rules.remove(0); - let before = rules.clone(); - let mut store = Fake { - rules, - add_fails: true, - ..Fake::default() - }; + let mut store = Fake::new(&[on(PUBLIC)], rules); + let before = store.rules.clone(); + store.add_fails = true; assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::Add)); assert_eq!(store.rules, before); assert_eq!(store.calls, [format!("add {EXE}")]); @@ -625,35 +808,33 @@ mod tests { /// nothing that let traffic through went. #[test] fn a_failed_remove_is_a_failure() { - let mut store = Fake { - rules: dismissed_prompt(), - remove_fails: true, - ..Fake::default() - }; + let mut store = Fake::new(&[on(PUBLIC)], dismissed_prompt()); + let before = store.rules.clone(); + store.remove_fails = true; assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::Remove)); - assert_eq!(store.rules, dismissed_prompt()); + assert_eq!(store.rules, before); } /// A remove that reports success but leaves the block, or an add that - /// leaves no Allow rule, is caught when the rules are read again. + /// leaves no Allow rule, is caught when everything is read again. #[test] fn a_fix_that_did_not_take_is_a_failure() { struct Stubborn(Fake); impl RuleStore for Stubborn { - fn rules(&mut self) -> Result, String> { + fn profiles(&mut self) -> Result, String> { + self.0.profiles() + } + fn rules(&mut self) -> Result, String> { self.0.rules() } fn add_allow(&mut self, exe: &str) -> Result<(), String> { self.0.add_allow(exe) } - fn remove(&mut self, _name: &str) -> Result<(), String> { + fn remove(&mut self, _id: usize) -> Result<(), String> { Ok(()) } } - let mut store = Stubborn(Fake { - rules: dismissed_prompt(), - ..Fake::default() - }); + let mut store = Stubborn(Fake::new(&[on(PUBLIC)], dismissed_prompt())); assert_eq!( fix_rules(&mut store, exe(), PORT), Err(Failure::StillBlocked) @@ -661,44 +842,46 @@ mod tests { struct NoAdd(Fake); impl RuleStore for NoAdd { - fn rules(&mut self) -> Result, String> { + fn profiles(&mut self) -> Result, String> { + self.0.profiles() + } + fn rules(&mut self) -> Result, String> { self.0.rules() } fn add_allow(&mut self, _exe: &str) -> Result<(), String> { Ok(()) } - fn remove(&mut self, name: &str) -> Result<(), String> { - self.0.remove(name) + fn remove(&mut self, id: usize) -> Result<(), String> { + self.0.remove(id) } } let mut rules = dismissed_prompt(); rules.remove(0); - let mut store = NoAdd(Fake { - rules, - ..Fake::default() - }); + let mut store = NoAdd(Fake::new(&[on(PUBLIC)], rules)); assert_eq!( fix_rules(&mut store, exe(), PORT), Err(Failure::StillBlocked) ); } - /// A block whose name another rule has too is left: removing it by name - /// could remove a rule that works. + /// A Windows prompt answered for one network type can leave an Allow + /// and a Block under one name: only the block goes. #[test] - fn a_block_sharing_its_name_is_left() { + fn a_block_sharing_its_name_goes_alone() { let mut rules = dismissed_prompt(); - rules.push(named( + rules.push(( PROMPT_UDP, - Rule::unscoped(r"C:\Tools\Other.exe", true, PUBLIC), + Rule::unscoped(&EXE.to_lowercase(), true, PRIVATE), )); - let before = rules.clone(); - let mut store = Fake { - rules, - ..Fake::default() - }; - assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::SharedName)); - assert_eq!(store.rules, before); + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + let left: Vec<_> = store + .rules + .iter() + .filter(|(name, _)| name == PROMPT_UDP) + .collect(); + assert_eq!(left.len(), 1); + assert!(left[0].1.as_ref().unwrap().allow, "the Allow rule stays"); } /// A scoped Block rule, a disabled one, or one for another port is the @@ -706,22 +889,22 @@ mod tests { #[test] fn blocks_the_verdict_does_not_count_are_left() { let rules = vec![ - named(ALLOW_NAME, installer_allow()), - named( + (ALLOW_NAME, installer_allow()), + ( "Scoped", Rule { remote_addresses: Some("10.0.0.0/8".into()), ..prompt_block(PUBLIC) }, ), - named( + ( "Off", Rule { enabled: false, ..prompt_block(PUBLIC) }, ), - named( + ( "Port", Rule { local_ports: Some("80".into()), @@ -729,29 +912,46 @@ mod tests { }, ), ]; - let mut store = Fake { - rules: rules.clone(), - ..Fake::default() - }; + let mut store = Fake::new(&[on(PUBLIC)], rules); + let before = store.rules.clone(); assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); - assert_eq!(store.rules, rules); + assert_eq!(store.rules, before); assert!(store.calls.is_empty()); } #[test] fn rules_that_cant_be_read_are_a_failure() { - let mut store = Fake { - read_fails: true, - ..Fake::default() - }; + let mut store = Fake::new(&[on(PUBLIC)], dismissed_prompt()); + store.read_fails = true; assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::Read)); assert!(store.calls.is_empty()); assert_eq!( - fix_rules(&mut Fake::default(), Path::new(""), PORT), + fix_rules(&mut store, Path::new(""), PORT), Err(Failure::Arguments) ); } + /// The store goes before the apartment, whatever the fix returns. + #[test] + fn the_store_is_released_before_the_apartment() { + struct Noted(&'static str, std::rc::Rc>>); + impl Drop for Noted { + fn drop(&mut self) { + self.1.borrow_mut().push(self.0); + } + } + for result in [Ok(()), Err(Failure::Remove)] { + let order = std::rc::Rc::new(RefCell::new(Vec::new())); + let got = in_apartment( + || Ok(Noted("apartment", order.clone())), + || Ok(Noted("store", order.clone())), + |_| result, + ); + assert_eq!(got, result); + assert_eq!(*order.borrow(), ["store", "apartment"]); + } + } + /// The elevated copy gets the fixed argument and a port, nothing else. #[test] fn the_elevated_copy_takes_only_a_port() { @@ -821,7 +1021,6 @@ mod tests { fn a_fix_that_fails_says_so() { for (answer, said) in [ (Elevated::Exited(Failure::Add.code()), NOT_CHANGED), - (Elevated::Exited(Failure::SharedName.code()), SHARED_NAME), (Elevated::Exited(Failure::Remove.code()), FAILED), (Elevated::Exited(Failure::StillBlocked.code()), FAILED), (Elevated::Exited(1), FAILED), diff --git a/host/src/gui/page_stream.rs b/host/src/gui/page_stream.rs index 57c696f6..8e904b87 100644 --- a/host/src/gui/page_stream.rs +++ b/host/src/gui/page_stream.rs @@ -1427,8 +1427,8 @@ pub(super) fn usb_cable_notice( } /// "Fix it" on the firewall notice: for this exe's own Block rule, which -/// Setup's firewall lines remove, not for "Blocks all incoming -/// connections", which only the user's switch in Windows Security turns off. +/// the fix removes, not for "Blocks all incoming connections", which only +/// the user's switch in Windows Security turns off. pub(super) fn firewall_fix_offered(report: Option<&crate::firewall::Report>) -> bool { matches!( report.and_then(crate::firewall::Report::certain_block), diff --git a/host/src/gui/tests.rs b/host/src/gui/tests.rs index 71859fe7..3584e9b6 100644 --- a/host/src/gui/tests.rs +++ b/host/src/gui/tests.rs @@ -45,7 +45,7 @@ fn the_firewall_notice_shows_only_for_a_block_it_is_sure_of() { ); let (title, body) = firewall_notice(Some(&blocked), false).unwrap(); assert_eq!(title, "Windows Firewall is blocking EternalMonitor"); - // This exe's own Block rule: Setup's firewall lines remove it. + // This exe's own Block rule: "Fix it" removes it. assert!(firewall_fix_offered(Some(&blocked))); assert!( body.contains("Run the EternalMonitor installer again"), diff --git a/host/src/main.rs b/host/src/main.rs index f9ec2ad9..ed1ff662 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -15,8 +15,8 @@ use tracing_subscriber::EnvFilter; fn main() -> Result<(), Box> { // The Stream page's "Fix it" for a Windows Firewall block starts this - // exe as administrator to run Setup's firewall lines, and nothing else: - // no settings, log, window or single-instance check. + // exe as administrator to change the firewall rules for this exe, and + // nothing else: no settings, log, window or single-instance check. eternal_host::firewall_fix::run_if_requested(); // Before anything below creates the settings folder or a file in it: a // folder that held nothing means a fresh install, which opens first-run From dd0fdd979378543ae7af61d767ad186f0b0f88f1 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 12:16:39 -0500 Subject: [PATCH 217/226] Mark the GDI start picture cached, and read it only when it can be right #228 added RawFrame::cached. The GDI start picture now sets it: no new picture is due after it, so an encoder that holds its last frame gets the shortest wait. The GDI read now also returns nothing for a duplication rotated 180 degrees (or a square mode turned 90), which the size check missed, and for an all-black read, as on the secure desktop, so the start picture is tried again with the next duplication instead of counting as real. A native test reads the primary display through GDI at its size, and nothing at another size or rotation. --- host/src/capture/dxgi.rs | 116 ++++++++++++++++++++++++++++++++------- host/src/capture/mod.rs | 7 ++- 2 files changed, 100 insertions(+), 23 deletions(-) diff --git a/host/src/capture/dxgi.rs b/host/src/capture/dxgi.rs index 360cffcd..22d9b70f 100644 --- a/host/src/capture/dxgi.rs +++ b/host/src/capture/dxgi.rs @@ -34,7 +34,10 @@ use windows::Win32::Graphics::Direct3D11::{ D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_READ, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING, }; -use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_SAMPLE_DESC}; +use windows::Win32::Graphics::Dxgi::Common::{ + DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_MODE_ROTATION, DXGI_MODE_ROTATION_IDENTITY, + DXGI_MODE_ROTATION_UNSPECIFIED, DXGI_SAMPLE_DESC, +}; use windows::Win32::Graphics::Dxgi::{ CreateDXGIFactory1, IDXGIAdapter1, IDXGIFactory1, IDXGIOutput, IDXGIOutput1, IDXGIOutputDuplication, IDXGIResource, DXGI_ADAPTER_DESC1, DXGI_ERROR_ACCESS_LOST, @@ -440,7 +443,10 @@ pub fn run_capture_loop( } if start_picture.due(picture.real(), second_screen, Instant::now()) { - if let Some(pixels) = read_screen_gdi(&output1, (tex_width, tex_height)) { + let rotation = duplication + .as_ref() + .map(|duplication| unsafe { duplication.GetDesc() }.Rotation); + if let Some(pixels) = read_screen_gdi(&output1, (tex_width, tex_height), rotation) { info!( device = %chosen.device_name, "Desktop duplication gave no image yet — starting from the screen's picture" @@ -460,6 +466,7 @@ pub fn run_capture_loop( data: Arc::clone(&last_frame), width: tex_width, height: tex_height, + cached: true, }); last_send = now; } @@ -1239,14 +1246,26 @@ fn publish_capture_geometry(output: &IDXGIOutput1, shared: &SharedControl) { /// The picture `output` shows, read through GDI, as `width`x`height` BGRA /// rows in a [`frame_buffer`]: for a duplication that gives no image (see -/// [`StartPicture`]). The pointer is not in it. None if the output's desktop -/// rect isn't that size (a rotated display) or GDI can't read it (no -/// interactive desktop). -fn read_screen_gdi(output: &IDXGIOutput1, (width, height): (u32, u32)) -> Option> { +/// [`StartPicture`]). The pointer is not in it. None if the display is +/// rotated (the duplication's `rotation`, or a desktop rect that isn't that +/// size, as for 90 or 270 degrees), if GDI can't read it (no interactive +/// desktop), or if it read only black, as on the secure desktop: the +/// duplication's own image is waited for then. +fn read_screen_gdi( + output: &IDXGIOutput1, + (width, height): (u32, u32), + rotation: Option, +) -> Option> { use windows::Win32::Graphics::Gdi::{ BitBlt, CreateCompatibleDC, CreateDIBSection, DeleteDC, DeleteObject, GdiFlush, GetDC, ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, DIB_RGB_COLORS, SRCCOPY, }; + if !matches!( + rotation, + Some(DXGI_MODE_ROTATION_IDENTITY | DXGI_MODE_ROTATION_UNSPECIFIED) + ) { + return None; + } let rect = unsafe { output.GetDesc() }.ok()?.DesktopCoordinates; if (rect.right - rect.left, rect.bottom - rect.top) != (width as i32, height as i32) { return None; @@ -1288,20 +1307,24 @@ fn read_screen_gdi(output: &IDXGIOutput1, (width, height): (u32, u32)) -> Option ) .is_ok(); let _ = GdiFlush(); - let pixels = copied.then(|| { - let mut buf = frame_buffer(row_bytes * height as usize); - std::ptr::copy_nonoverlapping( - bits as *const u8, - buf.as_mut_ptr(), - row_bytes * height as usize, - ); - // GDI leaves the alpha byte 0; Desktop Duplication's is opaque. - let (pixels, _) = buf[..row_bytes * height as usize].as_chunks_mut::<4>(); - for pixel in pixels { - pixel[3] = 255; - } - buf - }); + let pixels = copied + .then(|| { + let mut buf = frame_buffer(row_bytes * height as usize); + std::ptr::copy_nonoverlapping( + bits as *const u8, + buf.as_mut_ptr(), + row_bytes * height as usize, + ); + // GDI leaves the alpha byte 0; Desktop Duplication's is opaque. + let (pixels, _) = buf[..row_bytes * height as usize].as_chunks_mut::<4>(); + let mut lit = false; + for pixel in pixels { + lit |= pixel[..3] != [0, 0, 0]; + pixel[3] = 255; + } + lit.then_some(buf) + }) + .flatten(); SelectObject(memory, previous); let _ = DeleteObject(bitmap); pixels @@ -1605,6 +1628,59 @@ fn draw_monochrome_cursor( mod tests { use super::*; + /// Native: GDI reads the primary display at its own size, as opaque BGRA + /// in a capture buffer, and nothing at another size or rotation. Needs an + /// interactive desktop that isn't all black; skips with no output listed + /// (a non-interactive SSH session). + /// `cargo test -p eternal-host --lib the_screen_is_read_through_gdi -- --ignored --nocapture` + #[test] + #[ignore = "native: needs an interactive Windows desktop"] + fn the_screen_is_read_through_gdi_only_at_its_own_size_and_rotation() { + use windows::Win32::Graphics::Dxgi::Common::{ + DXGI_MODE_ROTATION_ROTATE180, DXGI_MODE_ROTATION_ROTATE90, + }; + let factory: IDXGIFactory1 = unsafe { CreateDXGIFactory1() }.expect("DXGI factory"); + let shared = SharedControl::new(0, 8_000_000); + let (listed, _) = super::super::listing::listed(&shared, &factory); + let Some(primary) = listed.into_iter().find(|listed| listed.info.is_primary) else { + eprintln!("skipping: no display output is listed in this session"); + return; + }; + let output: IDXGIOutput1 = primary.output.cast().expect("IDXGIOutput1"); + let (width, height) = (primary.info.width, primary.info.height); + let pixels = read_screen_gdi(&output, (width, height), Some(DXGI_MODE_ROTATION_IDENTITY)) + .expect("the desktop read through GDI"); + let picture_bytes = (width * height * 4) as usize; + assert_eq!(pixels.len(), picture_bytes + FRAME_PADDING); + let (read, _) = pixels[..picture_bytes].as_chunks::<4>(); + assert!(read.iter().all(|pixel| pixel[3] == 255), "not opaque"); + assert!(read.iter().any(|pixel| pixel[..3] != [0, 0, 0])); + for (size, rotation, what) in [ + ( + (width + 1, height), + Some(DXGI_MODE_ROTATION_IDENTITY), + "another size", + ), + ( + (width, height), + Some(DXGI_MODE_ROTATION_ROTATE180), + "upside down", + ), + ( + (width, height), + Some(DXGI_MODE_ROTATION_ROTATE90), + "a quarter turn", + ), + ((width, height), None, "no duplication"), + ] { + assert!( + read_screen_gdi(&output, size, rotation).is_none(), + "{what} was read" + ); + } + eprintln!("read {width}x{height} through GDI"); + } + /// Native: a real D3D11CreateDevice held past its limit. The caller gets /// the timeout on time, a second attempt is refused at once, the late /// device is released on its own thread, and then a device is created. diff --git a/host/src/capture/mod.rs b/host/src/capture/mod.rs index 23bbd9fa..1b27960a 100644 --- a/host/src/capture/mod.rs +++ b/host/src/capture/mod.rs @@ -261,7 +261,8 @@ pub struct RawFrame { /// A picture capture already had, sent again because the desktop gave /// no new one: the keepalive, a keyframe request's resend, the cached /// picture redrawn with or without the pointer, or a resend during - /// duplication recovery. + /// duplication recovery. Also a mirror's start picture read through GDI + /// (see `StartPicture`): no new picture is due after it either. pub cached: bool, } @@ -1142,8 +1143,8 @@ pub(crate) const START_PICTURE_GRACE: Duration = Duration::from_millis(100); /// A mirror's start picture when its duplication gives none. Desktop /// Duplication delivers an image only once the desktop presents one. A -/// display that went to sleep, or a desktop that changes nothing, presents -/// none: the encoder then opened only at the 750 ms keepalive, which sent the +/// display that went to sleep, and possibly a desktop that changes nothing, +/// presents none: the encoder then opened only at the 750 ms keepalive, which sent the /// black start frame, and the viewer saw black until the desktop changed. /// GDI still reads what the desktop shows. #[cfg(any(windows, test))] From 92db5fe5a459fedf621c7a3774a59673f372ee2d Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 11:15:53 -0500 Subject: [PATCH 218/226] Answer a loss after a recovery at once Measured on NVENC and AMF under injected Wi-Fi loss, the PC answers a keyframe request within about 15-130 ms, AMF's reopen included, and no requests pile up. The long recoveries at 3-10% loss were a frame lost soon after a recovery landed: the iPad sends one request per 500 ms and the PC grants one per 500 ms, so that loss waited for the decoder's next try 600 ms later. The PC now also grants a request that names a frame at or past the first keyframe it sent since its last grant: the iPad already had that recovery, so this is a new loss. That grant doesn't move the 500 ms window, and a request naming an older frame, or none, waits as before. The PC says so with HOSTCAP_KEYFRAME_PER_LOSS. To such a PC the iPad sends the first GapLoss after a keyframe inside its spacing, once per keyframe, without moving the spacing. An older PC gets the same requests as before. The loss measurement now reports each recovery, request and keyframe, models the iPad's request rules, and can model an older iPad. --- docs/decisions.md | 28 ++ host/src/transport/mod.rs | 3 + host/src/transport/session.rs | 198 ++++++++- host/tests/e2e_synthetic.rs | 375 ++++++++++++++++-- .../App/ConnectionManager.swift | 3 +- .../Network/ControlChannel.swift | 25 +- ios/EternalMonitor/Network/WireProtocol.swift | 3 + .../KeyframeRequestTests.swift | 62 ++- .../WireProtocolGoldenTests.swift | 1 + proto/src/v2/control.rs | 6 + proto/tests/golden.rs | 1 + 11 files changed, 658 insertions(+), 47 deletions(-) diff --git a/docs/decisions.md b/docs/decisions.md index 8f425469..6561517f 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -255,6 +255,34 @@ remaining drop in the simulation, but raised the p50 latency at 10% loss from 6.6 to 10.7 ms. It would also undo the repair-window rules above, which were tuned on a physical iPad, so it waits for a device run. +### Answer a loss after a recovery at once + +With keyframes repaired, the first run on real encoders under loss (NVENC and +AMF, headless hosts with synthetic capture and the fake iPad, 1-10% loss and +300 ms outages) put the longest recoveries somewhere else. The PC's IDR began +to arrive 15-130 ms after a request (medians), AMF's reopen included. But at +10% loss the next frame was often lost within half a second of a recovery +landing. The iPad sends one keyframe request per 500 ms and the PC grants one +per 500 ms, so that loss waited for the decoder's next try 600 ms later. Most +recoveries over half a second began that way. Keyframe size was not the cause: +a P-frame was 20-30 KB, an IDR 30-100 KB, sent within one frame interval. + +A loss after a recovery now gets its own IDR at once. When a keyframe has +arrived since the iPad's last request, its next GapLoss request goes inside +the 500 ms spacing, once per keyframe, without moving that spacing. The PC +grants it inside its own window when the frame the request names is at or +past the first keyframe the PC sent since its last grant, because then the +iPad already had that recovery. That grant doesn't move the window either. A +request that names an older frame, or none, as the decoder's requests do, +waits as before, so one loss still gets one IDR. A PC that does this sets +`HOSTCAP_KEYFRAME_PER_LOSS`, and the iPad asks early only such a PC. An older +PC moves its window for any grant, so an early request it granted could push +the iPad's next one out of its window. + +AMF still reopens for each forced keyframe, at most once per 500 ms, so on +AMF the IDR for such a loss can still wait for that limit. The 300 ms outage +rows, which end with one request after the outage, are unchanged. + ### Use Apple's usbmuxd tunnel for USB Apple's installed device service handles the cable/trust relationship. EMLINK diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 58a49eb9..5a41ac46 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -693,6 +693,9 @@ pub async fn start_sender( } let frag_count = frag_count_usize as u16; let capture_ts_us = crate::clock::instant_to_us(nal.timestamp); + if nal.is_keyframe { + shared.session.lock().note_keyframe_sent(session_id, stream_epoch, frame_seq); + } let mut send_failed = false; let keep_for_repair = shared.session.lock().client_supports_nack(); diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index be2f8475..cbea700f 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -15,8 +15,8 @@ use std::time::{Duration, Instant}; use eternal_wire::v2::control::{ ByeReason, ControlHeader, ControlMessage, FailedReason, HelloAck, HelloStatus, InputEvent, KeyframeRequest, Nack, ReceiverReport, StreamConfig, FEATURE_MEDIA_SIZE, FEATURE_POWER_BYE, - FEATURE_STANDBY_STATUS, FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_MEDIA_SIZE, - HOSTCAP_NACK, MIN_MEDIA_DGRAM, + FEATURE_STANDBY_STATUS, FEATURE_SUPPORTS_NACK, FEATURE_WANTS_INPUT, HOSTCAP_KEYFRAME_PER_LOSS, + HOSTCAP_MEDIA_SIZE, HOSTCAP_NACK, MIN_MEDIA_DGRAM, }; use tracing::{info, warn}; @@ -114,7 +114,8 @@ struct RefusalMinute { summarized: u32, left_out: u32, } -/// Honor at most one keyframe request per this window (PLI storm guard). +/// Honor at most one keyframe request per this window (PLI storm guard), +/// besides one for each loss after a recovery ([`ActiveSession::recovered`]). pub const KEYFRAME_REQUEST_MIN_INTERVAL: Duration = Duration::from_millis(500); /// Smaller media sizes per session whose keyframe skips that window: the /// iPad asks for two (1,200 then 576 bytes). Any later one is an ordinary, @@ -358,7 +359,12 @@ struct ActiveSession { client_clock: Option<(i64, Instant)>, late_input: LateInput, awaiting_takeover: bool, + /// The last IDR granted on [`KEYFRAME_REQUEST_MIN_INTERVAL`]'s schedule. last_keyframe_grant: Option, + /// The stream epoch and frame number of the first keyframe sent since + /// the last IDR was granted, on that schedule or not (see + /// [`ActiveSession::recovered`]). + keyframe_since_grant: Option<(u32, u32)>, last_report: Option, msg_seq_out: u32, /// When the first video went to this session. @@ -861,6 +867,18 @@ impl Session { } } + /// A keyframe went to `session_id`'s client as frame `frame_seq` of + /// `stream_epoch`. + pub fn note_keyframe_sent(&mut self, session_id: u32, stream_epoch: u32, frame_seq: u32) { + if let Some(session) = self.active.as_mut() { + if session.session_id == session_id { + session + .keyframe_since_grant + .get_or_insert((stream_epoch, frame_seq)); + } + } + } + pub fn session_id(&self) -> Option { self.active.as_ref().map(|s| s.session_id) } @@ -1319,6 +1337,7 @@ impl Session { late_input: LateInput::default(), awaiting_takeover: false, last_keyframe_grant: None, + keyframe_since_grant: None, last_report: None, msg_seq_out: 0, first_video_at: None, @@ -1415,6 +1434,7 @@ impl Session { ); if session.media_size_lowered <= MEDIA_SIZE_IDRS { session.last_keyframe_grant = Some(now); + session.keyframe_since_grant = None; actions.force_idr = true; return actions; } @@ -1424,6 +1444,14 @@ impl Session { if session.grant_keyframe(now) { info!(reason = ?request.reason, "Keyframe requested by client — forcing IDR"); actions.force_idr = true; + } else if session.recovered(&request) { + // A new loss: the client already had the last recovery. + session.keyframe_since_grant = None; + info!( + reason = ?request.reason, + "Keyframe requested by client after a recovery — forcing IDR" + ); + actions.force_idr = true; } actions } @@ -1991,6 +2019,7 @@ impl Session { host_name: config.host_name(), auth_token, host_caps: HOSTCAP_NACK + | HOSTCAP_KEYFRAME_PER_LOSS | config.host_caps() | if media_size { HOSTCAP_MEDIA_SIZE } else { 0 }, reason: reason.to_string(), @@ -2117,10 +2146,26 @@ impl ActiveSession { ); if granted { self.last_keyframe_grant = Some(now); + self.keyframe_since_grant = None; } granted } + /// Whether the client completed the first keyframe sent since the last + /// IDR was granted, or a later frame, before it lost the frame it asks + /// for now. Such a loss is new, so its IDR needn't wait out + /// [`KEYFRAME_REQUEST_MIN_INTERVAL`], and doesn't move it. A request + /// from before that recovery landed names an older frame and waits as + /// before. Only a request that names a frame can show it: the iPad's + /// `GapLoss` does; `DecodeError` and `Resume` name none. + fn recovered(&self, request: &KeyframeRequest) -> bool { + self.keyframe_since_grant.is_some_and(|(epoch, seq)| { + request.stream_epoch == epoch + && request.last_complete_seq != 0 + && request.last_complete_seq.wrapping_sub(seq) < u32::MAX / 2 + }) + } + /// `device=` stays last: uploads cut the iPad's name at the line's end. fn log_expiry(&self, now: Instant) { let (heard_at, heard) = self.heard; @@ -2890,6 +2935,148 @@ mod tests { assert!(resume.force_idr, "granted once the window has passed"); } + /// A keyframe request from `peer` at `now` + `millis`: whether it forces an IDR. + fn keyframe_asked( + session: &mut Session, + peer: PeerId, + now: Instant, + millis: u64, + (stream_epoch, last_complete_seq): (u32, u32), + reason: KeyframeReason, + ) -> bool { + session + .handle_control_authed( + peer, + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch, + last_complete_seq, + reason, + max_media_dgram: 0, + }), + &TestConfig, + now + Duration::from_millis(millis), + ) + .force_idr + } + + #[test] + fn a_loss_after_a_recovery_gets_its_keyframe_at_once() { + // Under Wi-Fi loss the next frame is often lost soon after a + // recovery IDR lands. A request naming a frame at or past that + // keyframe is a new loss: it gets its own IDR without waiting out + // the 500 ms window, and doesn't move it. A request from before the + // recovery waits, as does a second one for the same loss. + let mut session = Session::new(1234); + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + let id = session.session_id().unwrap(); + let gap = KeyframeReason::GapLoss; + let ask = |session: &mut Session, millis, last_complete_seq| { + keyframe_asked(session, peer, now, millis, (7, last_complete_seq), gap) + }; + + // The session's opening keyframe, then the first loss. + session.note_keyframe_sent(id, 7, 1); + assert!(ask(&mut session, 0, 10)); + assert!( + !ask(&mut session, 40, 11), + "no keyframe has gone out since the grant" + ); + session.note_keyframe_sent(id, 7, 12); + session.note_keyframe_sent(id, 7, 20); + assert!( + !ask(&mut session, 60, 11), + "a request from before the recovery waits" + ); + assert!(ask(&mut session, 150, 12), "the client had the recovery"); + assert!( + !ask(&mut session, 170, 14), + "the same loss asks again before its keyframe went out" + ); + session.note_keyframe_sent(id, 7, 15); + assert!( + !ask(&mut session, 200, 14), + "a request from before the second recovery waits" + ); + assert!(ask(&mut session, 260, 18), "a third loss"); + // The schedule still runs from the first grant. + assert!(!ask(&mut session, 450, 18)); + assert!( + keyframe_asked( + &mut session, + peer, + now, + 520, + (0, 0), + KeyframeReason::DecodeError + ), + "the window is 500 ms from the last grant on schedule" + ); + } + + #[test] + fn only_a_request_naming_a_frame_of_this_epoch_shows_a_recovery() { + let mut session = Session::new(1234); + let now = Instant::now(); + let peer = addr([10, 0, 0, 5], 50000); + session.handle_control_authed(peer, hello(1, 50000), &TestConfig, now); + let id = session.session_id().unwrap(); + assert!(keyframe_asked( + &mut session, + peer, + now, + 0, + (7, 10), + KeyframeReason::GapLoss + )); + session.note_keyframe_sent(id, 7, u32::MAX); + // The decoder's requests name no frame. + for reason in [KeyframeReason::DecodeError, KeyframeReason::Resume] { + assert!(!keyframe_asked( + &mut session, + peer, + now, + 100, + (0, 0), + reason + )); + } + assert!( + !keyframe_asked( + &mut session, + peer, + now, + 100, + (6, 5), + KeyframeReason::GapLoss + ), + "another epoch's frame" + ); + assert!( + !keyframe_asked( + &mut session, + peer, + now, + 100, + (7, u32::MAX - 1), + KeyframeReason::GapLoss + ), + "older, just below the wrap" + ); + assert!( + keyframe_asked( + &mut session, + peer, + now, + 100, + (7, 3), + KeyframeReason::GapLoss + ), + "newer, past the wrap" + ); + } + fn input_event(event_id: u32) -> ControlMessage { ControlMessage::InputEvent(InputEvent { input_ver: 1, @@ -3202,7 +3389,10 @@ mod tests { ); let ack = parse_ack(&accepted.replies[0].1); assert_eq!(ack.host_caps & HOSTCAP_MEDIA_SIZE != 0, offered); - assert_eq!(ack.host_caps & !HOSTCAP_MEDIA_SIZE, HOSTCAP_NACK); + assert_eq!( + ack.host_caps & !HOSTCAP_MEDIA_SIZE, + HOSTCAP_NACK | HOSTCAP_KEYFRAME_PER_LOSS + ); let id = session.session_id().unwrap(); for _ in 0..3 { assert!(session.may_send_media(id, 1000, now)); diff --git a/host/tests/e2e_synthetic.rs b/host/tests/e2e_synthetic.rs index 5a50b9eb..e8a2a0a0 100644 --- a/host/tests/e2e_synthetic.rs +++ b/host/tests/e2e_synthetic.rs @@ -27,6 +27,7 @@ use eternal_wire::reassembly::{AddOutcome, Reassembler, MAX_STALL_ALLOWANCE}; use eternal_wire::v2::control::{ encode_control, parse_control, ByeReason, ControlMessage, Hello2, HelloAck, HelloStatus, KeyframeReason, KeyframeRequest, Ping, ReceiverReport, CAP_DECODE_H264, + HOSTCAP_KEYFRAME_PER_LOSS, }; use eternal_wire::v2::media::MediaHeader; use eternal_wire::v2::{classify, Classified}; @@ -796,9 +797,21 @@ fn slow_usb_link_measurement() { /// The host's fault injection drops first transmissions only, so this /// receiver also loses that share of the repairs and of its own requests, /// as WiFi does. `ETERNAL_TEST_ENCODER` picks the encoder (default libx264), -/// `ETERNAL_TEST_SECONDS` the length (default 30). Timing on a busy machine -/// changes the numbers, so this only reports them; `wifi_repair_sim.rs` -/// holds the assertions. +/// `ETERNAL_TEST_SECONDS` the length (default 30), `ETERNAL_TEST_FPS` the +/// capture rate (default 60; the encoder's own IDR comes every 30 frames, so +/// a slower desktop waits longer for it). `ETERNAL_TEST_BURST_MS` +/// adds a total outage each way of that length every +/// `ETERNAL_TEST_BURST_EVERY_MS` (default 4000), after a 2 s start. +/// +/// It also reports each recovery: from the first sign of a broken picture +/// (a dropped frame, an undecodable one, or a burst's start) to the next +/// decodable frame, with the requests sent and the keyframes that began +/// meanwhile, and the size and send spread of keyframes and delta frames. +/// `ETERNAL_TEST_TRACE=1` prints each of those events with its time. +/// `ETERNAL_TEST_OLD_CLIENT=1` makes the receiver ask as iPads before 0.3.5 +/// did, without the request for a new loss after a keyframe. +/// Timing on a busy machine changes the numbers, so this only reports them; +/// `wifi_repair_sim.rs` holds the assertions. #[test] #[ignore = "measurement; run with --ignored --nocapture"] fn lossy_wifi_recovery_measurement() { @@ -809,6 +822,12 @@ fn lossy_wifi_recovery_measurement() { let loss: f64 = env("ETERNAL_TEST_LOSS").map_or(0.10, |v| v.parse().unwrap()); let seconds: u64 = env("ETERNAL_TEST_SECONDS").map_or(30, |v| v.parse().unwrap()); let encoder = env("ETERNAL_TEST_ENCODER").unwrap_or_else(|| "libx264".into()); + let fps: u32 = env("ETERNAL_TEST_FPS").map_or(60, |v| v.parse().unwrap()); + let burst = + Duration::from_millis(env("ETERNAL_TEST_BURST_MS").map_or(0, |v| v.parse().unwrap())); + let burst_every = Duration::from_millis( + env("ETERNAL_TEST_BURST_EVERY_MS").map_or(4000, |v| v.parse().unwrap()), + ); if env("ETERNAL_SYNTH_SIZE").is_none() { std::env::set_var("ETERNAL_SYNTH_SIZE", "1920x1080"); } @@ -824,6 +843,9 @@ fn lossy_wifi_recovery_measurement() { let shared = SharedControl::new(port, pipeline::DEFAULT_BITRATE_BPS); shared.pairing.lock().required = false; *shared.encoder_override.lock() = Some(encoder.clone()); + shared + .target_fps + .store(fps, std::sync::atomic::Ordering::SeqCst); let (tx, rx) = mpsc::channel(); let pipeline_shared = shared.clone(); let pipeline_tx = tx.clone(); @@ -845,11 +867,7 @@ fn lossy_wifi_recovery_measurement() { .set_read_timeout(Some(Duration::from_millis(1))) .unwrap(); let mut assembler = Reassembler::new(); - assembler.configure_repair( - true, - Duration::from_micros(16_667), - Duration::from_millis(5), - ); + assembler.configure_repair(true, Duration::from_secs(1) / fps, Duration::from_millis(5)); // The iPad's clock, as in `lossy_stream_recovers_and_adapts`: a pass // counts at most MAX_PASS, so a descheduled test thread expires nothing. const MAX_PASS: Duration = Duration::from_millis(20); @@ -862,10 +880,20 @@ fn lossy_wifi_recovery_measurement() { rng ^= rng << 17; ((rng >> 11) as f64 / (1u64 << 53) as f64) < loss }; - // The app's request spacing: the control channel sends one keyframe - // request per 500 ms, and a held picture asks again every 600 ms. + // The app's requests: the assembler asks when it drops a frame, then + // the decoder when a frame cannot be decoded, and again while it waits + // for a keyframe, at most every 600 ms (`VideoDecoder`); only the + // assembler's name the last frame. The control channel sends one + // request per 500 ms and drops the others, but to a host with + // `HOSTCAP_KEYFRAME_PER_LOSS` the first GapLoss after a keyframe goes at + // once without moving that spacing (`ControlChannel.sendKeyframeRequest`). + let old_client = env("ETERNAL_TEST_OLD_CLIENT").is_some(); let mut last_request: Option = None; - let mut wants_keyframe = false; + let mut keyframe_since_request = false; + let mut early_requests = 0u64; + let mut decoder_waiting: Option = None; + let mut decoder_asked: Option = None; + let mut requests_dropped = 0u64; let mut keyframes_seen = std::collections::HashSet::new(); let mut keyframes_complete = 0u64; let mut previous: Option<(u32, u32, bool)> = None; @@ -876,16 +904,74 @@ fn lossy_wifi_recovery_measurement() { let mut requests = 0u64; let mut bitrate = 0; let mut bytes = [0; 2048]; + // A broken picture until the next decodable frame. + #[derive(Debug)] + struct Episode { + since: Instant, + requests: u32, + delivered_requests: u32, + /// Requests sent while a newer keyframe was already arriving. + behind_keyframe: u32, + keyframes_begun: u32, + } + let begin = |since| Episode { + since, + requests: 0, + delivered_requests: 0, + behind_keyframe: 0, + keyframes_begun: 0, + }; + let mut episode: Option = None; + let mut episodes: Vec<(Duration, Episode)> = Vec::new(); + // The newest keyframe whose fragments began to arrive, until it is delivered. + let mut arriving_keyframe: Option<(u32, u32)> = None; + // First and last first-transmission arrival per frame, and keyframe flag. + let mut spread: HashMap<(u32, u32), (Instant, Instant, bool)> = HashMap::new(); + let mut keyframe_bytes = Vec::new(); + let mut delta_bytes = Vec::new(); + let mut was_in_burst = false; + // From a delivered request to the first fragment of the next keyframe. + let mut asked_at: Option = None; + let mut answer_us = Vec::new(); let started = Instant::now(); + let tracing = env("ETERNAL_TEST_TRACE").is_some(); + let trace = |what: std::fmt::Arguments| { + if tracing { + eprintln!("LOSSY_TRACE {:>6} {what}", started.elapsed().as_millis()); + } + }; while started.elapsed() < Duration::from_secs(seconds) { let woke = Instant::now(); ipad_now += (woke - pass_started).min(MAX_PASS); pass_started = woke; + let in_burst = !burst.is_zero() + && started.elapsed() >= Duration::from_secs(2) + && ((started.elapsed() - Duration::from_secs(2)).as_millis() % burst_every.as_millis()) + < burst.as_millis(); + if in_burst && !was_in_burst && episode.is_none() { + trace(format_args!("begin burst")); + episode = Some(begin(ipad_now)); + } + was_in_burst = in_burst; if let Ok((len, _)) = receiver.socket.recv_from(&mut bytes) { - if let Ok((header, payload)) = MediaHeader::decode(&bytes[..len]) { + if in_burst { + // Everything is lost each way during a burst. + } else if let Ok((header, payload)) = MediaHeader::decode(&bytes[..len]) { if header.session_id == receiver.session_id && !(header.is_retransmit && lost()) { - if header.is_keyframe { - keyframes_seen.insert((header.stream_epoch, header.frame_seq)); + let id = (header.stream_epoch, header.frame_seq); + if header.is_keyframe && keyframes_seen.insert(id) { + if let Some(episode) = episode.as_mut() { + episode.keyframes_begun += 1; + } + trace(format_args!("keyframe {id:?} first fragment")); + arriving_keyframe = Some(id); + if let Some(at) = asked_at.take() { + answer_us.push((woke - at).as_micros()); + } + } + if !header.is_retransmit { + let entry = spread.entry(id).or_insert((woke, woke, header.is_keyframe)); + entry.1 = woke; } assembler.add_media(header, payload, ipad_now); } @@ -895,46 +981,115 @@ fn lossy_wifi_recovery_measurement() { } assembler.tick(ipad_now); for nack in assembler.take_nacks() { - if !lost() { + if !lost() && !in_burst { receiver.send(&ControlMessage::Nack(nack)); } } - wants_keyframe |= assembler.take_keyframe_request().is_some(); - let held = last_decodable.is_some_and(|at| ipad_now - at >= Duration::from_millis(600)); - let spacing = if wants_keyframe { 500 } else { 600 }; - if (wants_keyframe || held) - && last_request.is_none_or(|at| ipad_now - at >= Duration::from_millis(spacing)) - { - wants_keyframe = false; - last_request = Some(ipad_now); - requests += 1; - let c = assembler.counters(); - if !lost() { - receiver.send(&ControlMessage::KeyframeRequest(KeyframeRequest { - stream_epoch: c.stream_epoch, - last_complete_seq: assembler.latest_completed_seq(), - reason: KeyframeReason::GapLoss, - max_media_dgram: 0, - })); + let mut asks = Vec::new(); + if assembler.take_keyframe_request().is_some() { + asks.push(KeyframeReason::GapLoss); + if episode.is_none() { + trace(format_args!("begin dropped frame")); } + episode.get_or_insert(begin(ipad_now)); } for frame in assembler.take_ready() { delivered += 1; keyframes_complete += u64::from(frame.is_keyframe); let epoch = assembler.counters().stream_epoch; + if frame.is_keyframe { + keyframe_since_request = true; + keyframe_bytes.push(frame.payload.len()); + if arriving_keyframe.is_some_and(|(e, seq)| e == epoch && seq <= frame.seq) { + arriving_keyframe = None; + } + } else { + delta_bytes.push(frame.payload.len()); + } // The host numbers sent frames contiguously per epoch, so a // delta decodes only right after the frame before it decoded. let decodes = frame.is_keyframe || previous.is_some_and(|(e, seq, ok)| ok && e == epoch && seq + 1 == frame.seq); previous = Some((epoch, frame.seq, decodes)); + if episode.is_some() || frame.is_keyframe { + trace(format_args!( + "deliver {} key={} decodes={decodes}", + frame.seq, frame.is_keyframe + )); + } if decodes { + decoder_waiting = None; decodable += 1; if let Some(at) = last_decodable { freezes.push(ipad_now - at); } last_decodable = Some(ipad_now); + if let Some(episode) = episode.take() { + trace(format_args!("end {:?}", ipad_now - episode.since)); + episodes.push((ipad_now - episode.since, episode)); + } + } else if last_decodable.is_some() { + if episode.is_none() { + trace(format_args!("begin undecodable frame")); + } + episode.get_or_insert(begin(ipad_now)); + if decoder_waiting.is_none() { + decoder_waiting = Some(ipad_now); + } + } + } + if let Some(since) = decoder_waiting { + if decoder_asked.is_none_or(|at| ipad_now - at >= Duration::from_millis(600)) { + decoder_asked = Some(ipad_now); + asks.push(if ipad_now - since >= Duration::from_secs(2) { + KeyframeReason::Resume + } else { + KeyframeReason::DecodeError + }); + } + } + for reason in asks { + let spaced = last_request.is_none_or(|at| ipad_now - at >= Duration::from_millis(500)); + let new_loss = !old_client + && receiver.host_caps & HOSTCAP_KEYFRAME_PER_LOSS != 0 + && reason == KeyframeReason::GapLoss + && keyframe_since_request; + if !spaced && !new_loss { + requests_dropped += 1; + } else { + trace(format_args!("request {reason:?} early={}", !spaced)); + if spaced { + last_request = Some(ipad_now); + } + keyframe_since_request = false; + early_requests += u64::from(!spaced); + requests += 1; + let c = assembler.counters(); + let delivered_request = !lost() && !in_burst; + if let Some(episode) = episode.as_mut() { + episode.requests += 1; + episode.delivered_requests += u32::from(delivered_request); + episode.behind_keyframe += u32::from(arriving_keyframe.is_some()); + } + if delivered_request { + asked_at.get_or_insert(woke); + let gap = reason == KeyframeReason::GapLoss; + receiver.send(&ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch: if gap { c.stream_epoch } else { 0 }, + last_complete_seq: if gap { + assembler.latest_completed_seq() + } else { + 0 + }, + reason, + max_media_dgram: 0, + })); + } } } + if in_burst { + continue; + } if receiver.last_report.elapsed() >= Duration::from_millis(500) { receiver.last_report = Instant::now(); let c = assembler.counters(); @@ -973,14 +1128,65 @@ fn lossy_wifi_recovery_measurement() { .count() }; let longest = freezes.last().copied().unwrap_or_default(); + let percentile = |values: &mut Vec, p: usize| { + values.sort_unstable(); + values + .get((values.len() * p / 100).min(values.len().saturating_sub(1))) + .copied() + .unwrap_or(0) + }; + let mut recovery_ms: Vec = episodes.iter().map(|(d, _)| d.as_millis()).collect(); + let storms = episodes + .iter() + .filter(|(_, e)| e.delivered_requests > 1) + .count(); + let extra_keyframes: u32 = episodes + .iter() + .map(|(_, e)| e.keyframes_begun.saturating_sub(1)) + .sum(); + let behind_keyframe: u32 = episodes.iter().map(|(_, e)| e.behind_keyframe).sum(); + let episode_requests: u32 = episodes.iter().map(|(_, e)| e.delivered_requests).sum(); + let mut key_sizes: Vec = keyframe_bytes.iter().map(|b| *b as u128).collect(); + let mut delta_sizes: Vec = delta_bytes.iter().map(|b| *b as u128).collect(); + let key_p50 = percentile(&mut key_sizes, 50); + let delta_p50 = percentile(&mut delta_sizes, 50); + let mut key_spread_us: Vec = spread + .values() + .filter(|(_, _, key)| *key) + .map(|(first, last, _)| (*last - *first).as_micros()) + .collect(); + let mut delta_spread_us: Vec = spread + .values() + .filter(|(_, _, key)| !*key) + .map(|(first, last, _)| (*last - *first).as_micros()) + .collect(); eprintln!( - "LOSSY_E2E encoder={encoder} loss={loss} seconds={seconds} keyframes={keyframes_complete}/{} \ + "LOSSY_E2E encoder={encoder} fps={fps} loss={loss} burst_ms={} seconds={seconds} keyframes={keyframes_complete}/{} \ decodable={decodable}/{delivered} sent={sent} retransmits={retransmits} requests={requests} \ - longest_freeze_ms={} freezes_over_250ms={} freezes_over_1s={} end_bitrate={bitrate} counters={:?}", + requests_dropped={requests_dropped} early_requests={early_requests} old_client={old_client} \ + longest_freeze_ms={} freezes_over_250ms={} freezes_over_1s={} end_bitrate={bitrate} \ + episodes={} recovery_p50_ms={} recovery_p90_ms={} recovery_max_ms={} \ + episode_requests={episode_requests} storm_episodes={storms} requests_behind_keyframe={behind_keyframe} \ + extra_keyframes={extra_keyframes} keyframe_p50_bytes={key_p50} keyframe_max_bytes={} \ + delta_p50_bytes={delta_p50} keyframe_spread_p50_us={} keyframe_spread_max_us={} \ + delta_spread_p50_us={} answers={} answer_p50_us={} answer_p90_us={} answer_max_us={} counters={:?}", + burst.as_millis(), keyframes_seen.len(), longest.as_millis(), over(250), over(1000), + episodes.len(), + percentile(&mut recovery_ms, 50), + percentile(&mut recovery_ms, 90), + recovery_ms.last().copied().unwrap_or(0), + key_sizes.last().copied().unwrap_or(0), + percentile(&mut key_spread_us, 50), + key_spread_us.last().copied().unwrap_or(0), + percentile(&mut delta_spread_us, 50), + answer_us.len(), + percentile(&mut answer_us, 50), + percentile(&mut answer_us, 90), + answer_us.last().copied().unwrap_or(0), assembler.counters() ); assert!(delivered > 0, "the stream must deliver frames"); @@ -1703,6 +1909,107 @@ fn an_ipad_gets_the_smaller_media_datagrams_it_asks_for() { } } +/// The first keyframe `receiver` gets after frame `after`, arriving before +/// `until`: its stream epoch and frame number. +fn keyframe_after(receiver: &mut FakeReceiver, after: u32, until: Instant) -> Option<(u32, u32)> { + let mut buffer = [0; 2048]; + while Instant::now() < until { + receiver.maybe_report(0, 0); + let Ok((len, _)) = receiver.socket.recv_from(&mut buffer) else { + continue; + }; + if let Ok((header, _)) = MediaHeader::decode(&buffer[..len]) { + if header.is_keyframe && !header.is_retransmit && header.frame_seq > after { + return Some((header.stream_epoch, header.frame_seq)); + } + } + } + None +} + +/// Under Wi-Fi loss the next frame is often lost soon after a recovery IDR +/// lands. The iPad then asks again at once, naming a frame from that +/// keyframe on, and the PC sends that loss its own IDR inside its 500 ms +/// request window rather than after it. Each round asks for a keyframe, +/// waits for it, and asks again naming it; the second IDR must arrive less +/// than 500 ms after the first request, which no IDR granted on the window +/// can. At 10 fps the encoder's own IDR comes 3 s after the last one, so +/// none falls in a round. +#[test] +fn a_loss_soon_after_a_recovery_gets_its_own_keyframe() { + let _env = test_env(); + init_test_tracing(); + ffmpeg_next::init().unwrap(); + std::env::set_var("ETERNAL_CAPTURE", "synthetic"); + std::env::set_var("ETERNAL_SYNTH_SIZE", format!("{SYNTH_W}x{SYNTH_H}")); + for name in ["ETERNAL_DROP", "ETERNAL_REORDER", "ETERNAL_JITTER_MS"] { + std::env::remove_var(name); + } + let listen_port = free_udp_port(); + let shared = SharedControl::new(listen_port, pipeline::DEFAULT_BITRATE_BPS); + shared.pairing.lock().required = false; + *shared.encoder_override.lock() = Some("libx264".into()); + shared + .target_fps + .store(10, std::sync::atomic::Ordering::SeqCst); + let (supervisor_tx, supervisor_rx) = mpsc::channel(); + let supervisor_shared = shared.clone(); + let supervisor_sender = supervisor_tx.clone(); + let supervisor = std::thread::spawn(move || { + eternal_host::supervisor::run( + listen_port, + supervisor_shared, + GpuInfo::software_fallback(), + supervisor_sender, + supervisor_rx, + ) + }); + + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut receiver = FakeReceiver::connect(listen_port); + let gap_loss = |stream_epoch, last_complete_seq| { + ControlMessage::KeyframeRequest(KeyframeRequest { + stream_epoch, + last_complete_seq, + reason: KeyframeReason::GapLoss, + max_media_dgram: 0, + }) + }; + let (epoch, mut last) = + keyframe_after(&mut receiver, 0, Instant::now() + Duration::from_secs(10)) + .expect("the opening keyframe"); + let mut in_window = Vec::new(); + for _ in 0..6 { + // Past the window of the round before. + let _ = keyframe_after( + &mut receiver, + u32::MAX, + Instant::now() + Duration::from_millis(700), + ); + let asked = Instant::now(); + receiver.send(&gap_loss(epoch, last)); + let (_, recovery) = keyframe_after(&mut receiver, last, asked + Duration::from_secs(2)) + .expect("the first loss gets an IDR"); + receiver.send(&gap_loss(epoch, recovery)); + let answer = + keyframe_after(&mut receiver, recovery, asked + Duration::from_millis(500)); + in_window.push(answer.is_some()); + last = answer.map_or(recovery, |(_, seq)| seq); + } + let answered = in_window.iter().filter(|answered| **answered).count(); + assert!( + answered >= 4, + "a new loss is answered inside the window: {in_window:?}" + ); + receiver.send(&ControlMessage::Bye(ByeReason::UserDisconnect)); + })); + let _ = supervisor_tx.send(SupervisorCommand::Shutdown); + supervisor.join().unwrap(); + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } +} + /// Native first-second audio measurement across repeated connects with the /// PC's real WASAPI loopback (inaudible: it only reads the mix). #[cfg(windows)] diff --git a/ios/EternalMonitor/App/ConnectionManager.swift b/ios/EternalMonitor/App/ConnectionManager.swift index 5598d4ef..52412d3a 100644 --- a/ios/EternalMonitor/App/ConnectionManager.swift +++ b/ios/EternalMonitor/App/ConnectionManager.swift @@ -795,8 +795,9 @@ final class ConnectionManager: ObservableObject { } } - assembler.onFrameAssembled = { [weak self, weak decoder, weak assembler] data, seq, captureTsUs, isKeyframe in + assembler.onFrameAssembled = { [weak self, weak decoder, weak assembler, weak controlChannelBox] data, seq, captureTsUs, isKeyframe in guard let self, self.currentAttempt.withLock({ $0 == connectionID }) else { return } + if isKeyframe { controlChannelBox?.value?.noteKeyframeAssembled() } self.streamCounters.recordAssembled(bytes: data.count) self.streamCounters.recordParsed() decoder?.decode(packet: FramePacket( diff --git a/ios/EternalMonitor/Network/ControlChannel.swift b/ios/EternalMonitor/Network/ControlChannel.swift index 9ea23a6e..43b03e75 100644 --- a/ios/EternalMonitor/Network/ControlChannel.swift +++ b/ios/EternalMonitor/Network/ControlChannel.swift @@ -85,12 +85,15 @@ final class ControlChannel { private static let pingIntervalMs = 2000 private static let pingBurstCount = 5 private static let pingBurstSpacingMs = 200 - /// The host grants one keyframe per 500 ms and ignores the rest. A WiFi + /// The host grants one keyframe per 500 ms, besides one for each loss + /// after a recovery (`sendKeyframeRequest`), and ignores the rest. A WiFi /// stall can damage many frames at once; ask once for all of them. static let keyframeRequestSpacingUs: UInt64 = 500_000 private let nowUs: () -> UInt64 private var lastKeyframeRequestUs: UInt64? + /// A keyframe was assembled since the last keyframe request went out. + private var keyframeSinceRequest = false init(queue: DispatchQueue, nowUs: @escaping () -> UInt64 = ControlChannel.clientNowUs, send: @escaping (Data) -> Void) { @@ -335,6 +338,11 @@ final class ControlChannel { timer.resume() } + /// A keyframe arrived whole: the next frame lost is a new loss. + func noteKeyframeAssembled() { + queue.async { [self] in keyframeSinceRequest = true } + } + /// `maxMediaDgram` asks the PC for media datagrams no larger, for the /// rest of the session; only a PC with `HelloAck.hostCapMediaSize` gets it. func sendKeyframeRequest(streamEpoch: UInt32, lastCompleteSeq: UInt32, reason: KeyframeReason, @@ -345,9 +353,20 @@ final class ControlChannel { // Startup comes at most every 2 s, while no frame completes. The // dropped frames' own requests then keep the spacing busy, and // would always hide it from the PC's log. - if reason != .startup, let last = lastKeyframeRequestUs, now &- last < Self.keyframeRequestSpacingUs { return } + let spaced = reason == .startup + || lastKeyframeRequestUs.map { now &- $0 >= Self.keyframeRequestSpacingUs } ?? true + // A frame lost after a keyframe arrived is a new loss, which a PC + // with `hostCapKeyframePerLoss` answers at once when the request + // names a frame from that keyframe on; only GapLoss names one. It + // goes once per keyframe and leaves the spacing as it was. An + // older PC moves its 500 ms window for any grant, so it isn't + // asked early. + let newLoss = reason == .gapLoss && keyframeSinceRequest + && hostCaps & HelloAck.hostCapKeyframePerLoss != 0 + guard spaced || newLoss else { return } let maxMediaDgram = hostCaps & HelloAck.hostCapMediaSize != 0 ? maxMediaDgram : 0 - lastKeyframeRequestUs = now + if spaced { lastKeyframeRequestUs = now } + keyframeSinceRequest = false onDiagnostic?("Requesting keyframe (\(reason))" + (maxMediaDgram == 0 ? "" : " with media datagrams up to \(maxMediaDgram) bytes")) sendMessage(.keyframeRequest(KeyframeRequest( streamEpoch: streamEpoch, diff --git a/ios/EternalMonitor/Network/WireProtocol.swift b/ios/EternalMonitor/Network/WireProtocol.swift index 14182f09..e702207c 100644 --- a/ios/EternalMonitor/Network/WireProtocol.swift +++ b/ios/EternalMonitor/Network/WireProtocol.swift @@ -332,6 +332,9 @@ struct HelloAck: Equatable { /// session, and its heartbeats carry `framesSent`. Only an iPad that /// sets `Hello2.featureMediaSize` hears it. static let hostCapMediaSize: UInt16 = 1 << 7 + /// The PC answers a GapLoss request at once when it names a frame from + /// the PC's last recovery keyframe on (`ControlChannel.sendKeyframeRequest`). + static let hostCapKeyframePerLoss: UInt16 = 1 << 8 var status: HelloStatus var acceptedVersion: UInt8 var clientNonce: UInt32 diff --git a/ios/EternalMonitorTests/KeyframeRequestTests.swift b/ios/EternalMonitorTests/KeyframeRequestTests.swift index d572bc21..e523f36e 100644 --- a/ios/EternalMonitorTests/KeyframeRequestTests.swift +++ b/ios/EternalMonitorTests/KeyframeRequestTests.swift @@ -19,13 +19,60 @@ final class KeyframeRequestTests: XCTestCase { XCTAssertEqual(requests, [.gapLoss, .startup, .gapLoss]) } + private let lossAfterKeyframes: [(UInt64, Event)] = [ + (1_000_000, .request(.gapLoss)), (1_050_000, .keyframe), + (1_100_000, .request(.gapLoss)), + (1_120_000, .request(.gapLoss)), // the same loss + (1_300_000, .request(.decodeError)), + (1_500_000, .request(.gapLoss)), // 500 ms after the first + (1_600_000, .keyframe), + (1_650_000, .request(.decodeError)), + (1_700_000, .request(.gapLoss)), + ] + + /// A frame lost after a keyframe arrived is a new loss: to a PC that + /// answers those, its GapLoss goes at once, once per keyframe, and leaves + /// the 500 ms spacing as it was. The decoder's requests name no frame, + /// so they keep the spacing. + func testALossAfterAKeyframeAsksAtOnce() async throws { + let requests = try await sent(lossAfterKeyframes, hostCaps: HelloAck.hostCapKeyframePerLoss) + XCTAssertEqual(requests.map(\.at), [1_000_000, 1_100_000, 1_500_000, 1_700_000]) + XCTAssertTrue(requests.allSatisfy { $0.reason == .gapLoss }) + } + + /// An older PC moves its window for any grant, so it gets one request + /// per 500 ms, as before. + func testAnOlderPCIsNotAskedEarly() async throws { + let requests = try await sent(lossAfterKeyframes, hostCaps: 0) + XCTAssertEqual(requests.map(\.at), [1_000_000, 1_500_000]) + } + + private enum Event { + case request(KeyframeReason) + /// The assembler completed a keyframe. + case keyframe + } + /// The keyframe requests a session's channel sends for `requests`, each /// made at its time on the channel's clock. private func sent(_ requests: [(UInt64, KeyframeReason)]) async throws -> [KeyframeReason] { + try await sent(requests.map { ($0.0, Event.request($0.1)) }).map(\.reason) + } + + /// The keyframe requests a session's channel sends for `events`, each at + /// its time on the channel's clock, with the time each was sent. + private func sent(_ events: [(UInt64, Event)], hostCaps: UInt16 = 0) async throws + -> [(at: UInt64, reason: KeyframeReason)] { let queue = DispatchQueue(label: "keyframe-request-test") let clock = OSAllocatedUnfairLock(initialState: 1_000_000) let (stream, continuation) = AsyncStream.makeStream() - let channel = ControlChannel(queue: queue, nowUs: { clock.withLock { $0 } }) { continuation.yield($0) } + let sentAt = OSAllocatedUnfairLock<[UInt64]>(initialState: []) + let channel = ControlChannel(queue: queue, nowUs: { clock.withLock { $0 } }) { data in + if case .keyframeRequest = Wire.parseControl(data)?.message { + sentAt.withLock { $0.append(clock.withLock { $0 }) } + } + continuation.yield(data) + } defer { channel.stop(); continuation.finish() } channel.startHandshake(listenPort: 50000, identity: ControlChannel.ClientIdentity( deviceName: "Test iPad", screenPxW: 640, screenPxH: 360, screenPtW: 640, screenPtH: 360, refreshHz: 60)) @@ -39,11 +86,16 @@ final class KeyframeRequestTests: XCTestCase { channel.handleControl(Wire.encodeControl(sessionId: 0, msgSeq: 1, message: .helloAck(HelloAck( status: .ok, acceptedVersion: 2, clientNonce: hello.clientNonce, sessionId: 41, heartbeatIntervalMs: 1000, reportIntervalMs: 500, livenessTimeoutMs: 3000, - streamConfig: StreamConfig(), hostName: "Test PC", authToken: Data(17...32), hostCaps: 0)))) + streamConfig: StreamConfig(), hostName: "Test PC", authToken: Data(17...32), hostCaps: hostCaps)))) } - for (at, reason) in requests { + for (at, event) in events { clock.withLock { $0 = at } - channel.sendKeyframeRequest(streamEpoch: 1, lastCompleteSeq: 7, reason: reason) + switch event { + case .request(let reason): + channel.sendKeyframeRequest(streamEpoch: 1, lastCompleteSeq: 7, reason: reason) + case .keyframe: + channel.noteKeyframeAssembled() + } queue.sync {} } channel.stop() @@ -52,6 +104,6 @@ final class KeyframeRequestTests: XCTestCase { while let data = await messages.next() { if case .keyframeRequest(let request) = Wire.parseControl(data)?.message { reasons.append(request.reason) } } - return reasons + return Array(zip(sentAt.withLock { $0 }, reasons)) } } diff --git a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift index 6abf3d04..3fc98061 100644 --- a/ios/EternalMonitorTests/WireProtocolGoldenTests.swift +++ b/ios/EternalMonitorTests/WireProtocolGoldenTests.swift @@ -501,6 +501,7 @@ final class WireProtocolGoldenTests: XCTestCase { /// earlier versions read, so those read them as before. func testMediaSizeVectors() throws { XCTAssertEqual(HelloAck.hostCapMediaSize, 0x0080) + XCTAssertEqual(HelloAck.hostCapKeyframePerLoss, 0x0100) XCTAssertEqual(Hello2.featureMediaSize, 0x0040) XCTAssertEqual(KeyframeRequest.minMediaDgram, 576) diff --git a/proto/src/v2/control.rs b/proto/src/v2/control.rs index 05610c21..482cc7aa 100644 --- a/proto/src/v2/control.rs +++ b/proto/src/v2/control.rs @@ -78,6 +78,12 @@ pub const HOSTCAP_POINTER: u16 = 1 << 6; /// for an iPad whose HELLO2 has [`FEATURE_MEDIA_SIZE`], so an older iPad /// gets the same bytes as before. Bit 6 is [`HOSTCAP_POINTER`]. pub const HOSTCAP_MEDIA_SIZE: u16 = 1 << 7; +/// The host answers a keyframe request inside its 500 ms window when the +/// request names a frame from the first keyframe it sent since its last +/// grant on: a loss after that recovery. Older iPads ignore the bit; an iPad +/// asks early only a host that sets it, since an older one moves its window +/// for any grant. +pub const HOSTCAP_KEYFRAME_PER_LOSS: u16 = 1 << 8; /// The smallest media datagram an iPad may ask for: the IPv4 minimum /// reassembly size, also the smallest packet size the PC's own setting takes. pub const MIN_MEDIA_DGRAM: u16 = 576; diff --git a/proto/tests/golden.rs b/proto/tests/golden.rs index ee270cd3..30af3809 100644 --- a/proto/tests/golden.rs +++ b/proto/tests/golden.rs @@ -918,6 +918,7 @@ fn media_size_vectors_round_trip_and_extend_the_old_bodies() { HOSTCAP_NACK | HOSTCAP_CONTIGUOUS_FRAME_SEQ | HOSTCAP_MEDIA_SIZE ); assert_eq!(HOSTCAP_MEDIA_SIZE, 0x0080); + assert_eq!(HOSTCAP_KEYFRAME_PER_LOSS, 0x0100); assert_eq!(FEATURE_MEDIA_SIZE, 0x0040); assert_eq!(MIN_MEDIA_DGRAM, 576); From ae7d63945053dde2a9cd37a330f444443af609aa Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 12:43:45 -0500 Subject: [PATCH 219/226] Read firewall rule paths with the machine's own values only A rule's %name% path now means what the machine's settings say, from the known folders and the system environment, never from the environment of the process, which any user can change for their own programs. The Stream page's verdict and the elevated fix share this, so another program's Block rule can never be taken for this exe's and removed. A system value that names another variable, or one only a user has, stays unknown. The Windows prompt is now started synchronously, as a thread with no message loop needs. --- host/src/firewall.rs | 109 +++++++++++++++++++++++- host/src/firewall_fix.rs | 174 ++++++++++++++++++++++++++++++++------- 2 files changed, 251 insertions(+), 32 deletions(-) diff --git a/host/src/firewall.rs b/host/src/firewall.rs index 13192118..f57bd41c 100644 --- a/host/src/firewall.rs +++ b/host/src/firewall.rs @@ -421,6 +421,26 @@ pub fn profile_name(bit: u32) -> &'static str { } } +/// What `%name%` in a rule's program path stands for, from the machine's +/// own settings only: the known folders for Program Files, the Windows +/// folder and ProgramData, then the system environment in +/// `HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment`. +/// Never this process's environment, which any user can change for their +/// own programs: a rule for another program must not look like this exe's, +/// to the verdict or to the elevated fix. A system value that itself names +/// a variable, or one only a user has, is unknown, and the path keeps it as +/// written. +#[cfg(windows)] +pub fn machine_env(name: &str) -> Option { + windows_policy::machine_env(name) +} + +/// Off Windows no rule has a path to expand. +#[cfg(not(windows))] +pub fn machine_env(_name: &str) -> Option { + None +} + /// `%NAME%` replaced with `lookup(NAME)`, as Windows stores a rule made in /// its own window (`%ProgramFiles%\...`). An unknown name stays as written. pub fn expand(path: &str, lookup: impl Fn(&str) -> Option) -> String { @@ -674,9 +694,72 @@ pub(crate) mod windows_policy { unsafe { CoCreateInstance(&NetFwPolicy2, None, CLSCTX_INPROC_SERVER)? }; let profiles = current_profiles(&policy)?; let rules = inbound_rules(&policy)?; - Ok(judge(&profiles, &rules, exe, port, lan, |name| { - std::env::var(name).ok() - })) + Ok(judge(&profiles, &rules, exe, port, lan, super::machine_env)) + } + + /// See [`super::machine_env`]. + pub(crate) fn machine_env(name: &str) -> Option { + use windows::Win32::UI::Shell::{ + FOLDERID_ProgramData, FOLDERID_ProgramFiles, FOLDERID_ProgramFilesCommon, + FOLDERID_ProgramFilesCommonX64, FOLDERID_ProgramFilesCommonX86, + FOLDERID_ProgramFilesX64, FOLDERID_ProgramFilesX86, FOLDERID_Windows, + }; + let folder = match name.to_ascii_lowercase().as_str() { + "programfiles" => FOLDERID_ProgramFiles, + "programw6432" => FOLDERID_ProgramFilesX64, + "programfiles(x86)" => FOLDERID_ProgramFilesX86, + "commonprogramfiles" => FOLDERID_ProgramFilesCommon, + "commonprogramw6432" => FOLDERID_ProgramFilesCommonX64, + "commonprogramfiles(x86)" => FOLDERID_ProgramFilesCommonX86, + "systemroot" | "windir" => FOLDERID_Windows, + "programdata" | "allusersprofile" => FOLDERID_ProgramData, + "systemdrive" => { + return known_folder(&FOLDERID_Windows) + .and_then(|windows| windows.get(..2).map(str::to_string)) + .filter(|drive| drive.ends_with(':')) + } + _ => return system_variable(name), + }; + known_folder(&folder) + } + + fn known_folder(id: &windows::core::GUID) -> Option { + use windows::Win32::Foundation::HANDLE; + use windows::Win32::System::Com::CoTaskMemFree; + use windows::Win32::UI::Shell::{SHGetKnownFolderPath, KF_FLAG_DEFAULT}; + let path = unsafe { SHGetKnownFolderPath(id, KF_FLAG_DEFAULT, HANDLE::default()) }.ok()?; + let text = unsafe { path.to_string() }.ok(); + unsafe { CoTaskMemFree(Some(path.0 as *const _)) }; + text + } + + /// A value of the system environment as stored, never expanded: the + /// expansion would use this process's environment. + fn system_variable(name: &str) -> Option { + use windows::core::{w, PCWSTR}; + use windows::Win32::Foundation::ERROR_SUCCESS; + use windows::Win32::System::Registry::{ + RegGetValueW, HKEY_LOCAL_MACHINE, RRF_NOEXPAND, RRF_RT_REG_EXPAND_SZ, RRF_RT_REG_SZ, + }; + let name: Vec = name.encode_utf16().chain(Some(0)).collect(); + let mut value = [0u16; 1024]; + let mut bytes = std::mem::size_of_val(&value) as u32; + let result = unsafe { + RegGetValueW( + HKEY_LOCAL_MACHINE, + w!(r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment"), + PCWSTR(name.as_ptr()), + RRF_RT_REG_SZ | RRF_RT_REG_EXPAND_SZ | RRF_NOEXPAND, + None, + Some(value.as_mut_ptr().cast()), + Some(&mut bytes), + ) + }; + if result != ERROR_SUCCESS { + return None; + } + let length = value.iter().position(|&c| c == 0).unwrap_or(value.len()); + Some(String::from_utf16_lossy(&value[..length])).filter(|value| !value.contains('%')) } /// The profiles of the networks in use now, as the verdict sees them. @@ -814,6 +897,26 @@ mod tests { const EXE: &str = r"C:\Program Files\EternalMonitor\EternalMonitor-host.exe"; const PORT: u16 = 9876; + /// The machine's values come from the machine: a variable only this + /// process has, or only a user has, is unknown. + #[cfg(windows)] + #[test] + fn machine_values_never_come_from_this_process() { + const OWN: &str = "EM_FIREWALL_MACHINE_TEST"; + std::env::set_var(OWN, r"C:\Program Files"); + assert_eq!(machine_env(OWN), None); + std::env::remove_var(OWN); + for name in ["USERPROFILE", "LOCALAPPDATA", "APPDATA"] { + assert_eq!(machine_env(name), None, "{name}"); + } + for name in ["ProgramFiles", "SystemRoot", "windir", "ProgramData"] { + let value = machine_env(name).unwrap_or_else(|| panic!("{name}")); + assert!(Path::new(&value).is_dir(), "{name}"); + } + let drive = machine_env("SystemDrive").unwrap(); + assert!(machine_env("SystemRoot").unwrap().starts_with(&drive)); + } + fn env(name: &str) -> Option { match name.to_lowercase().as_str() { "programfiles" => Some(r"C:\Program Files".into()), diff --git a/host/src/firewall_fix.rs b/host/src/firewall_fix.rs index 442f016c..8ef2a4d6 100644 --- a/host/src/firewall_fix.rs +++ b/host/src/firewall_fix.rs @@ -26,7 +26,10 @@ //! exe allowed, and not blocked by its own rule, on every network in use. //! //! The exe's path comes from the elevated process itself, and the port is -//! only ever a number. A user who answers No to the Windows prompt changes +//! only ever a number. A rule's path is this exe's only as written, or with +//! `%name%` read from the machine's own settings +//! ([`crate::firewall::machine_env`]), never from the environment the +//! elevated copy inherits from the user who clicked. A user who answers No to the Windows prompt changes //! nothing: the notice stays as it was. After the copy ran, the firewall //! is read again at once. @@ -86,9 +89,10 @@ impl Failure { } } -fn env(name: &str) -> Option { - std::env::var(name).ok() -} +/// What `%name%` in a rule's path stands for: [`crate::firewall::machine_env`] +/// in the elevated copy, never its environment, which the user who clicked +/// "Fix it" could have set. +type Lookup<'a> = &'a dyn Fn(&str) -> Option; /// The bits of the profiles in use with Windows Firewall on: where the /// verdict counts rules. @@ -102,12 +106,12 @@ fn in_use(profiles: &[Profile]) -> u32 { /// This exe's own Block rules the verdict counts: enabled, reaching every /// datagram to `port`, on a network in use. All the fix removes. -fn is_target(rule: &Rule, exe: &Path, port: u16, in_use: u32) -> bool { +fn is_target(rule: &Rule, exe: &Path, port: u16, in_use: u32, lookup: Lookup) -> bool { !rule.allow && rule.enabled && rule.udp && rule.profiles & in_use != 0 - && crate::firewall::ours(rule, exe, env) + && crate::firewall::ours(rule, exe, lookup) && reach(rule, port) == Reach::All } @@ -115,9 +119,15 @@ fn is_target(rule: &Rule, exe: &Path, port: u16, in_use: u32) -> bool { /// its own rule, on every network in use. A network that blocks all /// incoming connections or has the firewall off is the user's switch, not /// a rule the fix could change. -fn allowed_everywhere(profiles: &[Profile], rules: &[&Rule], exe: &Path, port: u16) -> bool { +fn allowed_everywhere( + profiles: &[Profile], + rules: &[&Rule], + exe: &Path, + port: u16, + lookup: Lookup, +) -> bool { let rules: Vec = rules.iter().map(|rule| (*rule).clone()).collect(); - let report = judge(profiles, &rules, exe, port, true, env); + let report = judge(profiles, &rules, exe, port, true, lookup); report.profiles.iter().all(|(_, verdict)| { matches!( verdict, @@ -133,7 +143,12 @@ fn inbound(found: &[Found]) -> impl Iterator { } /// The fix, on `store`, for this exe at `exe` and the stream's `port`. -pub fn fix_rules(store: &mut impl RuleStore, exe: &Path, port: u16) -> Result<(), Failure> { +pub fn fix_rules( + store: &mut impl RuleStore, + exe: &Path, + port: u16, + lookup: Lookup, +) -> Result<(), Failure> { let program = exe .to_str() .filter(|path| !path.is_empty()) @@ -142,14 +157,14 @@ pub fn fix_rules(store: &mut impl RuleStore, exe: &Path, port: u16) -> Result<() let found = store.rules().map_err(|_| Failure::Read)?; let bits = in_use(&profiles); let targets: Vec = inbound(&found) - .filter(|(_, rule)| is_target(rule, exe, port, bits)) + .filter(|(_, rule)| is_target(rule, exe, port, bits, lookup)) .map(|(id, _)| id) .collect(); let without_targets: Vec<&Rule> = inbound(&found) .filter(|(id, _)| !targets.contains(id)) .map(|(_, rule)| rule) .collect(); - if !allowed_everywhere(&profiles, &without_targets, exe, port) { + if !allowed_everywhere(&profiles, &without_targets, exe, port, lookup) { store.add_allow(program).map_err(|_| Failure::Add)?; } for id in targets { @@ -158,13 +173,24 @@ pub fn fix_rules(store: &mut impl RuleStore, exe: &Path, port: u16) -> Result<() let profiles = store.profiles().map_err(|_| Failure::Read)?; let after = store.rules().map_err(|_| Failure::Read)?; let after: Vec<&Rule> = inbound(&after).map(|(_, rule)| rule).collect(); - if allowed_everywhere(&profiles, &after, exe, port) { + if allowed_everywhere(&profiles, &after, exe, port, lookup) { Ok(()) } else { Err(Failure::StillBlocked) } } +/// [`fix_rules`] as the elevated copy runs it, with the machine's own +/// values for `%name%` in rule paths. +#[cfg(any(windows, test))] +pub(crate) fn fix_on_this_machine( + store: &mut impl RuleStore, + exe: &Path, + port: u16, +) -> Result<(), Failure> { + fix_rules(store, exe, port, &crate::firewall::machine_env) +} + /// Runs `fix` on a store opened inside an apartment, and drops the store /// before the apartment: COM objects released after `CoUninitialize` may /// call into a library Windows has already unloaded. @@ -358,7 +384,9 @@ mod windows_firewall { }; use windows::Win32::System::Com::{CoCreateGuid, CoCreateInstance, CLSCTX_INPROC_SERVER}; - use super::{fix_rules, in_apartment, Failure, Found, RuleStore, ALLOW_NAME, ALLOW_PROFILES}; + use super::{ + fix_on_this_machine, in_apartment, Failure, Found, RuleStore, ALLOW_NAME, ALLOW_PROFILES, + }; use crate::firewall::windows_policy::{current_profiles, for_each_rule, Apartment}; use crate::firewall::{rule_from, Profile}; @@ -442,7 +470,7 @@ mod windows_firewall { found: Vec::new(), }) }, - |store| fix_rules(store, exe, port), + |store| fix_on_this_machine(store, exe, port), ) .map_or_else(Failure::code, |()| 0) } @@ -466,7 +494,9 @@ mod windows_elevate { CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED, COINIT_DISABLE_OLE1DDE, }; use windows::Win32::System::Threading::{GetExitCodeProcess, WaitForSingleObject}; - use windows::Win32::UI::Shell::{ShellExecuteExW, SEE_MASK_NOCLOSEPROCESS, SHELLEXECUTEINFOW}; + use windows::Win32::UI::Shell::{ + ShellExecuteExW, SEE_MASK_NOASYNC, SEE_MASK_NOCLOSEPROCESS, SHELLEXECUTEINFOW, + }; use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, SW_HIDE}; use super::{Elevate, Elevated}; @@ -501,7 +531,9 @@ mod windows_elevate { let params = wide(args.as_ref()); let mut info = SHELLEXECUTEINFOW { cbSize: std::mem::size_of::() as u32, - fMask: SEE_MASK_NOCLOSEPROCESS, + // No message loop here, and the thread blocks below: the + // call must finish before it returns. + fMask: SEE_MASK_NOCLOSEPROCESS | SEE_MASK_NOASYNC, // The window just clicked: the Windows prompt belongs to it, // not to whatever is behind. hwnd: unsafe { GetForegroundWindow() }, @@ -575,6 +607,12 @@ mod tests { Rule::unscoped(EXE, true, ALLOW_PROFILES) } + /// The machine's own values. + fn machine(name: &str) -> Option { + name.eq_ignore_ascii_case("ProgramFiles") + .then(|| r"C:\Program Files".to_string()) + } + /// A firewall in memory. Removes take the rule they name and no other. struct Fake { profiles: Vec, @@ -677,7 +715,7 @@ mod tests { #[test] fn the_fix_removes_only_this_exes_block_and_keeps_the_rest() { let mut store = Fake::new(&[on(PUBLIC)], dismissed_prompt()); - assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); assert_eq!( store.names(), [ @@ -697,7 +735,7 @@ mod tests { let mut rules = dismissed_prompt(); rules.push(("Block EternalMonitor on Public", prompt_block(PUBLIC))); let mut store = Fake::new(&[on(PRIVATE)], rules); - assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); assert!(store.names().contains(&"Block EternalMonitor on Public")); assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")]); } @@ -722,7 +760,7 @@ mod tests { }, ]; let mut store = Fake::new(&[switches[0], switches[1], on(PUBLIC)], rules); - assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); assert!(store.names().contains(&"Block EternalMonitor on Domain")); assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")]); } @@ -734,7 +772,7 @@ mod tests { let mut rules = dismissed_prompt(); rules.remove(0); let mut store = Fake::new(&[on(PUBLIC)], rules); - assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); assert_eq!( store.calls, [format!("add {EXE}"), format!("remove {PROMPT_UDP}")] @@ -756,7 +794,11 @@ mod tests { &[on(PRIVATE)], vec![("Mine", allow.clone()), (PROMPT_UDP, prompt_block(PRIVATE))], ); - assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(()), "{allow:?}"); + assert_eq!( + fix_rules(&mut store, exe(), PORT, &machine), + Ok(()), + "{allow:?}" + ); assert_eq!(store.calls, [format!("remove {PROMPT_UDP}")], "{allow:?}"); } } @@ -799,7 +841,10 @@ mod tests { let mut store = Fake::new(&[on(PUBLIC)], rules); let before = store.rules.clone(); store.add_fails = true; - assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::Add)); + assert_eq!( + fix_rules(&mut store, exe(), PORT, &machine), + Err(Failure::Add) + ); assert_eq!(store.rules, before); assert_eq!(store.calls, [format!("add {EXE}")]); } @@ -811,7 +856,10 @@ mod tests { let mut store = Fake::new(&[on(PUBLIC)], dismissed_prompt()); let before = store.rules.clone(); store.remove_fails = true; - assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::Remove)); + assert_eq!( + fix_rules(&mut store, exe(), PORT, &machine), + Err(Failure::Remove) + ); assert_eq!(store.rules, before); } @@ -836,7 +884,7 @@ mod tests { } let mut store = Stubborn(Fake::new(&[on(PUBLIC)], dismissed_prompt())); assert_eq!( - fix_rules(&mut store, exe(), PORT), + fix_rules(&mut store, exe(), PORT, &machine), Err(Failure::StillBlocked) ); @@ -859,7 +907,7 @@ mod tests { rules.remove(0); let mut store = NoAdd(Fake::new(&[on(PUBLIC)], rules)); assert_eq!( - fix_rules(&mut store, exe(), PORT), + fix_rules(&mut store, exe(), PORT, &machine), Err(Failure::StillBlocked) ); } @@ -874,7 +922,7 @@ mod tests { Rule::unscoped(&EXE.to_lowercase(), true, PRIVATE), )); let mut store = Fake::new(&[on(PUBLIC)], rules); - assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); let left: Vec<_> = store .rules .iter() @@ -914,19 +962,87 @@ mod tests { ]; let mut store = Fake::new(&[on(PUBLIC)], rules); let before = store.rules.clone(); - assert_eq!(fix_rules(&mut store, exe(), PORT), Ok(())); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); assert_eq!(store.rules, before); assert!(store.calls.is_empty()); } + /// A rule's `%name%` means what the machine says, not what this + /// process's environment says: a user can set their own value, and the + /// elevated copy inherits it. Another program's rule never goes because + /// of it. + #[test] + fn only_the_machines_values_make_a_rule_this_exes() { + const USERS_OWN: &str = "EM_FIREWALL_FIX_TEST_ROOT"; + // A value only this process has, set as a user would. + std::env::set_var(USERS_OWN, r"C:\Program Files"); + let installed = r"D:\Apps\EternalMonitor\EternalMonitor-host.exe"; + let rules = vec![ + (ALLOW_NAME, Rule::unscoped(installed, true, ALLOW_PROFILES)), + ( + "Program Files copy", + Rule::unscoped( + r"%ProgramFiles%\EternalMonitor\EternalMonitor-host.exe", + false, + PUBLIC, + ), + ), + ( + "Users own", + Rule::unscoped( + &format!(r"%{USERS_OWN}%\EternalMonitor\EternalMonitor-host.exe"), + false, + PUBLIC, + ), + ), + ]; + // The process would say D:\Apps for ProgramFiles; the machine says + // C:\Program Files. + let mut store = Fake::new(&[on(PUBLIC)], rules.clone()); + assert_eq!( + fix_rules(&mut store, Path::new(installed), PORT, &machine), + Ok(()) + ); + assert!(store.calls.is_empty(), "{:?}", store.calls); + + // The same rules, for the copy the machine's value names: its + // block goes, and it gets its own Allow rule. + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!(fix_rules(&mut store, exe(), PORT, &machine), Ok(())); + assert_eq!( + store.calls, + [format!("add {EXE}"), "remove Program Files copy".into()] + ); + + // And as the elevated copy runs it. + let rules = vec![ + (ALLOW_NAME, installer_allow()), + ( + "Users own", + Rule::unscoped( + &format!(r"%{USERS_OWN}%\EternalMonitor\EternalMonitor-host.exe"), + false, + PUBLIC, + ), + ), + ]; + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!(fix_on_this_machine(&mut store, exe(), PORT), Ok(())); + assert!(store.calls.is_empty(), "{:?}", store.calls); + std::env::remove_var(USERS_OWN); + } + #[test] fn rules_that_cant_be_read_are_a_failure() { let mut store = Fake::new(&[on(PUBLIC)], dismissed_prompt()); store.read_fails = true; - assert_eq!(fix_rules(&mut store, exe(), PORT), Err(Failure::Read)); + assert_eq!( + fix_rules(&mut store, exe(), PORT, &machine), + Err(Failure::Read) + ); assert!(store.calls.is_empty()); assert_eq!( - fix_rules(&mut store, Path::new(""), PORT), + fix_rules(&mut store, Path::new(""), PORT, &machine), Err(Failure::Arguments) ); } From 5ca3f4aaeda15271bdd5c976f01c2bfef7f77fad Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 13:29:48 -0500 Subject: [PATCH 220/226] Expand machine values that name others, and never call an unreadable path allowed A system environment value stored to be expanded, such as one built on %SystemDrive%, is now expanded from the machine's own values too, up to eight levels deep. A value only a user has, or values that name each other in a cycle, stay unknown. A REG_SZ value is taken as written, as Windows does. A Block rule with this exe's file name whose path still can't be resolved may be this exe's. The verdict now calls that network uncertain instead of allowed, so no notice claims a block is certain, and the fix never removes such a rule and reports that it didn't finish instead of success. --- host/src/firewall.rs | 245 +++++++++++++++++++++++++++++++++------ host/src/firewall_fix.rs | 61 ++++++++-- 2 files changed, 257 insertions(+), 49 deletions(-) diff --git a/host/src/firewall.rs b/host/src/firewall.rs index f57bd41c..11c102dc 100644 --- a/host/src/firewall.rs +++ b/host/src/firewall.rs @@ -257,6 +257,9 @@ enum Whose { AllPrograms, /// The program could not be read. Unknown, + /// A path with this exe's file name that names a variable the machine + /// has no value for: it may be this exe's. + Unresolved, Other, } @@ -268,8 +271,11 @@ fn whose(rule: &Rule, exe: &Path, lookup: impl Fn(&str) -> Option) -> Wh match rule.program.as_deref() { None => Whose::Unknown, Some(program) if program.trim().is_empty() => Whose::AllPrograms, - Some(program) if same_program(program, exe, lookup) => Whose::Ours, - Some(_) => Whose::Other, + Some(program) => match expand_known(program, lookup) { + Some(path) if same_path(&path, exe) => Whose::Ours, + None if file_name(program) == file_name(&exe.to_string_lossy()) => Whose::Unresolved, + _ => Whose::Other, + }, } } @@ -289,6 +295,9 @@ pub enum Verdict { NoRule, /// Windows Firewall is off on this profile: another firewall may be on. Off, + /// An unscoped Block rule that may be this exe's: its path has this + /// exe's file name but names a variable the machine has no value for. + Uncertain, } impl Verdict { @@ -300,6 +309,7 @@ impl Verdict { Verdict::BlocksAll => "blocks_all", Verdict::NoRule => "no_rule", Verdict::Off => "off", + Verdict::Uncertain => "uncertain", } } @@ -349,7 +359,8 @@ impl Report { /// The whole picture in one word for the log. pub fn verdict(&self) -> &'static str { let blocking = self.profiles.iter().filter(|(_, v)| v.blocks()).count(); - if blocking > 0 && self.bypass_rules > 0 { + let unsure = self.profiles.iter().any(|(_, v)| *v == Verdict::Uncertain); + if unsure || (blocking > 0 && self.bypass_rules > 0) { "uncertain" } else if !self.profiles.is_empty() && blocking == self.profiles.len() { "blocked" @@ -427,12 +438,12 @@ pub fn profile_name(bit: u32) -> &'static str { /// `HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment`. /// Never this process's environment, which any user can change for their /// own programs: a rule for another program must not look like this exe's, -/// to the verdict or to the elevated fix. A system value that itself names -/// a variable, or one only a user has, is unknown, and the path keeps it as -/// written. +/// to the verdict or to the elevated fix. A system value that names other +/// variables is expanded from these values too ([`resolve`]); one only a +/// user has is unknown. #[cfg(windows)] pub fn machine_env(name: &str) -> Option { - windows_policy::machine_env(name) + resolve(name, &windows_policy::machine_value) } /// Off Windows no rule has a path to expand. @@ -441,9 +452,50 @@ pub fn machine_env(_name: &str) -> Option { None } -/// `%NAME%` replaced with `lookup(NAME)`, as Windows stores a rule made in -/// its own window (`%ProgramFiles%\...`). An unknown name stays as written. -pub fn expand(path: &str, lookup: impl Fn(&str) -> Option) -> String { +/// A machine value as stored. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Stored { + /// A known folder, or a REG_SZ value: as written, `%` and all. + Literal(String), + /// A REG_EXPAND_SZ value: its `%name%`s are machine values too. + Expand(String), +} + +/// How deep REG_EXPAND_SZ values may name one another. +pub const EXPAND_DEPTH: usize = 8; + +/// `name`'s value among the machine values `stored`, each REG_EXPAND_SZ +/// value's own `%name%`s resolved the same way. `None` when `name`, or a +/// value it names, has no machine value, when values name each other in a +/// cycle, or when more than [`EXPAND_DEPTH`] of them nest. +pub fn resolve(name: &str, stored: &impl Fn(&str) -> Option) -> Option { + resolve_within(name, stored, &mut Vec::new()) +} + +fn resolve_within( + name: &str, + stored: &impl Fn(&str) -> Option, + expanding: &mut Vec, +) -> Option { + match stored(name)? { + Stored::Literal(value) => Some(value), + Stored::Expand(value) => { + let key = name.to_lowercase(); + if expanding.len() >= EXPAND_DEPTH || expanding.contains(&key) { + return None; + } + expanding.push(key); + let value = expand_known(&value, |inner| resolve_within(inner, stored, expanding)); + expanding.pop(); + value + } + } +} + +/// `path` with every `%NAME%` replaced with `lookup(NAME)`, as Windows +/// stores a rule made in its own window (`%ProgramFiles%\...`), or `None` if +/// one has no value. `%%` and a lone `%` stay as written. +fn expand_known(path: &str, mut lookup: impl FnMut(&str) -> Option) -> Option { let mut out = String::new(); let mut rest = path; while let Some(start) = rest.find('%') { @@ -452,28 +504,44 @@ pub fn expand(path: &str, lookup: impl Fn(&str) -> Option) -> String { }; let name = &rest[start + 1..start + 1 + len]; out.push_str(&rest[..start]); - match lookup(name).filter(|_| !name.is_empty()) { - Some(value) => out.push_str(&value), - None => out.push_str(&rest[start..start + len + 2]), + if name.is_empty() { + out.push_str("%%"); + } else { + out.push_str(&lookup(name)?); } rest = &rest[start + len + 2..]; } out.push_str(rest); - out + Some(out) +} + +/// A path as Windows compares it: no quotes or `\\?\`, backslashes, no case. +fn plain(path: &str) -> String { + let path = path + .trim() + .trim_matches('"') + .replace('/', "\\") + .to_lowercase(); + path.strip_prefix(r"\\?\") + .map(str::to_string) + .unwrap_or(path) +} + +fn same_path(path: &str, exe: &Path) -> bool { + let exe = plain(&exe.to_string_lossy()); + !exe.is_empty() && plain(path) == exe +} + +/// The last part of a path, as Windows compares it. +fn file_name(path: &str) -> String { + let path = plain(path); + path.rsplit('\\').next().unwrap_or_default().to_string() } /// Whether a rule's program is `exe`. Windows compares paths without case, /// and its prompt stores them in lower case. pub fn same_program(program: &str, exe: &Path, lookup: impl Fn(&str) -> Option) -> bool { - let plain = |path: &str| { - path.trim() - .trim_matches('"') - .replace('/', "\\") - .to_lowercase() - }; - let exe = plain(&exe.to_string_lossy()); - let exe = exe.strip_prefix(r"\\?\").unwrap_or(&exe); - !exe.is_empty() && plain(&expand(program, lookup)) == exe + expand_known(program, lookup).is_some_and(|path| same_path(&path, exe)) } /// The verdict on each profile for inbound UDP to `exe` on `port`. Windows @@ -509,8 +577,11 @@ pub fn judge( let ours = |r: &Rule, whose, reach| !r.allow && whose == Whose::Ours && reach == Reach::All; let everyone = |r: &Rule, whose, reach| !r.allow && whose == Whose::AllPrograms && reach == Reach::All; - let unsure = - |r: &Rule, whose, reach| !r.allow && (whose == Whose::Unknown || reach == Reach::Some); + let unsure = |r: &Rule, whose, reach| { + !r.allow && (matches!(whose, Whose::Unknown | Whose::Unresolved) || reach == Reach::Some) + }; + let maybe_ours = + |r: &Rule, whose, reach| !r.allow && whose == Whose::Unresolved && reach == Reach::All; // An Allow rule that asks for IPsec (SecureFlags) can come with // "Override block rules": IPsec-authenticated traffic then gets past // Block rules for the exe. iPads never authenticate, so it changes @@ -538,6 +609,8 @@ pub fn judge( Verdict::BlockedForAllApps } else if count(p.bit, &ours) > 0 { Verdict::Blocked + } else if count(p.bit, &maybe_ours) > 0 { + Verdict::Uncertain } else if count(p.bit, &allows) > 0 { Verdict::Allowed } else { @@ -667,7 +740,7 @@ pub(crate) mod windows_policy { use windows::Win32::System::Ole::IEnumVARIANT; use super::{ - judge, rule_from, Field, Profile, Report, Rule, RuleSource, DOMAIN, PRIVATE, PUBLIC, + judge, rule_from, Field, Profile, Report, Rule, RuleSource, Stored, DOMAIN, PRIVATE, PUBLIC, }; /// VT_DISPATCH: what the rule enumerator hands out. @@ -697,8 +770,8 @@ pub(crate) mod windows_policy { Ok(judge(&profiles, &rules, exe, port, lan, super::machine_env)) } - /// See [`super::machine_env`]. - pub(crate) fn machine_env(name: &str) -> Option { + /// One machine value as stored, for [`super::resolve`]. + pub(crate) fn machine_value(name: &str) -> Option { use windows::Win32::UI::Shell::{ FOLDERID_ProgramData, FOLDERID_ProgramFiles, FOLDERID_ProgramFilesCommon, FOLDERID_ProgramFilesCommonX64, FOLDERID_ProgramFilesCommonX86, @@ -717,10 +790,11 @@ pub(crate) mod windows_policy { return known_folder(&FOLDERID_Windows) .and_then(|windows| windows.get(..2).map(str::to_string)) .filter(|drive| drive.ends_with(':')) + .map(Stored::Literal) } _ => return system_variable(name), }; - known_folder(&folder) + known_folder(&folder).map(Stored::Literal) } fn known_folder(id: &windows::core::GUID) -> Option { @@ -733,24 +807,26 @@ pub(crate) mod windows_policy { text } - /// A value of the system environment as stored, never expanded: the - /// expansion would use this process's environment. - fn system_variable(name: &str) -> Option { + /// A value of the system environment as stored, never expanded by + /// Windows: that would use this process's environment. + fn system_variable(name: &str) -> Option { use windows::core::{w, PCWSTR}; use windows::Win32::Foundation::ERROR_SUCCESS; use windows::Win32::System::Registry::{ - RegGetValueW, HKEY_LOCAL_MACHINE, RRF_NOEXPAND, RRF_RT_REG_EXPAND_SZ, RRF_RT_REG_SZ, + RegGetValueW, HKEY_LOCAL_MACHINE, REG_EXPAND_SZ, REG_SZ, REG_VALUE_TYPE, RRF_NOEXPAND, + RRF_RT_REG_EXPAND_SZ, RRF_RT_REG_SZ, }; let name: Vec = name.encode_utf16().chain(Some(0)).collect(); let mut value = [0u16; 1024]; let mut bytes = std::mem::size_of_val(&value) as u32; + let mut kind = REG_VALUE_TYPE::default(); let result = unsafe { RegGetValueW( HKEY_LOCAL_MACHINE, w!(r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment"), PCWSTR(name.as_ptr()), RRF_RT_REG_SZ | RRF_RT_REG_EXPAND_SZ | RRF_NOEXPAND, - None, + Some(&mut kind), Some(value.as_mut_ptr().cast()), Some(&mut bytes), ) @@ -759,7 +835,12 @@ pub(crate) mod windows_policy { return None; } let length = value.iter().position(|&c| c == 0).unwrap_or(value.len()); - Some(String::from_utf16_lossy(&value[..length])).filter(|value| !value.contains('%')) + let value = String::from_utf16_lossy(&value[..length]); + match kind { + REG_SZ => Some(Stored::Literal(value)), + REG_EXPAND_SZ => Some(Stored::Expand(value)), + _ => None, + } } /// The profiles of the networks in use now, as the verdict sees them. @@ -1477,9 +1558,97 @@ mod tests { } #[test] - fn expansion_leaves_unknown_and_unpaired_names() { - assert_eq!(expand(r"%nope%\a%", env), r"%nope%\a%"); - assert_eq!(expand(r"%%\%ProgramFiles%", env), r"%%\C:\Program Files"); + fn expansion_refuses_unknown_names_and_keeps_unpaired_ones() { + assert_eq!(expand_known(r"%nope%\a", env), None); + assert_eq!( + expand_known(r"%%\%ProgramFiles%\a%", env).as_deref(), + Some(r"%%\C:\Program Files\a%") + ); + } + + /// Machine values as Windows stores them: known folders and REG_SZ + /// values as written, REG_EXPAND_SZ values naming others. + fn stored(name: &str) -> Option { + let expand = |value: &str| Some(Stored::Expand(value.into())); + match name.to_lowercase().as_str() { + "systemdrive" => Some(Stored::Literal("C:".into())), + "programfiles" => Some(Stored::Literal(r"C:\Program Files".into())), + "emroot" => expand(r"%SystemDrive%\Apps"), + "literal" => Some(Stored::Literal(r"C:\100%\%SystemDrive%".into())), + "userish" => expand(r"%USERPROFILE%\Apps"), + "loop_a" => expand(r"%loop_b%\a"), + "loop_b" => expand(r"%LOOP_A%\b"), + "self" => expand(r"%self%"), + deep => { + // deep0 is a folder; each deepN names deepN-1. + let n: usize = deep.strip_prefix("deep")?.parse().ok()?; + if n == 0 { + Some(Stored::Literal(r"C:\Deep".into())) + } else { + expand(&format!(r"%deep{}%\{n}", n - 1)) + } + } + } + } + + #[test] + fn machine_values_naming_others_are_expanded_from_machine_values_only() { + let value = |name| resolve(name, &stored); + assert_eq!(value("EMROOT").as_deref(), Some(r"C:\Apps")); + assert_eq!(value("ProgramFiles").as_deref(), Some(r"C:\Program Files")); + // A REG_SZ value is never expanded, as Windows doesn't. + assert_eq!(value("literal").as_deref(), Some(r"C:\100%\%SystemDrive%")); + // A value only a user has, or no value at all. + assert_eq!(value("userish"), None); + assert_eq!(value("nope"), None); + // Values naming each other, or themselves. + assert_eq!(value("loop_a"), None); + assert_eq!(value("self"), None); + // Up to EXPAND_DEPTH nested values, and no deeper. + let deepest = format!("deep{EXPAND_DEPTH}"); + assert_eq!(value(&deepest).as_deref(), Some(r"C:\Deep\1\2\3\4\5\6\7\8")); + assert_eq!(value(&format!("deep{}", EXPAND_DEPTH + 1)), None); + } + + /// A Block rule whose path names a machine value that names another + /// (`EMROOT=%SystemDrive%\Apps`) is this exe's block. One whose path + /// can't be resolved but has this exe's file name may be: never + /// "allowed", and never certain. + #[test] + fn a_block_through_nested_machine_values_counts_and_an_unresolved_one_is_uncertain() { + let exe = r"C:\Apps\EternalMonitor\EternalMonitor-host.exe"; + let allow = Rule::unscoped(exe, true, 0x7fff_ffff); + let nested = Rule::unscoped( + r"%EMROOT%\EternalMonitor\EternalMonitor-host.exe", + false, + PUBLIC, + ); + let judged = |rules: &[Rule]| { + judge(&[on(PUBLIC)], rules, Path::new(exe), PORT, true, |name| { + resolve(name, &stored) + }) + }; + let blocked = judged(&[allow.clone(), nested]); + assert_eq!(blocked.profiles, [(PUBLIC, Verdict::Blocked)]); + assert!(blocked.certain_block().is_some()); + + let unresolved = Rule::unscoped( + r"%USERISH%\EternalMonitor\EternalMonitor-host.exe", + false, + PUBLIC, + ); + let unsure = judged(&[allow.clone(), unresolved]); + assert_eq!(unsure.profiles, [(PUBLIC, Verdict::Uncertain)]); + assert_eq!(unsure.verdict(), "uncertain"); + assert_eq!(unsure.certain_block(), None); + assert_eq!(unsure.scoped_block_rules, 1); + + // Another file name is another program's rule. + let other = Rule::unscoped(r"%USERISH%\Other\other.exe", false, PUBLIC); + assert_eq!( + judged(&[allow, other]).profiles, + [(PUBLIC, Verdict::Allowed)] + ); } /// A read that fails drops the last reading, so a block that is gone diff --git a/host/src/firewall_fix.rs b/host/src/firewall_fix.rs index 8ef2a4d6..048681ae 100644 --- a/host/src/firewall_fix.rs +++ b/host/src/firewall_fix.rs @@ -24,6 +24,9 @@ //! networks not in use and other programs' rules stay as they are. //! 3. It reads everything again and fails unless the verdict now finds the //! exe allowed, and not blocked by its own rule, on every network in use. +//! A Block rule that may be this exe's, with its file name and a path the +//! machine's values can't resolve, is never removed and keeps the fix +//! from claiming success. //! //! The exe's path comes from the elevated process itself, and the port is //! only ever a number. A rule's path is this exe's only as written, or with @@ -987,14 +990,6 @@ mod tests { PUBLIC, ), ), - ( - "Users own", - Rule::unscoped( - &format!(r"%{USERS_OWN}%\EternalMonitor\EternalMonitor-host.exe"), - false, - PUBLIC, - ), - ), ]; // The process would say D:\Apps for ProgramFiles; the machine says // C:\Program Files. @@ -1014,7 +1009,9 @@ mod tests { [format!("add {EXE}"), "remove Program Files copy".into()] ); - // And as the elevated copy runs it. + // A variable only this process has, as the elevated copy runs it: + // the rule is never removed, and as it may be this exe's, the fix + // doesn't claim success. let rules = vec![ (ALLOW_NAME, installer_allow()), ( @@ -1027,11 +1024,53 @@ mod tests { ), ]; let mut store = Fake::new(&[on(PUBLIC)], rules); - assert_eq!(fix_on_this_machine(&mut store, exe(), PORT), Ok(())); - assert!(store.calls.is_empty(), "{:?}", store.calls); + assert_eq!( + fix_on_this_machine(&mut store, exe(), PORT), + Err(Failure::StillBlocked) + ); + assert!(store.names().contains(&"Users own")); + assert!(!store.calls.iter().any(|call| call.starts_with("remove"))); std::env::remove_var(USERS_OWN); } + /// A machine value naming another (`EMROOT=%SystemDrive%\Apps`) makes + /// the rule this exe's: it goes. One that can't be resolved, with this + /// exe's file name, stays, and the fix fails rather than claim success. + #[test] + fn a_block_through_nested_machine_values_goes_and_an_unresolved_one_fails_the_fix() { + use crate::firewall::{resolve, Stored}; + let stored = |name: &str| match name.to_lowercase().as_str() { + "systemdrive" => Some(Stored::Literal("C:".into())), + "emroot" => Some(Stored::Expand(r"%SystemDrive%\Apps".into())), + "userish" => Some(Stored::Expand(r"%USERPROFILE%\Apps".into())), + _ => None, + }; + let lookup = |name: &str| resolve(name, &stored); + let exe = Path::new(r"C:\Apps\EternalMonitor\EternalMonitor-host.exe"); + let allow = Rule::unscoped(&exe.to_string_lossy(), true, ALLOW_PROFILES); + let block = |root: &str| { + Rule::unscoped( + &format!(r"%{root}%\EternalMonitor\EternalMonitor-host.exe"), + false, + PUBLIC, + ) + }; + + let rules = vec![(ALLOW_NAME, allow.clone()), ("Nested", block("EMROOT"))]; + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!(fix_rules(&mut store, exe, PORT, &lookup), Ok(())); + assert_eq!(store.calls, ["remove Nested"]); + + let rules = vec![(ALLOW_NAME, allow), ("Unresolved", block("USERISH"))]; + let mut store = Fake::new(&[on(PUBLIC)], rules); + assert_eq!( + fix_rules(&mut store, exe, PORT, &lookup), + Err(Failure::StillBlocked) + ); + assert!(store.names().contains(&"Unresolved")); + assert!(!store.calls.iter().any(|call| call.starts_with("remove"))); + } + #[test] fn rules_that_cant_be_read_are_a_failure() { let mut store = Fake::new(&[on(PUBLIC)], dismissed_prompt()); From 9972aec0701ba4c2286a5bc87630f53ef0d0a243 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 13:43:00 -0500 Subject: [PATCH 221/226] Read a missing display device through the confirmed device query The startup and quit Disable now use the same read as the second screen's attach wait. It confirms there is no device only when every present root device was read and none is the driver's. A device list that can't be read now counts as uncertain there, instead of as no device. So does a device that can't be read, and the Disable task runs for both, as before. The separate absence check is gone. --- host/src/vdd.rs | 34 ++++++++------- host/src/vdd_device.rs | 94 +++++++++++++++++++++--------------------- 2 files changed, 66 insertions(+), 62 deletions(-) diff --git a/host/src/vdd.rs b/host/src/vdd.rs index 6aca9a52..2cbd3f4e 100644 --- a/host/src/vdd.rs +++ b/host/src/vdd.rs @@ -604,13 +604,14 @@ const DEVICE_READ_LIMIT: Duration = Duration::from_secs(2); /// there is none, said in the log. #[cfg(windows)] fn device_off() -> bool { - off_within(crate::vdd_device::query_present, DEVICE_READ_LIMIT) + off_within(crate::vdd_device::query_confirmed, DEVICE_READ_LIMIT) } /// Whether `query`, run on its own thread, reads the device disabled, or -/// confirms there is no device (`None`), within `limit`. A read that doesn't -/// answer in time, or whose thread can't start, counts as not disabled, so -/// the Disable task runs under its own limit; a late answer is dropped. +/// confirms there is no device (`Missing`), within `limit`. A read that isn't +/// certain (`None`), doesn't answer in time, or whose thread can't start +/// counts as not disabled, so the Disable task runs under its own limit; a +/// late answer is dropped. #[cfg_attr(not(windows), allow(dead_code))] // Driver tasks exist only on Windows. fn off_within( query: impl FnOnce() -> Option + Send + 'static, @@ -623,7 +624,7 @@ fn off_within( let _ = answer.send(query()); }); match spawned.ok().map(|_| answered.recv_timeout(limit)) { - Some(Ok(None)) => { + Some(Ok(Some(crate::vdd_device::DeviceState::Missing))) => { tracing::info!("No virtual display device — not running the Disable task"); true } @@ -634,6 +635,7 @@ fn off_within( } off } + Some(Ok(None)) => false, _ => { tracing::warn!( limit_ms = limit.as_millis() as u64, @@ -932,18 +934,15 @@ mod tests { |_, _| panic!("ran the Disable task for a display that reads disabled") )); assert!(started.elapsed() < DEVICE_READ_LIMIT); - for device in [DeviceState::Ready, DeviceState::Missing] { - assert!( - !off_within(move || Some(device), DEVICE_READ_LIMIT), - "{device:?}" - ); + for device in [Some(DeviceState::Ready), None] { + assert!(!off_within(move || device, DEVICE_READ_LIMIT), "{device:?}"); } } /// Windows 10, where Setup installs no driver: Windows confirms there is - /// no device, so the Disable at startup and quit runs no task, and no - /// missing task is logged at every launch. A device list that couldn't - /// be read (`Missing`) still runs it. + /// no device (`Missing`), so the Disable at startup and quit runs no + /// task, and no missing task is logged at every launch. A read that isn't + /// certain (`None`) still runs it. #[test] fn a_confirmed_absent_device_runs_no_disable_task() { use crate::vdd_device::DeviceState; @@ -951,19 +950,22 @@ mod tests { let tasks = DriverTasks::new(); assert!(tasks.disable_unless_off( || {}, - || off_within(|| None, DEVICE_READ_LIMIT), + || off_within(|| Some(DeviceState::Missing), DEVICE_READ_LIMIT), |_, _| panic!("ran the Disable task with no device to turn off") )); let ran = Cell::new(false); tasks.disable_unless_off( || {}, - || off_within(|| Some(DeviceState::Missing), DEVICE_READ_LIMIT), + || off_within(|| None, DEVICE_READ_LIMIT), |_, _| { ran.set(true); TaskOutcome::Missing }, ); - assert!(ran.get(), "an unread device list skipped the Disable task"); + assert!( + ran.get(), + "an uncertain device read skipped the Disable task" + ); assert_eq!(tasks.health(), VddTaskHealth::TasksMissing); } diff --git a/host/src/vdd_device.rs b/host/src/vdd_device.rs index 65c9cb06..86694fee 100644 --- a/host/src/vdd_device.rs +++ b/host/src/vdd_device.rs @@ -187,30 +187,11 @@ pub fn from_task_stderr(stderr: &str) -> Option { DeviceState::from_task_result(digits.parse().ok()?) } -/// As [`query`], but `None` when Windows lists the present root devices, -/// reads each of them, and none is the driver's: there is no device to turn -/// off, as on Windows 10, where Setup installs none. [`query`] reads that as -/// `Missing`, as it does a list it couldn't get. -#[cfg(windows)] -pub fn query_present() -> Option { - match root_devnodes() { - Some(devnodes) if confirmed_absent(&devnodes) => None, - _ => Some(query()), - } -} - -/// Whether every present root device was read and none is the driver's. -#[cfg(any(windows, test))] -fn confirmed_absent(devnodes: &[Devnode]) -> bool { - devnodes - .iter() - .all(|devnode| matches!(devnode, Devnode::Other)) -} - /// Read the driver's device from Configuration Manager. #[cfg(windows)] pub fn query() -> DeviceState { - DeviceState::best(read_devices().into_iter().filter_map(|read| { + let reads = read_devices().unwrap_or_default(); + DeviceState::best(reads.into_iter().filter_map(|read| { let (flags, problem, has_driver, monitors) = read?; Some(DeviceState::from_devnode( flags, problem, has_driver, monitors, @@ -220,18 +201,24 @@ pub fn query() -> DeviceState { /// As `query`, for the attach wait: `NoMonitor` only when every running /// device of the driver has confirmed no monitor child (see -/// `DeviceState::confirmed`). `None` when that isn't certain: a device or -/// its children couldn't be read, or a monitor hasn't started yet. +/// `DeviceState::confirmed`). `None` when that isn't certain: the devices, +/// one of them or its children couldn't be read, or a monitor hasn't +/// started yet. `Missing` only when every present root device was read and +/// none is the driver's, as on Windows 10, where Setup installs none. #[cfg(windows)] pub fn query_confirmed() -> Option { - confirmed_from(read_devices()) + confirmed_from(read_devices()?) } +/// One device of the driver as read: its status flags and problem code, +/// whether a driver is bound, and its monitor children. +type DeviceRead = (u32, u32, bool, Monitors); + /// `query_confirmed` from each driver device's read (`None` when its status /// couldn't be read). A device with a monitor wins; otherwise anything /// uncertain makes the whole read uncertain. #[cfg_attr(not(windows), allow(dead_code))] // Read from Windows only. -fn confirmed_from(reads: Vec>) -> Option { +fn confirmed_from(reads: Vec>) -> Option { let states: Vec> = reads .into_iter() .map(|read| { @@ -248,14 +235,13 @@ fn confirmed_from(reads: Vec>) -> Option Vec> { +fn read_devices() -> Option>> { use windows::Win32::Devices::DeviceAndDriverInstallation::CM_DRP_DRIVER; - let Some(devnodes) = root_devnodes() else { - return Vec::new(); - }; - reads_from(&devnodes, |devinst, (flags, problem)| { + let devnodes = root_devnodes()?; + Some(reads_from(&devnodes, |devinst, (flags, problem)| { let has_driver = matches!(registry_property(devinst, CM_DRP_DRIVER), Ok(Some(_))); let children = children(devinst); ( @@ -264,7 +250,7 @@ fn read_devices() -> Vec> { has_driver, Monitors::from_children(children.as_deref()), ) - }) + })) } /// `read_devices` from the present root devices, with `read` reading one @@ -275,8 +261,8 @@ fn read_devices() -> Vec> { #[cfg(any(windows, test))] fn reads_from( devnodes: &[Devnode], - mut read: impl FnMut(u32, (u32, u32)) -> (u32, u32, bool, Monitors), -) -> Vec> { + mut read: impl FnMut(u32, (u32, u32)) -> DeviceRead, +) -> Vec> { devnodes .iter() .filter_map(|devnode| match *devnode { @@ -603,22 +589,38 @@ pub fn query() -> DeviceState { mod tests { use super::*; - /// Only a list where every root device was read, and none is the - /// driver's, says there is no device to turn off. + /// Setup installs no driver on Windows 10: a list where every root + /// device was read, and none is the driver's, reads as a confirmed + /// `Missing`. A device that couldn't be read keeps it uncertain. #[test] - fn only_a_fully_read_list_without_the_driver_is_an_absent_device() { + fn only_a_fully_read_list_without_the_driver_confirms_it_missing() { + let unreachable = |_, _| unreachable!("no device of the driver"); + assert_eq!( + confirmed_from(reads_from(&[Devnode::Other, Devnode::Other], unreachable)), + Some(DeviceState::Missing) + ); + for uncertain in [ + Devnode::Unread, + Devnode::Driver { + devinst: 7, + status: None, + }, + ] { + assert_eq!( + confirmed_from(reads_from(&[Devnode::Other, uncertain], unreachable)), + None, + "{uncertain:?}" + ); + } + let off = |_, (flags, problem)| (flags, problem, true, Monitors::None); let driver = Devnode::Driver { devinst: 7, - status: Some((0, 22)), + status: Some((DN_HAS_PROBLEM, 22)), }; - assert!(confirmed_absent(&[])); - assert!(confirmed_absent(&[Devnode::Other, Devnode::Other])); - assert!(!confirmed_absent(&[Devnode::Other, driver])); - assert!(!confirmed_absent(&[Devnode::Other, Devnode::Unread])); - assert!(!confirmed_absent(&[Devnode::Driver { - devinst: 7, - status: None - }])); + assert_eq!( + confirmed_from(reads_from(&[Devnode::Other, driver], off)), + Some(DeviceState::Problem(22)) + ); } /// Review r6: a root device whose location or hardware IDs couldn't be From 5b4f65170568d0bfa69406a77682ffead00dc683 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 13:59:24 -0500 Subject: [PATCH 222/226] Bound how much work expanding machine values can take Each machine value is looked up once per expansion, and an expansion stops, as unknown, after 256 references or 32 KB of text. Values that name each other many times over can no longer stall the firewall check or the fix. --- host/src/firewall.rs | 147 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 125 insertions(+), 22 deletions(-) diff --git a/host/src/firewall.rs b/host/src/firewall.rs index 11c102dc..7ec55b41 100644 --- a/host/src/firewall.rs +++ b/host/src/firewall.rs @@ -464,37 +464,70 @@ pub enum Stored { /// How deep REG_EXPAND_SZ values may name one another. pub const EXPAND_DEPTH: usize = 8; +/// How many `%name%`s one [`resolve`] may follow, counting repeats. +pub const MAX_REFERENCES: usize = 256; + +/// The most text one expansion may make, in bytes: more than any path +/// Windows takes. +pub const MAX_EXPANDED: usize = 32 * 1024; + /// `name`'s value among the machine values `stored`, each REG_EXPAND_SZ -/// value's own `%name%`s resolved the same way. `None` when `name`, or a -/// value it names, has no machine value, when values name each other in a -/// cycle, or when more than [`EXPAND_DEPTH`] of them nest. +/// value's own `%name%`s resolved the same way, each name once. `None` when +/// `name`, or a value it names, has no machine value, when values name each +/// other in a cycle, when more than [`EXPAND_DEPTH`] of them nest, or when +/// it takes more than [`MAX_REFERENCES`] or makes more than +/// [`MAX_EXPANDED`]. pub fn resolve(name: &str, stored: &impl Fn(&str) -> Option) -> Option { - resolve_within(name, stored, &mut Vec::new()) + Resolving { + stored, + known: std::collections::HashMap::new(), + expanding: Vec::new(), + references: 0, + } + .value(name) } -fn resolve_within( - name: &str, - stored: &impl Fn(&str) -> Option, - expanding: &mut Vec, -) -> Option { - match stored(name)? { - Stored::Literal(value) => Some(value), - Stored::Expand(value) => { - let key = name.to_lowercase(); - if expanding.len() >= EXPAND_DEPTH || expanding.contains(&key) { - return None; - } - expanding.push(key); - let value = expand_known(&value, |inner| resolve_within(inner, stored, expanding)); - expanding.pop(); - value +struct Resolving<'a, F> { + stored: &'a F, + /// Each name resolved so far. Any `None` fails the whole resolve, so + /// remembering one is never wrong. + known: std::collections::HashMap>, + expanding: Vec, + references: usize, +} + +impl Option> Resolving<'_, F> { + fn value(&mut self, name: &str) -> Option { + self.references += 1; + if self.references > MAX_REFERENCES { + return None; } + let key = name.to_lowercase(); + if let Some(known) = self.known.get(&key) { + return known.clone(); + } + let value = match (self.stored)(name) { + None => None, + Some(Stored::Literal(value)) => Some(value), + Some(Stored::Expand(value)) => { + if self.expanding.len() >= EXPAND_DEPTH || self.expanding.contains(&key) { + return None; + } + self.expanding.push(key.clone()); + let expanded = expand_known(&value, |inner| self.value(inner)); + self.expanding.pop(); + expanded + } + }; + self.known.insert(key, value.clone()); + value } } /// `path` with every `%NAME%` replaced with `lookup(NAME)`, as Windows /// stores a rule made in its own window (`%ProgramFiles%\...`), or `None` if -/// one has no value. `%%` and a lone `%` stay as written. +/// one has no value or it would make more than [`MAX_EXPANDED`]. `%%` and a +/// lone `%` stay as written. fn expand_known(path: &str, mut lookup: impl FnMut(&str) -> Option) -> Option { let mut out = String::new(); let mut rest = path; @@ -509,10 +542,13 @@ fn expand_known(path: &str, mut lookup: impl FnMut(&str) -> Option) -> O } else { out.push_str(&lookup(name)?); } + if out.len() > MAX_EXPANDED { + return None; + } rest = &rest[start + len + 2..]; } out.push_str(rest); - Some(out) + Some(out).filter(|out| out.len() <= MAX_EXPANDED) } /// A path as Windows compares it: no quotes or `\\?\`, backslashes, no case. @@ -1610,6 +1646,73 @@ mod tests { assert_eq!(value(&format!("deep{}", EXPAND_DEPTH + 1)), None); } + /// Values that each name the next several times would grow fourfold a + /// level: each name is looked up once, and the expansion stops at its + /// size or reference limit. + #[test] + fn a_branching_expansion_stops_at_its_limits() { + let branching = |name: &str| { + // b0 is a folder; each bN names bN-1 four times. + let n: usize = name.strip_prefix('b')?.parse().ok()?; + Some(if n == 0 { + Stored::Literal("x".repeat(20)) + } else { + Stored::Expand(format!("%b{0}%", n - 1).repeat(4)) + }) + }; + let lookups = std::cell::Cell::new(0); + let counted = |name: &str| { + lookups.set(lookups.get() + 1); + branching(name) + }; + // 4^8 × 20 bytes, 1.3 MB. + assert_eq!(resolve(&format!("b{EXPAND_DEPTH}"), &counted), None); + assert!(lookups.get() <= EXPAND_DEPTH + 1, "{}", lookups.get()); + // 4^4 × 20 bytes is within the limits. + assert_eq!( + resolve("b4", &branching).map(|value| value.len()), + Some(4usize.pow(4) * 20) + ); + // One value naming another too many times. + let wide = |name: &str| match name { + "wide" => Some(Stored::Expand("%leaf%".repeat(MAX_REFERENCES))), + "leaf" => Some(Stored::Literal("a".into())), + _ => None, + }; + assert_eq!(resolve("wide", &wide), None); + let within = |name: &str| match name { + "wide" => Some(Stored::Expand("%leaf%".repeat(MAX_REFERENCES - 1))), + _ => wide(name), + }; + assert_eq!( + resolve("wide", &within).map(|value| value.len()), + Some(MAX_REFERENCES - 1) + ); + // Past the size limit, nothing more is looked up. + let after = std::cell::Cell::new(false); + let big = |name: &str| match name { + "both" => Some(Stored::Expand("%big%%after%".into())), + "big" => Some(Stored::Literal("x".repeat(MAX_EXPANDED + 1))), + _ => { + after.set(true); + Some(Stored::Literal(String::new())) + } + }; + assert_eq!(resolve("both", &big), None); + assert!(!after.get()); + // Text after the last name counts too. + let tail = |name: &str| match name { + "tail" => Some(Stored::Expand("%full%x".into())), + "full" => Some(Stored::Literal("x".repeat(MAX_EXPANDED))), + _ => None, + }; + assert_eq!( + resolve("full", &tail).map(|value| value.len()), + Some(MAX_EXPANDED) + ); + assert_eq!(resolve("tail", &tail), None); + } + /// A Block rule whose path names a machine value that names another /// (`EMROOT=%SystemDrive%\Apps`) is this exe's block. One whose path /// can't be resolved but has this exe's file name may be: never From f2717216caad1414ec3faced38a0e5df5d4a4a59 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 14:51:46 -0500 Subject: [PATCH 223/226] Wake a laptop's discrete card when an iPad looks for PCs A laptop powers its discrete card off entirely (D3cold) about 20 s after its last use: on an AMD hybrid laptop the card and every PCIe port above it, up to the root port, went to D3. The next D3D11 device or AMF open on it then took about 800 ms instead of 160, and an AMF session's first picture after 40 s idle took about 870 ms instead of 130. An iPad looks for PCs over mDNS on its connect screen, seconds before it sends HELLO. The host now reads mDNS queries alongside its daemon, and a PTR or ANY question for the service from another device (not the PC's own daemon, which probes at each registration and re-advertisement) creates a D3D11 device on the encoder's adapter on a thread of its own and drops it. Nothing is kept open, and nothing wakes the card on a timer: a wake comes at most every 15 s, never while a viewer is connected, and each one no session followed doubles the wait for the next, up to 10 min. --- host/src/discovery.rs | 197 +++++++++++++++++++++++++++++++++- host/src/gpu_wake.rs | 184 +++++++++++++++++++++++++++++++ host/src/lib.rs | 1 + host/src/main.rs | 20 +++- host/src/transport/session.rs | 5 + 5 files changed, 405 insertions(+), 2 deletions(-) create mode 100644 host/src/gpu_wake.rs diff --git a/host/src/discovery.rs b/host/src/discovery.rs index f9dcac4c..8b3cb9b6 100644 --- a/host/src/discovery.rs +++ b/host/src/discovery.rs @@ -46,8 +46,15 @@ impl ReAdvertLog { /// documented upsert in mdns-sd, so the refresh never takes the service off /// the air (the old unregister-first cycle opened a ~1s discovery hole). /// +/// `on_browse` runs each time another device on the network looks for PCs +/// running EternalMonitor, as an iPad does on its connect screen (see +/// [`watch_browsers`] and [`crate::gpu_wake`]). +/// /// Returns the `ServiceDaemon` handle — call [`say_goodbye`] on it at exit. -pub fn advertise_service(port: u16) -> Option { +pub fn advertise_service( + port: u16, + on_browse: impl FnMut() + Send + 'static, +) -> Option { let mdns = match ServiceDaemon::new() { Ok(d) => d, Err(e) => { @@ -78,6 +85,7 @@ pub fn advertise_service(port: u16) -> Option { } info!("mDNS service registered — iPad can now discover this host"); PIPELINE_STATS.lock().mdns_active = true; + watch_browsers(on_browse); let mdns_clone = mdns.clone(); let host_fqdn_for_thread = host_fqdn.clone(); @@ -107,6 +115,131 @@ pub fn advertise_service(port: u16) -> Option { Some(mdns) } +/// Calls `on_browse` on a thread of its own each time a device that isn't +/// this PC asks the network for the service: a PTR or ANY question for +/// [`SERVICE_TYPE`]. The daemon's own events can't tell: it answers its own +/// probes too, at each registration and re-advertisement. Reads alongside +/// the daemon on the shared mDNS port and sends nothing. +#[cfg(windows)] +fn watch_browsers(mut on_browse: impl FnMut() + Send + 'static) { + use socket2::{Domain, Protocol, Socket, Type}; + let group = std::net::Ipv4Addr::new(224, 0, 0, 251); + let socket = (|| -> std::io::Result { + let socket = Socket::new(Domain::IPV4, Type::DGRAM, Some(Protocol::UDP))?; + socket.set_reuse_address(true)?; + socket.bind(&std::net::SocketAddr::from(([0, 0, 0, 0], 5353)).into())?; + let mut joined = 0; + for address in own_ipv4() { + joined += usize::from(socket.join_multicast_v4(&group, &address).is_ok()); + } + if joined == 0 { + socket.join_multicast_v4(&group, &std::net::Ipv4Addr::UNSPECIFIED)?; + } + Ok(socket.into()) + })(); + let socket = match socket { + Ok(socket) => socket, + Err(error) => { + info!(%error, "Not watching for iPads looking for this PC"); + return; + } + }; + let _ = thread::Builder::new() + .name("mdns-browsers".into()) + .spawn(move || { + let mut packet = [0u8; 1500]; + while let Ok((len, from)) = socket.recv_from(&mut packet) { + if asks_for_service(&packet[..len]) && !is_own(from.ip()) { + on_browse(); + } + } + }); +} + +#[cfg(not(windows))] +fn watch_browsers(_on_browse: impl FnMut() + Send + 'static) {} + +/// This PC's IPv4 addresses. +#[cfg(windows)] +fn own_ipv4() -> Vec { + crate::network::adapters() + .unwrap_or_default() + .into_iter() + .filter(|adapter| adapter.up) + .flat_map(|adapter| adapter.ipv4) + .collect() +} + +/// Whether a query came from this PC: its own daemon, or another program on it. +#[cfg(windows)] +fn is_own(address: IpAddr) -> bool { + address.is_loopback() + || match address { + IpAddr::V4(address) => own_ipv4().contains(&address), + IpAddr::V6(_) => false, + } +} + +/// Whether `packet` is a DNS query (not a response) with a PTR or ANY +/// question for [`SERVICE_TYPE`], as a device browsing for EternalMonitor +/// PCs sends. +#[cfg(any(windows, test))] +pub(crate) fn asks_for_service(packet: &[u8]) -> bool { + const PTR: u16 = 12; + const ANY: u16 = 255; + let u16_at = |at: usize| -> Option { + Some(u16::from_be_bytes([*packet.get(at)?, *packet.get(at + 1)?])) + }; + let (Some(flags), Some(questions)) = (u16_at(2), u16_at(4)) else { + return false; + }; + if flags & 0x8000 != 0 { + return false; + } + let wanted = SERVICE_TYPE.trim_end_matches('.'); + let mut at = 12; + for _ in 0..questions { + let Some((name, next)) = dns_name(packet, at) else { + return false; + }; + let Some(kind) = u16_at(next) else { + return false; + }; + if (kind == PTR || kind == ANY) && name.eq_ignore_ascii_case(wanted) { + return true; + } + at = next + 4; + } + false +} + +/// The dotted name at `at`, following compression pointers, and where the +/// record goes on after it. None if it is malformed or loops. +#[cfg(any(windows, test))] +fn dns_name(packet: &[u8], mut at: usize) -> Option<(String, usize)> { + let mut name = String::new(); + let mut after = None; + for _ in 0..32 { + let len = *packet.get(at)? as usize; + if len == 0 { + return Some((name, after.unwrap_or(at + 1))); + } + if len & 0xC0 == 0xC0 { + let target = ((len & 0x3F) << 8) | *packet.get(at + 1)? as usize; + after.get_or_insert(at + 2); + at = target; + continue; + } + let label = packet.get(at + 1..at + 1 + len)?; + if !name.is_empty() { + name.push('.'); + } + name.push_str(std::str::from_utf8(label).ok()?); + at += 1 + len; + } + None +} + /// Send mDNS goodbye packets and stop the daemon. Called on the way out so /// iPads drop the host from their scan list promptly instead of waiting for /// the TTL to lapse. The short grace period lets the goodbyes hit the wire — @@ -153,6 +286,68 @@ fn hostname() -> Option { mod tests { use super::*; + /// A DNS packet with one question per `(name, type)`. + fn query(flags: u16, questions: &[(&str, u16)]) -> Vec { + let mut packet = vec![0, 0]; + packet.extend_from_slice(&flags.to_be_bytes()); + packet.extend_from_slice(&(questions.len() as u16).to_be_bytes()); + packet.extend_from_slice(&[0; 6]); + for (name, kind) in questions { + for label in name.split('.') { + packet.push(label.len() as u8); + packet.extend_from_slice(label.as_bytes()); + } + packet.push(0); + packet.extend_from_slice(&kind.to_be_bytes()); + packet.extend_from_slice(&[0x80, 1]); + } + packet + } + + #[test] + fn a_browse_for_the_service_is_told_from_other_mdns_traffic() { + let ours = "_eternaldisplay._udp.local"; + assert!(asks_for_service(&query(0, &[(ours, 12)]))); + assert!(asks_for_service(&query( + 0, + &[("_airplay._tcp.local", 12), (ours, 12)] + ))); + assert!(asks_for_service(&query( + 0, + &[("_EternalDisplay._UDP.local", 255)] + ))); + assert!(!asks_for_service(&query(0, &[("_airplay._tcp.local", 12)]))); + assert!(!asks_for_service(&query(0, &[(ours, 1)])), "an A question"); + assert!( + !asks_for_service(&query(0x8400, &[(ours, 12)])), + "a response" + ); + // A second question that points back into the first name. + let mut packet = query(0, &[("_airplay._tcp.local", 12)]); + packet[5] = 2; + packet.extend_from_slice(&[15]); + packet.extend_from_slice(b"_eternaldisplay"); + packet.extend_from_slice(&[4]); + packet.extend_from_slice(b"_udp"); + // "local" is at offset 12 + 1 + 8 + 1 + 4 = 26. + packet.extend_from_slice(&[0xC0, 26, 0, 12, 0, 1]); + assert!(asks_for_service(&packet)); + } + + #[test] + fn a_malformed_or_looping_packet_is_not_a_browse() { + assert!(!asks_for_service(&[])); + assert!(!asks_for_service(&[0; 11])); + let full = query(0, &[("_eternaldisplay._udp.local", 12)]); + for cut in 12..full.len() - 4 { + assert!(!asks_for_service(&full[..cut]), "cut at {cut}"); + } + // A pointer to itself. + let mut looping = vec![0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0]; + looping.extend_from_slice(&[0xC0, 12, 0, 12, 0, 1]); + assert!(!asks_for_service(&looping)); + } + /// The service pins no address: mdns-sd advertises every interface's, /// Wi-Fi and Tailscale alike. The Address row's ranking never reaches it. #[test] diff --git a/host/src/gpu_wake.rs b/host/src/gpu_wake.rs new file mode 100644 index 00000000..8cd1bf1e --- /dev/null +++ b/host/src/gpu_wake.rs @@ -0,0 +1,184 @@ +//! Wakes the encoder's graphics card when an iPad is probably about to +//! connect. A laptop's discrete card powers off entirely (D3cold) about 20 s +//! after its last use: its PCIe link and the ports above it go down too. On +//! an AMD hybrid laptop the next D3D11 device or AMF open on it then took +//! about 800 ms instead of 160, all of it inside the first session's first +//! picture. An iPad looks for PCs over mDNS on its connect screen, seconds +//! before it sends HELLO, so its query (see `discovery::watch_browsers`) +//! starts the card waking: one tiny D3D11 device on it, dropped at once. +//! Nothing is kept open, and nothing wakes it on a timer. The first AMF open +//! then took about 280 ms instead of 830; holding the device for 10 s +//! changed nothing measurable, so it isn't held. + +use std::time::{Duration, Instant}; + +/// The shortest time between two wakes. A card woken this recently is still +/// in D3hot at worst, which wakes in about 160 ms. +pub const WAKE_EVERY: Duration = Duration::from_secs(15); +/// The longest time between two wakes after many wasted ones. +const WAKE_EVERY_AT_MOST: Duration = Duration::from_secs(600); + +/// When a browse may wake the card. Devices browse without connecting too +/// (an iPad left on its connect screen, another app's scan), so each wake no +/// session followed doubles the time to the next, up to +/// [`WAKE_EVERY_AT_MOST`]: a few wakes an hour at most, never a keep-alive. +/// A session resets it. +#[derive(Debug)] +pub struct WakeGate { + last: Option, + every: Duration, + /// `Session::sessions_started` at the last wake. + sessions_at_wake: Option, +} + +impl Default for WakeGate { + fn default() -> Self { + Self { + last: None, + every: WAKE_EVERY, + sessions_at_wake: None, + } + } +} + +impl WakeGate { + /// A device browsed for the service at `now`. Whether to wake the card: + /// never while a viewer is connected (`in_session`, the card is in use). + /// `sessions` is `Session::sessions_started`: a session since the last + /// wake means the wake paid off. + pub fn browse(&mut self, now: Instant, in_session: bool, sessions: u32) -> bool { + if in_session { + return false; + } + if let Some(last) = self.last { + if now.saturating_duration_since(last) < self.every { + return false; + } + } + let paid = self.sessions_at_wake.is_none_or(|at| at != sessions); + self.every = if paid { + WAKE_EVERY + } else { + (self.every * 2).min(WAKE_EVERY_AT_MOST) + }; + self.sessions_at_wake = Some(sessions); + self.last = Some(now); + true + } +} + +/// Starts waking the graphics card at DXGI adapter `adapter_index`, on a +/// thread of its own, unless a wake is still running. +#[cfg(windows)] +pub fn wake(adapter_index: u32) { + use std::sync::atomic::{AtomicBool, Ordering}; + static RUNNING: AtomicBool = AtomicBool::new(false); + if RUNNING.swap(true, Ordering::SeqCst) { + return; + } + let spawned = std::thread::Builder::new() + .name("gpu-wake".into()) + .spawn(move || { + let started = Instant::now(); + match touch(adapter_index) { + Ok(()) => tracing::debug!( + adapter_index, + took_ms = started.elapsed().as_millis() as u64, + "Woke the encoder's graphics card for a viewer that may connect" + ), + Err(error) => { + tracing::debug!(adapter_index, %error, "Could not wake the graphics card") + } + } + RUNNING.store(false, Ordering::SeqCst); + }); + if spawned.is_err() { + RUNNING.store(false, Ordering::SeqCst); + } +} + +#[cfg(not(windows))] +pub fn wake(_adapter_index: u32) {} + +/// A D3D11 device on the adapter, dropped at once: enough to power it up. +#[cfg(windows)] +fn touch(adapter_index: u32) -> windows::core::Result<()> { + use windows::core::Interface; + use windows::Win32::Graphics::Direct3D::D3D_DRIVER_TYPE_UNKNOWN; + use windows::Win32::Graphics::Direct3D11::{D3D11CreateDevice, D3D11_SDK_VERSION}; + use windows::Win32::Graphics::Dxgi::{CreateDXGIFactory1, IDXGIAdapter, IDXGIFactory1}; + let factory: IDXGIFactory1 = unsafe { CreateDXGIFactory1() }?; + let adapter: IDXGIAdapter = unsafe { factory.EnumAdapters1(adapter_index) }?.cast()?; + let mut device = None; + unsafe { + D3D11CreateDevice( + &adapter, + D3D_DRIVER_TYPE_UNKNOWN, + None, + Default::default(), + None, + D3D11_SDK_VERSION, + Some(&mut device), + None, + None, + ) + }?; + drop(device); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at(start: Instant, s: u64) -> Instant { + start + Duration::from_secs(s) + } + + #[test] + fn a_browse_wakes_the_card_at_most_once_per_interval_and_never_in_a_session() { + let start = Instant::now(); + let mut gate = WakeGate::default(); + assert!(gate.browse(at(start, 0), false, 7)); + assert!( + !gate.browse(at(start, 5), false, 7), + "a scan's burst of queries" + ); + assert!(!gate.browse(at(start, 14), false, 7)); + assert!(!gate.browse(at(start, 40), true, 8), "the card is in use"); + // The iPad connected and left: the wake paid off. + assert!(gate.browse(at(start, 60), false, 8)); + assert!(gate.browse(at(start, 75), false, 9)); + } + + #[test] + fn wakes_no_session_followed_come_ever_less_often() { + let start = Instant::now(); + let mut gate = WakeGate::default(); + // A device that browses every 20 s and never connects. + let mut wakes = Vec::new(); + for s in (0..3600).step_by(20) { + if gate.browse(at(start, s), false, 7) { + wakes.push(s); + } + } + let gaps: Vec = wakes.windows(2).map(|pair| pair[1] - pair[0]).collect(); + assert!(gaps.windows(2).all(|pair| pair[0] <= pair[1]), "{gaps:?}"); + // Then one every ten minutes: six an hour. + let settled = &gaps[gaps.len() - 3..]; + assert!( + settled + .iter() + .all(|&gap| gap >= WAKE_EVERY_AT_MOST.as_secs()), + "{gaps:?}" + ); + assert!( + wakes.len() <= 12, + "{} wakes in the first hour: {wakes:?}", + wakes.len() + ); + // An iPad connects: back to the short interval. + assert!(gate.browse(at(start, 4000), false, 8)); + assert!(gate.browse(at(start, 4015), false, 9)); + } +} diff --git a/host/src/lib.rs b/host/src/lib.rs index 4b7be678..f57c0ed6 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -14,6 +14,7 @@ pub mod encoder; pub mod feedback; pub mod firewall; pub mod gpu_preference; +pub mod gpu_wake; pub mod gui; pub mod handoff; pub mod i18n; diff --git a/host/src/main.rs b/host/src/main.rs index 8415cda5..d095a2c8 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -260,11 +260,29 @@ fn main() -> Result<(), Box> { } } + // An iPad looking for PCs starts waking the encoder's graphics card, which + // a laptop powers off when idle (see `gpu_wake`). + let wake_card = (gpu_info.encoder_name != "libx264").then_some(gpu_info.adapter_index); + let wake_shared = shared.clone(); + let mut wake_gate = eternal_host::gpu_wake::WakeGate::default(); + let on_browse = move || { + if let Some(adapter_index) = wake_card { + let sessions = wake_shared.session.lock().sessions_started(); + if wake_gate.browse( + std::time::Instant::now(), + wake_shared.client_connected(), + sessions, + ) { + eternal_host::gpu_wake::wake(adapter_index); + } + } + }; + let supervisor_thread = std::thread::spawn(move || { eternal_host::supervisor::run(listen_port, shared, gpu_info, supervisor_tx, supervisor_rx); }); - let mdns = discovery::advertise_service(listen_port); + let mdns = discovery::advertise_service(listen_port, on_browse); // The quit path, once: after the window closes (or the headless loop // ends), or from the first-paint watcher when the user quits while the diff --git a/host/src/transport/session.rs b/host/src/transport/session.rs index be2f8475..a2288121 100644 --- a/host/src/transport/session.rs +++ b/host/src/transport/session.rs @@ -865,6 +865,11 @@ impl Session { self.active.as_ref().map(|s| s.session_id) } + /// Changes each time a session starts: a count, from a random start. + pub fn sessions_started(&self) -> u32 { + self.next_session_id + } + pub fn link_lost(&mut self, link: LinkId) -> Actions { let on_link = self .active From 2ea4a2c270c96cbae64a48437c1498552c7100a3 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 15:09:11 -0500 Subject: [PATCH 224/226] Keep the browse watch running, and wake only when the card will be used - The browse watch read into a 1500-byte buffer and stopped at its first receive error: one oversized mDNS datagram (WSAEMSGSIZE) ended warming until the app restarted. It now reads up to mDNS's 9000 bytes, goes on at once after errors that pass, and waits longer after repeated or other errors, up to a minute, without stopping. - A HELLO could start a session between the browse's check and the wake's device. The wake thread now checks again, just before it creates the device, that no session has started since (the session count), and drops the wake if one has. - After a long backoff a session didn't restore the short wait until the old ten-minute one had passed: the gate checked the wait before noticing the session. It now resets the wait first, keeping the 15 s minimum. - Whether to wake now follows the encoder that will stream: the override in Settings, and no wake after a fallback to the CPU, nor for an encoder on another card. --- host/src/discovery.rs | 74 +++++++++++++++++++++++++-- host/src/gpu_wake.rs | 115 ++++++++++++++++++++++++++++++++++++++---- host/src/main.rs | 33 ++++++++---- 3 files changed, 198 insertions(+), 24 deletions(-) diff --git a/host/src/discovery.rs b/host/src/discovery.rs index 8b3cb9b6..e33eeb5d 100644 --- a/host/src/discovery.rs +++ b/host/src/discovery.rs @@ -147,15 +147,57 @@ fn watch_browsers(mut on_browse: impl FnMut() + Send + 'static) { let _ = thread::Builder::new() .name("mdns-browsers".into()) .spawn(move || { - let mut packet = [0u8; 1500]; - while let Ok((len, from)) = socket.recv_from(&mut packet) { - if asks_for_service(&packet[..len]) && !is_own(from.ip()) { - on_browse(); + let mut packet = vec![0u8; MDNS_PACKET_MAX]; + let mut errors = 0u32; + loop { + match socket.recv_from(&mut packet) { + Ok((len, from)) => { + errors = 0; + if asks_for_service(&packet[..len]) && !is_own(from.ip()) { + on_browse(); + } + } + Err(error) => { + let pause = after_receive_error(&error, errors); + errors = errors.saturating_add(1); + if errors.is_power_of_two() { + tracing::debug!(%error, errors, "mDNS browse watch: receive failed"); + } + thread::sleep(pause); + } } } }); } +/// The largest mDNS message (RFC 6762 section 17). A longer datagram fails +/// the receive with WSAEMSGSIZE, and is dropped. +#[cfg(windows)] +const MDNS_PACKET_MAX: usize = 9000; + +/// How long the browse watch pauses after a receive error, the `in_a_row`th +/// since the last packet. It never stops: one oversized or reset datagram +/// had ended it until the app restarted. Errors that pass (an oversized +/// datagram, a reset from an ICMP reply, an interruption) go on at once, a +/// few in a row; anything else, or more, waits longer each time, up to a +/// minute. +#[cfg(any(windows, test))] +fn after_receive_error(error: &std::io::Error, in_a_row: u32) -> Duration { + use std::io::ErrorKind; + const WSAEMSGSIZE: i32 = 10040; + let passing = matches!( + error.kind(), + ErrorKind::Interrupted + | ErrorKind::WouldBlock + | ErrorKind::TimedOut + | ErrorKind::ConnectionReset + ) || error.raw_os_error() == Some(WSAEMSGSIZE); + if passing && in_a_row < 8 { + return Duration::ZERO; + } + (Duration::from_millis(100) * 2u32.pow(in_a_row.min(10))).min(Duration::from_secs(60)) +} + #[cfg(not(windows))] fn watch_browsers(_on_browse: impl FnMut() + Send + 'static) {} @@ -334,6 +376,30 @@ mod tests { assert!(asks_for_service(&packet)); } + #[test] + fn a_receive_error_never_ends_the_browse_watch() { + use std::io::{Error, ErrorKind}; + let oversized = Error::from_raw_os_error(10040); + let reset = Error::from(ErrorKind::ConnectionReset); + let interrupted = Error::from(ErrorKind::Interrupted); + for error in [&oversized, &reset, &interrupted] { + assert_eq!(after_receive_error(error, 0), Duration::ZERO, "{error}"); + assert_eq!(after_receive_error(error, 7), Duration::ZERO, "{error}"); + assert!( + after_receive_error(error, 8) > Duration::ZERO, + "{error} keeps coming" + ); + } + let other = Error::from(ErrorKind::PermissionDenied); + let pauses: Vec = (0..20).map(|n| after_receive_error(&other, n)).collect(); + assert!(pauses[0] > Duration::ZERO); + assert!( + pauses.windows(2).all(|pair| pair[0] <= pair[1]), + "{pauses:?}" + ); + assert_eq!(*pauses.last().unwrap(), Duration::from_secs(60)); + } + #[test] fn a_malformed_or_looping_packet_is_not_a_browse() { assert!(!asks_for_service(&[])); diff --git a/host/src/gpu_wake.rs b/host/src/gpu_wake.rs index 8cd1bf1e..3718178d 100644 --- a/host/src/gpu_wake.rs +++ b/host/src/gpu_wake.rs @@ -50,27 +50,58 @@ impl WakeGate { if in_session { return false; } + // A session since the last wake restores the short wait first: after + // a long backoff, the next viewer's browse mustn't wait it out. + let paid = self.sessions_at_wake.is_none_or(|at| at != sessions); + if paid { + self.every = WAKE_EVERY; + } if let Some(last) = self.last { if now.saturating_duration_since(last) < self.every { return false; } } - let paid = self.sessions_at_wake.is_none_or(|at| at != sessions); - self.every = if paid { - WAKE_EVERY - } else { - (self.every * 2).min(WAKE_EVERY_AT_MOST) - }; + if !paid { + self.every = (self.every * 2).min(WAKE_EVERY_AT_MOST); + } self.sessions_at_wake = Some(sessions); self.last = Some(now); true } } +/// Whether the encoder that will stream runs on the card detection chose +/// (`detected`): the override in Settings (`encoder_override`) if any, and +/// not a software fallback. Only that card is woken; a CPU encoder, or one +/// on another card, needs none. +pub fn wakes_card(encoder_override: Option<&str>, detected: &str, software_fallback: bool) -> bool { + let family = |name: &str| { + name.strip_prefix("h264_") + .or_else(|| name.strip_prefix("hevc_")) + .map(str::to_owned) + }; + let effective = encoder_override.unwrap_or(detected); + !software_fallback && family(effective).is_some() && family(effective) == family(detected) +} + +/// Runs `touch` unless `still_wanted`, checked just before, says a session +/// has started since the wake was decided: the card is then the session's, +/// and a device of the wake's during encoder or capture startup would only +/// compete with it. Returns whether it ran. +#[cfg(any(windows, test))] +fn run_wake(still_wanted: impl FnOnce() -> bool, touch: impl FnOnce()) -> bool { + if !still_wanted() { + return false; + } + touch(); + true +} + /// Starts waking the graphics card at DXGI adapter `adapter_index`, on a -/// thread of its own, unless a wake is still running. +/// thread of its own, unless a wake is still running. The thread drops it +/// if `still_wanted` returns false when it starts (see [`run_wake`]). #[cfg(windows)] -pub fn wake(adapter_index: u32) { +pub fn wake(adapter_index: u32, still_wanted: impl FnOnce() -> bool + Send + 'static) { use std::sync::atomic::{AtomicBool, Ordering}; static RUNNING: AtomicBool = AtomicBool::new(false); if RUNNING.swap(true, Ordering::SeqCst) { @@ -80,7 +111,7 @@ pub fn wake(adapter_index: u32) { .name("gpu-wake".into()) .spawn(move || { let started = Instant::now(); - match touch(adapter_index) { + let ran = run_wake(still_wanted, || match touch(adapter_index) { Ok(()) => tracing::debug!( adapter_index, took_ms = started.elapsed().as_millis() as u64, @@ -89,6 +120,12 @@ pub fn wake(adapter_index: u32) { Err(error) => { tracing::debug!(adapter_index, %error, "Could not wake the graphics card") } + }); + if !ran { + tracing::debug!( + adapter_index, + "A session started first: not waking the graphics card" + ); } RUNNING.store(false, Ordering::SeqCst); }); @@ -98,7 +135,7 @@ pub fn wake(adapter_index: u32) { } #[cfg(not(windows))] -pub fn wake(_adapter_index: u32) {} +pub fn wake(_adapter_index: u32, _still_wanted: impl FnOnce() -> bool + Send + 'static) {} /// A D3D11 device on the adapter, dropped at once: enough to power it up. #[cfg(windows)] @@ -151,6 +188,64 @@ mod tests { assert!(gate.browse(at(start, 75), false, 9)); } + #[test] + fn only_the_encoder_that_will_stream_on_the_detected_card_wakes_it() { + assert!(wakes_card(None, "h264_amf", false)); + assert!( + wakes_card(Some("hevc_amf"), "h264_amf", false), + "its HEVC sibling" + ); + assert!( + !wakes_card(Some("libx264"), "h264_amf", false), + "a CPU override" + ); + assert!(!wakes_card(None, "h264_amf", true), "a software fallback"); + assert!(!wakes_card(None, "libx264", false)); + assert!( + !wakes_card(Some("h264_qsv"), "h264_amf", false), + "another card" + ); + assert!( + !wakes_card(Some("h264_nvenc"), "h264_amf", false), + "another card" + ); + } + + #[test] + fn a_wake_is_dropped_when_a_session_started_first() { + let touched = std::cell::Cell::new(false); + assert!(!run_wake(|| false, || touched.set(true))); + assert!( + !touched.get(), + "the card was touched during the session's startup" + ); + assert!(run_wake(|| true, || touched.set(true))); + assert!(touched.get()); + } + + #[test] + fn a_session_after_a_long_backoff_restores_the_short_interval_at_once() { + let start = Instant::now(); + let mut gate = WakeGate::default(); + // Browses with nobody connecting, until the wait is at its longest. + let mut last = 0; + for s in (0..7200).step_by(20) { + if gate.browse(at(start, s), false, 7) { + last = s; + } + } + assert_eq!(gate.every, WAKE_EVERY_AT_MOST); + // An iPad connects, leaves, and browses again 20 s after that wake. + assert!( + gate.browse(at(start, last + 20), false, 8), + "the session's browse waited out the old ten-minute wait" + ); + assert!( + !gate.browse(at(start, last + 30), false, 8), + "the 15 s minimum" + ); + } + #[test] fn wakes_no_session_followed_come_ever_less_often() { let start = Instant::now(); diff --git a/host/src/main.rs b/host/src/main.rs index d095a2c8..e25333f0 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -262,19 +262,32 @@ fn main() -> Result<(), Box> { // An iPad looking for PCs starts waking the encoder's graphics card, which // a laptop powers off when idle (see `gpu_wake`). - let wake_card = (gpu_info.encoder_name != "libx264").then_some(gpu_info.adapter_index); + let (wake_adapter, detected_encoder) = (gpu_info.adapter_index, gpu_info.encoder_name.clone()); let wake_shared = shared.clone(); let mut wake_gate = eternal_host::gpu_wake::WakeGate::default(); let on_browse = move || { - if let Some(adapter_index) = wake_card { - let sessions = wake_shared.session.lock().sessions_started(); - if wake_gate.browse( - std::time::Instant::now(), - wake_shared.client_connected(), - sessions, - ) { - eternal_host::gpu_wake::wake(adapter_index); - } + // The encoder that will stream: the override, unless the pipeline + // fell back to the CPU. + let encoder_override = wake_shared.encoder_override.lock().clone(); + let software_fallback = stats::PIPELINE_STATS.lock().using_software_fallback; + if !eternal_host::gpu_wake::wakes_card( + encoder_override.as_deref(), + &detected_encoder, + software_fallback, + ) { + return; + } + let sessions = wake_shared.session.lock().sessions_started(); + if wake_gate.browse( + std::time::Instant::now(), + wake_shared.client_connected(), + sessions, + ) { + // A HELLO between here and the wake's device takes priority. + let shared = wake_shared.clone(); + eternal_host::gpu_wake::wake(wake_adapter, move || { + !shared.client_connected() && shared.session.lock().sessions_started() == sessions + }); } }; From 25b5fbc2528b3c6284a3a7a7792eb94ea767ae15 Mon Sep 17 00:00:00 2001 From: Ali Younes Date: Sun, 8 Nov 2026 15:29:15 -0500 Subject: [PATCH 225/226] Wake the card of the encoder that streams, as the pipeline last opened it The browse callback judged a snapshot of startup: the detected encoder, the Settings override and the CPU-fallback flag. A hardware fallback (NVENC or AMF failing over to Quick Sync, which clears that flag) still woke the card no one used, and a late GPU detection the supervisor adopted never became wakeable: startup's libx264 stayed in the callback. SharedControl::wake_adapter now holds the card to wake. Each encoder open, fallbacks included, sets it from the encoder that opened and the pipeline's GPU: that GPU's adapter for a hardware encoder of its family, None for the CPU or another card's encoder. Adopting a late detection sets it too, until the new pipeline's encoder opens. The callback reads only that. --- host/src/control.rs | 6 +++++ host/src/encoder/mod.rs | 31 +++++++++++++++++++---- host/src/gpu_wake.rs | 54 +++++++++++++++++++++++------------------ host/src/main.rs | 16 +++--------- host/src/supervisor.rs | 38 ++++++++++++++++++++++++++++- 5 files changed, 103 insertions(+), 42 deletions(-) diff --git a/host/src/control.rs b/host/src/control.rs index 55cda40d..52acfc58 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -30,6 +30,11 @@ pub struct SharedControl { /// Encoder name override from the Settings tab, applied on next pipeline restart. /// `None` means use auto-detected encoder from GpuInfo. pub encoder_override: Arc>>, + /// The DXGI adapter of the encoder that streams, for waking it when an + /// iPad looks for PCs (see `gpu_wake`): set at each encoder open and + /// when a late detection is adopted, None for a CPU encoder or one on + /// a card detection didn't choose. + pub wake_adapter: Arc>>, /// Which display output the capture loop should duplicate, applied on next pipeline /// restart. `PrimaryAuto` mirrors the primary monitor (today's behavior). pub capture_target: Arc>, @@ -199,6 +204,7 @@ impl SharedControl { ))))), force_next_idr: Arc::new(AtomicBool::new(false)), encoder_override: Arc::new(Mutex::new(None)), + wake_adapter: Arc::new(Mutex::new(None)), capture_target: Arc::new(Mutex::new(CaptureTarget::PrimaryAuto)), vdd_status: Arc::new(Mutex::new(VddStatus::Inactive)), vdd_problem: Arc::new(Mutex::new(None)), diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index 25b80c82..64b4fb7b 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -352,7 +352,7 @@ fn run_encode_loop( { PIPELINE_STATS.lock().set_software_fallback(false); } - note_encoder_opened(&shared, &desired_encoder); + note_encoder_opened(&shared, &desired_encoder, &gpu); } // No picture yet, and the iPad would give up waiting for // it: libx264 streams while the open goes on. @@ -382,7 +382,7 @@ fn run_encode_loop( PIPELINE_STATS .lock() .set_software_fallback(encoder_name != "libx264"); - note_encoder_opened(&shared, "libx264"); + note_encoder_opened(&shared, "libx264", &gpu); } Err(e) if is_hevc_encoder(&desired_encoder) => { warn!( @@ -455,7 +455,7 @@ fn run_encode_loop( PIPELINE_STATS .lock() .set_software_fallback(name == "libx264"); - note_encoder_opened(&shared, name); + note_encoder_opened(&shared, name, &gpu); } Err(e) => return Err(e), } @@ -609,7 +609,7 @@ fn run_encode_loop( { PIPELINE_STATS.lock().set_software_fallback(false); } - note_encoder_opened(&shared, &reopen.encoder); + note_encoder_opened(&shared, &reopen.encoder, &gpu); } // The old session cannot take a frame of the new size. A // pipeline restart opens a fresh encoder, with the software @@ -2604,7 +2604,8 @@ fn codec_display_name_for(encoder_name: &str) -> &str { /// After every successful encoder open: report the live codec on the wire /// (HELLO_ACK/heartbeat `StreamConfig.codec`) and in the GUI. -fn note_encoder_opened(shared: &SharedControl, encoder_name: &str) { +fn note_encoder_opened(shared: &SharedControl, encoder_name: &str, gpu: &GpuInfo) { + *shared.wake_adapter.lock() = crate::gpu_wake::wake_adapter_for(encoder_name, gpu); let codec = if is_hevc_encoder(encoder_name) { eternal_wire::v2::control::CODEC_HEVC } else { @@ -6104,6 +6105,26 @@ mod tests { /// Automatic's BGRA open failing on NVENC opens YUV420P at once and keeps /// it for later opens; a picked input, a stall, or a YUV420P open that /// fails is not tried again. + /// NVENC failing over to Quick Sync streams from the Intel card: the + /// NVIDIA card detection chose is unused and is not woken for iPads. + #[test] + fn an_encoder_open_publishes_the_card_to_wake() { + let shared = SharedControl::new(0, 8_000_000); + let gpu = GpuInfo { + encoder_name: "h264_nvenc".to_string(), + adapter_index: 1, + ..GpuInfo::software_fallback() + }; + note_encoder_opened(&shared, "h264_nvenc", &gpu); + assert_eq!(*shared.wake_adapter.lock(), Some(1)); + note_encoder_opened(&shared, "h264_qsv", &gpu); + assert_eq!(*shared.wake_adapter.lock(), None, "a cross-vendor fallback"); + note_encoder_opened(&shared, "libx264", &gpu); + assert_eq!(*shared.wake_adapter.lock(), None, "the CPU"); + note_encoder_opened(&shared, "hevc_nvenc", &gpu); + assert_eq!(*shared.wake_adapter.lock(), Some(1), "back on the card"); + } + #[test] fn a_failed_automatic_bgra_open_falls_back_to_yuv420_and_stays_there() { use ffmpeg_next::format::Pixel; diff --git a/host/src/gpu_wake.rs b/host/src/gpu_wake.rs index 3718178d..5828f715 100644 --- a/host/src/gpu_wake.rs +++ b/host/src/gpu_wake.rs @@ -70,18 +70,20 @@ impl WakeGate { } } -/// Whether the encoder that will stream runs on the card detection chose -/// (`detected`): the override in Settings (`encoder_override`) if any, and -/// not a software fallback. Only that card is woken; a CPU encoder, or one -/// on another card, needs none. -pub fn wakes_card(encoder_override: Option<&str>, detected: &str, software_fallback: bool) -> bool { +/// The adapter to wake for `opened`, the encoder that opened in a pipeline +/// using `gpu`: the card detection chose, when `opened` is a hardware encoder +/// of its family (its HEVC sibling too). None for a CPU encoder, or for a +/// fallback to another card's encoder (NVENC or AMF failing over to Quick +/// Sync): the card detection chose is then unused, and waking it costs power +/// for nothing. +pub fn wake_adapter_for(opened: &str, gpu: &crate::gpu::GpuInfo) -> Option { let family = |name: &str| { name.strip_prefix("h264_") .or_else(|| name.strip_prefix("hevc_")) .map(str::to_owned) }; - let effective = encoder_override.unwrap_or(detected); - !software_fallback && family(effective).is_some() && family(effective) == family(detected) + (family(opened).is_some() && family(opened) == family(&gpu.encoder_name)) + .then_some(gpu.adapter_index) } /// Runs `touch` unless `still_wanted`, checked just before, says a session @@ -188,27 +190,31 @@ mod tests { assert!(gate.browse(at(start, 75), false, 9)); } + fn gpu(encoder: &str, adapter_index: u32) -> crate::gpu::GpuInfo { + crate::gpu::GpuInfo { + encoder_name: encoder.to_string(), + adapter_index, + ..crate::gpu::GpuInfo::software_fallback() + } + } + #[test] - fn only_the_encoder_that_will_stream_on_the_detected_card_wakes_it() { - assert!(wakes_card(None, "h264_amf", false)); - assert!( - wakes_card(Some("hevc_amf"), "h264_amf", false), + fn only_the_encoder_that_streams_on_the_detected_card_wakes_it() { + let nvenc = gpu("h264_nvenc", 1); + assert_eq!(wake_adapter_for("h264_nvenc", &nvenc), Some(1)); + assert_eq!( + wake_adapter_for("hevc_nvenc", &nvenc), + Some(1), "its HEVC sibling" ); - assert!( - !wakes_card(Some("libx264"), "h264_amf", false), - "a CPU override" - ); - assert!(!wakes_card(None, "h264_amf", true), "a software fallback"); - assert!(!wakes_card(None, "libx264", false)); - assert!( - !wakes_card(Some("h264_qsv"), "h264_amf", false), - "another card" - ); - assert!( - !wakes_card(Some("h264_nvenc"), "h264_amf", false), - "another card" + assert_eq!( + wake_adapter_for("h264_qsv", &nvenc), + None, + "a Quick Sync fallback" ); + assert_eq!(wake_adapter_for("h264_amf", &nvenc), None, "another card"); + assert_eq!(wake_adapter_for("libx264", &nvenc), None, "the CPU"); + assert_eq!(wake_adapter_for("libx264", &gpu("libx264", 0)), None); } #[test] diff --git a/host/src/main.rs b/host/src/main.rs index e25333f0..b5934d11 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -262,21 +262,13 @@ fn main() -> Result<(), Box> { // An iPad looking for PCs starts waking the encoder's graphics card, which // a laptop powers off when idle (see `gpu_wake`). - let (wake_adapter, detected_encoder) = (gpu_info.adapter_index, gpu_info.encoder_name.clone()); let wake_shared = shared.clone(); let mut wake_gate = eternal_host::gpu_wake::WakeGate::default(); let on_browse = move || { - // The encoder that will stream: the override, unless the pipeline - // fell back to the CPU. - let encoder_override = wake_shared.encoder_override.lock().clone(); - let software_fallback = stats::PIPELINE_STATS.lock().using_software_fallback; - if !eternal_host::gpu_wake::wakes_card( - encoder_override.as_deref(), - &detected_encoder, - software_fallback, - ) { + // The card of the encoder that streams, as its last open published. + let Some(adapter_index) = *wake_shared.wake_adapter.lock() else { return; - } + }; let sessions = wake_shared.session.lock().sessions_started(); if wake_gate.browse( std::time::Instant::now(), @@ -285,7 +277,7 @@ fn main() -> Result<(), Box> { ) { // A HELLO between here and the wake's device takes priority. let shared = wake_shared.clone(); - eternal_host::gpu_wake::wake(wake_adapter, move || { + eternal_host::gpu_wake::wake(adapter_index, move || { !shared.client_connected() && shared.session.lock().sessions_started() == sessions }); } diff --git a/host/src/supervisor.rs b/host/src/supervisor.rs index 7f79733b..076238d8 100644 --- a/host/src/supervisor.rs +++ b/host/src/supervisor.rs @@ -471,7 +471,7 @@ pub fn run( machine.note_started(generation); settle(&machine, &shared, &mut standby, listen_port); healthy_since = Instant::now(); - if take_late_detection(&mut gpu_info, crate::gpu::take_late_detection) { + if adopt_late_detection(&mut gpu_info, &shared, crate::gpu::take_late_detection) { let mut stats = PIPELINE_STATS.lock(); stats.set_gpu_name(gpu_info.name.clone()); stats.set_codec_name(gpu_info.codec_display_name.clone()); @@ -526,6 +526,21 @@ fn take_late_detection(gpu: &mut GpuInfo, late: impl FnOnce() -> Option true } +/// [`take_late_detection`], and the card it found becomes the one to wake +/// for an iPad that looks for PCs, until the new pipeline's encoder opens +/// and says which it uses. +fn adopt_late_detection( + gpu: &mut GpuInfo, + shared: &SharedControl, + late: impl FnOnce() -> Option, +) -> bool { + if !take_late_detection(gpu, late) { + return false; + } + *shared.wake_adapter.lock() = crate::gpu_wake::wake_adapter_for(&gpu.encoder_name, gpu); + true +} + /// Makes what iPads hear match `machine`: the session's answer to a HELLO2, /// and the standby responder on the port while no generation streams. fn settle( @@ -839,6 +854,27 @@ mod tests { assert!(!take_late_detection(&mut gpu, late)); assert_eq!(gpu.name, "Software"); } + #[test] + fn an_adopted_late_detection_becomes_the_card_to_wake() { + let shared = SharedControl::new(0, 8_000_000); + // Startup gave up and encodes on the CPU: nothing to wake. + let mut gpu = GpuInfo { + detection_stalled: true, + ..GpuInfo::software_fallback() + }; + assert!(!adopt_late_detection(&mut gpu, &shared, || None)); + assert_eq!(*shared.wake_adapter.lock(), None); + let late = || { + Some(GpuInfo { + encoder_name: "h264_amf".to_string(), + adapter_index: 1, + ..GpuInfo::software_fallback() + }) + }; + assert!(adopt_late_detection(&mut gpu, &shared, late)); + assert_eq!(*shared.wake_adapter.lock(), Some(1)); + } + #[test] fn audio_failures_never_restart_or_exhaust_the_video_failure_budget() { let mut machine = Machine::new(7); From a968a80a49efcc699d92ac0adde129bf7bf57273 Mon Sep 17 00:00:00 2001 From: Ali Younes <159104116+whoisaldo@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:11:31 -0500 Subject: [PATCH 226/226] Change NVENC's bitrate on the open session, and step up less often where a change reopens (#254) Most hardware encoder reopens came right after an ABR bitrate step, and each reopen costs a stutter and an IDR. Many of those steps were ABR moving back and forth on a link just below the next rung. - NVENC now changes its bitrate on the open session, through FFmpeg's reconfigure path, with the VBV sized as a fresh open would size it. The change stands only when FFmpeg reports that the reconfigure ran and did not fail; anything else reopens the encoder at the new bitrate, and later changes on that card reopen as before. - FFmpeg's verbose lines reach the encode thread only while it watches for that report, through an atomic flag in the log shim; FFmpeg's own log level is set once at startup. - For encoders that still reopen on a bitrate change (AMF, libx265), ABR waits 30 s after a change before stepping up, doubling to at most 120 s when a step up is undone, and resetting when one holds. Steps down are unchanged, and a Max bitrate change with ABR off applies at once. - Unit tests for the encoder table, NVENC confirmation and refusal on forced and periodic IDRs, the log bridge, the ABR holds, and a closed-loop simulation; native NVENC and AMF suites. --- host/src/control.rs | 5 + host/src/diagnostics.rs | 57 ++++ host/src/encoder/mod.rs | 651 ++++++++++++++++++++++++++++++++++---- host/src/ffmpeg_log.c | 29 +- host/src/transport/abr.rs | 277 +++++++++++++++- host/src/transport/mod.rs | 3 + 6 files changed, 964 insertions(+), 58 deletions(-) diff --git a/host/src/control.rs b/host/src/control.rs index 55cda40d..109ef90b 100644 --- a/host/src/control.rs +++ b/host/src/control.rs @@ -17,6 +17,10 @@ pub struct SharedControl { /// always <= `bitrate_bps`. The encoder reopens its session when this /// changes for hardware encoders. libx264 reconfigures the open context. pub abr_current_bps: Arc, + /// The open encoder session takes a new bitrate only by reopening (AMF, + /// libx265, or a session that refused a change in place), which costs a + /// stutter and an IDR, so ABR steps up less often for it. + pub encoder_reopens_for_bitrate: Arc, pub target_fps: Arc, pub client_preferred_fps: Arc, pub encoder_input: Arc>, @@ -189,6 +193,7 @@ impl SharedControl { running: Arc::new(AtomicBool::new(false)), bitrate_bps: Arc::new(AtomicU32::new(initial_bitrate_bps)), abr_current_bps: Arc::new(AtomicU32::new(initial_bitrate_bps)), + encoder_reopens_for_bitrate: Arc::new(AtomicBool::new(false)), target_fps: Arc::new(AtomicU32::new(DEFAULT_TARGET_FPS)), client_preferred_fps: Arc::new(AtomicU32::new(0)), encoder_input: Arc::new(Mutex::new(Default::default())), diff --git a/host/src/diagnostics.rs b/host/src/diagnostics.rs index af06d138..b5ef117e 100644 --- a/host/src/diagnostics.rs +++ b/host/src/diagnostics.rs @@ -34,6 +34,7 @@ pub fn log_system_report() { extern "C" { fn eternal_install_ffmpeg_logger(sink: unsafe extern "C" fn(i32, *const libc::c_char)); + fn eternal_ffmpeg_verbose(wanted: libc::c_int); } /// Install once, before encoder threads start. The C shim owns the platform's va_list ABI. @@ -45,6 +46,53 @@ pub fn install_ffmpeg_logging() { thread_local! { /// Set while this thread opens an encoder that is one of several to try. static PROBING: std::cell::Cell = const { std::cell::Cell::new(false) }; + /// FFmpeg's lines from this thread while it watches them (see + /// [`FfmpegWatch`]), verbose ones included. + static WATCHED: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; +} + +/// Threads watching FFmpeg's verbose lines. The shim forwards them while +/// any watches; FFmpeg's own level never changes after it is installed. +static WATCHERS: std::sync::Mutex = std::sync::Mutex::new(0); + +/// Keeps FFmpeg's lines from this thread, its verbose ones included, from +/// [`FfmpegWatch::start`] to [`FfmpegWatch::finish`]. Warnings and errors +/// are logged as always; verbose lines are only kept, never logged. Lines +/// from other threads meanwhile are not kept, and their verbose lines are +/// dropped as before. +pub struct FfmpegWatch(()); + +impl FfmpegWatch { + pub fn start() -> Self { + install_ffmpeg_logging(); + WATCHED.with(|watched| *watched.borrow_mut() = Some(Vec::new())); + let mut watchers = WATCHERS.lock().unwrap_or_else(|e| e.into_inner()); + *watchers += 1; + if *watchers == 1 { + unsafe { eternal_ffmpeg_verbose(1) }; + } + Self(()) + } + + /// The lines this thread logged since the watch started. + pub fn finish(self) -> Vec { + let lines = WATCHED + .with(|watched| watched.borrow_mut().take()) + .unwrap_or_default(); + drop(self); + lines + } +} + +impl Drop for FfmpegWatch { + fn drop(&mut self) { + WATCHED.with(|watched| watched.borrow_mut().take()); + let mut watchers = WATCHERS.lock().unwrap_or_else(|e| e.into_inner()); + *watchers = watchers.saturating_sub(1); + if *watchers == 0 { + unsafe { eternal_ffmpeg_verbose(0) }; + } + } } /// Runs `open`, an encoder open with another encoder to try if it fails, @@ -74,6 +122,15 @@ unsafe extern "C" fn ffmpeg_message(level: i32, message: *const libc::c_char) { if message.is_empty() { return; } + WATCHED.with(|watched| { + if let Some(lines) = watched.borrow_mut().as_mut() { + lines.push(message.to_string()); + } + }); + // Only a watching thread wants verbose lines (see `FfmpegWatch`). + if level > ffmpeg_next::ffi::AV_LOG_WARNING { + return; + } // `ffmpeg_level` keeps FFmpeg's own level either way. if level <= ffmpeg_next::ffi::AV_LOG_ERROR && !PROBING.with(std::cell::Cell::get) { tracing::error!(target: "ffmpeg", ffmpeg_level = level, "{message}"); diff --git a/host/src/encoder/mod.rs b/host/src/encoder/mod.rs index 25b80c82..5987f2b2 100644 --- a/host/src/encoder/mod.rs +++ b/host/src/encoder/mod.rs @@ -688,6 +688,10 @@ fn run_encode_loop( .expect("Encoder state must be initialized"); let current_bitrate = encoder.opened_bitrate; let opened_is_hevc = is_hevc_encoder(&encoder.opened_encoder); + shared.encoder_reopens_for_bitrate.store( + !takes_bitrate_in_place(&encoder.opened_encoder, refused_this_launch), + Ordering::Relaxed, + ); let convert_start = Instant::now(); encoder.convert(&raw_frame.data, (raw_frame.width * 4) as usize)?; @@ -721,12 +725,21 @@ fn run_encode_loop( let encode_start = Instant::now(); // A bitrate Quick Sync took in place applies with this frame: an // error now is its refusal. - let mut reset_refused = None; + let qsv_change = encoder + .bitrate_change + .is_some_and(|change| change.live == LiveBitrate::QsvReset); + // NVENC reconfigures for a new bitrate while it takes this frame, + // and says so only in its verbose lines. + let nvenc_watch = encoder + .bitrate_change + .is_some_and(|change| change.live == LiveBitrate::NvencReconfig) + .then(crate::diagnostics::FfmpegWatch::start); + let mut reset_refused: Option = None; if let Err(error) = encoder.send_frame(pts, raw_frame.timestamp, raw_frame.frame_number) { - if !encoder.bitrate_reset_unconfirmed { + if !qsv_change { return Err(error.into()); } - reset_refused = Some(error); + reset_refused = Some(error.to_string()); } let mut keyframe_out = false; @@ -837,17 +850,27 @@ fn run_encode_loop( } } Err(e) if is_eagain(&e) => break, - Err(e) if encoder.bitrate_reset_unconfirmed => reset_refused = Some(e), + Err(e) if qsv_change => reset_refused = Some(e.to_string()), Err(e) => return Err(e.into()), } } + if let Some(watch) = nvenc_watch { + if nvenc_reconfigured(&watch.finish()) { + encoder.bitrate_change = None; + } else if reset_refused.is_none() { + reset_refused = Some("NVENC did not reconfigure the session's bitrate".to_string()); + } + } if let Some(error) = reset_refused { - QSV_RESET_REFUSED.store(true, Ordering::Relaxed); + let live = encoder + .bitrate_change + .map_or(LiveBitrate::QsvReset, |change| change.live); + live.refused().store(true, Ordering::Relaxed); warn!( %error, encoder = %encoder.opened_encoder, bitrate = encoder.opened_bitrate, - "Quick Sync did not take the new bitrate on the open session; reopening it, as for its bitrate changes from now on" + "The encoder did not take the new bitrate on the open session; reopening it, as for its bitrate changes from now on" ); encoder_state = None; pending_keyframe = true; @@ -856,7 +879,9 @@ fn run_encode_loop( owed_keyframe = None; continue; } - encoder.bitrate_reset_unconfirmed = false; + if qsv_change { + encoder.bitrate_change = None; + } owed_keyframe = OwedKeyframe::after_encode( owed_keyframe.take(), picture, @@ -901,27 +926,93 @@ struct EncoderState { /// time_base 1/60 (escape hatch until the real-PTS path is verified on /// NVENC/AMF hardware). legacy_pts: bool, - /// A Quick Sync bitrate change was made in place, and no frame has gone - /// through the session since. Quick Sync applies it with the next frame, - /// so an error there means it refused the change. - bitrate_reset_unconfirmed: bool, + /// A hardware bitrate change made in place that the session has not yet + /// been seen to take (see [`Unconfirmed`]). + bitrate_change: Option, +} + +/// How an encoder takes a new bitrate on the open session, without the +/// reopen that costs a stutter. FFmpeg 7.1.1 checks the context's bitrate +/// before each frame in three of its encoders: +/// - libx264 applies bitrate and VBV changes through x264_encoder_reconfig. +/// - Quick Sync resets the session's rate control to it (qsvenc.c, +/// `update_bitrate` and `MFXVideoENCODE_Reset`), with no new sequence, +/// instead of its slow open. +/// - NVENC reconfigures the session (nvenc.c, `reconfig_encoder` and +/// `nvEncReconfigureEncoder`) with an IDR, if the card can change its +/// bitrate live (NV_ENC_CAPS_SUPPORT_DYN_BITRATE_CHANGE), instead of a +/// new session. +/// +/// AMF sets its bitrate only when it opens (amfenc_h264.c and +/// amfenc_hevc.c), and libx265 has no reconfigure, so both reopen, and ABR +/// steps up less often for them (see `AbrController::encoder_reopens`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LiveBitrate { + X264, + QsvReset, + NvencReconfig, +} + +impl LiveBitrate { + fn of(encoder_name: &str) -> Option { + if encoder_name == "libx264" { + Some(Self::X264) + } else if encoder_name.ends_with("_qsv") { + Some(Self::QsvReset) + } else if encoder_name.ends_with("_nvenc") { + Some(Self::NvencReconfig) + } else { + None + } + } + + /// Set once a session refused a bitrate change in place: from then on, + /// this launch, that encoder's bitrate changes reopen it, as before. + fn refused(self) -> &'static std::sync::atomic::AtomicBool { + static QSV: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + static NVENC: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + static NEVER: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + match self { + Self::QsvReset => &QSV, + Self::NvencReconfig => &NVENC, + Self::X264 => &NEVER, + } + } +} + +/// Whether `encoder_name` takes a new bitrate on the open session, unless +/// `refused` says a session of its kind refused one. +fn takes_bitrate_in_place(encoder_name: &str, refused: impl Fn(LiveBitrate) -> bool) -> bool { + LiveBitrate::of(encoder_name).is_some_and(|live| live == LiveBitrate::X264 || !refused(live)) } -/// Set once a Quick Sync session refused a bitrate change in place: from -/// then on, this launch, its bitrate changes reopen it, as before. -static QSV_RESET_REFUSED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); - -/// Encoders that take a new bitrate on the open session, without the reopen -/// that costs a stutter and an IDR. FFmpeg 7.1's libx264 wrapper applies -/// bitrate and VBV changes through x264_encoder_reconfig. Its Quick Sync -/// encoders compare `bit_rate` with the value they opened at before each -/// frame, and reset the session's rate control to it (qsvenc.c, -/// `update_bitrate` and `MFXVideoENCODE_Reset`), with no new sequence. A -/// Quick Sync open takes 630 ms at the median in the field, against 144 ms -/// for NVENC and 43 ms for AMF, so each bitrate reopen froze the picture -/// that long. Unless `qsv_refused`: a session refused the change once. -fn takes_bitrate_in_place(encoder_name: &str, qsv_refused: bool) -> bool { - encoder_name == "libx264" || (encoder_name.ends_with("_qsv") && !qsv_refused) +fn refused_this_launch(live: LiveBitrate) -> bool { + live.refused().load(Ordering::Relaxed) +} + +/// A hardware bitrate change made in place, settled by the end of the frame +/// it goes in with: then the session took it, or it is replaced by one +/// opened at the new bitrate, so the session's bitrate never outlives an +/// unconfirmed change. Quick Sync applies it with that frame, and an error +/// there is its refusal. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Unconfirmed { + live: LiveBitrate, +} + +/// Whether NVENC reconfigured its session for a new bitrate, from FFmpeg's +/// lines while it took the change's frame. FFmpeg 7.1.1's nvenc.c +/// (`reconfig_encoder`) logs "avg bitrate change" at verbose level when it +/// calls nvEncReconfigureEncoder for one, and "failed to reconfigure nvenc" +/// when that fails; it returns no error either way, and skips the change +/// with no line on a card that cannot change its bitrate live. An IDR says +/// nothing: one the frame forced, or the GOP's own, comes out either way. +/// Anything but a reconfigure that ran and did not fail is a refusal. +fn nvenc_reconfigured(lines: &[String]) -> bool { + lines.iter().any(|line| line.contains("avg bitrate change")) + && !lines + .iter() + .any(|line| line.contains("failed to reconfigure nvenc")) } impl EncoderState { @@ -931,10 +1022,8 @@ impl EncoderState { bitrate_bps: u32, target_fps: u32, ) -> bool { - if !takes_bitrate_in_place( - &self.opened_encoder, - QSV_RESET_REFUSED.load(Ordering::Relaxed), - ) || desired_encoder != self.opened_encoder + if !takes_bitrate_in_place(&self.opened_encoder, refused_this_launch) + || desired_encoder != self.opened_encoder || target_fps != self.opened_fps || bitrate_bps == self.opened_bitrate { @@ -958,9 +1047,19 @@ impl EncoderState { "Updated software encoder bitrate" ); } else { + let live = LiveBitrate::of(&self.opened_encoder).expect("takes it in place"); + if live == LiveBitrate::NvencReconfig { + // NVENC sizes the VBV to two seconds of the bitrate when it + // opens, and reconfigures it with the bitrate, as a session + // opened at it would have. + unsafe { + (*self.encoder.as_mut_ptr()).rc_buffer_size = + (bitrate_bps.max(500_000) as i32).saturating_mul(2) + }; + } // Quick Sync opened in VBR with no maximum rate or buffer size, // so only the target changes. - self.bitrate_reset_unconfirmed = true; + self.bitrate_change = Some(Unconfirmed { live }); info!( from = self.opened_bitrate, to = bitrate_bps, @@ -1183,7 +1282,7 @@ impl EncoderState { last_pts: -1, in_flight: InFlightFrames::default(), legacy_pts, - bitrate_reset_unconfirmed: false, + bitrate_change: None, }) } @@ -4169,6 +4268,14 @@ mod tests { in_place: usize, /// A Quick Sync session refused a bitrate change in place. qsv_refused: bool, + /// An NVENC session did: a card that cannot change its bitrate live. + nvenc_refused: bool, + /// Whether FFmpeg reconfigures an NVENC session for a change made + /// in place (see `nvenc_reconfigured`). + nvenc_reconfigures: bool, + /// The session was dropped after a refused change; the next frame + /// opens one, as the encode loop's first open does. + dropped: bool, /// The user's preference. hevc_wanted: bool, bitrate: u32, @@ -4196,6 +4303,11 @@ mod tests { pacer: BitratePacer::default(), in_place: 0, qsv_refused: false, + // As on a card that cannot change its bitrate live: these + // tests reopen NVENC sessions with bitrate changes. + nvenc_refused: true, + nvenc_reconfigures: true, + dropped: false, hevc_wanted: is_hevc_encoder(opened), bitrate: 15_000_000, fps: 60, @@ -4230,11 +4342,28 @@ mod tests { self.session.is_some() && !client_hevc && is_hevc_encoder(&self.opened.0); self.undecodable_retry .frame(undecodable, &mut self.failed_reopen, self.now); - // As `update_bitrate_without_reopening`. - if takes_bitrate_in_place(&self.opened.0, self.qsv_refused) - && (self.opened.0.as_str(), self.opened.2) == (desired.as_str(), self.fps) + if std::mem::take(&mut self.dropped) { + self.opened = (desired.clone(), self.bitrate, self.fps); + self.pacer.opened(self.now); + self.failed_reopen = None; + return Some(desired); + } + // As `update_bitrate_without_reopening`, settled at the end of + // the frame: a refused NVENC change drops the session. + if takes_bitrate_in_place(&self.opened.0, |live| match live { + LiveBitrate::QsvReset => self.qsv_refused, + LiveBitrate::NvencReconfig => self.nvenc_refused, + LiveBitrate::X264 => false, + }) && (self.opened.0.as_str(), self.opened.2) == (desired.as_str(), self.fps) && self.opened.1 != self.bitrate { + if LiveBitrate::of(&self.opened.0) == Some(LiveBitrate::NvencReconfig) + && !self.nvenc_reconfigures + { + self.nvenc_refused = true; + self.dropped = true; + return None; + } self.opened.1 = self.bitrate; self.failed_reopen = None; self.in_place += 1; @@ -5076,19 +5205,13 @@ mod tests { #[test] fn quick_sync_takes_every_bitrate_change_in_place_until_it_refuses_one() { for name in ["libx264", "h264_qsv", "hevc_qsv"] { - assert!(takes_bitrate_in_place(name, false), "{name}"); + assert!(takes_bitrate_in_place(name, |_| false), "{name}"); } - for name in [ - "h264_nvenc", - "hevc_nvenc", - "h264_amf", - "hevc_amf", - "libx265", - ] { - assert!(!takes_bitrate_in_place(name, false), "{name}"); + for name in ["h264_amf", "hevc_amf", "libx265"] { + assert!(!takes_bitrate_in_place(name, |_| false), "{name}"); } - assert!(!takes_bitrate_in_place("h264_qsv", true)); - assert!(takes_bitrate_in_place("libx264", true)); + assert!(!takes_bitrate_in_place("h264_qsv", |_| true)); + assert!(takes_bitrate_in_place("libx264", |_| true)); // As libx264: rises and drops, some within a second of each other, // each on the frame it arrives, and no reopen. The field's worst PC // reopened Quick Sync 148 times, about 0.6 s of frozen picture each. @@ -5997,6 +6120,276 @@ mod tests { assert_eq!(attempts, ["libx264"]); } + #[test] + fn nvenc_and_quick_sync_take_a_bitrate_in_place_and_amf_and_x265_reopen() { + use LiveBitrate::{NvencReconfig, QsvReset, X264}; + for (name, live) in [ + ("libx264", Some(X264)), + ("h264_qsv", Some(QsvReset)), + ("hevc_qsv", Some(QsvReset)), + ("h264_nvenc", Some(NvencReconfig)), + ("hevc_nvenc", Some(NvencReconfig)), + ("h264_amf", None), + ("hevc_amf", None), + ("libx265", None), + ("h264_videotoolbox", None), + ] { + assert_eq!(LiveBitrate::of(name), live, "{name}"); + assert_eq!( + takes_bitrate_in_place(name, |_| false), + live.is_some(), + "{name}" + ); + } + // A refusal stops one kind only, and libx264 never refuses. + let nvenc_refused = |live| live == NvencReconfig; + assert!(!takes_bitrate_in_place("h264_nvenc", nvenc_refused)); + assert!(takes_bitrate_in_place("h264_qsv", nvenc_refused)); + assert!(takes_bitrate_in_place("libx264", |_| true)); + // The harness's NVENC: in place on a card that changes it live. + let t0 = Instant::now(); + let mut p = Pipeline::new("h264_nvenc", "h264_nvenc", t0); + p.nvenc_refused = false; + assert_eq!( + play( + &mut p, + &[ + (0, Some(12_000_000)), + (100, Some(20_000_000)), + (200, Some(8_000_000)) + ], + 5_000 + ), + [] + ); + assert_eq!((p.in_place, p.opened.1), (3, 8_000_000)); + } + + #[test] + fn an_nvenc_change_is_settled_by_its_reconfigure_and_never_by_an_idr() { + let lines = |lines: &[&str]| { + lines + .iter() + .map(|line| line.to_string()) + .collect::>() + }; + // What FFmpeg 7.1.1's nvenc.c prints while it takes the change's + // frame. A reconfigure that ran: + assert!(nvenc_reconfigured(&lines(&[ + "[h264_nvenc @ 0x1] avg bitrate change: 8000000 -> 6000000", + "[h264_nvenc @ 0x1] vbv buffer size change: 16000000 -> 12000000", + ]))); + // One that failed: FFmpeg returns no error, so the line decides. + assert!(!nvenc_reconfigured(&lines(&[ + "[h264_nvenc @ 0x1] avg bitrate change: 8000000 -> 6000000", + "[h264_nvenc @ 0x1] failed to reconfigure nvenc: invalid param (8): ", + ]))); + // A card that cannot change its bitrate live: FFmpeg skips it and + // prints nothing, though the frame may come out as an IDR anyway, + // forced or the GOP's own. Before, that IDR confirmed it. + assert!(!nvenc_reconfigured(&[])); + assert!(!nvenc_reconfigured(&lines(&[ + "[h264_nvenc @ 0x1] aspect ratio change (DAR): 16:9 -> 16:9" + ]))); + } + + #[test] + fn ffmpeg_verbose_lines_reach_only_the_watching_thread_and_are_not_logged() { + ffmpeg_next::init().unwrap(); + crate::diagnostics::install_ffmpeg_logging(); + let say = |level: i32, text: &str| { + let message = std::ffi::CString::new(format!("{text}\n")).unwrap(); + unsafe { ffmpeg_next::ffi::av_log(std::ptr::null_mut(), level, message.as_ptr()) }; + }; + let (kept, logged_lines) = logged(|| { + let watch = crate::diagnostics::FfmpegWatch::start(); + say( + ffmpeg_next::ffi::AV_LOG_VERBOSE, + "avg bitrate change: 1 -> 2", + ); + say(ffmpeg_next::ffi::AV_LOG_WARNING, "a warning while watching"); + // Another thread's verbose line meanwhile is not kept here. + let other = std::thread::spawn(move || { + say( + ffmpeg_next::ffi::AV_LOG_VERBOSE, + "another thread's verbose line", + ); + }); + other.join().unwrap(); + let kept = watch.finish(); + // After the watch, verbose lines are dropped again. + say(ffmpeg_next::ffi::AV_LOG_VERBOSE, "verbose after the watch"); + kept + }); + assert_eq!(kept.len(), 2, "{kept:?}"); + assert!(kept[0].contains("avg bitrate change: 1 -> 2"), "{kept:?}"); + assert!(kept[1].contains("a warning while watching"), "{kept:?}"); + // The warning is logged as always; no verbose line ever is. + assert!( + logged_lines + .iter() + .any(|line| line.contains("a warning while watching")), + "{logged_lines:?}" + ); + assert!( + !logged_lines + .iter() + .any(|line| line.contains("verbose") || line.contains("avg bitrate change")), + "{logged_lines:?}" + ); + // A watch never touches FFmpeg's own level, which other threads + // read as they log. + assert_eq!( + unsafe { ffmpeg_next::ffi::av_log_get_level() }, + ffmpeg_next::ffi::AV_LOG_VERBOSE + ); + } + + #[test] + fn a_refused_nvenc_change_reopens_at_the_new_bitrate_even_with_abr_off() { + // ETERNAL_ABR=0: the target is the user's Max bitrate, and nothing + // changes it again for as long as the user leaves it. A change NVENC + // did not take must not leave the session on the old bitrate. + let mut abr = crate::transport::abr::AbrController::new(8_000_000, false); + let mut p = Pipeline::new("h264_nvenc", "h264_nvenc", Instant::now()); + p.nvenc_refused = false; + p.nvenc_reconfigures = false; + p.opened.1 = abr.current_bps(); + p.bitrate = abr.current_bps(); + let raised = abr.set_ceiling(20_000_000); + assert!(raised.changed); + p.bitrate = raised.target_bps; + // The frame it went in with: refused, the session is dropped. + assert_eq!(p.frame(Reopen::Opens), None); + assert!(p.nvenc_refused && p.dropped); + // The next frame opens one at the new bitrate. + p.later(16); + assert_eq!(p.frame(Reopen::Opens).as_deref(), Some("h264_nvenc")); + assert_eq!(p.opened.1, 20_000_000); + assert_eq!(p.in_place, 0); + // From then on its changes reopen, as after a Quick Sync refusal. + let lowered = abr.set_ceiling(10_000_000); + p.later(2_000); + p.bitrate = lowered.target_bps; + assert_eq!(p.frame(Reopen::Opens).as_deref(), Some("h264_nvenc")); + assert_eq!(p.opened.1, 10_000_000); + // A card that reconfigures takes the same changes in place. + let mut p = Pipeline::new("h264_nvenc", "h264_nvenc", Instant::now()); + p.nvenc_refused = false; + for bitrate in [20_000_000, 10_000_000] { + p.bitrate = bitrate; + assert_eq!(p.frame(Reopen::Opens), None); + assert_eq!(p.opened.1, bitrate); + } + assert_eq!(p.in_place, 2); + } + + /// Native, where NVENC opens: a bitrate change on the open session takes + /// effect with no new session, FFmpeg's reconfigure confirms each one, + /// including on a GOP's own IDR (frames 120 and 240) and on a frame that + /// forces an IDR (250), and the stream decodes throughout. Skips + /// elsewhere. + #[test] + fn nvenc_changes_its_bitrate_on_the_open_session() { + ffmpeg_next::init().unwrap(); + let Ok(mut state) = EncoderState::new( + "h264_nvenc", + 1280, + 720, + 8_000_000, + 60, + &mut EncoderInput::Yuv420, + Opening::First, + &mut || {}, + ) else { + eprintln!("skipping: h264_nvenc does not open here"); + return; + }; + let context = unsafe { state.encoder.as_ptr() }; + let codec = ffmpeg_next::decoder::find(ffmpeg_next::codec::Id::H264).unwrap(); + let mut decoder = ffmpeg_next::codec::Context::new_with_codec(codec) + .decoder() + .video() + .unwrap(); + let mut decoded = ffmpeg_next::frame::Video::empty(); + let mut decoded_frames = 0; + let mut sizes = [0usize; 3]; + let mut keyframes = Vec::new(); + let mut random = 1u32; + let mut confirmed = Vec::new(); + for number in 0..360u64 { + let change = match number { + 120 => Some(1_000_000), + 240 => Some(8_000_000), + 250 => Some(6_000_000), + _ => None, + }; + if let Some(bitrate) = change { + assert!(state.update_bitrate_without_reopening("h264_nvenc", bitrate, 60)); + assert_eq!( + state.bitrate_change.take(), + Some(Unconfirmed { + live: LiveBitrate::NvencReconfig + }) + ); + } + for value in state.frame.data_mut(0) { + random = random.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + *value = (random >> 24) as u8; + } + state.frame.data_mut(1).fill(128); + state.frame.data_mut(2).fill(128); + state.frame.set_pts(Some(number as i64 * 1_000_000 / 60)); + state.frame.set_kind(if number == 250 { + ffmpeg_next::picture::Type::I + } else { + ffmpeg_next::picture::Type::None + }); + let watch = change.map(|_| crate::diagnostics::FfmpegWatch::start()); + state.encoder.send_frame(&state.frame).unwrap(); + loop { + match state.encoder.receive_packet(&mut state.packet) { + Ok(()) => { + let out = state.packet.pts().unwrap(); + if state.packet.is_key() { + keyframes.push(out * 60 / 1_000_000); + } + if number % 120 >= 60 { + sizes[(number / 120) as usize] += state.packet.size(); + } + decoder.send_packet(&state.packet).unwrap(); + loop { + match decoder.receive_frame(&mut decoded) { + Ok(()) => decoded_frames += 1, + Err(error) if is_eagain(&error) => break, + Err(error) => panic!("decode failed: {error}"), + } + } + } + Err(error) if is_eagain(&error) => break, + Err(error) => panic!("encode failed: {error}"), + } + } + if let Some(watch) = watch { + let lines = watch.finish(); + assert!(nvenc_reconfigured(&lines), "frame {number}: {lines:?}"); + confirmed.push(number); + } + } + eprintln!("NVENC sizes {sizes:?}, keyframes at {keyframes:?}, confirmed {confirmed:?}"); + assert_eq!(confirmed, [120, 240, 250]); + assert_eq!(decoded_frames, 360); + assert_eq!(unsafe { state.encoder.as_ptr() }, context); + assert!( + sizes[1] * 2 < sizes[0], + "lower bitrate was not applied: {sizes:?}" + ); + assert!( + sizes[2] > sizes[1] * 2, + "higher bitrate was not applied: {sizes:?}" + ); + } + #[test] fn software_bitrate_changes_preserve_decoding_and_reduce_packet_size() { ffmpeg_next::init().unwrap(); @@ -6075,16 +6468,37 @@ mod tests { // FPS and codec changes must still use the existing reopen path. assert!(!state.update_bitrate_without_reopening("libx264", 4_000_000, 120)); assert!(!state.update_bitrate_without_reopening("libx265", 4_000_000, 60)); - for name in ["h264_nvenc", "h264_amf", "libx265"] { + for name in ["h264_amf", "hevc_amf", "libx265"] { state.opened_encoder = name.to_string(); assert!(!state.update_bitrate_without_reopening(name, 4_000_000, 60)); } + // NVENC takes it in place, with the two-second VBV a session opened + // at it gets, to be confirmed by the IDR its reconfigure forces. + state.opened_encoder = "h264_nvenc".to_string(); + assert!(state.update_bitrate_without_reopening("h264_nvenc", 5_000_000, 60)); + assert_eq!(state.opened_bitrate, 5_000_000); + assert_eq!( + unsafe { (*state.encoder.as_ptr()).rc_buffer_size }, + 10_000_000 + ); + assert_eq!( + state.bitrate_change, + Some(Unconfirmed { + live: LiveBitrate::NvencReconfig + }) + ); + state.bitrate_change = None; // Quick Sync takes it in place, to be confirmed by its next frame. state.opened_encoder = "h264_qsv".to_string(); - assert!(!state.bitrate_reset_unconfirmed); + assert!(state.bitrate_change.is_none()); assert!(state.update_bitrate_without_reopening("h264_qsv", 4_000_000, 60)); assert_eq!(state.opened_bitrate, 4_000_000); - assert!(state.bitrate_reset_unconfirmed); + assert_eq!( + state.bitrate_change, + Some(Unconfirmed { + live: LiveBitrate::QsvReset + }) + ); assert!(!state.update_bitrate_without_reopening("h264_qsv", 4_000_000, 60)); } @@ -6610,6 +7024,8 @@ mod tests { collector: std::thread::JoinHandle<()>, /// When each packet arrived, from `started`. packets: std::sync::Arc>>, + /// When each keyframe arrived, from `started`. + keyframes: std::sync::Arc>>, started: Instant, number: u64, } @@ -6617,6 +7033,10 @@ mod tests { #[cfg(windows)] impl NativeStage { fn start() -> Self { + Self::start_with("h264_nvenc") + } + + fn start_with(encoder: &str) -> Self { use std::sync::Arc; let lines = Arc::new(parking_lot::Mutex::new(Vec::::new())); struct Sink(Arc>>); @@ -6645,8 +7065,8 @@ mod tests { let shared = SharedControl::new(0, 8_000_000); shared.running.store(true, Ordering::SeqCst); let gpu = GpuInfo { - encoder_name: "h264_nvenc".to_string(), - codec_display_name: "H.264 (NVENC)".to_string(), + encoder_name: encoder.to_string(), + codec_display_name: codec_display_name_for(encoder).to_string(), ..GpuInfo::software_fallback() }; let generation = crate::supervisor::CURRENT_GENERATION.load(Ordering::SeqCst); @@ -6657,11 +7077,17 @@ mod tests { }); let started = Instant::now(); let packets = Arc::new(parking_lot::Mutex::new(Vec::new())); + let keyframes = Arc::new(parking_lot::Mutex::new(Vec::new())); let collector = { let packets = Arc::clone(&packets); + let keyframes = Arc::clone(&keyframes); std::thread::spawn(move || { - while rx.blocking_recv().is_some() { - packets.lock().push(started.elapsed()); + while let Some(nal) = rx.blocking_recv() { + let at = started.elapsed(); + packets.lock().push(at); + if nal.is_keyframe { + keyframes.lock().push(at); + } } }) }; @@ -6672,6 +7098,7 @@ mod tests { stage, collector, packets, + keyframes, started, number: 0, } @@ -6735,6 +7162,119 @@ mod tests { } } + /// Native, for measurements: the real encode stage on + /// `ETERNAL_MEASURE_ENCODER` (h264_nvenc unless set), steered by the real + /// ABR controller for `ETERNAL_MEASURE_MINUTES` (4) on a link that loses + /// whatever the encoder sends above 11 Mbps: 10 Mbps holds, and each + /// probe to 12 is lost. One receiver report a second. + /// Prints a MEASURE line: steps, reopens and in-place changes a minute, + /// and for each change the longest gap between packets in the next 2 s + /// (the hitch) and the time to the next keyframe. + /// `cargo test -p eternal-host --lib bitrate_steps_on_a_lossy_link -- --ignored --nocapture` + #[cfg(windows)] + #[test] + #[ignore = "native: measures a real encoder for minutes"] + fn bitrate_steps_on_a_lossy_link() { + use std::time::Duration; + let encoder = + std::env::var("ETERNAL_MEASURE_ENCODER").unwrap_or_else(|_| "h264_nvenc".to_string()); + let minutes: u64 = std::env::var("ETERNAL_MEASURE_MINUTES") + .ok() + .and_then(|m| m.parse().ok()) + .unwrap_or(4); + const CAPACITY: u32 = 11_000_000; + let mut stage = NativeStage::start_with(&encoder); + let mut abr = crate::transport::abr::AbrController::new(50_000_000, true); + stage + .shared + .abr_current_bps + .store(abr.current_bps(), Ordering::SeqCst); + let (mut received, mut lost) = (0u32, 0u32); + let mut changes = Vec::new(); + let end = stage.started + Duration::from_secs(minutes * 60); + let mut next_report = stage.started + Duration::from_secs(1); + while Instant::now() < end { + stage.publish(); + if Instant::now() < next_report { + continue; + } + next_report += Duration::from_secs(1); + // What the open session sends, as the encode loop records it. + let rate = PIPELINE_STATS.lock().bitrate_bps.max(1); + let over = rate.saturating_sub(CAPACITY) / 1_000; + received += rate / 1_000 - over; + lost += over; + let report = eternal_wire::v2::control::ReceiverReport { + frags_received: received, + frags_lost: lost, + ..Default::default() + }; + // AFTER-ONLY begin: the encoder's cost of a step, which ABR reads. + abr.encoder_reopens( + stage + .shared + .encoder_reopens_for_bitrate + .load(Ordering::Relaxed), + ); + // AFTER-ONLY end + let decision = abr.on_report(&report, None, Instant::now()); + if decision.changed { + stage + .shared + .abr_current_bps + .store(decision.target_bps, Ordering::SeqCst); + changes.push(stage.started.elapsed()); + } + } + for _ in 0..30 { + stage.publish(); + } + let packets = stage.packets.lock().clone(); + let keyframes = stage.keyframes.lock().clone(); + let text = stage.finish(); + let reopens = text.matches("Reopened encoder session").count(); + let in_place = text.matches("bitrate without reopening").count() + + text.matches("Updated software encoder bitrate").count(); + let mut hitches: Vec = Vec::new(); + let mut to_keyframe: Vec = Vec::new(); + for &change in &changes { + let window: Vec = packets + .iter() + .copied() + .filter(|&at| { + at + Duration::from_millis(100) >= change + && at <= change + Duration::from_secs(2) + }) + .collect(); + if let Some(gap) = window.windows(2).map(|pair| pair[1] - pair[0]).max() { + hitches.push(gap.as_millis()); + } + if let Some(key) = keyframes.iter().find(|&&at| at >= change) { + to_keyframe.push((*key - change).as_millis()); + } + } + let summary = |values: &mut Vec| { + values.sort(); + let at = |q: f64| { + values + .get(((values.len() as f64 - 1.0) * q).round() as usize) + .copied() + }; + (at(0.5), at(0.9), values.last().copied()) + }; + let per_minute = |count: usize| count as f64 / minutes as f64; + eprintln!( + "MEASURE encoder={encoder} minutes={minutes} steps={} steps_per_min={:.2} reopens={reopens} reopens_per_min={:.2} in_place={in_place} hitch_ms_p50_p90_max={:?} keyframe_after_change_ms_p50_p90_max={:?} packets={}", + changes.len(), + per_minute(changes.len()), + per_minute(reopens), + summary(&mut hitches), + summary(&mut to_keyframe), + packets.len(), + ); + assert!(!changes.is_empty(), "ABR never stepped"); + } + /// Native, on a PC with NVENC: the encode stage's first NVENC open is /// held for twice its limit (`ETERNAL_SIMULATE_STALL`). The stage streams /// on software once the first picture is due, gives the open up at its @@ -6885,6 +7425,11 @@ mod tests { #[ignore = "native: needs an NVIDIA GPU"] fn a_bitrate_change_after_a_stalled_nvenc_reopen_clears_is_prompt() { use std::time::Duration; + // As on an NVIDIA card that cannot change its bitrate live: there an + // ABR step reopens NVENC, the reopen this test holds. + LiveBitrate::NvencReconfig + .refused() + .store(true, Ordering::Relaxed); let mut stage = NativeStage::start(); // NVENC streams before the ABR step. A first open slower than the // first picture's wait streams libx264 until NVENC's session, opened diff --git a/host/src/ffmpeg_log.c b/host/src/ffmpeg_log.c index 75f7dabc..b4b34a1a 100644 --- a/host/src/ffmpeg_log.c +++ b/host/src/ffmpeg_log.c @@ -3,8 +3,26 @@ /* Keep va_list in C: its ABI differs between MSVC, Intel and Apple ARM. */ static void (*log_sink)(int, const char *); +/* Set while a thread watches FFmpeg's verbose lines (see + `diagnostics::FfmpegWatch`); the sink gives them only to that thread. Read + from every thread that logs, so it is atomic. Relaxed is enough: a watching + thread sets it before its own lines, and other threads' lines only decide + what the sink drops. */ +#ifdef _MSC_VER +#include +static volatile long verbose_wanted; +static int verbose_is_wanted(void) { return _InterlockedCompareExchange(&verbose_wanted, 0, 0) != 0; } +static void set_verbose_wanted(int wanted) { _InterlockedExchange(&verbose_wanted, wanted ? 1 : 0); } +#else +#include +static atomic_int verbose_wanted; +static int verbose_is_wanted(void) { return atomic_load_explicit(&verbose_wanted, memory_order_relaxed) != 0; } +static void set_verbose_wanted(int wanted) { atomic_store_explicit(&verbose_wanted, wanted ? 1 : 0, memory_order_relaxed); } +#endif + static void forward_log(void *context, int level, const char *format, va_list args) { - if (level > AV_LOG_WARNING || !log_sink) return; + if (!log_sink) return; + if (level > AV_LOG_WARNING && !(level <= AV_LOG_VERBOSE && verbose_is_wanted())) return; char line[4096]; int prefix = 1; int length = av_log_format_line2(context, level, format, args, line, sizeof(line), &prefix); @@ -13,6 +31,13 @@ static void forward_log(void *context, int level, const char *format, va_list ar void eternal_install_ffmpeg_logger(void (*sink)(int, const char *)) { log_sink = sink; - av_log_set_level(AV_LOG_WARNING); + /* Set once, before FFmpeg is used: FFmpeg's level is a plain int that + every logging thread reads, so it is never changed afterwards. The + callback drops what is not wanted. */ + av_log_set_level(AV_LOG_VERBOSE); av_log_set_callback(forward_log); } + +void eternal_ffmpeg_verbose(int wanted) { + set_verbose_wanted(wanted); +} diff --git a/host/src/transport/abr.rs b/host/src/transport/abr.rs index 1827e614..83a8d6b3 100644 --- a/host/src/transport/abr.rs +++ b/host/src/transport/abr.rs @@ -16,6 +16,16 @@ const LADDER_BPS: [u32; 11] = [ const DOWN_COOLDOWN: Duration = Duration::from_secs(3); const UP_COOLDOWN: Duration = Duration::from_secs(15); +/// The wait after any change before a step up, while the encoder takes a +/// new bitrate only by reopening (AMF, libx265): each step there costs a +/// stutter and an IDR. On a link just below the next rung, a step up is +/// undone by a step down within seconds, so a probe that fails reopens the +/// encoder twice. A probe undone within [`PROBE_HOLDS`] doubles the wait, +/// up to [`REOPEN_UP_COOLDOWN_MAX`]; one that holds that long resets it. +const REOPEN_UP_COOLDOWN: Duration = Duration::from_secs(30); +const REOPEN_UP_COOLDOWN_MAX: Duration = Duration::from_secs(120); +/// A step up held this long, with no step down, worked. +const PROBE_HOLDS: Duration = Duration::from_secs(30); /// This long continuously clean before stepping up. const CLEAN_WINDOW: Duration = Duration::from_secs(15); /// Freeze the controller when reports stop (liveness machinery owns that). @@ -50,6 +60,11 @@ pub struct AbrController { /// Whether the tunnel dropped frames in each of the latest reports, /// newest in bit 0. tunnel_drop_history: u8, + /// The encoder takes a new bitrate only by reopening (see + /// [`Self::encoder_reopens`]). + encoder_reopens: bool, + /// The wait before a step up while it does. + reopen_up_cooldown: Duration, } impl AbrController { @@ -73,6 +88,50 @@ impl AbrController { prev_cumulative: None, prev_tunnel_drops: None, tunnel_drop_history: 0, + encoder_reopens: false, + reopen_up_cooldown: REOPEN_UP_COOLDOWN, + } + } + + /// Whether the open encoder session takes a new bitrate only by + /// reopening it, as the transport reads it before each report. Then a + /// step up waits [`REOPEN_UP_COOLDOWN`] or more after any change instead + /// of [`UP_COOLDOWN`]. Steps down are not held: they relieve a link that + /// is losing frames. + pub fn encoder_reopens(&mut self, reopens: bool) { + self.encoder_reopens = reopens; + } + + /// The wait after a change before a step up. + fn up_cooldown(&self) -> Duration { + if self.encoder_reopens { + self.reopen_up_cooldown + } else { + UP_COOLDOWN + } + } + + /// A step down at `now`. One soon after a step up undoes that probe: + /// the next probe waits twice as long, while the encoder reopens. + fn note_down(&mut self, now: Instant) { + let undoes_probe = matches!( + self.last_change, + Some((when, false)) if now.duration_since(when) < PROBE_HOLDS + ); + if undoes_probe && self.encoder_reopens { + self.reopen_up_cooldown = (self.reopen_up_cooldown * 2).min(REOPEN_UP_COOLDOWN_MAX); + } + self.last_change = Some((now, true)); + } + + /// A step up that has held for [`PROBE_HOLDS`] worked: the next waits + /// the shortest time again. + fn note_probe_held(&mut self, now: Instant) { + if matches!( + self.last_change, + Some((when, false)) if now.duration_since(when) >= PROBE_HOLDS + ) { + self.reopen_up_cooldown = REOPEN_UP_COOLDOWN; } } @@ -123,6 +182,7 @@ impl AbrController { if !self.enabled { return unchanged; } + self.note_probe_held(now); // Interval deltas from cumulative counters. let (delta_received, delta_lost, delta_repaired) = match self.prev_cumulative { @@ -200,7 +260,7 @@ impl AbrController { "ABR stepping bitrate down" ); self.current_bps = target; - self.last_change = Some((now, true)); + self.note_down(now); return AbrDecision { target_bps: target, changed: true, @@ -220,7 +280,7 @@ impl AbrController { { let clean_since = *self.clean_since.get_or_insert(now); let up_cooldown_over = match self.last_change { - Some((when, _)) => now.duration_since(when) >= UP_COOLDOWN, + Some((when, _)) => now.duration_since(when) >= self.up_cooldown(), None => true, }; if now.duration_since(clean_since) >= CLEAN_WINDOW && up_cooldown_over { @@ -278,7 +338,7 @@ impl AbrController { "ABR stepping bitrate down" ); self.current_bps = target; - self.last_change = Some((now, true)); + self.note_down(now); AbrDecision { target_bps: target, changed: true, @@ -675,4 +735,215 @@ mod tests { let d = abr.on_send_refused(t0 + Duration::from_millis(3_100)); assert_eq!((d.target_bps, d.changed), (10_000_000, true)); } + + /// Feeds one clean report a second from `from` to `to` (in s from + /// `t0`), with counters continuing from `counter`. The changes, as + /// (second, target). + fn clean_reports( + abr: &mut AbrController, + t0: Instant, + counter: &mut u32, + seconds: std::ops::Range, + ) -> Vec<(u64, u32)> { + let mut changes = Vec::new(); + for second in seconds { + *counter += 1_000; + let d = abr.on_report( + &report(*counter, 0, 0), + None, + t0 + Duration::from_secs(second), + ); + if d.changed { + changes.push((second, d.target_bps)); + } + } + changes + } + + #[test] + fn a_step_up_waits_30_s_after_a_change_while_the_encoder_reopens() { + // In place: 15 s clean from the first clean report, at 1 s; then a + // rung per 15 s. Reopening: 30 s after the change, then a rung per + // 30 s. + for (reopens, first_up, wait) in [(false, 16, 15), (true, 30, 30)] { + let mut abr = AbrController::new(50_000_000, true); + abr.encoder_reopens(reopens); + let t0 = Instant::now(); + // Loss steps 15 -> 12 Mbps at 0 s; clean from then on. + assert!(abr.on_report(&report(980, 30, 0), None, t0).changed); + let mut counter = 980; + let changes = clean_reports(&mut abr, t0, &mut counter, 1..80); + assert_eq!(changes[0], (first_up, 15_000_000), "reopens={reopens}"); + // Then one rung per wait, the clean window being shorter. + let gaps: Vec = changes + .windows(2) + .map(|pair| pair[1].0 - pair[0].0) + .collect(); + assert!( + gaps.iter().all(|&gap| gap == wait), + "reopens={reopens}: {changes:?}" + ); + } + } + + #[test] + fn a_probe_undone_soon_doubles_the_wait_and_one_that_holds_resets_it() { + let mut abr = AbrController::new(50_000_000, true); + abr.encoder_reopens(true); + let t0 = Instant::now(); + let at = |s: u64| t0 + Duration::from_secs(s); + let mut counter = 0; + // Steps down at 0 s, then probes up once the wait is over, and each + // probe is lost 4 s later, on a link that cannot carry it. + assert!(abr.on_report(&report(980, 30, 0), None, at(0)).changed); + let mut downs = vec![0u64]; + let mut ups = Vec::new(); + let mut second = 1; + while second < 600 { + let since_down = second - downs.last().unwrap(); + let lossy = ups + .last() + .is_some_and(|&up: &u64| up > *downs.last().unwrap() && second - up == 4); + counter += 1_000; + let lost = if lossy { 50 } else { 0 }; + let d = abr.on_report(&report(counter, lost, 0), None, at(second)); + if d.changed { + if lossy { + downs.push(second); + } else { + ups.push(second); + } + } + let _ = since_down; + second += 1; + } + let waits: Vec = ups + .iter() + .map(|up| up - downs.iter().rev().find(|down| *down < up).unwrap()) + .collect(); + assert_eq!(waits, [30, 60, 120, 120, 120, 120], "{ups:?} {downs:?}"); + // A probe that holds 30 s: the next one waits 30 s again. + let mut abr = AbrController::new(50_000_000, true); + abr.encoder_reopens(true); + assert!(abr.on_report(&report(980, 30, 0), None, at(0)).changed); + let mut counter = 980; + let ups = clean_reports(&mut abr, t0, &mut counter, 1..31); + assert_eq!(ups, [(30, 15_000_000)]); + assert!( + abr.on_report(&report(counter + 900, 60, 0), None, at(32)) + .changed + ); + counter += 900; + // Undone at 32 s: the wait doubles to 60 s. + assert_eq!(clean_reports(&mut abr, t0, &mut counter, 33..95)[0].0, 92); + // That probe holds; after 30 s the wait is 30 s again. + let ups = clean_reports(&mut abr, t0, &mut counter, 95..160); + assert_eq!(ups, [(122, 20_000_000), (152, 25_000_000)]); + } + + #[test] + fn steps_down_and_in_place_encoders_are_not_held() { + // A reopening encoder steps down as before: at once, then 3 s apart. + let mut abr = AbrController::new(50_000_000, true); + abr.encoder_reopens(true); + let t0 = Instant::now(); + let at = |ms: u64| t0 + Duration::from_millis(ms); + let (mut counter, mut lost) = (0, 0); + let mut downs = Vec::new(); + for ms in (0..10_000).step_by(1_000) { + counter += 1_000; + lost += 50; + if abr + .on_report(&report(counter, lost, 0), None, at(ms)) + .changed + { + downs.push((ms, abr.current_bps())); + } + } + assert_eq!( + downs, + [ + (0, 12_000_000), + (3_000, 10_000_000), + (6_000, 8_000_000), + (9_000, 6_000_000) + ] + ); + // A refused send too. + let d = abr.on_send_refused(at(12_000)); + assert_eq!((d.target_bps, d.changed), (4_000_000, true)); + // An encoder that takes the bitrate in place keeps the 15 s wait: + // 15 s clean from the first clean report. + abr.encoder_reopens(false); + let ups = clean_reports(&mut abr, at(12_000), &mut counter, 1..17); + assert_eq!(ups, [(16, 6_000_000)]); + } + + /// A link that carries `capacity` bps: each second, the fragments over + /// it are lost. `capacity_at` gives it by second. The steps, by second. + fn on_a_link(reopens: bool, seconds: u64, capacity_at: impl Fn(u64) -> u32) -> Vec<(u64, u32)> { + let mut abr = AbrController::new(50_000_000, true); + abr.encoder_reopens(reopens); + let t0 = Instant::now(); + let (mut received, mut lost) = (0u32, 0u32); + let mut steps = Vec::new(); + for second in 0..seconds { + let rate = abr.current_bps(); + let capacity = capacity_at(second); + // A fragment a kilobit: rate in kbit per second. + let sent = rate / 1_000; + let over = rate.saturating_sub(capacity) / 1_000; + received += sent - over; + lost += over; + let d = abr.on_report( + &report(received, lost, 0), + None, + t0 + Duration::from_secs(second), + ); + if d.changed { + steps.push((second, d.target_bps)); + } + } + steps + } + + #[test] + fn a_link_just_below_the_next_rung_probes_less_often_while_the_encoder_reopens() { + // 11 Mbps: 10 Mbps holds, every probe to 12 Mbps is lost. + let minutes = 30; + let steady = on_a_link(false, minutes * 60, |_| 11_000_000); + let reopening = on_a_link(true, minutes * 60, |_| 11_000_000); + let per_minute = |steps: &[(u64, u32)]| steps.len() as f64 / minutes as f64; + eprintln!( + "steps a minute at 11 Mbps: {:.2} in place, {:.2} reopening", + per_minute(&steady), + per_minute(&reopening) + ); + // In place: a probe and its undoing every 15 s plus a few. + assert!(per_minute(&steady) >= 6.0, "{steady:?}"); + // Reopening: about one probe and its undoing every 2 minutes once + // the wait reaches its cap. + assert!(per_minute(&reopening) <= 1.2, "{reopening:?}"); + // Both settle on 10 Mbps between probes and never stay above the + // link. + for steps in [&steady, &reopening] { + assert!(steps.iter().all(|&(_, bps)| bps <= 12_000_000), "{steps:?}"); + } + // Capacity that comes back: the reopening encoder still climbs, a + // rung per 30 s. + let recovered = on_a_link( + true, + 400, + |second| if second < 60 { 7_000_000 } else { 60_000_000 }, + ); + let after: Vec<_> = recovered + .iter() + .filter(|(second, _)| *second >= 60) + .collect(); + assert_eq!( + after.last().map(|(_, bps)| *bps), + Some(50_000_000), + "{recovered:?}" + ); + } } diff --git a/host/src/transport/mod.rs b/host/src/transport/mod.rs index 70b808ea..9ba55d34 100644 --- a/host/src/transport/mod.rs +++ b/host/src/transport/mod.rs @@ -1125,6 +1125,9 @@ async fn act( let clamped = state.abr.set_ceiling(ceiling); let tunnel_drops = (shared.target_addr.lock().link != LinkId::Udp).then(link::tunnel_congestion_drops); + state + .abr + .encoder_reopens(shared.encoder_reopens_for_bitrate.load(Ordering::Relaxed)); let reported = state.abr.on_report(&report, tunnel_drops, Instant::now()); let decision = abr::AbrDecision { target_bps: reported.target_bps,