From d4e86fceb1934b38efa8c5cb12f015b1a32be71a Mon Sep 17 00:00:00 2001 From: JellyfishKnight Date: Sun, 27 Sep 2026 21:28:00 +0800 Subject: [PATCH 1/4] feat(foveation): support external combined gaze over OSC --- alvr/server_core/src/tracking/eye_gaze.rs | 142 ++++++++++++++++++++++ alvr/server_core/src/tracking/mod.rs | 38 +++++- alvr/server_openvr/src/lib.rs | 2 +- alvr/session/src/settings.rs | 11 +- 4 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 alvr/server_core/src/tracking/eye_gaze.rs diff --git a/alvr/server_core/src/tracking/eye_gaze.rs b/alvr/server_core/src/tracking/eye_gaze.rs new file mode 100644 index 0000000000..c4f1321b23 --- /dev/null +++ b/alvr/server_core/src/tracking/eye_gaze.rs @@ -0,0 +1,142 @@ +use alvr_common::{ + anyhow::Result, + glam::{Quat, Vec3}, +}; +use rosc::{OscPacket, OscType}; +use std::{ + io::ErrorKind, + net::{Ipv4Addr, UdpSocket}, + time::{Duration, Instant}, +}; + +const GAZE_ADDRESS: &str = "/alvr/eye/combined/v1"; +const MAX_SAMPLE_AGE: Duration = Duration::from_millis(100); +const SEQUENCE_RESET_TIMEOUT: Duration = Duration::from_secs(1); +const MAX_DATAGRAM_SIZE: usize = 256; +const MAX_PACKETS_PER_POLL: usize = 64; + +pub struct EyeGazeReceiver { + socket: UdpSocket, + last_packet: Option<(u32, Instant)>, + last_tracking_timestamp: Option, + last_polled_at: Instant, + discard_queued: bool, +} + +impl EyeGazeReceiver { + pub fn new(port: u16) -> Result { + let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, port))?; + socket.set_nonblocking(true)?; + + Ok(Self { + socket, + last_packet: None, + last_tracking_timestamp: None, + last_polled_at: Instant::now(), + discard_queued: false, + }) + } + + // Consume each observation once. Missing input must not refresh the foveation hold timer. + pub fn receive(&mut self, tracking_timestamp: Duration) -> Result> { + if self + .last_tracking_timestamp + .is_some_and(|previous| tracking_timestamp <= previous) + { + return Ok(None); + } + self.last_tracking_timestamp = Some(tracking_timestamp); + + let now = Instant::now(); + // After a tracking stall, drain the socket before accepting fresh observations again. + self.discard_queued |= now.duration_since(self.last_polled_at) > MAX_SAMPLE_AGE; + self.last_polled_at = now; + let mut gaze = None; + // One extra byte lets us reject oversized datagrams even if recv truncates them. + let mut buffer = [0; MAX_DATAGRAM_SIZE + 1]; + + for _ in 0..MAX_PACKETS_PER_POLL { + let size = match self.socket.recv(&mut buffer) { + Ok(size) => size, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + self.discard_queued = false; + + return Ok(gaze); + } + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + // Windows reports an oversized datagram instead of returning truncated bytes. + Err(error) if error.raw_os_error() == Some(10040) => continue, + Err(error) => return Err(error.into()), + }; + if self.discard_queued || size > MAX_DATAGRAM_SIZE { + continue; + } + let Some(packet) = buffer.get(..size) else { + continue; + }; + // This endpoint accepts single messages, not scheduled or recursive OSC bundles. + if packet.first() != Some(&b'/') { + continue; + } + let Ok((remaining, OscPacket::Message(message))) = rosc::decoder::decode_udp(packet) + else { + continue; + }; + if !remaining.is_empty() || message.addr != GAZE_ADDRESS { + continue; + } + let [ + OscType::Int(sequence), + OscType::Int(valid), + OscType::Int(age_us), + OscType::Float(x), + OscType::Float(y), + OscType::Float(z), + OscType::Float(w), + ] = message.args.as_slice() + else { + continue; + }; + if !matches!(valid, 0 | 1) + || *age_us < 0 + || Duration::from_micros(*age_us as u64) > MAX_SAMPLE_AGE + { + continue; + } + + let orientation = if *valid == 1 { + let orientation = Quat::from_xyzw(*x, *y, *z, *w); + let length_squared = orientation.length_squared(); + if !length_squared.is_finite() + || length_squared <= f32::EPSILON + || (orientation.normalize() * Vec3::NEG_Z).z >= -f32::EPSILON + { + continue; + } + + Some(orientation) + } else { + None + }; + // Interpret the OSC signed integer as the bit pattern of a wrapping u32 counter. + let sequence = *sequence as u32; + if let Some((previous, received_at)) = self.last_packet { + let distance = sequence.wrapping_sub(previous); + if distance == 0 + || (now.duration_since(received_at) <= SEQUENCE_RESET_TIMEOUT + && distance >= 1 << 31) + { + continue; + } + } + + self.last_packet = Some((sequence, now)); + gaze = orientation; + } + + // Do not block head/controller tracking or use a potentially backlogged sample in a flood. + self.discard_queued = true; + + Ok(None) + } +} diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index 07e8f53453..173e1b4880 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -1,4 +1,5 @@ mod body; +mod eye_gaze; mod face; mod vmc; @@ -15,15 +16,16 @@ use crate::{ use alvr_common::{ ConnectionError, DEVICE_ID_TO_PATH, DeviceMotion, Pose, ViewParams, glam::{Quat, Vec3}, - inputs as inp, + inputs as inp, warn, }; use alvr_events::{EventType, TrackingEvent}; use alvr_packets::TrackingData; use alvr_session::{ - BodyTrackingConfig, HeadsetConfig, RecenteringMode, Settings, VMCConfig, + BodyTrackingConfig, GazeInputSource, HeadsetConfig, RecenteringMode, Settings, VMCConfig, settings_schema::Switch, }; use alvr_sockets::StreamReceiver; +use eye_gaze::EyeGazeReceiver; use std::{ cmp::Ordering, collections::{HashMap, VecDeque}, @@ -322,6 +324,19 @@ pub fn tracking_loop( .into_option() .and_then(|config| VMCSink::new(config).ok()); + let gaze_input_source = initial_settings + .video + .foveated_encoding + .as_option() + .map_or(GazeInputSource::None, |config| config.gaze_input_source); + let mut eye_gaze_receiver = if let GazeInputSource::ExternalOsc { port } = gaze_input_source { + EyeGazeReceiver::new(port) + .inspect_err(|error| warn!("Failed to bind external gaze OSC port {port}: {error}")) + .ok() + } else { + None + }; + while is_streaming() { let data = match tracking_receiver.recv(STREAMING_RECV_TIMEOUT) { Ok(tracking) => tracking, @@ -334,6 +349,22 @@ pub fn tracking_loop( let timestamp = tracking.poll_timestamp; + let combined_eye_gaze = match (gaze_input_source, &mut eye_gaze_receiver) { + (GazeInputSource::ExternalOsc { .. }, Some(receiver)) => { + match receiver.receive(timestamp) { + Ok(gaze) => gaze, + Err(error) => { + warn!("External gaze OSC receiver stopped: {error}"); + eye_gaze_receiver = None; + + None + } + } + } + (GazeInputSource::ExternalOsc { .. }, None) => None, + (GazeInputSource::None | GazeInputSource::Headset, _) => tracking.face.eyes_combined, + }; + if let Some(stats) = &mut *ctx.statistics_manager.write() { stats.report_tracking_received(timestamp); } @@ -393,7 +424,8 @@ pub fn tracking_loop( tracking_manager_lock.report_hand_skeleton(HandType::Right, timestamp, skeleton); } - tracking_manager_lock.report_combined_eye_gaze(timestamp, tracking.face.eyes_combined); + // Select foveation input without replacing the client's face/OSC output data. + tracking_manager_lock.report_combined_eye_gaze(timestamp, combined_eye_gaze); if let Some(sink) = &mut face_tracking_sink { sink.send_tracking(&tracking.face); diff --git a/alvr/server_openvr/src/lib.rs b/alvr/server_openvr/src/lib.rs index 8ecc54a254..12d577df63 100644 --- a/alvr/server_openvr/src/lib.rs +++ b/alvr/server_openvr/src/lib.rs @@ -258,7 +258,7 @@ fn spawn_event_loop(events_receiver: mpsc::Receiver) { .foveated_encoding .as_option() .is_some_and(|config| { - config.gaze_input_source == GazeInputSource::Headset + config.gaze_input_source != GazeInputSource::None }) }) .map(|params| { diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index c0806d3e2c..5b5a220b79 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -474,7 +474,7 @@ pub struct FoveatedEncodingConfig { #[schema(strings( display_name = "Center shift", - help = "Horizontal and vertical shift, in X/Y order. Used as a static fallback; overridden, not added to, when headset eye-tracking centers are available." + help = "Horizontal and vertical shift, in X/Y order. Used as a static fallback; overridden, not added to, when eye-tracking centers are available." ))] #[schema(gui(slider(min = -1.0, max = 1.0, step = 0.01)))] #[schema(flag = "steamvr-restart")] @@ -492,6 +492,14 @@ pub enum GazeInputSource { None, #[schema(strings(display_name = "Headset eye tracking"))] Headset, + #[schema(strings( + display_name = "External OSC", + help = "Receive head-local combined gaze over OSC/UDP on localhost." + ))] + ExternalOsc { + #[schema(strings(display_name = "UDP port"))] + port: u16, + }, } #[repr(C)] @@ -1882,6 +1890,7 @@ pub fn session_settings_default() -> SettingsDefault { gui_collapsed: true, force_enable: false, gaze_input_source: GazeInputSourceDefault { + ExternalOsc: GazeInputSourceExternalOscDefault { port: 9945 }, variant: GazeInputSourceDefaultVariant::None, }, center_size: ArrayDefault { From 0533e0e6dc1ce8bb0f19dcfadfe8b906dd5eff3f Mon Sep 17 00:00:00 2001 From: JellyfishKnight Date: Tue, 29 Sep 2026 02:38:39 +0800 Subject: [PATCH 2/4] refactor(tracking): share OSC eye input across foveation and social output --- alvr/client_openxr/src/interaction.rs | 20 ++++----- alvr/server_core/src/connection.rs | 10 ++--- alvr/server_core/src/tracking/eye_gaze.rs | 3 +- alvr/server_core/src/tracking/face.rs | 8 ++-- alvr/server_core/src/tracking/mod.rs | 49 ++++++++++++----------- alvr/server_openvr/src/lib.rs | 6 +-- alvr/session/src/settings.rs | 49 ++++++++++++----------- 7 files changed, 77 insertions(+), 68 deletions(-) diff --git a/alvr/client_openxr/src/interaction.rs b/alvr/client_openxr/src/interaction.rs index db32dd75c5..df67bd3456 100644 --- a/alvr/client_openxr/src/interaction.rs +++ b/alvr/client_openxr/src/interaction.rs @@ -11,9 +11,7 @@ use alvr_common::{ }; use alvr_graphics::HandData; use alvr_packets::{ButtonEntry, ButtonValue, ClientStreamConfig, FaceData, FaceExpressions}; -use alvr_session::{ - BodyTrackingBDConfig, BodyTrackingSourcesConfig, FaceTrackingSourcesConfig, GazeInputSource, -}; +use alvr_session::{BodyTrackingBDConfig, BodyTrackingSourcesConfig, FaceTrackingSourcesConfig}; use openxr as xr; use std::{ collections::{HashMap, HashSet}, @@ -150,12 +148,12 @@ pub struct InteractionSourcesConfig { impl InteractionSourcesConfig { pub fn new(config: &ClientStreamConfig) -> Self { + let face_tracking = config.settings.headset.face_tracking.as_option(); + let eye_tracking_osc_override = + face_tracking.is_some_and(|c| c.eye_tracking_osc_override.enabled()); + Self { - face_tracking: config - .settings - .headset - .face_tracking - .as_option() + face_tracking: face_tracking .map(|c| c.sources.clone()) .or_else(|| { // Foveation requests eye input locally, without enabling face tracking output. @@ -164,8 +162,12 @@ impl InteractionSourcesConfig { .video .foveated_encoding .as_option() - .is_some_and(|c| c.gaze_input_source == GazeInputSource::Headset) + .is_some_and(|c| c.eye_tracking) .then_some(FaceTrackingSourcesConfig::PreferEyeTrackingOnly) + }) + .filter(|sources| { + !eye_tracking_osc_override + || *sources == FaceTrackingSourcesConfig::PreferFullFaceTracking }), body_tracking: config .settings diff --git a/alvr/server_core/src/connection.rs b/alvr/server_core/src/connection.rs index 29ef0d8bc1..d935c7fcca 100644 --- a/alvr/server_core/src/connection.rs +++ b/alvr/server_core/src/connection.rs @@ -26,8 +26,8 @@ use alvr_packets::{ VIDEO, VideoPacketHeader, }; use alvr_session::{ - BodyTrackingSinkConfig, CodecType, ControllersEmulationMode, FrameSize, GazeInputSource, - H264Profile, Settings, SocketProtocol, SteamvrHmdInitConfig, + BodyTrackingSinkConfig, CodecType, ControllersEmulationMode, FrameSize, H264Profile, Settings, + SocketProtocol, SteamvrHmdInitConfig, }; use alvr_sockets::{ CONTROL_PORT, KEEPALIVE_INTERVAL, KEEPALIVE_TIMEOUT, ProtoControlSocket, SocketConnection, @@ -142,7 +142,7 @@ pub fn compute_restart_settings_hash( .map(|c| c.sources.meta.prefer_full_body) .unwrap_or(false); - let mut foveation_gaze_input_source = GazeInputSource::None; + let mut foveation_eye_tracking = false; let mut foveation_center_size_x = 0.0_f32; let mut foveation_center_size_y = 0.0_f32; let mut foveation_center_shift_x = 0.0_f32; @@ -151,7 +151,7 @@ pub fn compute_restart_settings_hash( let mut foveation_edge_ratio_y = 0.0_f32; let enable_foveated_encoding = if let Switch::Enabled(config) = &settings.video.foveated_encoding { - foveation_gaze_input_source = config.gaze_input_source; + foveation_eye_tracking = config.eye_tracking; [foveation_center_size_x, foveation_center_size_y] = config.center_size; [foveation_center_shift_x, foveation_center_shift_y] = config.center_shift; [foveation_edge_ratio_x, foveation_edge_ratio_y] = config.edge_ratio; @@ -244,7 +244,7 @@ pub fn compute_restart_settings_hash( nvenc.enable_weighted_prediction.hash(&mut h); // Foveated encoding enable_foveated_encoding.hash(&mut h); - foveation_gaze_input_source.hash(&mut h); + foveation_eye_tracking.hash(&mut h); foveation_center_size_x.to_bits().hash(&mut h); foveation_center_size_y.to_bits().hash(&mut h); foveation_center_shift_x.to_bits().hash(&mut h); diff --git a/alvr/server_core/src/tracking/eye_gaze.rs b/alvr/server_core/src/tracking/eye_gaze.rs index c4f1321b23..0e462c21c7 100644 --- a/alvr/server_core/src/tracking/eye_gaze.rs +++ b/alvr/server_core/src/tracking/eye_gaze.rs @@ -114,7 +114,8 @@ impl EyeGazeReceiver { continue; } - Some(orientation) + // Social output also consumes this quaternion and expects a unit rotation. + Some(orientation.normalize()) } else { None }; diff --git a/alvr/server_core/src/tracking/face.rs b/alvr/server_core/src/tracking/face.rs index 9b9f345091..09831c6499 100644 --- a/alvr/server_core/src/tracking/face.rs +++ b/alvr/server_core/src/tracking/face.rs @@ -15,8 +15,9 @@ pub struct FaceTrackingSink { } impl FaceTrackingSink { - pub fn new(config: FaceTrackingSinkConfig, local_osc_port: u16) -> Result { + pub fn new(config: FaceTrackingSinkConfig, local_osc_port: u16) -> Result> { let port = match config { + FaceTrackingSinkConfig::None => return Ok(None), FaceTrackingSinkConfig::VrchatEyeOsc { port } => port, FaceTrackingSinkConfig::VrcFaceTracking => VRCFT_PORT, }; @@ -24,11 +25,11 @@ impl FaceTrackingSink { let socket = UdpSocket::bind(format!("127.0.0.1:{local_osc_port}"))?; socket.connect(format!("127.0.0.1:{port}"))?; - Ok(Self { + Ok(Some(Self { config, socket, packet_buffer: vec![], - }) + })) } fn send_osc_message(&self, path: &str, args: Vec) { @@ -53,6 +54,7 @@ impl FaceTrackingSink { pub fn send_tracking(&mut self, face_data: &FaceData) { match self.config { + FaceTrackingSinkConfig::None => (), FaceTrackingSinkConfig::VrchatEyeOsc { .. } => { if let [Some(left), Some(right)] = face_data.eyes_social { let (left_pitch, left_yaw, _) = left.to_euler(EulerRot::XYZ); diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index 173e1b4880..64155bec47 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -21,7 +21,7 @@ use alvr_common::{ use alvr_events::{EventType, TrackingEvent}; use alvr_packets::TrackingData; use alvr_session::{ - BodyTrackingConfig, GazeInputSource, HeadsetConfig, RecenteringMode, Settings, VMCConfig, + BodyTrackingConfig, HeadsetConfig, RecenteringMode, Settings, VMCConfig, settings_schema::Switch, }; use alvr_sockets::StreamReceiver; @@ -302,12 +302,26 @@ pub fn tracking_loop( ) }); + let eye_tracking_osc_port = initial_settings + .headset + .face_tracking + .as_option() + .and_then(|config| config.eye_tracking_osc_override.as_option()) + .map(|config| config.port); + let mut eye_gaze_receiver = eye_tracking_osc_port.and_then(|port| { + EyeGazeReceiver::new(port) + .inspect_err(|error| warn!("Failed to bind external gaze OSC port {port}: {error}")) + .ok() + }); + let mut face_tracking_sink = initial_settings .headset .face_tracking .into_option() .and_then(|config| { - FaceTrackingSink::new(config.sink, initial_settings.connection.osc_local_port).ok() + FaceTrackingSink::new(config.sink, initial_settings.connection.osc_local_port) + .ok() + .flatten() }); let mut body_tracking_sink = initial_settings @@ -324,19 +338,6 @@ pub fn tracking_loop( .into_option() .and_then(|config| VMCSink::new(config).ok()); - let gaze_input_source = initial_settings - .video - .foveated_encoding - .as_option() - .map_or(GazeInputSource::None, |config| config.gaze_input_source); - let mut eye_gaze_receiver = if let GazeInputSource::ExternalOsc { port } = gaze_input_source { - EyeGazeReceiver::new(port) - .inspect_err(|error| warn!("Failed to bind external gaze OSC port {port}: {error}")) - .ok() - } else { - None - }; - while is_streaming() { let data = match tracking_receiver.recv(STREAMING_RECV_TIMEOUT) { Ok(tracking) => tracking, @@ -349,8 +350,11 @@ pub fn tracking_loop( let timestamp = tracking.poll_timestamp; - let combined_eye_gaze = match (gaze_input_source, &mut eye_gaze_receiver) { - (GazeInputSource::ExternalOsc { .. }, Some(receiver)) => { + if eye_tracking_osc_port.is_some() { + // Select one eye input before foveation and social output consume it. Do not fall + // back to headset eyes when OSC input is missing, invalid or unavailable. + tracking.face.eyes_social = [None; 2]; + tracking.face.eyes_combined = if let Some(receiver) = &mut eye_gaze_receiver { match receiver.receive(timestamp) { Ok(gaze) => gaze, Err(error) => { @@ -360,10 +364,10 @@ pub fn tracking_loop( None } } - } - (GazeInputSource::ExternalOsc { .. }, None) => None, - (GazeInputSource::None | GazeInputSource::Headset, _) => tracking.face.eyes_combined, - }; + } else { + None + }; + } if let Some(stats) = &mut *ctx.statistics_manager.write() { stats.report_tracking_received(timestamp); @@ -424,8 +428,7 @@ pub fn tracking_loop( tracking_manager_lock.report_hand_skeleton(HandType::Right, timestamp, skeleton); } - // Select foveation input without replacing the client's face/OSC output data. - tracking_manager_lock.report_combined_eye_gaze(timestamp, combined_eye_gaze); + tracking_manager_lock.report_combined_eye_gaze(timestamp, tracking.face.eyes_combined); if let Some(sink) = &mut face_tracking_sink { sink.send_tracking(&tracking.face); diff --git a/alvr/server_openvr/src/lib.rs b/alvr/server_openvr/src/lib.rs index 12d577df63..068d2b300b 100644 --- a/alvr/server_openvr/src/lib.rs +++ b/alvr/server_openvr/src/lib.rs @@ -28,7 +28,7 @@ use alvr_server_core::{ HandType, ServerCoreContext, ServerCoreEvent, ServerNegotiatedStreamingConfig, }; use alvr_session::{ - BodyTrackingSinkConfig, CodecType, ControllersConfig, ControllersEmulationMode, GazeInputSource, + BodyTrackingSinkConfig, CodecType, ControllersConfig, ControllersEmulationMode, }; use foveated_encoding::EyeTrackedFoveation; use std::{ @@ -257,9 +257,7 @@ fn spawn_event_loop(events_receiver: mpsc::Receiver) { .video .foveated_encoding .as_option() - .is_some_and(|config| { - config.gaze_input_source != GazeInputSource::None - }) + .is_some_and(|config| config.eye_tracking) }) .map(|params| { EyeTrackedFoveation::new(params, config.transcoding_view_resolution) diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 5b5a220b79..56b2ef7a9f 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -460,9 +460,12 @@ pub struct FoveatedEncodingConfig { #[schema(strings(help = "Force enable on smartphone clients"))] pub force_enable: bool, - #[schema(strings(display_name = "Eye tracking input"))] + #[schema(strings( + display_name = "Use eye tracking", + help = "Move the foveation center using eye tracking. Turn this off to keep fixed foveated encoding. Configure external input under Headset > Face tracking > Eye tracking OSC override." + ))] #[schema(flag = "steamvr-restart")] - pub gaze_input_source: GazeInputSource, + pub eye_tracking: bool, #[schema(strings( display_name = "Center region size", @@ -474,7 +477,7 @@ pub struct FoveatedEncodingConfig { #[schema(strings( display_name = "Center shift", - help = "Horizontal and vertical shift, in X/Y order. Used as a static fallback; overridden, not added to, when eye-tracking centers are available." + help = "Horizontal and vertical shift, in X/Y order. Used when Use eye tracking is off, or as a fallback when eye tracking is unavailable. Eye-tracking centers replace this shift rather than adding to it." ))] #[schema(gui(slider(min = -1.0, max = 1.0, step = 0.01)))] #[schema(flag = "steamvr-restart")] @@ -486,22 +489,6 @@ pub struct FoveatedEncodingConfig { pub edge_ratio: [f32; 2], } -#[derive(SettingsSchema, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Hash)] -pub enum GazeInputSource { - #[schema(strings(help = "Use the configured static foveation center"))] - None, - #[schema(strings(display_name = "Headset eye tracking"))] - Headset, - #[schema(strings( - display_name = "External OSC", - help = "Receive head-local combined gaze over OSC/UDP on localhost." - ))] - ExternalOsc { - #[schema(strings(display_name = "UDP port"))] - port: u16, - }, -} - #[repr(C)] #[derive(SettingsSchema, Clone, Copy, Serialize, Deserialize, Pod, Zeroable)] pub struct ColorCorrectionConfig { @@ -927,16 +914,31 @@ pub enum FaceTrackingSourcesConfig { #[derive(SettingsSchema, Serialize, Deserialize, Clone)] pub enum FaceTrackingSinkConfig { + #[schema(strings(display_name = "Disabled"))] + None, #[schema(strings(display_name = "VRChat Eye OSC"))] VrchatEyeOsc { port: u16 }, #[schema(strings(display_name = "VRCFaceTracking"))] VrcFaceTracking, } +#[derive(SettingsSchema, Serialize, Deserialize, Clone)] +pub struct OscPort { + pub port: u16, +} + #[derive(SettingsSchema, Serialize, Deserialize, Clone)] #[schema(collapsible)] pub struct FaceTrackingConfig { pub sources: FaceTrackingSourcesConfig, + #[schema(strings( + display_name = "Eye tracking OSC override", + help = "Use external combined gaze received over OSC/UDP on localhost instead of headset eye directions. This input is shared by foveated encoding and social output. Turn this off to use headset eye tracking." + ))] + pub eye_tracking_osc_override: Switch, + #[schema(strings( + help = "Forward eye and face tracking to another application. Select Disabled to turn off forwarding without disabling eye tracking for foveated encoding." + ))] pub sink: FaceTrackingSinkConfig, } @@ -1889,10 +1891,7 @@ pub fn session_settings_default() -> SettingsDefault { content: FoveatedEncodingConfigDefault { gui_collapsed: true, force_enable: false, - gaze_input_source: GazeInputSourceDefault { - ExternalOsc: GazeInputSourceExternalOscDefault { port: 9945 }, - variant: GazeInputSourceDefaultVariant::None, - }, + eye_tracking: false, center_size: ArrayDefault { gui_collapsed: false, content: [0.45, 0.4], @@ -2005,6 +2004,10 @@ pub fn session_settings_default() -> SettingsDefault { sources: FaceTrackingSourcesConfigDefault { variant: FaceTrackingSourcesConfigDefaultVariant::PreferFullFaceTracking, }, + eye_tracking_osc_override: SwitchDefault { + enabled: false, + content: OscPortDefault { port: 9945 }, + }, sink: FaceTrackingSinkConfigDefault { VrchatEyeOsc: FaceTrackingSinkConfigVrchatEyeOscDefault { port: 9000 }, variant: FaceTrackingSinkConfigDefaultVariant::VrchatEyeOsc, From a7be4e8abbdc487a0f5fe60a8cdd8f93bdd5b7d1 Mon Sep 17 00:00:00 2001 From: JellyfishKnight Date: Tue, 29 Sep 2026 13:19:06 +0800 Subject: [PATCH 3/4] refactor(tracking): group social and foveation outputs --- alvr/client_openxr/src/interaction.rs | 20 ++------- .../presets/builtin_schema.rs | 4 +- alvr/server_core/src/connection.rs | 6 ++- alvr/server_core/src/tracking/face.rs | 29 +++++++------ alvr/server_core/src/tracking/mod.rs | 19 ++++++--- alvr/server_openvr/src/lib.rs | 6 +-- alvr/session/src/settings.rs | 42 ++++++++++++------- 7 files changed, 71 insertions(+), 55 deletions(-) diff --git a/alvr/client_openxr/src/interaction.rs b/alvr/client_openxr/src/interaction.rs index df67bd3456..af4ecc5689 100644 --- a/alvr/client_openxr/src/interaction.rs +++ b/alvr/client_openxr/src/interaction.rs @@ -153,22 +153,10 @@ impl InteractionSourcesConfig { face_tracking.is_some_and(|c| c.eye_tracking_osc_override.enabled()); Self { - face_tracking: face_tracking - .map(|c| c.sources.clone()) - .or_else(|| { - // Foveation requests eye input locally, without enabling face tracking output. - config - .settings - .video - .foveated_encoding - .as_option() - .is_some_and(|c| c.eye_tracking) - .then_some(FaceTrackingSourcesConfig::PreferEyeTrackingOnly) - }) - .filter(|sources| { - !eye_tracking_osc_override - || *sources == FaceTrackingSourcesConfig::PreferFullFaceTracking - }), + face_tracking: face_tracking.map(|c| c.sources.clone()).filter(|sources| { + !eye_tracking_osc_override + || *sources == FaceTrackingSourcesConfig::PreferFullFaceTracking + }), body_tracking: config .settings .headset diff --git a/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs b/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs index bb9f2de6c4..76876978ff 100644 --- a/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs +++ b/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs @@ -466,7 +466,7 @@ pub fn eye_face_tracking_schema() -> PresetSchemaNode { modifiers: vec![ bool_modifier("session_settings.headset.face_tracking.enabled", true), string_modifier( - "session_settings.headset.face_tracking.content.sink.variant", + "session_settings.headset.face_tracking.content.sink.social_presence.variant", "VrchatEyeOsc", ), ], @@ -477,7 +477,7 @@ pub fn eye_face_tracking_schema() -> PresetSchemaNode { modifiers: vec![ bool_modifier("session_settings.headset.face_tracking.enabled", true), string_modifier( - "session_settings.headset.face_tracking.content.sink.variant", + "session_settings.headset.face_tracking.content.sink.social_presence.variant", "VrcFaceTracking", ), ], diff --git a/alvr/server_core/src/connection.rs b/alvr/server_core/src/connection.rs index d935c7fcca..6fe17ad15d 100644 --- a/alvr/server_core/src/connection.rs +++ b/alvr/server_core/src/connection.rs @@ -151,7 +151,11 @@ pub fn compute_restart_settings_hash( let mut foveation_edge_ratio_y = 0.0_f32; let enable_foveated_encoding = if let Switch::Enabled(config) = &settings.video.foveated_encoding { - foveation_eye_tracking = config.eye_tracking; + foveation_eye_tracking = settings + .headset + .face_tracking + .as_option() + .is_some_and(|config| config.sink.eye_tracked_foveated_encoding); [foveation_center_size_x, foveation_center_size_y] = config.center_size; [foveation_center_shift_x, foveation_center_shift_y] = config.center_shift; [foveation_edge_ratio_x, foveation_edge_ratio_y] = config.edge_ratio; diff --git a/alvr/server_core/src/tracking/face.rs b/alvr/server_core/src/tracking/face.rs index 09831c6499..4fb1840f18 100644 --- a/alvr/server_core/src/tracking/face.rs +++ b/alvr/server_core/src/tracking/face.rs @@ -1,6 +1,9 @@ -use alvr_common::{anyhow::Result, glam::EulerRot}; +use alvr_common::{ + anyhow::{Result, bail}, + glam::EulerRot, +}; use alvr_packets::{FaceData, FaceExpressions}; -use alvr_session::FaceTrackingSinkConfig; +use alvr_session::FaceTrackingSocialPresenceSinkConfig; use rosc::{OscMessage, OscPacket, OscType}; use std::{f32::consts::PI, net::UdpSocket}; @@ -9,27 +12,29 @@ const RAD_TO_DEG: f32 = 180.0 / PI; const VRCFT_PORT: u16 = 0xA1F7; pub struct FaceTrackingSink { - config: FaceTrackingSinkConfig, + config: FaceTrackingSocialPresenceSinkConfig, socket: UdpSocket, packet_buffer: Vec, } impl FaceTrackingSink { - pub fn new(config: FaceTrackingSinkConfig, local_osc_port: u16) -> Result> { + pub fn new(config: FaceTrackingSocialPresenceSinkConfig, local_osc_port: u16) -> Result { let port = match config { - FaceTrackingSinkConfig::None => return Ok(None), - FaceTrackingSinkConfig::VrchatEyeOsc { port } => port, - FaceTrackingSinkConfig::VrcFaceTracking => VRCFT_PORT, + FaceTrackingSocialPresenceSinkConfig::None => { + bail!("Cannot create a disabled social presence sink") + } + FaceTrackingSocialPresenceSinkConfig::VrchatEyeOsc { port } => port, + FaceTrackingSocialPresenceSinkConfig::VrcFaceTracking => VRCFT_PORT, }; let socket = UdpSocket::bind(format!("127.0.0.1:{local_osc_port}"))?; socket.connect(format!("127.0.0.1:{port}"))?; - Ok(Some(Self { + Ok(Self { config, socket, packet_buffer: vec![], - })) + }) } fn send_osc_message(&self, path: &str, args: Vec) { @@ -54,8 +59,8 @@ impl FaceTrackingSink { pub fn send_tracking(&mut self, face_data: &FaceData) { match self.config { - FaceTrackingSinkConfig::None => (), - FaceTrackingSinkConfig::VrchatEyeOsc { .. } => { + FaceTrackingSocialPresenceSinkConfig::None => (), + FaceTrackingSocialPresenceSinkConfig::VrchatEyeOsc { .. } => { if let [Some(left), Some(right)] = face_data.eyes_social { let (left_pitch, left_yaw, _) = left.to_euler(EulerRot::XYZ); let (right_pitch, right_yaw, _) = right.to_euler(EulerRot::XYZ); @@ -102,7 +107,7 @@ impl FaceTrackingSink { ); } } - FaceTrackingSinkConfig::VrcFaceTracking => { + FaceTrackingSocialPresenceSinkConfig::VrcFaceTracking => { self.packet_buffer.clear(); if let [Some(left_quat), Some(right_quat)] = face_data.eyes_social { diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index 64155bec47..9a3208e928 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -21,8 +21,8 @@ use alvr_common::{ use alvr_events::{EventType, TrackingEvent}; use alvr_packets::TrackingData; use alvr_session::{ - BodyTrackingConfig, HeadsetConfig, RecenteringMode, Settings, VMCConfig, - settings_schema::Switch, + BodyTrackingConfig, FaceTrackingSocialPresenceSinkConfig, HeadsetConfig, RecenteringMode, + Settings, VMCConfig, settings_schema::Switch, }; use alvr_sockets::StreamReceiver; use eye_gaze::EyeGazeReceiver; @@ -318,10 +318,19 @@ pub fn tracking_loop( .headset .face_tracking .into_option() + .filter(|config| { + !matches!( + config.sink.social_presence, + FaceTrackingSocialPresenceSinkConfig::None + ) + }) .and_then(|config| { - FaceTrackingSink::new(config.sink, initial_settings.connection.osc_local_port) - .ok() - .flatten() + FaceTrackingSink::new( + config.sink.social_presence, + initial_settings.connection.osc_local_port, + ) + .inspect_err(|error| warn!("Failed to initialize social presence sink: {error}")) + .ok() }); let mut body_tracking_sink = initial_settings diff --git a/alvr/server_openvr/src/lib.rs b/alvr/server_openvr/src/lib.rs index 068d2b300b..f0ad382318 100644 --- a/alvr/server_openvr/src/lib.rs +++ b/alvr/server_openvr/src/lib.rs @@ -254,10 +254,10 @@ fn spawn_event_loop(events_receiver: mpsc::Receiver) { .foveated_encoding .filter(|_| { alvr_server_core::settings() - .video - .foveated_encoding + .headset + .face_tracking .as_option() - .is_some_and(|config| config.eye_tracking) + .is_some_and(|config| config.sink.eye_tracked_foveated_encoding) }) .map(|params| { EyeTrackedFoveation::new(params, config.transcoding_view_resolution) diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 56b2ef7a9f..6a3df2e1fe 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -460,13 +460,6 @@ pub struct FoveatedEncodingConfig { #[schema(strings(help = "Force enable on smartphone clients"))] pub force_enable: bool, - #[schema(strings( - display_name = "Use eye tracking", - help = "Move the foveation center using eye tracking. Turn this off to keep fixed foveated encoding. Configure external input under Headset > Face tracking > Eye tracking OSC override." - ))] - #[schema(flag = "steamvr-restart")] - pub eye_tracking: bool, - #[schema(strings( display_name = "Center region size", help = "Width and height, in X/Y order" @@ -477,7 +470,7 @@ pub struct FoveatedEncodingConfig { #[schema(strings( display_name = "Center shift", - help = "Horizontal and vertical shift, in X/Y order. Used when Use eye tracking is off, or as a fallback when eye tracking is unavailable. Eye-tracking centers replace this shift rather than adding to it." + help = "Horizontal and vertical shift, in X/Y order. Used for fixed foveated encoding and when eye tracking is unavailable. Eye-tracking centers replace this shift rather than adding to it." ))] #[schema(gui(slider(min = -1.0, max = 1.0, step = 0.01)))] #[schema(flag = "steamvr-restart")] @@ -702,7 +695,8 @@ pub struct VideoConfig { #[schema(strings( notice = r"Disabling foveated encoding may result in significantly higher encode/decode times and stuttering, or even crashing. -If you want to reduce the amount of pixelation on the edges, increase the center region width and height" +If you want to reduce the amount of pixelation on the edges, increase the center region width and height. +For eye-tracked foveation, enable Headset > Face tracking > Sink > Eye-tracked foveated encoding." ))] #[schema(flag = "steamvr-restart")] pub foveated_encoding: Switch, @@ -913,7 +907,7 @@ pub enum FaceTrackingSourcesConfig { } #[derive(SettingsSchema, Serialize, Deserialize, Clone)] -pub enum FaceTrackingSinkConfig { +pub enum FaceTrackingSocialPresenceSinkConfig { #[schema(strings(display_name = "Disabled"))] None, #[schema(strings(display_name = "VRChat Eye OSC"))] @@ -922,6 +916,20 @@ pub enum FaceTrackingSinkConfig { VrcFaceTracking, } +#[derive(SettingsSchema, Serialize, Deserialize, Clone)] +pub struct FaceTrackingSinkConfig { + #[schema(strings( + help = "Forward eye and face tracking to another application. Disabled turns off forwarding without disabling eye-tracked foveated encoding." + ))] + pub social_presence: FaceTrackingSocialPresenceSinkConfig, + #[schema(strings( + display_name = "Eye-tracked foveated encoding", + help = "Use the selected eye tracking input to move the foveation center. Requires Video > Foveated encoding to be enabled. Turn this off to keep fixed foveated encoding." + ))] + #[schema(flag = "steamvr-restart")] + pub eye_tracked_foveated_encoding: bool, +} + #[derive(SettingsSchema, Serialize, Deserialize, Clone)] pub struct OscPort { pub port: u16, @@ -936,9 +944,6 @@ pub struct FaceTrackingConfig { help = "Use external combined gaze received over OSC/UDP on localhost instead of headset eye directions. This input is shared by foveated encoding and social output. Turn this off to use headset eye tracking." ))] pub eye_tracking_osc_override: Switch, - #[schema(strings( - help = "Forward eye and face tracking to another application. Select Disabled to turn off forwarding without disabling eye tracking for foveated encoding." - ))] pub sink: FaceTrackingSinkConfig, } @@ -1891,7 +1896,6 @@ pub fn session_settings_default() -> SettingsDefault { content: FoveatedEncodingConfigDefault { gui_collapsed: true, force_enable: false, - eye_tracking: false, center_size: ArrayDefault { gui_collapsed: false, content: [0.45, 0.4], @@ -2009,8 +2013,14 @@ pub fn session_settings_default() -> SettingsDefault { content: OscPortDefault { port: 9945 }, }, sink: FaceTrackingSinkConfigDefault { - VrchatEyeOsc: FaceTrackingSinkConfigVrchatEyeOscDefault { port: 9000 }, - variant: FaceTrackingSinkConfigDefaultVariant::VrchatEyeOsc, + social_presence: FaceTrackingSocialPresenceSinkConfigDefault { + VrchatEyeOsc: FaceTrackingSocialPresenceSinkConfigVrchatEyeOscDefault { + port: 9000, + }, + variant: + FaceTrackingSocialPresenceSinkConfigDefaultVariant::VrchatEyeOsc, + }, + eye_tracked_foveated_encoding: false, }, }, }, From 3ab677cfaa764784c70e233f83e0b835cd97976b Mon Sep 17 00:00:00 2001 From: JellyfishKnight Date: Tue, 29 Sep 2026 20:40:17 +0800 Subject: [PATCH 4/4] refactor(tracking): make social presence output optional --- .../presets/builtin_schema.rs | 12 +++++++++-- alvr/server_core/src/tracking/face.rs | 9 +-------- alvr/server_core/src/tracking/mod.rs | 20 ++++++------------- alvr/session/src/settings.rs | 20 ++++++++++--------- 4 files changed, 28 insertions(+), 33 deletions(-) diff --git a/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs b/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs index 76876978ff..a3038fb6b6 100644 --- a/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs +++ b/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs @@ -465,8 +465,12 @@ pub fn eye_face_tracking_schema() -> PresetSchemaNode { display_name: "VRChat Eye OSC".into(), modifiers: vec![ bool_modifier("session_settings.headset.face_tracking.enabled", true), + bool_modifier( + "session_settings.headset.face_tracking.content.sink.social_presence.set", + true, + ), string_modifier( - "session_settings.headset.face_tracking.content.sink.social_presence.variant", + "session_settings.headset.face_tracking.content.sink.social_presence.content.variant", "VrchatEyeOsc", ), ], @@ -476,8 +480,12 @@ pub fn eye_face_tracking_schema() -> PresetSchemaNode { display_name: "VRCFaceTracking".into(), modifiers: vec![ bool_modifier("session_settings.headset.face_tracking.enabled", true), + bool_modifier( + "session_settings.headset.face_tracking.content.sink.social_presence.set", + true, + ), string_modifier( - "session_settings.headset.face_tracking.content.sink.social_presence.variant", + "session_settings.headset.face_tracking.content.sink.social_presence.content.variant", "VrcFaceTracking", ), ], diff --git a/alvr/server_core/src/tracking/face.rs b/alvr/server_core/src/tracking/face.rs index 4fb1840f18..643bd34006 100644 --- a/alvr/server_core/src/tracking/face.rs +++ b/alvr/server_core/src/tracking/face.rs @@ -1,7 +1,4 @@ -use alvr_common::{ - anyhow::{Result, bail}, - glam::EulerRot, -}; +use alvr_common::{anyhow::Result, glam::EulerRot}; use alvr_packets::{FaceData, FaceExpressions}; use alvr_session::FaceTrackingSocialPresenceSinkConfig; use rosc::{OscMessage, OscPacket, OscType}; @@ -20,9 +17,6 @@ pub struct FaceTrackingSink { impl FaceTrackingSink { pub fn new(config: FaceTrackingSocialPresenceSinkConfig, local_osc_port: u16) -> Result { let port = match config { - FaceTrackingSocialPresenceSinkConfig::None => { - bail!("Cannot create a disabled social presence sink") - } FaceTrackingSocialPresenceSinkConfig::VrchatEyeOsc { port } => port, FaceTrackingSocialPresenceSinkConfig::VrcFaceTracking => VRCFT_PORT, }; @@ -59,7 +53,6 @@ impl FaceTrackingSink { pub fn send_tracking(&mut self, face_data: &FaceData) { match self.config { - FaceTrackingSocialPresenceSinkConfig::None => (), FaceTrackingSocialPresenceSinkConfig::VrchatEyeOsc { .. } => { if let [Some(left), Some(right)] = face_data.eyes_social { let (left_pitch, left_yaw, _) = left.to_euler(EulerRot::XYZ); diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index 9a3208e928..5352b9fe20 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -21,8 +21,8 @@ use alvr_common::{ use alvr_events::{EventType, TrackingEvent}; use alvr_packets::TrackingData; use alvr_session::{ - BodyTrackingConfig, FaceTrackingSocialPresenceSinkConfig, HeadsetConfig, RecenteringMode, - Settings, VMCConfig, settings_schema::Switch, + BodyTrackingConfig, HeadsetConfig, RecenteringMode, Settings, VMCConfig, + settings_schema::Switch, }; use alvr_sockets::StreamReceiver; use eye_gaze::EyeGazeReceiver; @@ -318,19 +318,11 @@ pub fn tracking_loop( .headset .face_tracking .into_option() - .filter(|config| { - !matches!( - config.sink.social_presence, - FaceTrackingSocialPresenceSinkConfig::None - ) - }) + .and_then(|config| config.sink.social_presence) .and_then(|config| { - FaceTrackingSink::new( - config.sink.social_presence, - initial_settings.connection.osc_local_port, - ) - .inspect_err(|error| warn!("Failed to initialize social presence sink: {error}")) - .ok() + FaceTrackingSink::new(config, initial_settings.connection.osc_local_port) + .inspect_err(|error| warn!("Failed to initialize social presence sink: {error}")) + .ok() }); let mut body_tracking_sink = initial_settings diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 6a3df2e1fe..e0c2fb946f 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -908,8 +908,6 @@ pub enum FaceTrackingSourcesConfig { #[derive(SettingsSchema, Serialize, Deserialize, Clone)] pub enum FaceTrackingSocialPresenceSinkConfig { - #[schema(strings(display_name = "Disabled"))] - None, #[schema(strings(display_name = "VRChat Eye OSC"))] VrchatEyeOsc { port: u16 }, #[schema(strings(display_name = "VRCFaceTracking"))] @@ -919,9 +917,9 @@ pub enum FaceTrackingSocialPresenceSinkConfig { #[derive(SettingsSchema, Serialize, Deserialize, Clone)] pub struct FaceTrackingSinkConfig { #[schema(strings( - help = "Forward eye and face tracking to another application. Disabled turns off forwarding without disabling eye-tracked foveated encoding." + help = "Forward eye and face tracking to another application. Select Default to disable forwarding without disabling eye-tracked foveated encoding." ))] - pub social_presence: FaceTrackingSocialPresenceSinkConfig, + pub social_presence: Option, #[schema(strings( display_name = "Eye-tracked foveated encoding", help = "Use the selected eye tracking input to move the foveation center. Requires Video > Foveated encoding to be enabled. Turn this off to keep fixed foveated encoding." @@ -2013,12 +2011,16 @@ pub fn session_settings_default() -> SettingsDefault { content: OscPortDefault { port: 9945 }, }, sink: FaceTrackingSinkConfigDefault { - social_presence: FaceTrackingSocialPresenceSinkConfigDefault { - VrchatEyeOsc: FaceTrackingSocialPresenceSinkConfigVrchatEyeOscDefault { - port: 9000, + social_presence: OptionalDefault { + set: true, + content: FaceTrackingSocialPresenceSinkConfigDefault { + VrchatEyeOsc: + FaceTrackingSocialPresenceSinkConfigVrchatEyeOscDefault { + port: 9000, + }, + variant: + FaceTrackingSocialPresenceSinkConfigDefaultVariant::VrchatEyeOsc, }, - variant: - FaceTrackingSocialPresenceSinkConfigDefaultVariant::VrchatEyeOsc, }, eye_tracked_foveated_encoding: false, },