diff --git a/alvr/client_openxr/src/interaction.rs b/alvr/client_openxr/src/interaction.rs index db32dd75c5..af4ecc5689 100644 --- a/alvr/client_openxr/src/interaction.rs +++ b/alvr/client_openxr/src/interaction.rs @@ -11,9 +11,7 @@ use alvr_common::{ }; use alvr_graphics::HandData; use alvr_packets::{ButtonEntry, ButtonValue, ClientStreamConfig, FaceData, FaceExpressions}; -use alvr_session::{ - BodyTrackingBDConfig, BodyTrackingSourcesConfig, FaceTrackingSourcesConfig, GazeInputSource, -}; +use alvr_session::{BodyTrackingBDConfig, BodyTrackingSourcesConfig, FaceTrackingSourcesConfig}; use openxr as xr; use std::{ collections::{HashMap, HashSet}, @@ -150,23 +148,15 @@ pub struct InteractionSourcesConfig { impl InteractionSourcesConfig { pub fn new(config: &ClientStreamConfig) -> Self { + let face_tracking = config.settings.headset.face_tracking.as_option(); + let eye_tracking_osc_override = + face_tracking.is_some_and(|c| c.eye_tracking_osc_override.enabled()); + Self { - face_tracking: config - .settings - .headset - .face_tracking - .as_option() - .map(|c| c.sources.clone()) - .or_else(|| { - // Foveation requests eye input locally, without enabling face tracking output. - config - .settings - .video - .foveated_encoding - .as_option() - .is_some_and(|c| c.gaze_input_source == GazeInputSource::Headset) - .then_some(FaceTrackingSourcesConfig::PreferEyeTrackingOnly) - }), + face_tracking: face_tracking.map(|c| c.sources.clone()).filter(|sources| { + !eye_tracking_osc_override + || *sources == FaceTrackingSourcesConfig::PreferFullFaceTracking + }), body_tracking: config .settings .headset diff --git a/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs b/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs index bb9f2de6c4..a3038fb6b6 100644 --- a/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs +++ b/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs @@ -465,8 +465,12 @@ pub fn eye_face_tracking_schema() -> PresetSchemaNode { display_name: "VRChat Eye OSC".into(), modifiers: vec![ bool_modifier("session_settings.headset.face_tracking.enabled", true), + bool_modifier( + "session_settings.headset.face_tracking.content.sink.social_presence.set", + true, + ), string_modifier( - "session_settings.headset.face_tracking.content.sink.variant", + "session_settings.headset.face_tracking.content.sink.social_presence.content.variant", "VrchatEyeOsc", ), ], @@ -476,8 +480,12 @@ pub fn eye_face_tracking_schema() -> PresetSchemaNode { display_name: "VRCFaceTracking".into(), modifiers: vec![ bool_modifier("session_settings.headset.face_tracking.enabled", true), + bool_modifier( + "session_settings.headset.face_tracking.content.sink.social_presence.set", + true, + ), string_modifier( - "session_settings.headset.face_tracking.content.sink.variant", + "session_settings.headset.face_tracking.content.sink.social_presence.content.variant", "VrcFaceTracking", ), ], diff --git a/alvr/server_core/src/connection.rs b/alvr/server_core/src/connection.rs index 29ef0d8bc1..6fe17ad15d 100644 --- a/alvr/server_core/src/connection.rs +++ b/alvr/server_core/src/connection.rs @@ -26,8 +26,8 @@ use alvr_packets::{ VIDEO, VideoPacketHeader, }; use alvr_session::{ - BodyTrackingSinkConfig, CodecType, ControllersEmulationMode, FrameSize, GazeInputSource, - H264Profile, Settings, SocketProtocol, SteamvrHmdInitConfig, + BodyTrackingSinkConfig, CodecType, ControllersEmulationMode, FrameSize, H264Profile, Settings, + SocketProtocol, SteamvrHmdInitConfig, }; use alvr_sockets::{ CONTROL_PORT, KEEPALIVE_INTERVAL, KEEPALIVE_TIMEOUT, ProtoControlSocket, SocketConnection, @@ -142,7 +142,7 @@ pub fn compute_restart_settings_hash( .map(|c| c.sources.meta.prefer_full_body) .unwrap_or(false); - let mut foveation_gaze_input_source = GazeInputSource::None; + let mut foveation_eye_tracking = false; let mut foveation_center_size_x = 0.0_f32; let mut foveation_center_size_y = 0.0_f32; let mut foveation_center_shift_x = 0.0_f32; @@ -151,7 +151,11 @@ pub fn compute_restart_settings_hash( let mut foveation_edge_ratio_y = 0.0_f32; let enable_foveated_encoding = if let Switch::Enabled(config) = &settings.video.foveated_encoding { - foveation_gaze_input_source = config.gaze_input_source; + foveation_eye_tracking = settings + .headset + .face_tracking + .as_option() + .is_some_and(|config| config.sink.eye_tracked_foveated_encoding); [foveation_center_size_x, foveation_center_size_y] = config.center_size; [foveation_center_shift_x, foveation_center_shift_y] = config.center_shift; [foveation_edge_ratio_x, foveation_edge_ratio_y] = config.edge_ratio; @@ -244,7 +248,7 @@ pub fn compute_restart_settings_hash( nvenc.enable_weighted_prediction.hash(&mut h); // Foveated encoding enable_foveated_encoding.hash(&mut h); - foveation_gaze_input_source.hash(&mut h); + foveation_eye_tracking.hash(&mut h); foveation_center_size_x.to_bits().hash(&mut h); foveation_center_size_y.to_bits().hash(&mut h); foveation_center_shift_x.to_bits().hash(&mut h); diff --git a/alvr/server_core/src/tracking/eye_gaze.rs b/alvr/server_core/src/tracking/eye_gaze.rs new file mode 100644 index 0000000000..0e462c21c7 --- /dev/null +++ b/alvr/server_core/src/tracking/eye_gaze.rs @@ -0,0 +1,143 @@ +use alvr_common::{ + anyhow::Result, + glam::{Quat, Vec3}, +}; +use rosc::{OscPacket, OscType}; +use std::{ + io::ErrorKind, + net::{Ipv4Addr, UdpSocket}, + time::{Duration, Instant}, +}; + +const GAZE_ADDRESS: &str = "/alvr/eye/combined/v1"; +const MAX_SAMPLE_AGE: Duration = Duration::from_millis(100); +const SEQUENCE_RESET_TIMEOUT: Duration = Duration::from_secs(1); +const MAX_DATAGRAM_SIZE: usize = 256; +const MAX_PACKETS_PER_POLL: usize = 64; + +pub struct EyeGazeReceiver { + socket: UdpSocket, + last_packet: Option<(u32, Instant)>, + last_tracking_timestamp: Option, + last_polled_at: Instant, + discard_queued: bool, +} + +impl EyeGazeReceiver { + pub fn new(port: u16) -> Result { + let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, port))?; + socket.set_nonblocking(true)?; + + Ok(Self { + socket, + last_packet: None, + last_tracking_timestamp: None, + last_polled_at: Instant::now(), + discard_queued: false, + }) + } + + // Consume each observation once. Missing input must not refresh the foveation hold timer. + pub fn receive(&mut self, tracking_timestamp: Duration) -> Result> { + if self + .last_tracking_timestamp + .is_some_and(|previous| tracking_timestamp <= previous) + { + return Ok(None); + } + self.last_tracking_timestamp = Some(tracking_timestamp); + + let now = Instant::now(); + // After a tracking stall, drain the socket before accepting fresh observations again. + self.discard_queued |= now.duration_since(self.last_polled_at) > MAX_SAMPLE_AGE; + self.last_polled_at = now; + let mut gaze = None; + // One extra byte lets us reject oversized datagrams even if recv truncates them. + let mut buffer = [0; MAX_DATAGRAM_SIZE + 1]; + + for _ in 0..MAX_PACKETS_PER_POLL { + let size = match self.socket.recv(&mut buffer) { + Ok(size) => size, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + self.discard_queued = false; + + return Ok(gaze); + } + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + // Windows reports an oversized datagram instead of returning truncated bytes. + Err(error) if error.raw_os_error() == Some(10040) => continue, + Err(error) => return Err(error.into()), + }; + if self.discard_queued || size > MAX_DATAGRAM_SIZE { + continue; + } + let Some(packet) = buffer.get(..size) else { + continue; + }; + // This endpoint accepts single messages, not scheduled or recursive OSC bundles. + if packet.first() != Some(&b'/') { + continue; + } + let Ok((remaining, OscPacket::Message(message))) = rosc::decoder::decode_udp(packet) + else { + continue; + }; + if !remaining.is_empty() || message.addr != GAZE_ADDRESS { + continue; + } + let [ + OscType::Int(sequence), + OscType::Int(valid), + OscType::Int(age_us), + OscType::Float(x), + OscType::Float(y), + OscType::Float(z), + OscType::Float(w), + ] = message.args.as_slice() + else { + continue; + }; + if !matches!(valid, 0 | 1) + || *age_us < 0 + || Duration::from_micros(*age_us as u64) > MAX_SAMPLE_AGE + { + continue; + } + + let orientation = if *valid == 1 { + let orientation = Quat::from_xyzw(*x, *y, *z, *w); + let length_squared = orientation.length_squared(); + if !length_squared.is_finite() + || length_squared <= f32::EPSILON + || (orientation.normalize() * Vec3::NEG_Z).z >= -f32::EPSILON + { + continue; + } + + // Social output also consumes this quaternion and expects a unit rotation. + Some(orientation.normalize()) + } else { + None + }; + // Interpret the OSC signed integer as the bit pattern of a wrapping u32 counter. + let sequence = *sequence as u32; + if let Some((previous, received_at)) = self.last_packet { + let distance = sequence.wrapping_sub(previous); + if distance == 0 + || (now.duration_since(received_at) <= SEQUENCE_RESET_TIMEOUT + && distance >= 1 << 31) + { + continue; + } + } + + self.last_packet = Some((sequence, now)); + gaze = orientation; + } + + // Do not block head/controller tracking or use a potentially backlogged sample in a flood. + self.discard_queued = true; + + Ok(None) + } +} diff --git a/alvr/server_core/src/tracking/face.rs b/alvr/server_core/src/tracking/face.rs index 9b9f345091..643bd34006 100644 --- a/alvr/server_core/src/tracking/face.rs +++ b/alvr/server_core/src/tracking/face.rs @@ -1,6 +1,6 @@ use alvr_common::{anyhow::Result, glam::EulerRot}; use alvr_packets::{FaceData, FaceExpressions}; -use alvr_session::FaceTrackingSinkConfig; +use alvr_session::FaceTrackingSocialPresenceSinkConfig; use rosc::{OscMessage, OscPacket, OscType}; use std::{f32::consts::PI, net::UdpSocket}; @@ -9,16 +9,16 @@ const RAD_TO_DEG: f32 = 180.0 / PI; const VRCFT_PORT: u16 = 0xA1F7; pub struct FaceTrackingSink { - config: FaceTrackingSinkConfig, + config: FaceTrackingSocialPresenceSinkConfig, socket: UdpSocket, packet_buffer: Vec, } impl FaceTrackingSink { - pub fn new(config: FaceTrackingSinkConfig, local_osc_port: u16) -> Result { + pub fn new(config: FaceTrackingSocialPresenceSinkConfig, local_osc_port: u16) -> Result { let port = match config { - FaceTrackingSinkConfig::VrchatEyeOsc { port } => port, - FaceTrackingSinkConfig::VrcFaceTracking => VRCFT_PORT, + FaceTrackingSocialPresenceSinkConfig::VrchatEyeOsc { port } => port, + FaceTrackingSocialPresenceSinkConfig::VrcFaceTracking => VRCFT_PORT, }; let socket = UdpSocket::bind(format!("127.0.0.1:{local_osc_port}"))?; @@ -53,7 +53,7 @@ impl FaceTrackingSink { pub fn send_tracking(&mut self, face_data: &FaceData) { match self.config { - FaceTrackingSinkConfig::VrchatEyeOsc { .. } => { + FaceTrackingSocialPresenceSinkConfig::VrchatEyeOsc { .. } => { if let [Some(left), Some(right)] = face_data.eyes_social { let (left_pitch, left_yaw, _) = left.to_euler(EulerRot::XYZ); let (right_pitch, right_yaw, _) = right.to_euler(EulerRot::XYZ); @@ -100,7 +100,7 @@ impl FaceTrackingSink { ); } } - FaceTrackingSinkConfig::VrcFaceTracking => { + FaceTrackingSocialPresenceSinkConfig::VrcFaceTracking => { self.packet_buffer.clear(); if let [Some(left_quat), Some(right_quat)] = face_data.eyes_social { diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index 07e8f53453..5352b9fe20 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -1,4 +1,5 @@ mod body; +mod eye_gaze; mod face; mod vmc; @@ -15,7 +16,7 @@ use crate::{ use alvr_common::{ ConnectionError, DEVICE_ID_TO_PATH, DeviceMotion, Pose, ViewParams, glam::{Quat, Vec3}, - inputs as inp, + inputs as inp, warn, }; use alvr_events::{EventType, TrackingEvent}; use alvr_packets::TrackingData; @@ -24,6 +25,7 @@ use alvr_session::{ settings_schema::Switch, }; use alvr_sockets::StreamReceiver; +use eye_gaze::EyeGazeReceiver; use std::{ cmp::Ordering, collections::{HashMap, VecDeque}, @@ -300,12 +302,27 @@ pub fn tracking_loop( ) }); + let eye_tracking_osc_port = initial_settings + .headset + .face_tracking + .as_option() + .and_then(|config| config.eye_tracking_osc_override.as_option()) + .map(|config| config.port); + let mut eye_gaze_receiver = eye_tracking_osc_port.and_then(|port| { + EyeGazeReceiver::new(port) + .inspect_err(|error| warn!("Failed to bind external gaze OSC port {port}: {error}")) + .ok() + }); + let mut face_tracking_sink = initial_settings .headset .face_tracking .into_option() + .and_then(|config| config.sink.social_presence) .and_then(|config| { - FaceTrackingSink::new(config.sink, initial_settings.connection.osc_local_port).ok() + FaceTrackingSink::new(config, initial_settings.connection.osc_local_port) + .inspect_err(|error| warn!("Failed to initialize social presence sink: {error}")) + .ok() }); let mut body_tracking_sink = initial_settings @@ -334,6 +351,25 @@ pub fn tracking_loop( let timestamp = tracking.poll_timestamp; + if eye_tracking_osc_port.is_some() { + // Select one eye input before foveation and social output consume it. Do not fall + // back to headset eyes when OSC input is missing, invalid or unavailable. + tracking.face.eyes_social = [None; 2]; + tracking.face.eyes_combined = if let Some(receiver) = &mut eye_gaze_receiver { + match receiver.receive(timestamp) { + Ok(gaze) => gaze, + Err(error) => { + warn!("External gaze OSC receiver stopped: {error}"); + eye_gaze_receiver = None; + + None + } + } + } else { + None + }; + } + if let Some(stats) = &mut *ctx.statistics_manager.write() { stats.report_tracking_received(timestamp); } diff --git a/alvr/server_openvr/src/lib.rs b/alvr/server_openvr/src/lib.rs index 8ecc54a254..f0ad382318 100644 --- a/alvr/server_openvr/src/lib.rs +++ b/alvr/server_openvr/src/lib.rs @@ -28,7 +28,7 @@ use alvr_server_core::{ HandType, ServerCoreContext, ServerCoreEvent, ServerNegotiatedStreamingConfig, }; use alvr_session::{ - BodyTrackingSinkConfig, CodecType, ControllersConfig, ControllersEmulationMode, GazeInputSource, + BodyTrackingSinkConfig, CodecType, ControllersConfig, ControllersEmulationMode, }; use foveated_encoding::EyeTrackedFoveation; use std::{ @@ -254,12 +254,10 @@ fn spawn_event_loop(events_receiver: mpsc::Receiver) { .foveated_encoding .filter(|_| { alvr_server_core::settings() - .video - .foveated_encoding + .headset + .face_tracking .as_option() - .is_some_and(|config| { - config.gaze_input_source == GazeInputSource::Headset - }) + .is_some_and(|config| config.sink.eye_tracked_foveated_encoding) }) .map(|params| { EyeTrackedFoveation::new(params, config.transcoding_view_resolution) diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index c0806d3e2c..e0c2fb946f 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -460,10 +460,6 @@ pub struct FoveatedEncodingConfig { #[schema(strings(help = "Force enable on smartphone clients"))] pub force_enable: bool, - #[schema(strings(display_name = "Eye tracking input"))] - #[schema(flag = "steamvr-restart")] - pub gaze_input_source: GazeInputSource, - #[schema(strings( display_name = "Center region size", help = "Width and height, in X/Y order" @@ -474,7 +470,7 @@ pub struct FoveatedEncodingConfig { #[schema(strings( display_name = "Center shift", - help = "Horizontal and vertical shift, in X/Y order. Used as a static fallback; overridden, not added to, when headset eye-tracking centers are available." + help = "Horizontal and vertical shift, in X/Y order. Used for fixed foveated encoding and when eye tracking is unavailable. Eye-tracking centers replace this shift rather than adding to it." ))] #[schema(gui(slider(min = -1.0, max = 1.0, step = 0.01)))] #[schema(flag = "steamvr-restart")] @@ -486,14 +482,6 @@ pub struct FoveatedEncodingConfig { pub edge_ratio: [f32; 2], } -#[derive(SettingsSchema, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Hash)] -pub enum GazeInputSource { - #[schema(strings(help = "Use the configured static foveation center"))] - None, - #[schema(strings(display_name = "Headset eye tracking"))] - Headset, -} - #[repr(C)] #[derive(SettingsSchema, Clone, Copy, Serialize, Deserialize, Pod, Zeroable)] pub struct ColorCorrectionConfig { @@ -707,7 +695,8 @@ pub struct VideoConfig { #[schema(strings( notice = r"Disabling foveated encoding may result in significantly higher encode/decode times and stuttering, or even crashing. -If you want to reduce the amount of pixelation on the edges, increase the center region width and height" +If you want to reduce the amount of pixelation on the edges, increase the center region width and height. +For eye-tracked foveation, enable Headset > Face tracking > Sink > Eye-tracked foveated encoding." ))] #[schema(flag = "steamvr-restart")] pub foveated_encoding: Switch, @@ -918,17 +907,41 @@ pub enum FaceTrackingSourcesConfig { } #[derive(SettingsSchema, Serialize, Deserialize, Clone)] -pub enum FaceTrackingSinkConfig { +pub enum FaceTrackingSocialPresenceSinkConfig { #[schema(strings(display_name = "VRChat Eye OSC"))] VrchatEyeOsc { port: u16 }, #[schema(strings(display_name = "VRCFaceTracking"))] VrcFaceTracking, } +#[derive(SettingsSchema, Serialize, Deserialize, Clone)] +pub struct FaceTrackingSinkConfig { + #[schema(strings( + help = "Forward eye and face tracking to another application. Select Default to disable forwarding without disabling eye-tracked foveated encoding." + ))] + pub social_presence: Option, + #[schema(strings( + display_name = "Eye-tracked foveated encoding", + help = "Use the selected eye tracking input to move the foveation center. Requires Video > Foveated encoding to be enabled. Turn this off to keep fixed foveated encoding." + ))] + #[schema(flag = "steamvr-restart")] + pub eye_tracked_foveated_encoding: bool, +} + +#[derive(SettingsSchema, Serialize, Deserialize, Clone)] +pub struct OscPort { + pub port: u16, +} + #[derive(SettingsSchema, Serialize, Deserialize, Clone)] #[schema(collapsible)] pub struct FaceTrackingConfig { pub sources: FaceTrackingSourcesConfig, + #[schema(strings( + display_name = "Eye tracking OSC override", + help = "Use external combined gaze received over OSC/UDP on localhost instead of headset eye directions. This input is shared by foveated encoding and social output. Turn this off to use headset eye tracking." + ))] + pub eye_tracking_osc_override: Switch, pub sink: FaceTrackingSinkConfig, } @@ -1881,9 +1894,6 @@ pub fn session_settings_default() -> SettingsDefault { content: FoveatedEncodingConfigDefault { gui_collapsed: true, force_enable: false, - gaze_input_source: GazeInputSourceDefault { - variant: GazeInputSourceDefaultVariant::None, - }, center_size: ArrayDefault { gui_collapsed: false, content: [0.45, 0.4], @@ -1996,9 +2006,23 @@ pub fn session_settings_default() -> SettingsDefault { sources: FaceTrackingSourcesConfigDefault { variant: FaceTrackingSourcesConfigDefaultVariant::PreferFullFaceTracking, }, + eye_tracking_osc_override: SwitchDefault { + enabled: false, + content: OscPortDefault { port: 9945 }, + }, sink: FaceTrackingSinkConfigDefault { - VrchatEyeOsc: FaceTrackingSinkConfigVrchatEyeOscDefault { port: 9000 }, - variant: FaceTrackingSinkConfigDefaultVariant::VrchatEyeOsc, + social_presence: OptionalDefault { + set: true, + content: FaceTrackingSocialPresenceSinkConfigDefault { + VrchatEyeOsc: + FaceTrackingSocialPresenceSinkConfigVrchatEyeOscDefault { + port: 9000, + }, + variant: + FaceTrackingSocialPresenceSinkConfigDefaultVariant::VrchatEyeOsc, + }, + }, + eye_tracked_foveated_encoding: false, }, }, },