feat(foveation): support external combined gaze over OSC

alvr/client_openxr/src/interaction.rs

@@ -11,9 +11,7 @@ use alvr_common::{
 };
 use alvr_graphics::HandData;
 use alvr_packets::{ButtonEntry, ButtonValue, ClientStreamConfig, FaceData, FaceExpressions};
-use alvr_session::{
-    BodyTrackingBDConfig, BodyTrackingSourcesConfig, FaceTrackingSourcesConfig, GazeInputSource,
-};
+use alvr_session::{BodyTrackingBDConfig, BodyTrackingSourcesConfig, FaceTrackingSourcesConfig};
 use openxr as xr;
 use std::{
     collections::{HashMap, HashSet},
@@ -150,23 +148,15 @@ pub struct InteractionSourcesConfig {
 
 impl InteractionSourcesConfig {
     pub fn new(config: &ClientStreamConfig) -> Self {
+        let face_tracking = config.settings.headset.face_tracking.as_option();
+        let eye_tracking_osc_override =
+            face_tracking.is_some_and(|c| c.eye_tracking_osc_override.enabled());
+
         Self {
-            face_tracking: config
-                .settings
-                .headset
-                .face_tracking
-                .as_option()
-                .map(|c| c.sources.clone())
-                .or_else(|| {
-                    // Foveation requests eye input locally, without enabling face tracking output.
-                    config
-                        .settings
-                        .video
-                        .foveated_encoding
-                        .as_option()
-                        .is_some_and(|c| c.gaze_input_source == GazeInputSource::Headset)
-                        .then_some(FaceTrackingSourcesConfig::PreferEyeTrackingOnly)
-                }),
+            face_tracking: face_tracking.map(|c| c.sources.clone()).filter(|sources| {
+                !eye_tracking_osc_override
+                    || *sources == FaceTrackingSourcesConfig::PreferFullFaceTracking
+            }),
             body_tracking: config
                 .settings
                 .headset

alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs

@@ -465,8 +465,12 @@ pub fn eye_face_tracking_schema() -> PresetSchemaNode {
                 display_name: "VRChat Eye OSC".into(),
                 modifiers: vec![
                     bool_modifier("session_settings.headset.face_tracking.enabled", true),
+                    bool_modifier(
+                        "session_settings.headset.face_tracking.content.sink.social_presence.set",
+                        true,
+                    ),
                     string_modifier(
-                        "session_settings.headset.face_tracking.content.sink.variant",
+                        "session_settings.headset.face_tracking.content.sink.social_presence.content.variant",
                         "VrchatEyeOsc",
                     ),
                 ],
@@ -476,8 +480,12 @@ pub fn eye_face_tracking_schema() -> PresetSchemaNode {
                 display_name: "VRCFaceTracking".into(),
                 modifiers: vec![
                     bool_modifier("session_settings.headset.face_tracking.enabled", true),
+                    bool_modifier(
+                        "session_settings.headset.face_tracking.content.sink.social_presence.set",
+                        true,
+                    ),
                     string_modifier(
-                        "session_settings.headset.face_tracking.content.sink.variant",
+                        "session_settings.headset.face_tracking.content.sink.social_presence.content.variant",
                         "VrcFaceTracking",
                     ),
                 ],

alvr/server_core/src/connection.rs

@@ -26,8 +26,8 @@ use alvr_packets::{
     VIDEO, VideoPacketHeader,
 };
 use alvr_session::{
-    BodyTrackingSinkConfig, CodecType, ControllersEmulationMode, FrameSize, GazeInputSource,
-    H264Profile, Settings, SocketProtocol, SteamvrHmdInitConfig,
+    BodyTrackingSinkConfig, CodecType, ControllersEmulationMode, FrameSize, H264Profile, Settings,
+    SocketProtocol, SteamvrHmdInitConfig,
 };
 use alvr_sockets::{
     CONTROL_PORT, KEEPALIVE_INTERVAL, KEEPALIVE_TIMEOUT, ProtoControlSocket, SocketConnection,
@@ -142,7 +142,7 @@ pub fn compute_restart_settings_hash(
         .map(|c| c.sources.meta.prefer_full_body)
         .unwrap_or(false);
 
-    let mut foveation_gaze_input_source = GazeInputSource::None;
+    let mut foveation_eye_tracking = false;
     let mut foveation_center_size_x = 0.0_f32;
     let mut foveation_center_size_y = 0.0_f32;
     let mut foveation_center_shift_x = 0.0_f32;
@@ -151,7 +151,11 @@ pub fn compute_restart_settings_hash(
     let mut foveation_edge_ratio_y = 0.0_f32;
     let enable_foveated_encoding =
         if let Switch::Enabled(config) = &settings.video.foveated_encoding {
-            foveation_gaze_input_source = config.gaze_input_source;
+            foveation_eye_tracking = settings
+                .headset
+                .face_tracking
+                .as_option()
+                .is_some_and(|config| config.sink.eye_tracked_foveated_encoding);
             [foveation_center_size_x, foveation_center_size_y] = config.center_size;
             [foveation_center_shift_x, foveation_center_shift_y] = config.center_shift;
             [foveation_edge_ratio_x, foveation_edge_ratio_y] = config.edge_ratio;
@@ -244,7 +248,7 @@ pub fn compute_restart_settings_hash(
     nvenc.enable_weighted_prediction.hash(&mut h);
     // Foveated encoding
     enable_foveated_encoding.hash(&mut h);
-    foveation_gaze_input_source.hash(&mut h);
+    foveation_eye_tracking.hash(&mut h);
     foveation_center_size_x.to_bits().hash(&mut h);
     foveation_center_size_y.to_bits().hash(&mut h);
     foveation_center_shift_x.to_bits().hash(&mut h);

alvr/server_core/src/tracking/eye_gaze.rs

@@ -0,0 +1,143 @@
+use alvr_common::{
+    anyhow::Result,
+    glam::{Quat, Vec3},
+};
+use rosc::{OscPacket, OscType};
+use std::{
+    io::ErrorKind,
+    net::{Ipv4Addr, UdpSocket},
+    time::{Duration, Instant},
+};
+
+const GAZE_ADDRESS: &str = "/alvr/eye/combined/v1";
+const MAX_SAMPLE_AGE: Duration = Duration::from_millis(100);
+const SEQUENCE_RESET_TIMEOUT: Duration = Duration::from_secs(1);
+const MAX_DATAGRAM_SIZE: usize = 256;
+const MAX_PACKETS_PER_POLL: usize = 64;
+
+pub struct EyeGazeReceiver {
+    socket: UdpSocket,
+    last_packet: Option<(u32, Instant)>,
+    last_tracking_timestamp: Option<Duration>,
+    last_polled_at: Instant,
+    discard_queued: bool,
+}
+
+impl EyeGazeReceiver {
+    pub fn new(port: u16) -> Result<Self> {
+        let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, port))?;
+        socket.set_nonblocking(true)?;
+
+        Ok(Self {
+            socket,
+            last_packet: None,
+            last_tracking_timestamp: None,
+            last_polled_at: Instant::now(),
+            discard_queued: false,
+        })
+    }
+
+    // Consume each observation once. Missing input must not refresh the foveation hold timer.
+    pub fn receive(&mut self, tracking_timestamp: Duration) -> Result<Option<Quat>> {
+        if self
+            .last_tracking_timestamp
+            .is_some_and(|previous| tracking_timestamp <= previous)
+        {
+            return Ok(None);
+        }
+        self.last_tracking_timestamp = Some(tracking_timestamp);
+
+        let now = Instant::now();
+        // After a tracking stall, drain the socket before accepting fresh observations again.
+        self.discard_queued |= now.duration_since(self.last_polled_at) > MAX_SAMPLE_AGE;
+        self.last_polled_at = now;
+        let mut gaze = None;
+        // One extra byte lets us reject oversized datagrams even if recv truncates them.
+        let mut buffer = [0; MAX_DATAGRAM_SIZE + 1];
+
+        for _ in 0..MAX_PACKETS_PER_POLL {
+            let size = match self.socket.recv(&mut buffer) {
+                Ok(size) => size,
+                Err(error) if error.kind() == ErrorKind::WouldBlock => {
+                    self.discard_queued = false;
+
+                    return Ok(gaze);
+                }
+                Err(error) if error.kind() == ErrorKind::Interrupted => continue,
+                // Windows reports an oversized datagram instead of returning truncated bytes.
+                Err(error) if error.raw_os_error() == Some(10040) => continue,
+                Err(error) => return Err(error.into()),
+            };
+            if self.discard_queued || size > MAX_DATAGRAM_SIZE {
+                continue;
+            }
+            let Some(packet) = buffer.get(..size) else {
+                continue;
+            };
+            // This endpoint accepts single messages, not scheduled or recursive OSC bundles.
+            if packet.first() != Some(&b'/') {
+                continue;
+            }
+            let Ok((remaining, OscPacket::Message(message))) = rosc::decoder::decode_udp(packet)
+            else {
+                continue;
+            };
+            if !remaining.is_empty() || message.addr != GAZE_ADDRESS {
+                continue;
+            }
+            let [
+                OscType::Int(sequence),
+                OscType::Int(valid),
+                OscType::Int(age_us),
+                OscType::Float(x),
+                OscType::Float(y),
+                OscType::Float(z),
+                OscType::Float(w),
+            ] = message.args.as_slice()
+            else {
+                continue;
+            };
+            if !matches!(valid, 0 | 1)
+                || *age_us < 0
+                || Duration::from_micros(*age_us as u64) > MAX_SAMPLE_AGE
+            {
+                continue;
+            }
+
+            let orientation = if *valid == 1 {
+                let orientation = Quat::from_xyzw(*x, *y, *z, *w);
+                let length_squared = orientation.length_squared();
+                if !length_squared.is_finite()
+                    || length_squared <= f32::EPSILON
+                    || (orientation.normalize() * Vec3::NEG_Z).z >= -f32::EPSILON
+                {
+                    continue;
+                }
+
+                // Social output also consumes this quaternion and expects a unit rotation.
+                Some(orientation.normalize())
+            } else {
+                None
+            };
+            // Interpret the OSC signed integer as the bit pattern of a wrapping u32 counter.
+            let sequence = *sequence as u32;
+            if let Some((previous, received_at)) = self.last_packet {
+                let distance = sequence.wrapping_sub(previous);
+                if distance == 0
+                    || (now.duration_since(received_at) <= SEQUENCE_RESET_TIMEOUT
+                        && distance >= 1 << 31)
+                {
+                    continue;
+                }
+            }
+
+            self.last_packet = Some((sequence, now));
+            gaze = orientation;
+        }
+
+        // Do not block head/controller tracking or use a potentially backlogged sample in a flood.
+        self.discard_queued = true;
+
+        Ok(None)
+    }
+}

alvr/server_core/src/tracking/face.rs

@@ -1,6 +1,6 @@
 use alvr_common::{anyhow::Result, glam::EulerRot};
 use alvr_packets::{FaceData, FaceExpressions};
-use alvr_session::FaceTrackingSinkConfig;
+use alvr_session::FaceTrackingSocialPresenceSinkConfig;
 use rosc::{OscMessage, OscPacket, OscType};
 use std::{f32::consts::PI, net::UdpSocket};
 
@@ -9,16 +9,16 @@ const RAD_TO_DEG: f32 = 180.0 / PI;
 const VRCFT_PORT: u16 = 0xA1F7;
 
 pub struct FaceTrackingSink {
-    config: FaceTrackingSinkConfig,
+    config: FaceTrackingSocialPresenceSinkConfig,
     socket: UdpSocket,
     packet_buffer: Vec<u8>,
 }
 
 impl FaceTrackingSink {
-    pub fn new(config: FaceTrackingSinkConfig, local_osc_port: u16) -> Result<Self> {
+    pub fn new(config: FaceTrackingSocialPresenceSinkConfig, local_osc_port: u16) -> Result<Self> {
         let port = match config {
-            FaceTrackingSinkConfig::VrchatEyeOsc { port } => port,
-            FaceTrackingSinkConfig::VrcFaceTracking => VRCFT_PORT,
+            FaceTrackingSocialPresenceSinkConfig::VrchatEyeOsc { port } => port,
+            FaceTrackingSocialPresenceSinkConfig::VrcFaceTracking => VRCFT_PORT,
         };
 
         let socket = UdpSocket::bind(format!("127.0.0.1:{local_osc_port}"))?;
@@ -53,7 +53,7 @@ impl FaceTrackingSink {
 
     pub fn send_tracking(&mut self, face_data: &FaceData) {
         match self.config {
-            FaceTrackingSinkConfig::VrchatEyeOsc { .. } => {
+            FaceTrackingSocialPresenceSinkConfig::VrchatEyeOsc { .. } => {
                 if let [Some(left), Some(right)] = face_data.eyes_social {
                     let (left_pitch, left_yaw, _) = left.to_euler(EulerRot::XYZ);
                     let (right_pitch, right_yaw, _) = right.to_euler(EulerRot::XYZ);
@@ -100,7 +100,7 @@ impl FaceTrackingSink {
                     );
                 }
             }
-            FaceTrackingSinkConfig::VrcFaceTracking => {
+            FaceTrackingSocialPresenceSinkConfig::VrcFaceTracking => {
                 self.packet_buffer.clear();
 
                 if let [Some(left_quat), Some(right_quat)] = face_data.eyes_social {

alvr/server_core/src/tracking/mod.rs

@@ -1,4 +1,5 @@
 mod body;
+mod eye_gaze;
 mod face;
 mod vmc;
 
@@ -15,7 +16,7 @@ use crate::{
 use alvr_common::{
     ConnectionError, DEVICE_ID_TO_PATH, DeviceMotion, Pose, ViewParams,
     glam::{Quat, Vec3},
-    inputs as inp,
+    inputs as inp, warn,
 };
 use alvr_events::{EventType, TrackingEvent};
 use alvr_packets::TrackingData;
@@ -24,6 +25,7 @@ use alvr_session::{
     settings_schema::Switch,
 };
 use alvr_sockets::StreamReceiver;
+use eye_gaze::EyeGazeReceiver;
 use std::{
     cmp::Ordering,
     collections::{HashMap, VecDeque},
@@ -300,12 +302,27 @@ pub fn tracking_loop(
                 )
             });
 
+    let eye_tracking_osc_port = initial_settings
+        .headset
+        .face_tracking
+        .as_option()
+        .and_then(|config| config.eye_tracking_osc_override.as_option())
+        .map(|config| config.port);
+    let mut eye_gaze_receiver = eye_tracking_osc_port.and_then(|port| {
+        EyeGazeReceiver::new(port)
+            .inspect_err(|error| warn!("Failed to bind external gaze OSC port {port}: {error}"))
+            .ok()
+    });
+
     let mut face_tracking_sink = initial_settings
         .headset
         .face_tracking
         .into_option()
+        .and_then(|config| config.sink.social_presence)
         .and_then(|config| {
-            FaceTrackingSink::new(config.sink, initial_settings.connection.osc_local_port).ok()
+            FaceTrackingSink::new(config, initial_settings.connection.osc_local_port)
+                .inspect_err(|error| warn!("Failed to initialize social presence sink: {error}"))
+                .ok()
         });
 
     let mut body_tracking_sink = initial_settings
@@ -334,6 +351,25 @@ pub fn tracking_loop(
 
         let timestamp = tracking.poll_timestamp;
 
+        if eye_tracking_osc_port.is_some() {
+            // Select one eye input before foveation and social output consume it. Do not fall
+            // back to headset eyes when OSC input is missing, invalid or unavailable.
+            tracking.face.eyes_social = [None; 2];
+            tracking.face.eyes_combined = if let Some(receiver) = &mut eye_gaze_receiver {
+                match receiver.receive(timestamp) {
+                    Ok(gaze) => gaze,
+                    Err(error) => {
+                        warn!("External gaze OSC receiver stopped: {error}");
+                        eye_gaze_receiver = None;
+
+                        None
+                    }
+                }
+            } else {
+                None
+            };
+        }
+
         if let Some(stats) = &mut *ctx.statistics_manager.write() {
             stats.report_tracking_received(timestamp);
         }