Adaptive interpolation (more responsive movement for players and NPCs)

Code/client/Components/InterpolationComponent.h

@@ -5,22 +5,30 @@
 #endif
 
 #include <Structs/AnimationVariables.h>
+#include <Systems/PlaybackClock.h>
 
 struct InterpolationComponent
 {
     struct TimePoint
     {
+        // The owner's capture tick
         uint64_t Tick{};
         glm::vec3 Position{};
         glm::vec3 Rotation{};
         AnimationVariables Variables{};
         float Direction{};
 
         TimePoint() = default;
-        TimePoint(const TimePoint&) = default;
-        TimePoint& operator=(const TimePoint&) = default;
     };
 
     List<TimePoint> TimePoints;
+    // Last displayed position
     glm::vec3 Position;
+
+    PlaybackClock Clock;
+
+    // Left over from correcting an extrapolation, decays over a few frames instead of popping the actor into place
+    glm::vec3 PositionError{};
+    bool IsExtrapolating{false};
+    bool HasNewTimePoint{false};
 };

Code/client/Components/LocalAnimationComponent.h

@@ -11,13 +11,5 @@ struct LocalAnimationComponent
     Vector<ActionEvent> Actions;
     ActionEvent LastProcessedAction;
 
-    [[nodiscard]] Outcome<ActionEvent, bool> GetLatestAction() const noexcept
-    {
-        if (Actions.empty())
-            return false;
-
-        return Actions[Actions.size() - 1];
-    }
-
     void Append(const ActionEvent& acEvent) noexcept { Actions.push_back(acEvent); }
 };

Code/client/Services/CharacterService.h

@@ -13,6 +13,7 @@ struct ActionEvent;
 struct AssignCharacterResponse;
 struct CharacterSpawnRequest;
 struct ServerReferencesMoveRequest;
+struct NotifyActionEvents;
 struct NotifyInventoryChanges;
 struct NotifyFactionsChanges;
 struct NotifyRemoveCharacter;
@@ -66,6 +67,7 @@ struct CharacterService
     void OnAssignCharacter(const AssignCharacterResponse& acMessage) noexcept;
     void OnCharacterSpawn(const CharacterSpawnRequest& acMessage) const noexcept;
     void OnReferencesMoveRequest(const ServerReferencesMoveRequest& acMessage) const noexcept;
+    void OnNotifyActionEvents(const NotifyActionEvents& acMessage) const noexcept;
     void OnActionEvent(const ActionEvent& acActionEvent) const noexcept;
     void OnFactionsChanges(const NotifyFactionsChanges& acEvent) const noexcept;
     void OnOwnershipTransfer(const NotifyOwnershipTransfer& acMessage) noexcept;
@@ -146,6 +148,7 @@ struct CharacterService
     entt::scoped_connection m_assignCharacterConnection;
     entt::scoped_connection m_characterSpawnConnection;
     entt::scoped_connection m_referenceMovementSnapshotConnection;
+    entt::scoped_connection m_actionEventsConnection;
     entt::scoped_connection m_mountConnection;
     entt::scoped_connection m_notifyMountConnection;
     entt::scoped_connection m_initPackageConnection;

Code/client/Services/Debug/Views/ComponentView.cpp

@@ -141,6 +141,19 @@ void DebugService::DrawComponentDebugView()
                 if (ImGui::CollapsingHeader("InterpolationComponent"))
                 {
                     ImGui::Text("%f,%f,%f\n", pComponent->Position.x, pComponent->Position.y, pComponent->Position.z);
+
+                    const auto& clock = pComponent->Clock;
+                    if (!pComponent->TimePoints.empty())
+                    {
+                        // How far playback runs behind the newest snapshot right now, negative while extrapolating
+                        const double bufferedTime = static_cast<double>(pComponent->TimePoints.back().Tick) - clock.GetRenderTick();
+
+                        ImGui::Text(
+                            "Buffered: %.1f ms (target %.1f ms)\nSnapshot interval: %.1f ms\nJitter: %.1f ms\nPlayback rate: %.3f\n"
+                            "Extrapolating: %s\nCorrection: %.1f",
+                            bufferedTime, clock.GetTargetDelay(), clock.GetSnapshotInterval(), clock.GetJitter(), clock.GetPlaybackRate(),
+                            pComponent->IsExtrapolating ? "yes" : "no", glm::length(pComponent->PositionError));
+                    }
                 }
             }
 

Code/client/Services/Debug/Views/NetworkView.cpp

@@ -2,6 +2,7 @@
 #include <Services/TransportService.h>
 
 #include <imgui.h>
+#include <steam/isteamnetworkingutils.h>
 
 void DebugService::DrawNetworkView()
 {
@@ -40,6 +41,39 @@ void DebugService::DrawNetworkView()
             ImGui::PlotLines("Lines", values, IM_ARRAYSIZE(values), values_offset, nullptr, 0.f, FLT_MAX, ImVec2(0, 160.0f));
         }
 
+        // Applies to everything this client sends and receives, to see how remote actors hold up on a bad connection
+        if (ImGui::CollapsingHeader("Simulate bad connection"))
+        {
+            static int s_lag = 0;
+            static float s_loss = 0.f;
+            static float s_delayedPackets = 0.f;
+            static int s_extraDelay = 0;
+
+            auto* pUtils = SteamNetworkingUtils();
+
+            if (ImGui::SliderInt("Lag each way (ms)", &s_lag, 0, 500))
+            {
+                pUtils->SetGlobalConfigValueInt32(k_ESteamNetworkingConfig_FakePacketLag_Send, s_lag);
+                pUtils->SetGlobalConfigValueInt32(k_ESteamNetworkingConfig_FakePacketLag_Recv, s_lag);
+            }
+
+            if (ImGui::SliderFloat("Packet loss (%)", &s_loss, 0.f, 50.f))
+            {
+                pUtils->SetGlobalConfigValueFloat(k_ESteamNetworkingConfig_FakePacketLoss_Send, s_loss);
+                pUtils->SetGlobalConfigValueFloat(k_ESteamNetworkingConfig_FakePacketLoss_Recv, s_loss);
+            }
+
+            // GameNetworkingSockets has no jitter setting, delaying a share of the packets is the closest to it
+            if (ImGui::SliderFloat("Delayed packets (%)", &s_delayedPackets, 0.f, 50.f))
+            {
+                pUtils->SetGlobalConfigValueFloat(k_ESteamNetworkingConfig_FakePacketReorder_Send, s_delayedPackets);
+                pUtils->SetGlobalConfigValueFloat(k_ESteamNetworkingConfig_FakePacketReorder_Recv, s_delayedPackets);
+            }
+
+            if (ImGui::SliderInt("Delayed packets extra lag (ms)", &s_extraDelay, 0, 300))
+                pUtils->SetGlobalConfigValueInt32(k_ESteamNetworkingConfig_FakePacketReorder_Time, s_extraDelay);
+        }
+
         /*
         float width = -ImGui::CalcItemWidth();
         ImGui::PushItemWidth(width);

Code/client/Services/Generic/CharacterService.cpp

@@ -42,6 +42,8 @@
 #include <Messages/AssignCharacterResponse.h>
 #include <Messages/ServerReferencesMoveRequest.h>
 #include <Messages/ClientReferencesMoveRequest.h>
+#include <Messages/RequestActionEvents.h>
+#include <Messages/NotifyActionEvents.h>
 #include <Messages/CharacterSpawnRequest.h>
 #include <Messages/RequestFactionsChanges.h>
 #include <Messages/NotifyFactionsChanges.h>
@@ -83,6 +85,7 @@ CharacterService::CharacterService(World& aWorld, entt::dispatcher& aDispatcher,
     m_assignCharacterConnection = m_dispatcher.sink<AssignCharacterResponse>().connect<&CharacterService::OnAssignCharacter>(this);
     m_characterSpawnConnection = m_dispatcher.sink<CharacterSpawnRequest>().connect<&CharacterService::OnCharacterSpawn>(this);
     m_referenceMovementSnapshotConnection = m_dispatcher.sink<ServerReferencesMoveRequest>().connect<&CharacterService::OnReferencesMoveRequest>(this);
+    m_actionEventsConnection = m_dispatcher.sink<NotifyActionEvents>().connect<&CharacterService::OnNotifyActionEvents>(this);
     m_factionsConnection = m_dispatcher.sink<NotifyFactionsChanges>().connect<&CharacterService::OnFactionsChanges>(this);
     m_ownershipTransferConnection = m_dispatcher.sink<NotifyOwnershipTransfer>().connect<&CharacterService::OnOwnershipTransfer>(this);
     m_removeCharacterConnection = m_dispatcher.sink<NotifyRemoveCharacter>().connect<&CharacterService::OnRemoveCharacter>(this);
@@ -640,18 +643,20 @@ void CharacterService::OnCharacterSpawn(const CharacterSpawnRequest& acMessage)
 
 void CharacterService::OnReferencesMoveRequest(const ServerReferencesMoveRequest& acMessage) const noexcept
 {
-    auto view = m_world.view<RemoteComponent, InterpolationComponent, RemoteAnimationComponent>();
+    // When it came off the network, not when this frame got to it. Otherwise a long frame on our side would look like
+    // network jitter and make every remote actor play further behind for seconds.
+    const double cArrivalTime = m_transport.GetMessageReceiveTime();
 
-    for (const auto& [serverId, update] : acMessage.Updates)
+    auto view = m_world.view<RemoteComponent, InterpolationComponent>();
+
+    for (const auto& [serverId, movement] : acMessage.Updates)
     {
         auto itor = std::find_if(std::begin(view), std::end(view), [serverId = serverId, view](entt::entity entity) { return view.get<RemoteComponent>(entity).Id == serverId; });
 
         if (itor == std::end(view))
             continue;
 
         auto& interpolationComponent = view.get<InterpolationComponent>(*itor);
-        auto& animationComponent = view.get<RemoteAnimationComponent>(*itor);
-        const auto& movement = update.UpdatedMovement;
 
         InterpolationComponent::TimePoint point;
         point.Tick = acMessage.Tick;
@@ -660,9 +665,25 @@ void CharacterService::OnReferencesMoveRequest(const ServerReferencesMoveRequest
         point.Variables = movement.Variables;
         point.Direction = movement.Direction;
 
-        InterpolationSystem::AddPoint(interpolationComponent, point);
+        InterpolationSystem::AddPoint(interpolationComponent, std::move(point), cArrivalTime);
+    }
+}
+
+void CharacterService::OnNotifyActionEvents(const NotifyActionEvents& acMessage) const noexcept
+{
+    auto view = m_world.view<RemoteComponent, RemoteAnimationComponent>();
+
+    for (const auto& [serverId, actions] : acMessage.Actions)
+    {
+        auto itor = std::find_if(std::begin(view), std::end(view), [serverId = serverId, view](entt::entity entity) { return view.get<RemoteComponent>(entity).Id == serverId; });
+
+        if (itor == std::end(view))
+            continue;
+
+        auto& animationComponent = view.get<RemoteAnimationComponent>(*itor);
 
-        for (const auto& action : update.ActionEvents)
+        // Played once the actor's playback clock reaches their tick, in step with its movement
+        for (const auto& action : actions)
         {
             animationComponent.TimePoints.push_back(action);
         }
@@ -816,8 +837,12 @@ void CharacterService::OnOwnershipTransfer(const NotifyOwnershipTransfer& acMess
     {
         m_world.emplace_or_replace<RemoteComponent>(cEntity, acMessage.ServerId, pFormIdComponent->Id, acMessage.OwnershipEpoch);
 
-        if (!m_world.all_of<InterpolationComponent>(cEntity))
-            InterpolationSystem::Setup(m_world, cEntity);
+        // The new owner carries on from its own, older view of the actor and stamps snapshots with its own clock, so the
+        // previous owner's snapshots and timing estimates would only drag the actor back. Start over from where it is now.
+        const auto* pOldInterpolationComponent = m_world.try_get<InterpolationComponent>(cEntity);
+        const glm::vec3 cPosition = pActor ? pActor->position : (pOldInterpolationComponent ? pOldInterpolationComponent->Position : glm::vec3{});
+        InterpolationSystem::Setup(m_world, cEntity).Position = cPosition;
+
         if (!m_world.all_of<RemoteAnimationComponent>(cEntity))
             AnimationSystem::Setup(m_world, cEntity);
     }
@@ -1724,18 +1749,30 @@ void CharacterService::ProcessLeveledConforms() noexcept
 
 void CharacterService::RunLocalUpdates() const noexcept
 {
-    static std::chrono::steady_clock::time_point lastSendTimePoint;
-    constexpr auto cDelayBetweenSnapshots = 100ms;
+    // Remote clients show actors about one snapshot interval late, so players, mounts and actors in combat get every snapshot
+    // and the rest every few. Assumes 60+ fps: 30ms keeps 60 fps gaps at two frames, with 33ms most were three (~25 Hz).
+    constexpr auto cDelayBetweenSnapshots = 30ms;
+    constexpr uint32_t cLowPrioritySnapshotDivider = 3;
+    // Movement is unreliable, keeping each message around a single packet (~120 bytes per humanoid) means a lost packet
+    // only drops a few actors
+    constexpr size_t cMaxActorsPerMovementMessage = 8;
+
+    static std::chrono::steady_clock::time_point lastSnapshotTimePoint;
+    static uint32_t snapshotCount = 0;
 
     const auto now = std::chrono::steady_clock::now();
-    if (now - lastSendTimePoint < cDelayBetweenSnapshots)
+    if (now - lastSnapshotTimePoint < cDelayBetweenSnapshots)
         return;
 
-    lastSendTimePoint = now;
+    lastSnapshotTimePoint = now;
+    const bool cIsFullSnapshot = ++snapshotCount % cLowPrioritySnapshotDivider == 0;
+
+    RequestActionEvents actionsMessage;
 
-    ClientReferencesMoveRequest message;
-    message.Tick = m_transport.GetClock().GetCurrentTick();
+    ClientReferencesMoveRequest movementMessage;
+    movementMessage.Tick = m_transport.GetClock().GetCurrentTick();
 
+    const auto* pPlayer = PlayerCharacter::Get();
     auto animatedLocalView = m_world.view<LocalComponent, LocalAnimationComponent, FormIdComponent>();
 
     for (auto entity : animatedLocalView)
@@ -1744,16 +1781,39 @@ void CharacterService::RunLocalUpdates() const noexcept
         auto& animationComponent = animatedLocalView.get<LocalAnimationComponent>(entity);
         auto& formIdComponent = animatedLocalView.get<FormIdComponent>(entity);
 
-        AnimationSystem::Serialize(m_world, message, localComponent, animationComponent, formIdComponent);
+        auto* pActor = Cast<Actor>(TESForm::GetById(formIdComponent.Id));
+        if (!pActor)
+            continue;
+
+        // Remote clients play actions on the same timeline as movement, by the time playback reaches an action it waits
+        // for the snapshot after it anyway, so sending actions along with that snapshot doesn't delay them
+        AnimationSystem::SerializeActions(actionsMessage, localComponent, animationComponent);
+
+        if (!cIsFullSnapshot && pActor != pPlayer && !pActor->IsMount() && !pActor->IsInCombat())
+            continue;
+
+        if (movementMessage.Updates.size() == cMaxActorsPerMovementMessage)
+        {
+            m_transport.Send(movementMessage, TiltedPhoques::kUnreliable);
+            movementMessage.Updates.clear();
+        }
+
+        AnimationSystem::SerializeMovement(m_world, pActor, movementMessage.Updates[localComponent.Id]);
     }
 
-    m_transport.Send(message);
+    if (!actionsMessage.Actions.empty())
+        m_transport.Send(actionsMessage);
+
+    // NoNagle on the last message sends everything queued this frame right away, in as few packets as possible
+    if (!movementMessage.Updates.empty())
+        m_transport.Send(movementMessage, TiltedPhoques::kUnreliableNoNagle);
 }
 
 void CharacterService::RunRemoteUpdates() noexcept
 {
-    // Delay by 300ms to let the interpolation system accumulate interpolation points
-    const auto tick = m_transport.GetClock().GetCurrentTick() - 300;
+    // Each remote actor is played back on its owner's timeline, only as far behind as its snapshots need, see PlaybackClock.
+    // Playback can only rely on snapshots that have been handled, so it runs on the time they were last picked up at.
+    const double cNow = m_transport.GetLastPollTime();
 
     // Interpolation has to keep running even if the actor is not in view, otherwise we will never know if we need to spawn it
     auto interpolatedEntities = m_world.view<RemoteComponent, InterpolationComponent>();
@@ -1770,10 +1830,10 @@ void CharacterService::RunRemoteUpdates() noexcept
             pActor = Cast<Actor>(pForm);
         }
 
-        InterpolationSystem::Update(pActor, interpolationComponent, tick);
+        InterpolationSystem::Update(pActor, interpolationComponent, cNow);
     }
 
-    auto animatedView = m_world.view<RemoteComponent, RemoteAnimationComponent, FormIdComponent>();
+    auto animatedView = m_world.view<RemoteComponent, RemoteAnimationComponent, InterpolationComponent, FormIdComponent>();
 
     for (auto entity : animatedView)
     {
@@ -1785,7 +1845,10 @@ void CharacterService::RunRemoteUpdates() noexcept
         if (!pActor)
             continue;
 
-        AnimationSystem::Update(m_world, pActor, animationComponent, tick);
+        // Actions run on the clock the interpolation pass above just advanced, so the body and its animations stay in step
+        const auto cTick = InterpolationSystem::GetPlaybackTick(animatedView.get<InterpolationComponent>(entity));
+
+        AnimationSystem::Update(m_world, pActor, animationComponent, cTick);
     }
 
     auto facegenView = m_world.view<FormIdComponent, FaceGenComponent>();

Code/client/Services/Generic/TransportService.cpp

@@ -21,6 +21,7 @@
 #include <Messages/ServerMessageFactory.h>
 #include <Messages/NotifySettingsChange.h>
 #include <Packet.hpp>
+#include <steam/isteamnetworkingutils.h>
 
 #include <ScriptExtender.h>
 #include <Services/DiscordService.h>
@@ -65,7 +66,7 @@ TransportService::TransportService(World& aWorld, entt::dispatcher& aDispatcher)
     };
 }
 
-bool TransportService::Send(const ClientMessage& acMessage) const noexcept
+bool TransportService::Send(const ClientMessage& acMessage, const TiltedPhoques::EPacketFlags aPacketFlags) const noexcept
 {
     static thread_local ScratchAllocator s_allocator(1 << 18);
 
@@ -85,7 +86,7 @@ bool TransportService::Send(const ClientMessage& acMessage) const noexcept
         acMessage.Serialize(writer);
         TiltedPhoques::PacketView packet(reinterpret_cast<char*>(buffer.GetWriteData()), writer.Size());
 
-        Client::Send(&packet);
+        Client::Send(&packet, aPacketFlags);
 
         return true;
     }
@@ -178,8 +179,16 @@ void TransportService::OnUpdate()
 {
 }
 
+double TransportService::GetMessageReceiveTime() const noexcept
+{
+    return static_cast<double>(GetCurrentMessageReceiveTime()) / 1000.0;
+}
+
 void TransportService::HandleUpdate(const UpdateEvent& acEvent) noexcept
 {
+    // Taken before polling, so that every message that arrived before this time gets handled by this update
+    m_lastPollTime = static_cast<double>(SteamNetworkingUtils()->GetLocalTimestamp()) / 1000.0;
+
     Update();
 }