diff --git a/code/framework/src/networking/network_peer.cpp b/code/framework/src/networking/network_peer.cpp index 40ad6fe26..925b1f646 100644 --- a/code/framework/src/networking/network_peer.cpp +++ b/code/framework/src/networking/network_peer.cpp @@ -12,6 +12,7 @@ #include "replication/replication_manager.h" #include +#include namespace Framework::Networking { NetworkPeer::NetworkPeer() { @@ -65,9 +66,11 @@ namespace Framework::Networking { return; } - // Rebuild the spatial index before ReplicaManager3 computes per-connection relevance. + // Rebuild the spatial index before ReplicaManager3 computes per-connection relevance, then + // elect syncers off the fresh positions. if (_replicationManager) { _replicationManager->RebuildInterest(); + _replicationManager->RunDelegation(MafiaNet::GetTime()); } for (_packet = _peer->Receive(); _packet; _peer->DeallocatePacket(_packet), _packet = _peer->Receive()) { diff --git a/code/framework/src/networking/replication/delegation_policy.h b/code/framework/src/networking/replication/delegation_policy.h new file mode 100644 index 000000000..336512dcb --- /dev/null +++ b/code/framework/src/networking/replication/delegation_policy.h @@ -0,0 +1,87 @@ +/* + * MafiaHub OSS license + * Copyright (c) 2021-2023, MafiaHub. All rights reserved. + * + * This file comes from MafiaHub, hosted at https://github.com/MafiaHub/Framework. + * See LICENSE file in the source repository for information regarding licensing. + */ + +#pragma once + +#include + +#include + +#include +#include + +namespace Framework::Networking::Replication { + // Tunables for proximity syncer election. Distances are ground-plane world units. acquireRange < + // dropRange gives hysteresis: a candidate must reach acquireRange to be elected, but the current + // owner is held until it drifts past dropRange, so a boundary owner doesn't thrash. + struct DelegationParams { + float acquireRange = 80.0f; + float dropRange = 130.0f; + uint32_t electionIntervalMs = 500; + int loadSoftCap = 0; // max delegated entities newly granted per client (0 = off) + }; + + struct DelegationCandidate { + MafiaNet::PeerGuid guid = MafiaNet::UNASSIGNED_PEER_GUID; + glm::vec3 position {0.0f}; + int ownedLoad = 0; + bool eligible = true; // caller-applied dimension + game veto + }; + + // Pure, networking-free election. Keeps the current owner within dropRange, else elects the + // least-loaded eligible candidate within acquireRange (nearest breaking ties), else UNASSIGNED. + // groundXY picks the second ground axis (Y for Z-up, else Z). + inline MafiaNet::PeerGuid ElectOwner(const glm::vec3 &entityPos, MafiaNet::PeerGuid currentOwner, const std::vector &candidates, const DelegationParams ¶ms, bool groundXY) { + const auto ground = [groundXY](const glm::vec3 &p) { + return groundXY ? p.y : p.z; + }; + const auto dist2 = [&](const glm::vec3 &p) { + const float dx = p.x - entityPos.x; + const float dv = ground(p) - ground(entityPos); + return dx * dx + dv * dv; + }; + + // Hysteresis: hold the current owner while eligible and within dropRange (soft cap ignored). + if (currentOwner != MafiaNet::UNASSIGNED_PEER_GUID) { + const float dropSq = params.dropRange * params.dropRange; + for (const auto &candidate : candidates) { + if (candidate.guid != currentOwner) { + continue; + } + if (candidate.eligible && dist2(candidate.position) <= dropSq) { + return currentOwner; + } + break; + } + } + + const float acquireSq = params.acquireRange * params.acquireRange; + MafiaNet::PeerGuid best = MafiaNet::UNASSIGNED_PEER_GUID; + int bestLoad = 0; + float bestDist = 0.0f; + for (const auto &candidate : candidates) { + if (!candidate.eligible || candidate.guid == MafiaNet::UNASSIGNED_PEER_GUID) { + continue; + } + const float d = dist2(candidate.position); + if (d > acquireSq) { + continue; + } + if (params.loadSoftCap > 0 && candidate.ownedLoad >= params.loadSoftCap) { + continue; + } + const bool better = best == MafiaNet::UNASSIGNED_PEER_GUID || candidate.ownedLoad < bestLoad || (candidate.ownedLoad == bestLoad && d < bestDist); + if (better) { + best = candidate.guid; + bestLoad = candidate.ownedLoad; + bestDist = d; + } + } + return best; + } +} // namespace Framework::Networking::Replication diff --git a/code/framework/src/networking/replication/network_entity.cpp b/code/framework/src/networking/replication/network_entity.cpp index 0c80ea036..0ee6cb88b 100644 --- a/code/framework/src/networking/replication/network_entity.cpp +++ b/code/framework/src/networking/replication/network_entity.cpp @@ -41,10 +41,22 @@ namespace Framework::Networking::Replication { } void NetworkEntity::AdoptIncomingOwner(MafiaNet::PeerGuid incomingOwner) { - // The server keeps its own authoritative owner assignment and must not let an owning client - // dictate it back; clients adopt whatever the server sends. - if (!IsServerPeer()) { - ownerGUID = incomingOwner; + SetOwnerFromServer(incomingOwner); + } + + void NetworkEntity::SetOwnerFromServer(MafiaNet::PeerGuid newOwner) { + if (IsServerPeer()) { + return; + } + const bool was = IsOwner(); + ownerGUID = newOwner; + // Construction defers the callback (see DeserializeConstruction) until the pose is read. + if (_constructing) { + return; + } + const bool now = IsOwner(); + if (now != was) { + OnOwnershipChanged(now); } } @@ -94,12 +106,18 @@ namespace Framework::Networking::Replication { uint8_t incomingEpoch = stateEpoch; constructionBitstream->Read(incomingEpoch); ApplyIncomingEpoch(incomingEpoch); + _constructing = true; FieldSerializer seed(constructionBitstream, false); SerializeBaseFields(seed); SerializeTransform(seed); OnSerializeConstruction(seed); SerializeFields(seed); + _constructing = false; OnConstructed(); + // Announce ownership if we arrive already owning it, now that the pose is populated. + if (!IsServerPeer() && IsOwner()) { + OnOwnershipChanged(true); + } return true; } diff --git a/code/framework/src/networking/replication/network_entity.h b/code/framework/src/networking/replication/network_entity.h index a498a4149..6f19e1474 100644 --- a/code/framework/src/networking/replication/network_entity.h +++ b/code/framework/src/networking/replication/network_entity.h @@ -137,6 +137,21 @@ namespace Framework::Networking::Replication { }; Streaming streaming; + // Fate of a delegatable entity when no client can take it. + enum class OrphanMode : uint8_t { + Freeze, // return to the server (UNASSIGNED), keep replicating its last pose + Destroy, // remove it + }; + + // Server-only, never replicated. Opt-in via delegatable; see ReplicationManager::RunDelegation. + struct Delegation { + bool delegatable = false; + OrphanMode orphanMode = OrphanMode::Freeze; + // Pin the syncer to a client, bypassing election. UNASSIGNED elects normally. + MafiaNet::PeerGuid pinnedOwner = MafiaNet::UNASSIGNED_PEER_GUID; + }; + Delegation delegation; + // --- Game extension points --- virtual void OnSerializeConstruction(FieldSerializer &fields) { (void)fields; @@ -164,6 +179,18 @@ namespace Framework::Networking::Replication { // Called on the owning client after SerializeForcedState has applied the forced fields. virtual void OnStateForced() {} + // Client-only: authority over this entity crossed our peer. On gain, the replicated pose is + // already current, so seed the local sim from it; on loss, tear it down. + virtual void OnOwnershipChanged(bool nowOwner) { + (void)nowOwner; + } + + // Server: veto electing `candidate` as syncer (proximity/dimension already applied). + virtual bool CanDelegateTo(MafiaNet::PeerGuid candidate) const { + (void)candidate; + return true; + } + // Server: push this entity's forced state to its owner. No-op for unowned (server-owned) // entities, which replicate to everyone normally. void ForceState(); @@ -174,6 +201,10 @@ namespace Framework::Networking::Replication { // MafiaNet::UNASSIGNED_PEER_GUID to return ownership to the server. void SetOwner(MafiaNet::PeerGuid guid); + // Client-side: adopt an owner from the server, firing OnOwnershipChanged on a flip. No-op on + // the server. The single sink for client owner changes so the callback can't be bypassed. + void SetOwnerFromServer(MafiaNet::PeerGuid newOwner); + // True on the peer with authority over this entity: the owning client, or the server for // server-owned entities. The game decides what owning means (bind the local avatar, drive // updates upstream, ...); this just answers who holds authority. @@ -226,6 +257,9 @@ namespace Framework::Networking::Replication { // applied. bool ApplyIncomingEpoch(uint8_t incomingEpoch); + // Set during DeserializeConstruction: defers OnOwnershipChanged to the end, once the pose is read. + bool _constructing = false; + // CRC32 of the registered name; stamped by EntityRegistry, not game-settable. uint32_t typeId = 0; friend class EntityRegistry; diff --git a/code/framework/src/networking/replication/replication_manager.cpp b/code/framework/src/networking/replication/replication_manager.cpp index ecf378568..fa101fe5d 100644 --- a/code/framework/src/networking/replication/replication_manager.cpp +++ b/code/framework/src/networking/replication/replication_manager.cpp @@ -66,8 +66,9 @@ namespace Framework::Networking::Replication { }); owner->RegisterRPC([this](const SetOwnerRPC &payload, MafiaNet::Packet *) { if (auto *entity = GetEntityByNetworkID(payload.networkId)) { - entity->ownerGUID = payload.ownerGUID; + // Epoch before the flip: a gained owner sends from inside OnOwnershipChanged. entity->stateEpoch = payload.stateEpoch; + entity->SetOwnerFromServer(payload.ownerGUID); } }); _clientRPCsRegistered = true; @@ -211,6 +212,83 @@ namespace Framework::Networking::Replication { _interest.CollectVisible(viewer, viewerGUID, out); } + void ReplicationManager::RunDelegation(uint64_t nowMs) { + if (!_isServer) { + return; + } + if (nowMs - _lastDelegationMs < _delegationParams.electionIntervalMs) { + return; + } + _lastDelegationMs = nowMs; + + // Delegated entities each client owns, for load balancing. + std::unordered_map load; + ForEachEntity([&load](NetworkEntity *entity) { + if (entity->delegation.delegatable && entity->ownerGUID != MafiaNet::UNASSIGNED_PEER_GUID) { + ++load[entity->ownerGUID]; + } + }); + + // Destroy deferred past the sweep: DestroyEntity mutates the index ForEachEntity walks. + std::vector toDestroy; + std::vector candidates; + ForEachEntity([&](NetworkEntity *entity) { + if (!entity->delegation.delegatable) { + return; + } + + // A pinned syncer bypasses election. + if (entity->delegation.pinnedOwner != MafiaNet::UNASSIGNED_PEER_GUID) { + if (entity->ownerGUID != entity->delegation.pinnedOwner) { + SetOwner(entity, entity->delegation.pinnedOwner); + } + return; + } + + candidates.clear(); + candidates.reserve(_viewers.size()); + const auto entityWorld = entity->GetVirtualWorld(); + for (const auto &[guid, viewer] : _viewers) { + if (!viewer) { + continue; + } + DelegationCandidate candidate; + candidate.guid = guid; + candidate.position = viewer->position; + const auto loadIt = load.find(guid); + candidate.ownedLoad = loadIt != load.end() ? loadIt->second : 0; + candidate.eligible = MafiaNet::VirtualWorldsCanSee(viewer->GetVirtualWorld(), entityWorld) && entity->CanDelegateTo(guid); + candidates.push_back(candidate); + } + + const MafiaNet::PeerGuid previous = entity->ownerGUID; + const MafiaNet::PeerGuid next = ElectOwner(entity->position, previous, candidates, _delegationParams, _groundXY); + if (next == previous) { + return; + } + + if (next == MafiaNet::UNASSIGNED_PEER_GUID && entity->delegation.orphanMode == NetworkEntity::OrphanMode::Destroy) { + toDestroy.push_back(entity); + return; + } + + SetOwner(entity, next); + // Keep load current within the pass so later entities balance against it. + if (previous != MafiaNet::UNASSIGNED_PEER_GUID) { + if (const auto it = load.find(previous); it != load.end() && --it->second <= 0) { + load.erase(it); + } + } + if (next != MafiaNet::UNASSIGNED_PEER_GUID) { + ++load[next]; + } + }); + + for (NetworkEntity *entity : toDestroy) { + DestroyEntity(entity); + } + } + void ReplicationManager::OnClosedConnection(const MafiaNet::SystemAddress &systemAddress, MafiaNet::RakNetGUID rakNetGUID, MafiaNet::PI2_LostConnectionReason lostConnectionReason) { // The player's avatar is server-created, so the base PopConnection (which only tears down // replicas a dropped peer itself created) leaves it behind. Notify the game while the avatar diff --git a/code/framework/src/networking/replication/replication_manager.h b/code/framework/src/networking/replication/replication_manager.h index 0902216cf..224b20ee9 100644 --- a/code/framework/src/networking/replication/replication_manager.h +++ b/code/framework/src/networking/replication/replication_manager.h @@ -8,6 +8,7 @@ #pragma once +#include "delegation_policy.h" #include "entity_registry.h" #include "interest_grid.h" #include "network_entity.h" @@ -106,6 +107,7 @@ namespace Framework::Networking::Replication { // InterestGrid::SetGroundPlaneXY. void SetInterestGroundPlaneXY(bool groundXY) { _interest.SetGroundPlaneXY(groundXY); + _groundXY = groundXY; // election measures distance on the same plane } // Rebuild the spatial index from current entity positions. Server only; call once per tick // before ReplicaManager3 serializes (driven from NetworkPeer::Update). @@ -116,6 +118,18 @@ namespace Framework::Networking::Replication { return _interest.Generation(); } + // --- Syncer delegation --- + // Server only. Runs proximity election over delegatable entities and calls SetOwner on a + // change, cadence-gated by DelegationParams::electionIntervalMs. Driven from NetworkPeer::Update + // after RebuildInterest; nowMs is MafiaNet::GetTime. + void RunDelegation(uint64_t nowMs); + void SetDelegationParams(const DelegationParams ¶ms) { + _delegationParams = params; + } + const DelegationParams &GetDelegationParams() const { + return _delegationParams; + } + // Server: invoked from OnClosedConnection just before the dropped peer's avatar is destroyed, // while it is still resolvable. The integration layer wires its player-disconnect notification // here. @@ -151,6 +165,9 @@ namespace Framework::Networking::Replication { NetworkPeer *_owner = nullptr; bool _clientRPCsRegistered = false; InterestGrid _interest; + DelegationParams _delegationParams; + uint64_t _lastDelegationMs = 0; // last election pass; gates the cadence + bool _groundXY = false; // election distance plane, mirrored from the interest grid std::unordered_map _viewers; fu2::function _onClientDisconnect; fu2::function _onEntityCreated; diff --git a/code/tests/framework_ut.cpp b/code/tests/framework_ut.cpp index db4cb0d29..c9fafd536 100644 --- a/code/tests/framework_ut.cpp +++ b/code/tests/framework_ut.cpp @@ -6,7 +6,7 @@ * See LICENSE file in the source repository for information regarding licensing. */ -#define UNIT_MAX_MODULES 12 +#define UNIT_MAX_MODULES 13 #include "logging/logger.h" #include "unit.h" @@ -17,6 +17,7 @@ #include "modules/network_packets_ut.h" #include "modules/state_machine_ut.h" #include "modules/persistent_config_ut.h" +#include "modules/delegation_ut.h" // Scripting tests #include "modules/engine_ut.h" @@ -36,6 +37,7 @@ int main() { UNIT_MODULE(network_packets); UNIT_MODULE(state_machine); UNIT_MODULE(persistent_config); + UNIT_MODULE(delegation); // Scripting tests UNIT_MODULE(engine); diff --git a/code/tests/modules/delegation_ut.h b/code/tests/modules/delegation_ut.h new file mode 100644 index 000000000..3e1c3df49 --- /dev/null +++ b/code/tests/modules/delegation_ut.h @@ -0,0 +1,143 @@ +/* + * MafiaHub OSS license + * Copyright (c) 2021-2023, MafiaHub. All rights reserved. + * + * This file comes from MafiaHub, hosted at https://github.com/MafiaHub/Framework. + * See LICENSE file in the source repository for information regarding licensing. + */ + +#pragma once + +#include "networking/replication/delegation_policy.h" + +#include +#include + +// Coverage for the pure syncer-election policy (ElectOwner). +MODULE(delegation, { + using namespace Framework::Networking::Replication; + + const MafiaNet::PeerGuid kNone = MafiaNet::UNASSIGNED_PEER_GUID; + const auto guid = [](uint64_t v) { + return static_cast(v); + }; + const auto raw = [](MafiaNet::PeerGuid g) { + return static_cast(g); + }; + const auto candidate = [](MafiaNet::PeerGuid g, float x, float z, int load, bool eligible) { + DelegationCandidate c; + c.guid = g; + c.position = glm::vec3(x, 0.0f, z); + c.ownedLoad = load; + c.eligible = eligible; + return c; + }; + + // Defaults: acquire 80, drop 130; entity at the origin, XZ plane unless a test overrides. + DelegationParams params; + const glm::vec3 origin(0.0f, 0.0f, 0.0f); + + IT("elects the nearest eligible candidate when the entity is unowned", { + std::vector c = { + candidate(guid(1), 60.0f, 0.0f, 0, true), + candidate(guid(2), 30.0f, 0.0f, 0, true), + }; + const auto next = ElectOwner(origin, kNone, c, params, false); + UEQUALS(raw(next), raw(guid(2))); + }); + + IT("prefers the least-loaded candidate over a nearer but busier one", { + std::vector c = { + candidate(guid(1), 20.0f, 0.0f, 3, true), // closest but heavily loaded + candidate(guid(2), 70.0f, 0.0f, 0, true), // farther but idle + }; + const auto next = ElectOwner(origin, kNone, c, params, false); + UEQUALS(raw(next), raw(guid(2))); + }); + + IT("keeps the current owner inside dropRange even past acquireRange (hysteresis)", { + std::vector c = { + candidate(guid(1), 100.0f, 0.0f, 0, true), // owner, in the hysteresis band (80..130) + candidate(guid(2), 40.0f, 0.0f, 0, true), // closer newcomer + }; + const auto next = ElectOwner(origin, guid(1), c, params, false); + UEQUALS(raw(next), raw(guid(1))); + }); + + IT("re-elects when the current owner drifts past dropRange", { + std::vector c = { + candidate(guid(1), 140.0f, 0.0f, 0, true), // owner, now out of drop range + candidate(guid(2), 50.0f, 0.0f, 0, true), // eligible taker within acquire + }; + const auto next = ElectOwner(origin, guid(1), c, params, false); + UEQUALS(raw(next), raw(guid(2))); + }); + + IT("returns UNASSIGNED when no candidate is within acquireRange", { + std::vector c = { + candidate(guid(1), 90.0f, 0.0f, 0, true), + candidate(guid(2), 200.0f, 0.0f, 0, true), + }; + const auto next = ElectOwner(origin, kNone, c, params, false); + UEQUALS(raw(next), raw(kNone)); + }); + + IT("orphans the entity when the drifted owner has no taker in range", { + // Owner past drop and the only other candidate is out of acquire: falls back to the server. + std::vector c = { + candidate(guid(1), 140.0f, 0.0f, 0, true), + candidate(guid(2), 120.0f, 0.0f, 0, true), + }; + const auto next = ElectOwner(origin, guid(1), c, params, false); + UEQUALS(raw(next), raw(kNone)); + }); + + IT("ignores ineligible candidates (dimension / game veto)", { + std::vector c = { + candidate(guid(1), 20.0f, 0.0f, 0, false), // closest but vetoed + candidate(guid(2), 70.0f, 0.0f, 0, true), + }; + const auto next = ElectOwner(origin, kNone, c, params, false); + UEQUALS(raw(next), raw(guid(2))); + }); + + IT("does not keep a current owner that has become ineligible", { + std::vector c = { + candidate(guid(1), 50.0f, 0.0f, 0, false), + candidate(guid(2), 60.0f, 0.0f, 0, true), + }; + const auto next = ElectOwner(origin, guid(1), c, params, false); + UEQUALS(raw(next), raw(guid(2))); + }); + + IT("skips candidates at the load soft cap when a lighter one exists", { + params.loadSoftCap = 2; + std::vector c = { + candidate(guid(1), 20.0f, 0.0f, 2, true), // closest but at the cap + candidate(guid(2), 70.0f, 0.0f, 1, true), // under the cap + }; + const auto next = ElectOwner(origin, kNone, c, params, false); + UEQUALS(raw(next), raw(guid(2))); + params.loadSoftCap = 0; + }); + + IT("keeps the current owner even when it is over the soft cap", { + params.loadSoftCap = 1; + std::vector c = { + candidate(guid(1), 100.0f, 0.0f, 5, true), // owner in hysteresis band, over cap + candidate(guid(2), 40.0f, 0.0f, 0, true), + }; + const auto next = ElectOwner(origin, guid(1), c, params, false); + UEQUALS(raw(next), raw(guid(1))); + params.loadSoftCap = 0; + }); + + IT("measures distance on the XY plane when groundXY is set (Z-up)", { + // XY distance 0 despite z=1000, so in range. + std::vector c = { + {guid(1), glm::vec3(0.0f, 0.0f, 1000.0f), 0, true}, + }; + const auto next = ElectOwner(origin, kNone, c, params, true); + UEQUALS(raw(next), raw(guid(1))); + }); +});