Minitest

Robust.Server/GameObjects/ServerEntityManager.cs

@@ -33,6 +33,10 @@ public sealed class ServerEntityManager : EntityManager, IServerEntityManager
             "robust_entities_count",
             "Amount of alive entities.");
 
+        private static readonly Gauge EntityMsgQueueSize = Metrics.CreateGauge(
+            "robust_entity_msg_queue_size",
+            "Number of queued MsgEntity messages on the server.");
+
         [Dependency] private readonly IReplayRecordingManager _replay = default!;
         [Dependency] private readonly IServerNetManager _networkManager = default!;
         [Dependency] private readonly IGameTiming _gameTiming = default!;
@@ -145,6 +149,8 @@ internal override void SetLifeStage(MetaDataComponent meta, EntityLifeStage stag
             new();
 
         private bool _logLateMsgs;
+        private int _entityMsgQueueLimit;
+        private int _entityMsgMaxFutureTicks;
 
         /// <inheritdoc />
         public void SetupNetworking()
@@ -154,6 +160,8 @@ public void SetupNetworking()
             _playerManager.PlayerStatusChanged += OnPlayerStatusChanged;
 
             _configurationManager.OnValueChanged(CVars.NetLogLateMsg, b => _logLateMsgs = b, true);
+            _configurationManager.OnValueChanged(CVars.NetEntityMsgQueueLimit, v => _entityMsgQueueLimit = Math.Max(0, v), true);
+            _configurationManager.OnValueChanged(CVars.NetEntityMsgMaxFutureTicks, v => _entityMsgMaxFutureTicks = Math.Max(0, v), true);
         }
 
         /// <inheritdoc />
@@ -170,6 +178,7 @@ public override void TickUpdate(float frameTime, bool noPredictions, Histogram?
             base.TickUpdate(frameTime, noPredictions, histogram);
 
             EntitiesCount.Set(Entities.Count);
+            EntityMsgQueueSize.Set(_queue.Count);
         }
 
         public uint GetLastMessageSequence(ICommonSession? session)
@@ -204,6 +213,33 @@ public void SendSystemNetworkMessage(EntityEventArgs message, INetChannel target
 
         private void HandleEntityNetworkMessage(MsgEntity message)
         {
+            if (_entityMsgQueueLimit > 0 && _queue.Count >= _entityMsgQueueLimit)
+            {
+                _netEntSawmill.Warning(
+                    "Dropping MsgEntity from {0} for exceeding MsgEntity queue limit ({1}).",
+                    message.MsgChannel,
+                    _entityMsgQueueLimit);
+                return;
+            }
+
+            if (_entityMsgMaxFutureTicks > 0)
+            {
+                var curTick = _gameTiming.CurTick.Value;
+                var msgTick = message.SourceTick.Value;
+                var maxAllowed = curTick + (uint) _entityMsgMaxFutureTicks;
+
+                if (msgTick > maxAllowed)
+                {
+                    _netEntSawmill.Warning(
+                        "Dropping MsgEntity from {0} for future tick. cur={1} msg={2} limit={3}",
+                        message.MsgChannel,
+                        curTick,
+                        msgTick,
+                        _entityMsgMaxFutureTicks);
+                    return;
+                }
+            }
+
             if (_logLateMsgs)
             {
                 var msgT = message.SourceTick;

Robust.Server/GameStates/PvsSystem.Ack.cs

@@ -30,7 +30,10 @@ private void OnClientAck(ICommonSession session, GameTick ackedTick)
             return;
 
         sessionData.LastReceivedAck = ackedTick;
-        PendingAcks.Add(session);
+        lock (PendingAcks)
+        {
+            PendingAcks.Add(session);
+        }
     }
 
     /// <summary>
@@ -39,18 +42,21 @@ private void OnClientAck(ICommonSession session, GameTick ackedTick)
     /// <param name="histogram"></param>
     private WaitHandle? ProcessQueuedAcks()
     {
-        if (PendingAcks.Count == 0)
-            return null;
+        lock (PendingAcks)
+        {
+            if (PendingAcks.Count == 0)
+                return null;
 
-        _toAck.Clear();
+            _toAck.Clear();
 
-        foreach (var session in PendingAcks)
-        {
-            if (session.Status != SessionStatus.Disconnected)
-                _toAck.Add(GetOrNewPvsSession(session));
-        }
+            foreach (var session in PendingAcks)
+            {
+                if (session.Status != SessionStatus.Disconnected)
+                    _toAck.Add(GetOrNewPvsSession(session));
+            }
 
-        PendingAcks.Clear();
+            PendingAcks.Clear();
+        }
 
         if (!_async)
         {

Robust.Server/GameStates/PvsSystem.GetStates.cs

@@ -22,50 +22,64 @@ internal sealed partial class PvsSystem
     /// <returns>New entity State for the given entity.</returns>
     private EntityState GetEntityState(ICommonSession? player, EntityUid entityUid, GameTick fromTick, MetaDataComponent meta)
     {
-        var changed = new List<ComponentChange>();
+        var changed = _componentChangeListPool.Get();
+        changed.Clear();
 
         bool sendCompList = meta.LastComponentRemoved > fromTick;
-        HashSet<ushort>? netComps = sendCompList ? new() : null;
-        var stateEv = new ComponentGetState(player, fromTick);
-
-        foreach (var (netId, component) in meta.NetComponents)
+        HashSet<ushort>? netComps = null;
+        if (sendCompList)
+        {
+            netComps = _netComponentSetPool.Get();
+            netComps.Clear();
+        }
+        try
         {
-            DebugTools.Assert(component.NetSyncEnabled);
+            var stateEv = new ComponentGetState(player, fromTick);
 
-            if (component.Deleted || !component.Initialized)
+            foreach (var (netId, component) in meta.NetComponents)
             {
-                Log.Error($"Entity manager returned deleted or uninitialized component of type {component.GetType()} on entity {ToPrettyString(entityUid)} while generating entity state data for {player?.Name ?? "replay"}");
-                continue;
-            }
+                DebugTools.Assert(component.NetSyncEnabled);
 
-            if (component.SendOnlyToOwner && player != null && player.AttachedEntity != entityUid)
-                continue;
+                if (component.Deleted || !component.Initialized)
+                {
+                    Log.Error($"Entity manager returned deleted or uninitialized component of type {component.GetType()} on entity {ToPrettyString(entityUid)} while generating entity state data for {player?.Name ?? "replay"}");
+                    continue;
+                }
 
-            if (component.LastModifiedTick <= fromTick)
-            {
-                if (sendCompList && (!component.SessionSpecific || player == null || EntityManager.CanGetComponentState(component, player)))
-                    netComps!.Add(netId);
-                continue;
-            }
+                if (component.SendOnlyToOwner && player != null && player.AttachedEntity != entityUid)
+                    continue;
 
-            if (component.SessionSpecific && player != null && !EntityManager.CanGetComponentState(component, player))
-                continue;
+                if (component.LastModifiedTick <= fromTick)
+                {
+                    if (sendCompList && (!component.SessionSpecific || player == null || EntityManager.CanGetComponentState(component, player)))
+                        netComps!.Add(netId);
+                    continue;
+                }
 
-            var state = ComponentState(entityUid, component, netId, ref stateEv);
-            changed.Add(new ComponentChange(netId, state, component.LastModifiedTick));
+                if (component.SessionSpecific && player != null && !EntityManager.CanGetComponentState(component, player))
+                    continue;
 
-            if (state != null)
-                DebugTools.Assert(fromTick > component.CreationTick || state is not IComponentDeltaState);
+                var state = ComponentState(entityUid, component, netId, ref stateEv);
+                changed.Add(new ComponentChange(netId, state, component.LastModifiedTick));
 
-            if (sendCompList)
-                netComps!.Add(netId);
-        }
+                if (state != null)
+                    DebugTools.Assert(fromTick > component.CreationTick || state is not IComponentDeltaState);
 
-        DebugTools.Assert(meta.EntityLastModifiedTick >= meta.LastComponentRemoved);
-        DebugTools.Assert(GetEntity(meta.NetEntity) == entityUid);
-        var entState = new EntityState(meta.NetEntity, changed, meta.EntityLastModifiedTick, netComps);
+                if (sendCompList)
+                    netComps!.Add(netId);
+            }
 
-        return entState;
+            DebugTools.Assert(meta.EntityLastModifiedTick >= meta.LastComponentRemoved);
+            DebugTools.Assert(GetEntity(meta.NetEntity) == entityUid);
+            return new EntityState(meta.NetEntity, changed, meta.EntityLastModifiedTick, netComps);
+        }
+        catch
+        {
+            _componentChangeListPool.Return(changed);
+            if (netComps != null)
+                _netComponentSetPool.Return(netComps);
+            throw;
+        }
     }
 
     private IComponentState? ComponentState(EntityUid uid, IComponent comp, ushort netId, ref ComponentGetState stateEv)
@@ -83,30 +97,39 @@ private EntityState GetEntityState(ICommonSession? player, EntityUid entityUid,
     private EntityState GetFullEntityState(ICommonSession player, EntityUid entityUid, MetaDataComponent meta)
     {
         var bus = EntityManager.EventBusInternal;
-        var changed = new List<ComponentChange>();
+        var changed = _componentChangeListPool.Get();
+        changed.Clear();
         var stateEv = new ComponentGetState(player, GameTick.Zero);
 
-        HashSet<ushort> netComps = new();
+        var netComps = _netComponentSetPool.Get();
+        netComps.Clear();
 
-        foreach (var (netId, component) in meta.NetComponents)
+        try
         {
-            DebugTools.Assert(component.NetSyncEnabled);
-
-            if (component.SendOnlyToOwner && player.AttachedEntity != entityUid)
-                continue;
+            foreach (var (netId, component) in meta.NetComponents)
+            {
+                DebugTools.Assert(component.NetSyncEnabled);
 
-            if (component.SessionSpecific && !EntityManager.CanGetComponentState(bus, component, player))
-                continue;
+                if (component.SendOnlyToOwner && player.AttachedEntity != entityUid)
+                    continue;
 
-            var state = ComponentState(entityUid, component, netId, ref stateEv);
-            DebugTools.Assert(state is not IComponentDeltaState);
-            changed.Add(new ComponentChange(netId, state, component.LastModifiedTick));
-            netComps.Add(netId);
-        }
+                if (component.SessionSpecific && !EntityManager.CanGetComponentState(bus, component, player))
+                    continue;
 
-        var entState = new EntityState(meta.NetEntity, changed, meta.EntityLastModifiedTick, netComps);
+                var state = ComponentState(entityUid, component, netId, ref stateEv);
+                DebugTools.Assert(state is not IComponentDeltaState);
+                changed.Add(new ComponentChange(netId, state, component.LastModifiedTick));
+                netComps.Add(netId);
+            }
 
-        return entState;
+            return new EntityState(meta.NetEntity, changed, meta.EntityLastModifiedTick, netComps);
+        }
+        catch
+        {
+            _componentChangeListPool.Return(changed);
+            _netComponentSetPool.Return(netComps);
+            throw;
+        }
     }
 
     /// <summary>

Robust.Server/GameStates/PvsSystem.Pooling.cs

@@ -22,6 +22,12 @@ private readonly ObjectPool<List<PvsIndex>> _entDataListPool
     private readonly ObjectPool<HashSet<EntityUid>> _uidSetPool
         = new DefaultObjectPool<HashSet<EntityUid>>(new SetPolicy<EntityUid>(), MaxVisPoolSize);
 
+    private readonly ObjectPool<List<ComponentChange>> _componentChangeListPool
+        = new DefaultObjectPool<List<ComponentChange>>(new ListPolicy<ComponentChange>(), MaxVisPoolSize);
+
+    private readonly ObjectPool<HashSet<ushort>> _netComponentSetPool
+        = new DefaultObjectPool<HashSet<ushort>>(new SetPolicy<ushort>(), MaxVisPoolSize);
+
     private readonly ObjectPool<PvsChunk> _chunkPool =
         new DefaultObjectPool<PvsChunk>(new PvsChunkPolicy(), 256);
 

Robust.Server/GameStates/PvsSystem.Send.cs

@@ -45,41 +45,46 @@ private void SendSessionState(PvsSession data, ZStdCompressionContext ctx)
     {
         DebugTools.AssertEqual(data.State, null);
 
-        // PVS benchmarks use dummy sessions.
-        // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
-        if (data.Session.Channel is not DummyChannel)
+        try
         {
-            DebugTools.AssertNotEqual(data.StateStream, null);
-            var msg = new MsgState
+            if (data.Session.Channel is { } channel && channel is not DummyChannel)
             {
-                StateStream = data.StateStream,
-                ForceSendReliably = data.ForceSendReliably,
-                CompressionContext = ctx
-            };
+                if (!channel.IsConnected)
+                    return;
+
+                DebugTools.AssertNotEqual(data.StateStream, null);
+                var msg = new MsgState
+                {
+                    StateStream = data.StateStream,
+                    ForceSendReliably = data.ForceSendReliably,
+                    CompressionContext = ctx
+                };
 
-            _netMan.ServerSendMessage(msg, data.Session.Channel);
-            if (msg.ShouldSendReliably())
+                _netMan.ServerSendMessage(msg, channel);
+                if (msg.ShouldSendReliably())
+                {
+                    data.RequestedFull = false;
+                    data.LastReceivedAck = _gameTiming.CurTick;
+                    lock (PendingAcks)
+                    {
+                        PendingAcks.Add(data.Session);
+                    }
+                }
+            }
+            else
             {
-                data.RequestedFull = false;
                 data.LastReceivedAck = _gameTiming.CurTick;
+                data.RequestedFull = false;
                 lock (PendingAcks)
                 {
                     PendingAcks.Add(data.Session);
                 }
             }
         }
-        else
+        finally
         {
-            // Always "ack" dummy sessions.
-            data.LastReceivedAck = _gameTiming.CurTick;
-            data.RequestedFull = false;
-            lock (PendingAcks)
-            {
-                PendingAcks.Add(data.Session);
-            }
+            data.StateStream?.Dispose();
+            data.StateStream = null;
         }
-
-        data.StateStream?.Dispose();
-        data.StateStream = null;
     }
 }