System to automatically restart server after certain uptime. (space-wizards#32814)

Author: PJB3005

Content.Server/ServerUpdates/ServerUpdateManager.cs

@@ -12,33 +12,57 @@
 namespace Content.Server.ServerUpdates;
 
 /// <summary>
-/// Responsible for restarting the server for update, when not disruptive.
+/// Responsible for restarting the server periodically or for update, when not disruptive.
 /// </summary>
-public sealed class ServerUpdateManager
+/// <remarks>
+/// This was originally only designed for restarting on *update*,
+/// but now also handles periodic restarting to keep server uptime via <see cref="CCVars.ServerUptimeRestartMinutes"/>.
+/// </remarks>
+public sealed class ServerUpdateManager : IPostInjectInit
 {
     [Dependency] private readonly IGameTiming _gameTiming = default!;
     [Dependency] private readonly IWatchdogApi _watchdog = default!;
     [Dependency] private readonly IPlayerManager _playerManager = default!;
     [Dependency] private readonly IChatManager _chatManager = default!;
     [Dependency] private readonly IBaseServer _server = default!;
     [Dependency] private readonly IConfigurationManager _cfg = default!;
+    [Dependency] private readonly ILogManager _logManager = default!;
+
+    private ISawmill _sawmill = default!;
 
     [ViewVariables]
     private bool _updateOnRoundEnd;
 
     private TimeSpan? _restartTime;
 
+    private TimeSpan _uptimeRestart;
+
     public void Initialize()
     {
         _watchdog.UpdateReceived += WatchdogOnUpdateReceived;
         _playerManager.PlayerStatusChanged += PlayerManagerOnPlayerStatusChanged;
+
+        _cfg.OnValueChanged(
+            CCVars.ServerUptimeRestartMinutes,
+            minutes => _uptimeRestart = TimeSpan.FromMinutes(minutes),
+            true);
     }
 
     public void Update()
     {
-        if (_restartTime != null && _restartTime < _gameTiming.RealTime)
+        if (_restartTime != null)
         {
-            DoShutdown();
+            if (_restartTime < _gameTiming.RealTime)
+            {
+                DoShutdown();
+            }
+        }
+        else
+        {
+            if (ShouldShutdownDueToUptime())
+            {
+                ServerEmptyUpdateRestartCheck("uptime");
+            }
         }
     }
 
@@ -48,7 +72,7 @@ public void Update()
     /// <returns>True if the server is going to restart.</returns>
     public bool RoundEnded()
     {
-        if (_updateOnRoundEnd)
+        if (_updateOnRoundEnd || ShouldShutdownDueToUptime())
         {
             DoShutdown();
             return true;
@@ -61,11 +85,14 @@ private void PlayerManagerOnPlayerStatusChanged(object? sender, SessionStatusEve
     {
         switch (e.NewStatus)
         {
-            case SessionStatus.Connecting:
+            case SessionStatus.Connected:
+                if (_restartTime != null)
+                    _sawmill.Debug("Aborting server restart timer due to player connection");
+
                 _restartTime = null;
                 break;
             case SessionStatus.Disconnected:
-                ServerEmptyUpdateRestartCheck();
+                ServerEmptyUpdateRestartCheck("last player disconnect");
                 break;
         }
     }
@@ -74,37 +101,51 @@ private void WatchdogOnUpdateReceived()
     {
         _chatManager.DispatchServerAnnouncement(Loc.GetString("server-updates-received"));
         _updateOnRoundEnd = true;
-        ServerEmptyUpdateRestartCheck();
+        ServerEmptyUpdateRestartCheck("update notification");
     }
 
     /// <summary>
     ///     Checks whether there are still players on the server,
     /// and if not starts a timer to automatically reboot the server if an update is available.
     /// </summary>
-    private void ServerEmptyUpdateRestartCheck()
+    private void ServerEmptyUpdateRestartCheck(string reason)
     {
         // Can't simple check the current connected player count since that doesn't update
         // before PlayerStatusChanged gets fired.
         // So in the disconnect handler we'd still see a single player otherwise.
         var playersOnline = _playerManager.Sessions.Any(p => p.Status != SessionStatus.Disconnected);
-        if (playersOnline || !_updateOnRoundEnd)
+        if (playersOnline || !(_updateOnRoundEnd || ShouldShutdownDueToUptime()))
         {
             // Still somebody online.
             return;
         }
 
         if (_restartTime != null)
         {
-            // Do nothing because I guess we already have a timer running..?
+            // Do nothing because we already have a timer running.
             return;
         }
 
         var restartDelay = TimeSpan.FromSeconds(_cfg.GetCVar(CCVars.UpdateRestartDelay));
         _restartTime = restartDelay + _gameTiming.RealTime;
+
+        _sawmill.Debug("Started server-empty restart timer due to {Reason}", reason);
     }
 
     private void DoShutdown()
     {
-        _server.Shutdown(Loc.GetString("server-updates-shutdown"));
+        _sawmill.Debug($"Shutting down via {nameof(ServerUpdateManager)}!");
+        var reason = _updateOnRoundEnd ? "server-updates-shutdown" : "server-updates-shutdown-uptime";
+        _server.Shutdown(Loc.GetString(reason));
+    }
+
+    private bool ShouldShutdownDueToUptime()
+    {
+        return _uptimeRestart != TimeSpan.Zero && _gameTiming.RealTime > _uptimeRestart;
+    }
+
+    void IPostInjectInit.PostInject()
+    {
+        _sawmill = _logManager.GetSawmill("restart");
     }
 }

Content.Shared/CCVar/CCVars.cs

@@ -32,6 +32,21 @@ public sealed class CCVars : CVars
         public static readonly CVarDef<string> DefaultGuide =
             CVarDef.Create("server.default_guide", "NewPlayer", CVar.REPLICATED | CVar.SERVER);
 
+        /// <summary>
+        /// If greater than 0, automatically restart the server after this many minutes of uptime.
+        /// </summary>
+        /// <remarks>
+        /// <para>
+        /// This is intended to work around various bugs and performance issues caused by long continuous server uptime.
+        /// </para>
+        /// <para>
+        /// This uses the same non-disruptive logic as update restarts,
+        /// i.e. the game will only restart at round end or when there is nobody connected.
+        /// </para>
+        /// </remarks>
+        public static readonly CVarDef<int> ServerUptimeRestartMinutes =
+            CVarDef.Create("server.uptime_restart_minutes", 0, CVar.SERVERONLY);
+
         /*
          * Ambience
          */

Resources/Locale/en-US/server-updates/server-updates.ftl

@@ -1,2 +1,3 @@
 server-updates-received = Update has been received, server will automatically restart for update at the end of this round.
 server-updates-shutdown = Server is shutting down for update and will automatically restart.
+server-updates-shutdown-uptime = Server is shutting down for periodic cleanup and will automatically restart.