feat: Updates serialization to use MMF (considerable memory savings) (#…

…1841)

### Summary
Updates the serialization strategy to use `MemoryMappedFile` instead of thick buffers. This has the benefit of being on-par with the current implementation (based on hardware/OS), however won't incur the double-memory issue.

> [!Important]
> **Developer Note**
> The `BinaryFileWriter` and `BinaryFileReader` has been removed in favor of `MemoryMapFileWriter` and `UnmanagedDataReader`

.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "modernuoschemagenerator": {
-      "version": "2.10.9",
+      "version": "2.11.3",
       "commands": [
         "ModernUOSchemaGenerator"
       ]

Projects/Server.Tests/Tests/Network/Packets/Outgoing/AccountPacketTests.cs

@@ -48,8 +48,6 @@ public Mobile this[int index]
         }
 
         public DateTime Created { get; set; }
-        public long SavePosition { get; set; }
-        public BufferWriter SaveBuffer { get; set; }
         public Serial Serial { get; }
         public void Deserialize(IGenericReader reader) => throw new NotImplementedException();
 

Projects/Server/Guild.cs

@@ -1,6 +1,6 @@
 /*************************************************************************
  * ModernUO                                                              *
- * Copyright 2019-2023 - ModernUO Development Team                       *
+ * Copyright 2019-2024 - ModernUO Development Team                       *
  * Email: [email protected]                                                *
  * File: Guild.cs                                                        *
  *                                                                       *
@@ -52,12 +52,6 @@ protected BaseGuild()
     [CommandProperty(AccessLevel.GameMaster, readOnly: true)]
     public DateTime Created { get; set; } = Core.Now;
 
-    [IgnoreDupe]
-    public long SavePosition { get; set; } = -1;
-
-    [IgnoreDupe]
-    public BufferWriter SaveBuffer { get; set; }
-
     public abstract void Serialize(IGenericWriter writer);
 
     public abstract void Deserialize(IGenericReader reader);

Projects/Server/IEntity.cs

@@ -1,6 +1,6 @@
 /*************************************************************************
  * ModernUO                                                              *
- * Copyright 2019-2023 - ModernUO Development Team                       *
+ * Copyright 2019-2024 - ModernUO Development Team                       *
  * Email: [email protected]                                                *
  * File: IEntity.cs                                                      *
  *                                                                       *
@@ -51,10 +51,6 @@ public Entity(Serial serial, Point3D loc, Map map) : this(serial)
 
     public DateTime Created { get; set; } = Core.Now;
 
-    public long SavePosition { get; set; } = -1;
-
-    public BufferWriter SaveBuffer { get; set; }
-
     public Serial Serial { get; }
 
     public Point3D Location { get; private set; }

Projects/Server/Items/Item.cs

@@ -1,3 +1,18 @@
+/*************************************************************************
+ * ModernUO                                                              *
+ * Copyright 2019-2024 - ModernUO Development Team                       *
+ * Email: [email protected]                                                *
+ * File: Item.cs                                                         *
+ *                                                                       *
+ * This program is free software: you can redistribute it and/or modify  *
+ * it under the terms of the GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3 of the License, or     *
+ * (at your option) any later version.                                   *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
 using System;
 using System.Collections.Generic;
 using System.Reflection;
@@ -783,12 +798,6 @@ public virtual void GetProperties(IPropertyList list)
     [CommandProperty(AccessLevel.GameMaster, readOnly: true)]
     public DateTime Created { get; set; } = Core.Now;
 
-    [IgnoreDupe]
-    public long SavePosition { get; set; } = -1;
-
-    [IgnoreDupe]
-    public BufferWriter SaveBuffer { get; set; }
-
     [IgnoreDupe]
     [CommandProperty(AccessLevel.Counselor)]
     public Serial Serial { get; }

Projects/Server/Main.cs

@@ -1,6 +1,6 @@
 /*************************************************************************
  * ModernUO                                                              *
- * Copyright 2019-2023 - ModernUO Development Team                       *
+ * Copyright 2019-2024 - ModernUO Development Team                       *
  * Email: [email protected]                                                *
  * File: Main.cs                                                         *
  *                                                                       *
@@ -40,6 +40,7 @@ public static class Core
     private static bool _performProcessKill;
     private static bool _restartOnKill;
     private static bool _performSnapshot;
+    private static string _snapshotPath;
     private static bool _crashed;
     private static string _baseDirectory;
 
@@ -411,7 +412,6 @@ private static void HandleClosed()
         logger.Information("Shutting down");
 
         World.WaitForWriteCompletion();
-
         World.ExitSerializationThreads();
 
         if (!_crashed)
@@ -553,12 +553,13 @@ public static void RunEventLoop()
                 if (_performSnapshot)
                 {
                     // Return value is the offset that can be used to fix timers that should drift
-                    World.Snapshot();
+                    World.Snapshot(_snapshotPath);
                     _performSnapshot = false;
                 }
 
                 if (_performProcessKill)
                 {
+                    World.WaitForWriteCompletion();
                     break;
                 }
 
@@ -594,7 +595,11 @@ public static void RunEventLoop()
         DoKill(_restartOnKill);
     }
 
-    internal static void RequestSnapshot() => _performSnapshot = true;
+    internal static void RequestSnapshot(string snapshotPath)
+    {
+        _performSnapshot = true;
+        _snapshotPath = snapshotPath;
+    }
 
     public static void VerifySerialization()
     {

Projects/Server/Mobiles/Mobile.cs

@@ -1,3 +1,18 @@
+/*************************************************************************
+ * ModernUO                                                              *
+ * Copyright 2019-2024 - ModernUO Development Team                       *
+ * Email: [email protected]                                                *
+ * File: Mobile.cs                                                       *
+ *                                                                       *
+ * This program is free software: you can redistribute it and/or modify  *
+ * it under the terms of the GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3 of the License, or     *
+ * (at your option) any later version.                                   *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
 using System;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
@@ -2262,12 +2277,6 @@ public virtual void GetProperties(IPropertyList list)
     [CommandProperty(AccessLevel.GameMaster, readOnly: true)]
     public DateTime Created { get; set; } = Core.Now;
 
-    [IgnoreDupe]
-    public long SavePosition { get; set; } = -1;
-
-    [IgnoreDupe]
-    public BufferWriter SaveBuffer { get; set; }
-
     [IgnoreDupe]
     [CommandProperty(AccessLevel.Counselor)]
     public Serial Serial { get; }

Projects/Server/Serialization/AdhocPersistence.cs

@@ -1,6 +1,6 @@
 /*************************************************************************
  * ModernUO                                                              *
- * Copyright 2019-2023 - ModernUO Development Team                       *
+ * Copyright 2019-2024 - ModernUO Development Team                       *
  * Email: [email protected]                                                *
  * File: AdhocPersistence.cs                                             *
  *                                                                       *
@@ -25,87 +25,91 @@ namespace Server;
 public static class AdhocPersistence
 {
     /**
-     * Serializes to memory synchronously. Optional buffer can be provided.
-     * Note: The buffer may not be the same after returning from the function if more data is written
-     * than the initial buffer can handle.
+     * Serializes to memory.
      */
-    public static BufferWriter Serialize(Action<IGenericWriter> serializer, ConcurrentQueue<Type> types)
+    public static IGenericWriter SerializeToBuffer(Action<IGenericWriter> serializer, ConcurrentQueue<Type> types = null)
     {
         var saveBuffer = new BufferWriter(true, types);
         serializer(saveBuffer);
         return saveBuffer;
     }
 
     /**
-     * Writes a buffer to disk. This function should be called asynchronously.
-     * Writes the filePath for the binary data, and an accompanying SerializedTypes.db file of all possible types.
+     * Deserializes from a buffer.
      */
-    public static void WriteSnapshot(FileInfo file, Span<byte> buffer)
+    public static IGenericReader DeserializeFromBuffer(
+        byte[] buffer, Action<IGenericReader> deserializer, Dictionary<ulong, string> typesDb = null
+    )
     {
-        var dirPath = file.DirectoryName;
-        PathUtility.EnsureDirectory(dirPath);
-
-        using var fs = new FileStream(file.FullName, FileMode.Create, FileAccess.Write);
-        fs.Write(buffer);
+        var reader = new BufferReader(buffer, typesDb);
+        deserializer(reader);
+        return reader;
     }
 
     /**
-     * Serializes to a memory buffer synchronously, then flushes to the path asynchronously.
-     * See WriteSnapshot for more info about how to snapshot.
+     * Serializes to a Memory Mapped file synchronously, then flushes to the file asynchronously.
      */
-    public static void SerializeAndSnapshot(string filePath, Action<IGenericWriter> serializer, ConcurrentQueue<Type> types = null)
+    public static void SerializeAndSnapshot(
+        string filePath, Action<IGenericWriter> serializer, long sizeHint = 1024 * 1024 * 32
+    )
     {
-        types ??= new ConcurrentQueue<Type>();
-        var saveBuffer = Serialize(serializer, types);
+        var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
+        PathUtility.EnsureDirectory(Path.GetDirectoryName(fullPath));
+        ConcurrentQueue<Type> types = [];
+        var writer = new MemoryMapFileWriter(new FileStream(filePath, FileMode.Create), sizeHint, types);
+        serializer(writer);
+
         Task.Run(
             () =>
             {
-                var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
-                var file = new FileInfo(fullPath);
+                var fs = writer.FileStream;
 
-                WriteSnapshot(file, saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position));
+                writer.Dispose();
+                fs.Dispose();
 
-                // TODO: Create a PooledHashSet if performance becomes an issue.
-                var typesSet = new HashSet<Type>();
+                HashSet<Type> typesSet = [];
 
                 // Dedupe the queue.
                 foreach (var type in types)
                 {
                     typesSet.Add(type);
                 }
 
-                Persistence.WriteSerializedTypesSnapshot(file.DirectoryName, typesSet);
-            });
+                Persistence.WriteSerializedTypesSnapshot(Path.GetDirectoryName(fullPath), typesSet);
+            },
+            Core.ClosingTokenSource.Token
+        );
     }
 
-    public static void Deserialize(string filePath, Action<IGenericReader> deserializer)
+    public static unsafe void Deserialize(string filePath, Action<IGenericReader> deserializer)
     {
         var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
         var file = new FileInfo(fullPath);
 
-        if (!file.Exists)
+        if (!file.Exists || file.Length == 0)
         {
             return;
         }
 
         var fileLength = file.Length;
-        if (fileLength == 0)
-        {
-            return;
-        }
 
         string error;
 
         try
         {
             using var mmf = MemoryMappedFile.CreateFromFile(fullPath, FileMode.Open);
-            using var stream = mmf.CreateViewStream();
-            using var br = new BinaryFileReader(stream);
-            deserializer(br);
+            using var accessor = mmf.CreateViewStream();
 
-            error = br.Position != fileLength
-                ? $"Serialized {fileLength} bytes, but {br.Position} bytes deserialized"
+            byte* ptr = null;
+            accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
+            UnmanagedDataReader dataReader = new UnmanagedDataReader(ptr, accessor.Length);
+            deserializer(dataReader);
+
+            error = dataReader.Position != fileLength
+                ? $"Serialized {fileLength} bytes, but {dataReader.Position} bytes deserialized"
                 : null;
+
+            accessor.SafeMemoryMappedViewHandle.ReleasePointer();
         }
         catch (Exception e)
         {