Hopefully optimize the DB and split the YAML schema out

ReplayBrowser/Controllers/DataController.cs

@@ -21,9 +21,9 @@ public DataController(ReplayDbContext context)
     public async Task<List<string>> GetUsernameCompletion(
         [FromQuery] string username)
     {
-        var completions = await _context.Players
-            .Where(p => p.PlayerOocName.ToLower().StartsWith(username.ToLower()))
-            .Select(p => p.PlayerOocName)
+        var completions = await _context.ReplayParticipants
+            .Where(p => p.Username.ToLower().StartsWith(username.ToLower()))
+            .Select(p => p.Username)
             .Distinct() // Remove duplicates
             .Take(10)
             .ToListAsync();

ReplayBrowser/Controllers/ReplayController.cs

@@ -3,6 +3,7 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
 using ReplayBrowser.Data;
+using ReplayBrowser.Data.Models;
 using ReplayBrowser.Helpers;
 using ReplayBrowser.Services;
 
@@ -16,14 +17,14 @@
     private readonly ReplayDbContext _dbContext;
     private readonly AccountService _accountService;
     private readonly ReplayHelper _replayHelper;
-    
+
     public ReplayController(ReplayDbContext dbContext, AccountService accountService, ReplayHelper replayHelper)
     {
         _dbContext = dbContext;
         _accountService = accountService;
         _replayHelper = replayHelper;
     }
-    
+
     [HttpGet("{replayId}")]
     [AllowAnonymous]
     public async Task<IActionResult> GetReplay(int replayId)
@@ -34,10 +35,10 @@
         {
             return NotFound();
         }
-        
+
         return Ok(replay);
     }
-    
+
     /// <summary>
     /// Deletes all stored profiles for a server group.
     /// </summary>
@@ -45,31 +46,31 @@
     [HttpDelete("profile/delete/{serverId}")]
     public async Task<IActionResult> DeleteProfile(string serverId)
     {
         if (!User.Identity.IsAuthenticated)
         {
             return Unauthorized();
         }
-        
+
         var guidRequestor = AccountHelper.GetAccountGuid(User);
-        
+
         var requestor = await _dbContext.Accounts
             .Include(a => a.Settings)
             .Include(a => a.History)
             .FirstOrDefaultAsync(a => a.Guid == guidRequestor);
-        
+
         if (requestor == null)
         {
             return NotFound("Account is null. This should not happen.");
         }
-        
-        if (!requestor.IsAdmin) 
+
+        if (!requestor.IsAdmin)
             return Unauthorized("You are not an admin.");
 
         var players = await _dbContext.Replays
             .Where(r => r.ServerId == serverId)
-            .Include(r => r.RoundEndPlayers)
-            .Where(r => r.RoundEndPlayers != null)
-            .SelectMany(r => r.RoundEndPlayers)
+            .Include(r => r.RoundParticipants)
+            .Where(r => r.RoundParticipants != null)
+            .SelectMany(r => r.RoundParticipants)
             .Select(p => p.PlayerGuid)
             .Distinct()
             .ToListAsync();
@@ -80,26 +81,26 @@
                                                       DELETE FROM "CharacterData"
                                                       WHERE "CollectedPlayerDataPlayerGuid" = '{player}';
                                                       """);
-            
+
             await _dbContext.Database.ExecuteSqlRawAsync($"""
                                                           DELETE FROM "JobCountData"
                                                           WHERE "CollectedPlayerDataPlayerGuid" = '{player}';
                                                           """);
         }
-        
+
         await _dbContext.PlayerProfiles
             .Where(p => players.Contains(p.PlayerGuid))
             .ExecuteDeleteAsync();
-        
+
         return Ok();
     }
-    
+
     [HttpGet("profile/{profileGuid:guid}")]
     public async Task<IActionResult> GetPlayerData(Guid profileGuid)
     {
         // ok very jank, we construct a AuthenticationState object from the current user
         var authState = new AuthenticationState(HttpContext.User);
-        
+
         try
         {
             return Ok(await _replayHelper.GetPlayerProfile(profileGuid, authState));
@@ -109,7 +110,7 @@
             return Unauthorized(e.Message);
         }
     }
-    
+
     /// <summary>
     /// Marks a profile "watched" for the current user.
     /// </summary>
@@ -122,25 +123,25 @@
             .Include(a => a.Settings)
             .Include(a => a.History)
             .FirstOrDefaultAsync(a => a.Guid == guid);
-        
+
         if (account == null)
         {
             return Unauthorized();
         }
 
         var isWatched = account.SavedProfiles.Contains(profileGuid);
-        
+
         if (!account.SavedProfiles.Remove(profileGuid))
         {
             account.SavedProfiles.Add(profileGuid);
         }
-        
+
         await _dbContext.SaveChangesAsync();
 
         return Ok(!isWatched);
     }
-    
-    
+
+
     /// <summary>
     /// Marks a replay as a favorite for the current user.
     /// </summary>
@@ -153,25 +154,25 @@
             .Include(a => a.Settings)
             .Include(a => a.History)
             .FirstOrDefaultAsync(a => a.Guid == guid);
-        
+
         if (account == null)
         {
             return Unauthorized();
         }
-        
+
         var replay = await _dbContext.Replays.FindAsync(replayId);
         if (replay == null)
         {
             return NotFound();
         }
 
         var isFavorited = account.FavoriteReplays.Contains(replayId);
-        
+
         if (!account.FavoriteReplays.Remove(replayId))
         {
             account.FavoriteReplays.Add(replayId);
         }
-        
+
         await _dbContext.SaveChangesAsync();
 
         return Ok(!isFavorited);

ReplayBrowser/Data/Migrations/20240820195835_JobDepartmentMap.cs

@@ -0,0 +1,88 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace Server.Migrations
+{
+    /// <inheritdoc />
+    public partial class JobDepartmentMap : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "JobDepartments",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "integer", nullable: false)
+                        .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
+                    Job = table.Column<string>(type: "text", nullable: false),
+                    Department = table.Column<string>(type: "text", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_JobDepartments", x => x.Id);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "IX_JobDepartments_Job",
+                table: "JobDepartments",
+                column: "Job",
+                unique: true);
+
+            migrationBuilder.Sql(
+                """
+                INSERT INTO "JobDepartments"
+                ( "Job", "Department" )
+                VALUES
+                ( 'Captain', 'Command' ),
+                ( 'HeadOfPersonnel', 'Command' ),
+                ( 'ChiefMedicalOfficer', 'Command' ),
+                ( 'ResearchDirector', 'Command' ),
+                ( 'HeadOfSecurity', 'Command' ),
+                ( 'ChiefEngineer', 'Command' ),
+                ( 'Quartermaster', 'Command' ),
+                ( 'Borg', 'Science' ),
+                ( 'Scientist', 'Science' ),
+                ( 'ResearchAssistant', 'Science' ),
+                ( 'Warden', 'Security' ),
+                ( 'Detective', 'Security' ),
+                ( 'SecurityOfficer', 'Security' ),
+                ( 'SecurityCadet', 'Security' ),
+                ( 'MedicalDoctor', 'Medical' ),
+                ( 'Chemist', 'Medical' ),
+                ( 'Paramedic', 'Medical' ),
+                ( 'Psychologist', 'Medical' ),
+                ( 'MedicalIntern', 'Medical' ),
+                ( 'StationEngineer', 'Engineering' ),
+                ( 'AtmosphericTechnician', 'Engineering' ),
+                ( 'TechnicalAssistant', 'Engineering' ),
+                ( 'Janitor', 'Service' ),
+                ( 'Chef', 'Service' ),
+                ( 'Botanist', 'Service' ),
+                ( 'Bartender', 'Service' ),
+                ( 'Chaplain', 'Service' ),
+                ( 'Lawyer', 'Service' ),
+                ( 'Musician', 'Service' ),
+                ( 'Reporter', 'Service' ),
+                ( 'Zookeeper', 'Service' ),
+                ( 'Librarian', 'Service' ),
+                ( 'ServiceWorker', 'Service' ),
+                ( 'Clown', 'Service' ),
+                ( 'Mime', 'Service' ),
+                ( 'CargoTechnician', 'Cargo' ),
+                ( 'SalvageSpecialist', 'Cargo' ),
+                ( 'Passenger', 'The tide' );
+                """
+            );
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "JobDepartments");
+        }
+    }
+}

ReplayBrowser/Data/Migrations/20240820204611_ReplayParticipant.cs

@@ -0,0 +1,105 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace Server.Migrations
+{
+    /// <inheritdoc />
+    public partial class ReplayParticipant : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropForeignKey(
+                name: "FK_Players_Replays_ReplayId",
+                table: "Players");
+
+            migrationBuilder.AlterColumn<int>(
+                name: "ReplayId",
+                table: "Players",
+                type: "integer",
+                nullable: false,
+                defaultValue: 0,
+                oldClrType: typeof(int),
+                oldType: "integer",
+                oldNullable: true);
+
+            migrationBuilder.CreateTable(
+                name: "ReplayParticipants",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "integer", nullable: false)
+                        .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
+                    PlayerGuid = table.Column<Guid>(type: "uuid", nullable: false),
+                    ReplayId = table.Column<int>(type: "integer", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_ReplayParticipants", x => x.Id);
+                    table.ForeignKey(
+                        name: "FK_ReplayParticipants_Replays_ReplayId",
+                        column: x => x.ReplayId,
+                        principalTable: "Replays",
+                        principalColumn: "Id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "IX_ReplayParticipants_PlayerGuid_ReplayId",
+                table: "ReplayParticipants",
+                columns: new[] { "PlayerGuid", "ReplayId" },
+                unique: true);
+
+            migrationBuilder.CreateIndex(
+                name: "IX_ReplayParticipants_ReplayId",
+                table: "ReplayParticipants",
+                column: "ReplayId");
+
+            migrationBuilder.AddForeignKey(
+                name: "FK_Players_Replays_ReplayId",
+                table: "Players",
+                column: "ReplayId",
+                principalTable: "Replays",
+                principalColumn: "Id",
+                onDelete: ReferentialAction.Cascade);
+
+            migrationBuilder.Sql(
+                """
+                insert into "ReplayParticipants" ("PlayerGuid", "ReplayId") (
+                    select DISTINCT p."PlayerGuid", p."ReplayId"
+                    from "Players" p
+                    GROUP BY p."PlayerGuid", p."ReplayId"
+                );
+                """
+            );
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropForeignKey(
+                name: "FK_Players_Replays_ReplayId",
+                table: "Players");
+
+            migrationBuilder.DropTable(
+                name: "ReplayParticipants");
+
+            migrationBuilder.AlterColumn<int>(
+                name: "ReplayId",
+                table: "Players",
+                type: "integer",
+                nullable: true,
+                oldClrType: typeof(int),
+                oldType: "integer");
+
+            migrationBuilder.AddForeignKey(
+                name: "FK_Players_Replays_ReplayId",
+                table: "Players",
+                column: "ReplayId",
+                principalTable: "Replays",
+                principalColumn: "Id");
+        }
+    }
+}

ReplayBrowser/Data/Migrations/20240820232455_PlayerTrim.cs

@@ -0,0 +1,185 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Server.Migrations
+{
+    /// <inheritdoc />
+    public partial class PlayerTrim : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            // FIXME: Merge with previous migration
+
+            migrationBuilder.Sql(
+                """
+                TRUNCATE "ReplayParticipants";
+                """
+            );
+
+            migrationBuilder.AlterColumn<string>(
+                name: "Link",
+                table: "Replays",
+                type: "text",
+                nullable: false,
+                defaultValue: "",
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldNullable: true);
+
+            migrationBuilder.AddColumn<int>(
+                name: "ParticipantId",
+                table: "Players",
+                type: "integer",
+                nullable: false,
+                defaultValue: 0);
+
+            migrationBuilder.CreateIndex(
+                name: "IX_Players_ParticipantId",
+                table: "Players",
+                column: "ParticipantId");
+
+            migrationBuilder.AddColumn<string>(
+                name: "Username",
+                table: "ReplayParticipants",
+                type: "text",
+                nullable: false,
+                defaultValue: "");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_ReplayParticipants_Username",
+                table: "ReplayParticipants",
+                column: "Username");
+
+            // SQL magic here
+
+            migrationBuilder.Sql(
+                """
+                insert into "ReplayParticipants" ("PlayerGuid", "ReplayId", "Username") (
+                    select DISTINCT p."PlayerGuid", p."ReplayId", p."PlayerOocName"
+                    from "Players" p
+                    GROUP BY p."PlayerGuid", p."ReplayId", p."PlayerOocName"
+                );
+                """
+            );
+
+            migrationBuilder.Sql(
+                """
+                UPDATE "Players" AS pl
+                SET "ParticipantId" = p."Id"
+                FROM "ReplayParticipants" AS p
+                WHERE p."ReplayId" = pl."ReplayId"
+                    AND p."PlayerGuid" = pl."PlayerGuid";
+                """
+            );
+
+            // Data must be valid by this point
+
+            migrationBuilder.AddForeignKey(
+                name: "FK_Players_ReplayParticipants_ParticipantId",
+                table: "Players",
+                column: "ParticipantId",
+                principalTable: "ReplayParticipants",
+                principalColumn: "Id",
+                onDelete: ReferentialAction.Cascade);
+
+            migrationBuilder.DropForeignKey(
+                name: "FK_Players_Replays_ReplayId",
+                table: "Players");
+
+            // Data lossy operations
+
+            migrationBuilder.DropIndex(
+                name: "IX_Players_PlayerGuid",
+                table: "Players");
+
+            migrationBuilder.DropIndex(
+                name: "IX_Players_PlayerOocName",
+                table: "Players");
+
+            migrationBuilder.DropIndex(
+                name: "IX_Players_ReplayId",
+                table: "Players");
+
+            migrationBuilder.DropColumn(
+                name: "PlayerGuid",
+                table: "Players");
+
+            migrationBuilder.DropColumn(
+                name: "PlayerOocName",
+                table: "Players");
+
+            migrationBuilder.DropColumn(
+                name: "ReplayId",
+                table: "Players");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropForeignKey(
+                name: "FK_Players_ReplayParticipants_ParticipantId",
+                table: "Players");
+
+            migrationBuilder.DropIndex(
+                name: "IX_ReplayParticipants_Username",
+                table: "ReplayParticipants");
+
+            migrationBuilder.DropColumn(
+                name: "Username",
+                table: "ReplayParticipants");
+
+            migrationBuilder.RenameColumn(
+                name: "ParticipantId",
+                table: "Players",
+                newName: "ReplayId");
+
+            migrationBuilder.RenameIndex(
+                name: "IX_Players_ParticipantId",
+                table: "Players",
+                newName: "IX_Players_ReplayId");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "Link",
+                table: "Replays",
+                type: "text",
+                nullable: true,
+                oldClrType: typeof(string),
+                oldType: "text");
+
+            migrationBuilder.AddColumn<Guid>(
+                name: "PlayerGuid",
+                table: "Players",
+                type: "uuid",
+                nullable: false,
+                defaultValue: new Guid("00000000-0000-0000-0000-000000000000"));
+
+            migrationBuilder.AddColumn<string>(
+                name: "PlayerOocName",
+                table: "Players",
+                type: "text",
+                nullable: false,
+                defaultValue: "");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_Players_PlayerGuid",
+                table: "Players",
+                column: "PlayerGuid");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_Players_PlayerOocName",
+                table: "Players",
+                column: "PlayerOocName");
+
+            migrationBuilder.AddForeignKey(
+                name: "FK_Players_Replays_ReplayId",
+                table: "Players",
+                column: "ReplayId",
+                principalTable: "Replays",
+                principalColumn: "Id",
+                onDelete: ReferentialAction.Cascade);
+        }
+    }
+}

ReplayBrowser/Data/Migrations/20240821134427_JobForeignKey.cs

@@ -0,0 +1,59 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Server.Migrations
+{
+    /// <inheritdoc />
+    public partial class JobForeignKey : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<int>(
+                name: "EffectiveJobId",
+                table: "Players",
+                type: "integer",
+                nullable: true);
+
+            migrationBuilder.CreateIndex(
+                name: "IX_Players_EffectiveJobId",
+                table: "Players",
+                column: "EffectiveJobId");
+
+            migrationBuilder.AddForeignKey(
+                name: "FK_Players_JobDepartments_EffectiveJobId",
+                table: "Players",
+                column: "EffectiveJobId",
+                principalTable: "JobDepartments",
+                principalColumn: "Id");
+
+            migrationBuilder.Sql(
+                """
+                UPDATE "Players"
+                SET "EffectiveJobId" = (
+                    SELECT j."Id" FROM "JobDepartments" j
+                    WHERE "Players"."JobPrototypes"[1] = j."Job"
+                )
+                WHERE "Players"."JobPrototypes" != '{}'
+                """
+            );
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropForeignKey(
+                name: "FK_Players_JobDepartments_EffectiveJobId",
+                table: "Players");
+
+            migrationBuilder.DropIndex(
+                name: "IX_Players_EffectiveJobId",
+                table: "Players");
+
+            migrationBuilder.DropColumn(
+                name: "EffectiveJobId",
+                table: "Players");
+        }
+    }
+}

ReplayBrowser/Data/Models/JobDepartment.cs

@@ -0,0 +1,20 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace ReplayBrowser.Data.Models;
+
+/// <summary>
+/// Maps a given job ID to a department
+/// </summary>
+public class JobDepartment : IEntityTypeConfiguration<JobDepartment>
+{
+    public int Id { get; set; }
+
+    public required string Job { get; set; }
+    public required string Department { get; set; }
+
+    public void Configure(EntityTypeBuilder<JobDepartment> builder)
+    {
+        builder.HasIndex(j => j.Job).IsUnique();
+    }
+}

ReplayBrowser/Data/Models/Player.cs

@@ -1,52 +1,51 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Metadata.Builders;
-using YamlDotNet.Serialization;
+using ReplayBrowser.Models.Ingested;
 
 namespace ReplayBrowser.Data.Models;
 
 public class Player : IEntityTypeConfiguration<Player>
 {
     public int Id { get; set; }
 
-    [YamlMember(Alias = "antagPrototypes")]
-    public List<string> AntagPrototypes { get; set; }
-    [YamlMember(Alias = "jobPrototypes")]
-    public List<string> JobPrototypes { get; set; }
-    [YamlMember(Alias = "playerGuid")]
-    public Guid PlayerGuid { get; set; }
-    [YamlMember(Alias = "playerICName")]
-    public string PlayerIcName { get; set; }
-    [YamlMember(Alias = "playerOOCName")]
-    public string PlayerOocName { get; set; }
-    [YamlMember(Alias = "antag")]
+    public List<string> AntagPrototypes { get; set; } = null!;
+    public List<string> JobPrototypes { get; set; } = null!;
+    public required string PlayerIcName { get; set; }
     public bool Antag { get; set; }
 
-    // Foreign key
-
-    public int? ReplayId { get; set; }
-    public Replay? Replay { get; set; }
+    public ReplayParticipant Participant { get; set; } = null!;
+    public int ParticipantId { get; set; }
 
+    public JobDepartment? EffectiveJob { get; set; }
+    public int? EffectiveJobId { get; set; }
 
     public void Configure(EntityTypeBuilder<Player> builder)
     {
-        builder.HasIndex(p => p.PlayerGuid);
         builder.HasIndex(p => p.PlayerIcName);
-        builder.HasIndex(p => p.PlayerOocName);
-        builder.HasIndex(p => p.ReplayId);
+        builder.HasIndex(p => p.ParticipantId);
+    }
+
+    public static Player FromYaml(YamlPlayer player)
+    {
+        return new Player {
+            PlayerIcName = player.PlayerIcName,
+
+            JobPrototypes = player.JobPrototypes,
+            AntagPrototypes = player.AntagPrototypes,
+
+            Antag = player.Antag
+        };
     }
 
     public void RedactInformation(bool wasGdpr = false)
     {
         if (wasGdpr)
         {
             PlayerIcName = "Removed by GDPR request";
-            PlayerOocName = "Removed by GDPR request";
         }
         else
         {
             PlayerIcName = "Redacted";
-            PlayerOocName = "Redacted";
         }
-        PlayerGuid = Guid.Empty;
     }
 }

ReplayBrowser/Data/Models/Replay.cs

@@ -4,6 +4,7 @@
 using Microsoft.EntityFrameworkCore.Metadata.Builders;
 using NpgsqlTypes;
 using ReplayBrowser.Models;
+using ReplayBrowser.Models.Ingested;
 using YamlDotNet.Serialization;
 
 namespace ReplayBrowser.Data.Models;
@@ -12,39 +13,25 @@ public class Replay : IEntityTypeConfiguration<Replay>
 {
     public int Id { get; set; }
 
-    public string? Link { get; set; }
+    public string Link { get; set; }
 
-    [YamlMember(Alias = "roundId")]
     public int? RoundId { get; set; }
 
-    [YamlMember(Alias = "server_name")]
     public string? ServerName { get; set; }
     public DateTime? Date { get; set; }
 
-    [YamlMember(Alias = "map")]
     public string? Map { get; set; }
 
-    [YamlMember(Alias = "maps")]
     public List<string>? Maps { get; set; }
-    [YamlMember(Alias = "gamemode")]
     public string Gamemode { get; set; }
-    [YamlMember(Alias = "roundEndPlayers")]
-    public List<Player>? RoundEndPlayers { get; set; }
-    [YamlMember(Alias = "roundEndText")]
+    public List<ReplayParticipant>? RoundParticipants { get; set; }
     public string? RoundEndText { get; set; }
-    [YamlMember(Alias = "server_id")]
     public string ServerId { get; set; }
-    [YamlMember(Alias = "endTick")]
     public int EndTick { get; set; }
-    [YamlMember(Alias = "duration")]
     public string Duration { get; set; }
-    [YamlMember(Alias = "fileCount")]
     public int FileCount { get; set; }
-    [YamlMember(Alias = "size")]
     public int Size { get; set; }
-    [YamlMember(Alias = "uncompressedSize")]
     public int UncompressedSize { get; set; }
-    [YamlMember(Alias = "endTime")]
     public string EndTime { get; set; }
 
     [JsonIgnore]
@@ -98,23 +85,84 @@ public ReplayResult ToResult()
         };
     }
 
-    public void RedactInformation(Guid? accountGuid)
+    public static Replay FromYaml(YamlReplay replay, string link)
+    {
+        var participants = replay.RoundEndPlayers?
+            .GroupBy(p => p.PlayerGuid)
+            .Select(pg => new ReplayParticipant {
+                PlayerGuid = pg.Key,
+                Players = pg.Select(yp => Player.FromYaml(yp)).ToList(),
+                Username = pg.First().PlayerOocName
+            })
+            .ToList();
+
+        return new Replay {
+            Link = link,
+            ServerId = replay.ServerId,
+            ServerName = replay.ServerName,
+            RoundId = replay.RoundId,
+            Gamemode = replay.Gamemode,
+            Map = replay.Map,
+            Maps = replay.Maps,
+
+            RoundParticipants = participants,
+            RoundEndText = replay.RoundEndText,
+
+            EndTick = replay.EndTick,
+            Duration = replay.Duration,
+            FileCount = replay.FileCount,
+            Size = replay.Size,
+            UncompressedSize = replay.UncompressedSize,
+            EndTime = replay.EndTime
+        };
+    }
+
+    public void RedactInformation(Guid? accountGuid, bool wasGdpr)
     {
         if (accountGuid == null)
-        {
             return;
-        }
+        if (RoundParticipants == null)
+            return;
 
-        if (RoundEndPlayers != null)
+        foreach (var participant in RoundParticipants)
         {
-            foreach (var player in RoundEndPlayers)
+            if (participant.PlayerGuid != accountGuid)
+                continue;
+
+            // FIXME: This can cause unique constraint failure! Take care when redacting
+            participant.PlayerGuid = Guid.Empty;
+            if (wasGdpr)
+            {
+                participant.Username = "Removed by GDPR request";
+            }
+            else
+            {
+                participant.Username = "Redacted";
+            }
+
+            if (participant.Players is null)
+                return;
+
+            // Note that the above might be null if entries were not .Included!
+            foreach (var character in participant.Players)
             {
-                if (player.PlayerGuid == accountGuid)
-                {
-                    player.PlayerOocName = "Redacted by user request";
-                    player.PlayerGuid = Guid.Empty;
-                }
+                character.RedactInformation();
             }
         }
     }
+
+    public void RedactCleanup()
+    {
+        if (RoundParticipants is null) return;
+        var empty = RoundParticipants.Where(p => p.PlayerGuid == Guid.Empty).ToList();
+        RoundParticipants!.RemoveAll(p => p.PlayerGuid == Guid.Empty);
+        RoundParticipants.Add(
+            new ReplayParticipant {
+                PlayerGuid = Guid.Empty,
+                // Collate everything into one more generic group
+                Username = "Redacted",
+                Players = empty.SelectMany(p => p.Players!).ToList()
+            }
+        );
+    }
 }

ReplayBrowser/Data/Models/ReplayParticipant.cs

@@ -0,0 +1,25 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace ReplayBrowser.Data.Models;
+
+/// <summary>
+/// Basic entry that says some player was in a given round
+/// </summary>
+public class ReplayParticipant : IEntityTypeConfiguration<ReplayParticipant>
+{
+    public int Id { get; set; }
+    public Guid PlayerGuid { get; set; }
+    public string Username { get; set; } = null!;
+
+    public Replay Replay { get; set; } = null!;
+    public int ReplayId { get; set; }
+
+    public List<Player>? Players { get; set; }
+
+    public void Configure(EntityTypeBuilder<ReplayParticipant> builder)
+    {
+        builder.HasIndex(p => new { p.PlayerGuid, p.ReplayId }).IsUnique();
+        builder.HasIndex(p => p.Username);
+    }
+}

ReplayBrowser/Data/ReplayDbContext.cs

@@ -43,4 +43,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
     /// </summary>
     /// <returns></returns>
     public DbSet<CollectedPlayerData> PlayerProfiles { get; set; }
+
+    public DbSet<JobDepartment> JobDepartments { get; set; }
+
+    public DbSet<ReplayParticipant> ReplayParticipants { get; set; }
 }

ReplayBrowser/Helpers/AccountHelper.cs

@@ -5,18 +5,19 @@ namespace ReplayBrowser.Helpers;
 
 public static class AccountHelper
 {
+
     public static Guid? GetAccountGuid(AuthenticationState authState)
     {
         if (authState.User.Identity == null)
         {
             return null;
         }
-        
+
         if (!authState.User.Identity.IsAuthenticated)
         {
             return null;
         }
-        
+
         return Guid.Parse(authState.User.Claims.First(c => c.Type == "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier").Value);
     }
 
@@ -26,12 +27,12 @@ public static class AccountHelper
         {
             return null;
         }
-        
+
         if (!authState.Identity.IsAuthenticated)
         {
             return null;
         }
-        
+
         return Guid.Parse(authState.Claims.First(c => c.Type == "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier").Value);
     }
 }

ReplayBrowser/Helpers/ReplayHelper.cs

@@ -104,14 +104,15 @@ public async Task<List<ReplayResult>> GetMostRecentReplays(AuthenticationState s
 
        var replayPlayers = await _context.Players
             .AsNoTracking()
-            .Where(p => p.PlayerGuid == playerGuid)
-            .Where(p => p.Replay!.Date != null)
-            .Include(p => p.Replay)
+            .Where(p => p.Participant.PlayerGuid == playerGuid)
+            .Where(p => p.Participant.Replay!.Date != null)
+            .Include(p => p.Participant)
+            .ThenInclude(p => p.Replay)
             .Select(p => new {
                 p.Id,
-                p.ReplayId,
-                Date = (DateTime) p.Replay!.Date,
-                p.Replay!.Duration,
+                p.Participant.ReplayId,
+                Date = (DateTime) p.Participant.Replay!.Date!,
+                p.Participant.Replay!.Duration,
                 p.JobPrototypes,
                 // .Antag becomes unnecessary because this always has at least an empty string,
                 p.AntagPrototypes,
@@ -181,7 +182,8 @@ public async Task<int> GetTotalReplayCount()
     {
         var replay = await _context.Replays
             .AsNoTracking()
-            .Include(r => r.RoundEndPlayers)
+            .Include(r => r.RoundParticipants!)
+            .ThenInclude(p => p.Players)
             .FirstOrDefaultAsync(r => r.Id == id);
 
         if (replay == null)
@@ -194,40 +196,26 @@ public async Task<int> GetTotalReplayCount()
 
     private Replay FilterReplay(Replay replay, Account? caller = null)
     {
-        if (replay.RoundEndPlayers == null)
+        if (replay.RoundParticipants == null)
             return replay;
 
-        var accountDictionary = new Dictionary<Guid, AccountSettings>();
-        foreach (var replayRoundEndPlayer in replay.RoundEndPlayers)
-        {
-            if (!accountDictionary.ContainsKey(replayRoundEndPlayer.PlayerGuid))
-            {
-                var accountForPlayerTemp = _accountService.GetAccountSettings(replayRoundEndPlayer.PlayerGuid);
-                if (accountForPlayerTemp == null)
-                {
-                    continue;
-                }
-
-                accountDictionary.Add(replayRoundEndPlayer.PlayerGuid, accountForPlayerTemp);
-            }
-            var accountForPlayer = accountDictionary[replayRoundEndPlayer.PlayerGuid];
+        if (caller is not null && caller.IsAdmin)
+            return replay;
 
-            if (!accountForPlayer.RedactInformation) continue;
+        var redactFor = replay.RoundParticipants!
+            .Where(p => p.PlayerGuid != Guid.Empty)
+            .Select(p => new { p.PlayerGuid, Redact = _accountService.GetAccountSettings(p.PlayerGuid)?.RedactInformation ?? false })
+            .Where(p => p.Redact)
+            .Select(p => p.PlayerGuid)
+            .ToList();
 
-            if (caller == null)
-            {
-                replayRoundEndPlayer.RedactInformation();
-                continue;
-            }
+        if (caller is not null)
+            redactFor.Remove(caller.Guid);
 
-            if (replayRoundEndPlayer.PlayerGuid == caller.Guid) continue;
+        foreach (var redact in redactFor)
+            replay.RedactInformation(redact, false);
 
-            // If the caller is an admin, we can show the information.
-            if (!caller.IsAdmin)
-            {
-                replayRoundEndPlayer.RedactInformation();
-            }
-        }
+        replay.RedactCleanup();
 
         return replay;
     }
@@ -358,8 +346,8 @@ public async Task<SearchResult> SearchReplays(List<SearchQueryItem> searchItems,
     {
         var accountGuid = AccountHelper.GetAccountGuid(state);
 
-        var player = await _context.Players
-            .FirstOrDefaultAsync(p => p.PlayerOocName.ToLower() == username.ToLower());
+        var player = await _context.ReplayParticipants
+            .FirstOrDefaultAsync(p => p.Username.ToLower() == username.ToLower());
         if (player == null)
         {
             return null;
@@ -383,7 +371,7 @@ public async Task<SearchResult> SearchReplays(List<SearchQueryItem> searchItems,
         return new PlayerData()
         {
             PlayerGuid = player.PlayerGuid,
-            Username = player.PlayerOocName
+            Username = player.Username
         };
     }
 
@@ -405,7 +393,9 @@ public async Task<SearchResult> SearchReplays(List<SearchQueryItem> searchItems,
 
         var queryable = _context.Replays
             .AsNoTracking()
-            .Include(r => r.RoundEndPlayers).AsQueryable();
+            .Include(r => r.RoundParticipants!)
+            .ThenInclude(r => r.Players)
+            .AsQueryable();
 
         foreach (var searchItem in searchItems)
         {
@@ -419,9 +409,9 @@ public async Task<SearchResult> SearchReplays(List<SearchQueryItem> searchItems,
                 ),
                 SearchMode.Gamemode => queryable.Where(x => x.Gamemode.ToLower().Contains(searchItem.SearchValue.ToLower())),
                 SearchMode.ServerId => queryable.Where(x => x.ServerId.ToLower().Contains(searchItem.SearchValue.ToLower())),
-                SearchMode.Guid => queryable.Where(r => r.RoundEndPlayers.Any(p => p.PlayerGuid.ToString().ToLower().Contains(searchItem.SearchValue.ToLower()))),
-                SearchMode.PlayerIcName => queryable.Where(r => r.RoundEndPlayers.Any(p => p.PlayerIcName.ToLower().Contains(searchItem.SearchValue.ToLower()))),
-                SearchMode.PlayerOocName => queryable.Where(r => r.RoundEndPlayers.Any(p => p.PlayerOocName.ToLower().Contains(searchItem.SearchValue.ToLower()))),
+                SearchMode.Guid => queryable.Where(r => r.RoundParticipants.Any(p => p.PlayerGuid.ToString().ToLower().Contains(searchItem.SearchValue.ToLower()))),
+                SearchMode.PlayerIcName => queryable.Where(r => r.RoundParticipants.Any(p => p.Players.Any(pl => pl.PlayerIcName.ToLower().Contains(searchItem.SearchValue.ToLower())))),
+                SearchMode.PlayerOocName => queryable.Where(r => r.RoundParticipants.Any(p => p.Username.ToLower().Contains(searchItem.SearchValue.ToLower()))),
                 // ReSharper disable once EntityFramework.UnsupportedServerSideFunctionCall (its lying, this works)
                 SearchMode.RoundEndText => queryable.Where(x => x.RoundEndTextSearchVector.Matches(searchItem.SearchValue)),
                 SearchMode.ServerName => queryable.Where(x => x.ServerName != null && x.ServerName.ToLower().Contains(searchItem.SearchValue.ToLower())),
@@ -464,7 +454,6 @@ public async Task<SearchResult> SearchReplays(List<SearchQueryItem> searchItems,
         }
 
         var replays = await _context.Replays
-            .Include(r => r.RoundEndPlayers)
             .Where(r => account.FavoriteReplays.Contains(r.Id))
             .Select(r => r.ToResult())
             .ToListAsync();

ReplayBrowser/Models/Ingested/YamlPlayer.cs

@@ -0,0 +1,15 @@
+using YamlDotNet.Serialization;
+
+namespace ReplayBrowser.Models.Ingested;
+
+public class YamlPlayer
+{
+    public required  List<string> AntagPrototypes { get; set; }
+    public required  List<string> JobPrototypes { get; set; }
+    public Guid PlayerGuid { get; set; }
+    [YamlMember(Alias = "playerICName", ApplyNamingConventions = false)]
+    public required  string PlayerIcName { get; set; }
+    [YamlMember(Alias = "playerOOCName", ApplyNamingConventions = false)]
+    public required string PlayerOocName { get; set; }
+    public bool Antag { get; set; }
+}

ReplayBrowser/Models/Ingested/YamlReplay.cs

@@ -0,0 +1,26 @@
+using YamlDotNet.Serialization;
+
+namespace ReplayBrowser.Models.Ingested;
+
+public class YamlReplay {
+    public int? RoundId { get; set; }
+
+    [YamlMember(Alias = "server_id", ApplyNamingConventions = false)]
+    public string ServerId { get; set; }
+    [YamlMember(Alias = "server_name", ApplyNamingConventions = false)]
+    public string? ServerName { get; set; }
+
+    public string Gamemode { get; set; }
+    public string? Map { get; set; }
+    public List<string>? Maps { get; set; }
+
+    public List<YamlPlayer>? RoundEndPlayers { get; set; }
+    public string? RoundEndText { get; set; }
+
+    public int EndTick { get; set; }
+    public string Duration { get; set; }
+    public int FileCount { get; set; }
+    public int Size { get; set; }
+    public int UncompressedSize { get; set; }
+    public string EndTime { get; set; }
+}

ReplayBrowser/Models/RangeOption.cs

@@ -54,4 +54,18 @@ public static string GetTimeSpan(this RangeOption rangeOption)
             _ => throw new ArgumentOutOfRangeException(nameof(rangeOption), rangeOption, null)
         };
     }
+
+    public static TimeSpan GetNormalTimeSpan(this RangeOption rangeOption)
+    {
+        return rangeOption switch
+        {
+            RangeOption.Last24Hours => new TimeSpan(TimeSpan.TicksPerDay),
+            RangeOption.Last7Days => new TimeSpan(7 * TimeSpan.TicksPerDay),
+            RangeOption.Last30Days => new TimeSpan(30 * TimeSpan.TicksPerDay),
+            RangeOption.Last90Days => new TimeSpan(90 * TimeSpan.TicksPerDay),
+            RangeOption.Last365Days => new TimeSpan(365 * TimeSpan.TicksPerDay),
+            RangeOption.AllTime => new TimeSpan(DateTime.UtcNow.Ticks),
+            _ => throw new ArgumentOutOfRangeException(nameof(rangeOption), rangeOption, null)
+        };
+    }
 }

ReplayBrowser/Pages/Profile.razor

@@ -11,8 +11,7 @@
 @inject ReplayHelper ReplayHelper
 @inject Ss14ApiHelper Ss14ApiHelper
 
-
-<PageTitle>Player info</PageTitle>
+<PageTitle>Player info - @(_playerData?.PlayerData.Username ?? "Unknown")</PageTitle>
 
 @if (_playerData is null)
 {

ReplayBrowser/Pages/Shared/MainLayout.razor

@@ -175,16 +175,25 @@
 
     // Once the modal is opened, we need to rerequest the round end players if they are null
     // This is because the modal doesn't load the data until it is opened
-    async function loadDetails(players, endTextElement, replayId) {
+    async function loadDetails(playersElemn, endTextElement, replayId) {
         let response = await fetch(`/api/Replay/${replayId}`);
         let data = await response.json();
 
         let playerList = "";
-        if (data.roundEndPlayers == null || data.roundEndPlayers.length == 0) {
-            players.innerHTML = "<p class='card-text text-danger'>Replay is incomplete. No players available.</p>";
+        if (data.roundParticipants == null || data.roundParticipants.length == 0) {
+            playersElemn.innerHTML = "<p class='card-text text-danger'>Replay is incomplete. No players available.</p>";
         } else {
+            let players = data.roundParticipants.flatMap(
+                pc => pc.players.map(
+                    pl => ({
+                        ...pl,
+                        playerGuid: pc.playerGuid,
+                        username: pc.username
+                    })
+                )
+            )
             // Sort the players so that antags are at the top
-            data.roundEndPlayers.sort((a, b) => {
+            players.sort((a, b) => {
                 if (a.antagPrototypes.length > 0 && b.antagPrototypes.length == 0) {
                     return -1;
                 } else if (a.antagPrototypes.length == 0 && b.antagPrototypes.length > 0) {
@@ -193,20 +202,19 @@
                 return 0;
             });
 
-            data.roundEndPlayers.forEach(player => {
+            playersElemn.innerHTML = players.map(player => {
                 let job = "Unknown";
                 if (player.jobPrototypes.length > 0) {
                     job = player.jobPrototypes[0];
                 }
-                let playerText = `<a href="/player/${player.playerGuid}"><span style="color: gray">${player.playerOocName}</span></a> was <bold>${player.playerIcName}</bold> playing role of <span style="color: orange"><bold>${job}</bold></span>`;
+                let playerText = `<a href="/player/${player.playerGuid}"><span style="color: gray">${player.username}</span></a> was <bold>${player.playerIcName}</bold> playing role of <span style="color: orange"><bold>${job}</bold></span>`;
                 if (player.antagPrototypes.length > 0) {
-                    playerText = `<a href="/player/${player.playerGuid}"><span style="color: red">${player.playerOocName}</span></a> was <span style="color:red"><bold>${player.playerIcName}</bold></span> playing role of <span style="color: orange"><bold>${job}</bold></span>`;
+                    playerText = `<a href="/player/${player.playerGuid}"><span style="color: red">${player.username}</span></a> was <span style="color:red"><bold>${player.playerIcName}</bold></span> playing role of <span style="color: orange"><bold>${job}</bold></span>`;
                 }
                 // Need to show the guid as well
                 playerText += `<br><span style="color: gray;font-size: x-small;"> ${player.playerGuid}</span>`;
-                playerList += playerText + "<br>";
-            });
-            players.innerHTML = playerList;
+                return playerText;
+            }).join("<br>\n");
         }
 
         if (!data.roundEndText || data.roundEndText.trim() == "") {

ReplayBrowser/Pages/Shared/ReplayDetails.razor

@@ -86,7 +86,7 @@
                      }
                 </div>
                 <div class="modal-body">
-                    @if (FullReplay is not null && FullReplay.RoundEndPlayers is not null)
+                    @if (FullReplay is not null && FullReplay.RoundParticipants is not null)
                     {
                         <p id="players-@Replay.Id">@ReplayPlayersFormatted</p>
                     } else
@@ -116,7 +116,7 @@
     <a href="@Replay.Link" target="_blank" class="btn btn-primary">Download</a>
 
 
-    @if (FullReplay is null || FullReplay.RoundEndPlayers is null || FullReplay.RoundEndText is null) {
+    @if (FullReplay is null || FullReplay.RoundParticipants is null || FullReplay.RoundEndText is null) {
     <script>
     document.addEventListener('DOMContentLoaded', function() {
         const modalPlayers = document.getElementById("buttonPlayers-@Replay.Id")
@@ -201,15 +201,18 @@
         }
 
         // Format the player list
-        if (FullReplay is not null && FullReplay.RoundEndPlayers is not null)
+        if (FullReplay is not null && FullReplay.RoundParticipants is not null)
         {
-            var players = FullReplay.RoundEndPlayers;
+            var players = FullReplay.RoundParticipants.SelectMany(p => p.Players.Select(c => new {
+                    Player = c, p.PlayerGuid, p.Username
+                })).ToList();
+
             players.Sort((a, b) =>
             {
-                if (a.AntagPrototypes.Count > 0 && b.AntagPrototypes.Count == 0)
+                if (a.Player.AntagPrototypes.Count > 0 && b.Player.AntagPrototypes.Count == 0)
                 {
                     return -1;
-                } else if (a.AntagPrototypes.Count == 0 && b.AntagPrototypes.Count > 0)
+                } else if (a.Player.AntagPrototypes.Count == 0 && b.Player.AntagPrototypes.Count > 0)
                 {
                     return 1;
                 }
@@ -219,15 +222,15 @@
             // Antags are placed at the top
             var playerList = players.Select(player => {
                 var job = "Unknown";
-                if (player.JobPrototypes.Count > 0)
+                if (player.Player.JobPrototypes.Count > 0)
                 {
-                    job = player.JobPrototypes[0];
+                    job = player.Player.JobPrototypes[0];
                 }
 
-                var playerText = $"<a href=\"/player/{player.PlayerGuid}\"><span style=\"color: gray\">{player.PlayerOocName}</span></a> was <bold>{player.PlayerIcName}</bold> playing role of <span style=\"color: orange\"><bold>{job}</bold></span>";
-                if (player.AntagPrototypes.Count > 0)
+                var playerText = $"<a href=\"/player/{player.PlayerGuid}\"><span style=\"color: gray\">{player.Username}</span></a> was <bold>{player.Player.PlayerIcName}</bold> playing role of <span style=\"color: orange\"><bold>{job}</bold></span>";
+                if (player.Player.AntagPrototypes.Count > 0)
                 {
-                    playerText = $"<a href=\"/player/{player.PlayerGuid}\"><span style=\"color: red\">{player.PlayerOocName}</span></a> was <span style=\"color:red\"><bold>{player.PlayerIcName}</bold></span> playing role of <span style=\"color: orange\"><bold>{job}</bold></span>";
+                    playerText = $"<a href=\"/player/{player.PlayerGuid}\"><span style=\"color: red\">{player.Username}</span></a> was <span style=\"color:red\"><bold>{player.Player.PlayerIcName}</bold></span> playing role of <span style=\"color: orange\"><bold>{job}</bold></span>";
                 }
                 // Need to show the guid as well
                 playerText += $"<br><span style=\"color: gray;font-size: x-small;\"> {{{player.PlayerGuid}}}</span>";

ReplayBrowser/Program.cs

@@ -1,3 +1,5 @@
+using ReplayBrowser.Services;
+using ReplayBrowser.Services.ReplayParser;
 using Serilog;
 using Serilog.Events;
 
@@ -11,7 +13,7 @@ public static void Main(string[] args)
             .WriteTo.Console()
             .WriteTo.File("logs/log.txt", rollingInterval: RollingInterval.Hour, fileSizeLimitBytes: null)
             .CreateLogger();
-        
+
         try
         {
             Log.Information("Starting up");
@@ -26,7 +28,7 @@ public static void Main(string[] args)
             Log.CloseAndFlush();
         }
     }
-    
+
     public static IHostBuilder CreateHostBuilder(string[] args) =>
         Host.CreateDefaultBuilder(args)
             .ConfigureAppConfiguration((context, builder) =>
@@ -50,5 +52,11 @@ public static IHostBuilder CreateHostBuilder(string[] args) =>
             {
                 webBuilder.UseKestrel();
                 webBuilder.UseStartup<Startup>();
+            })
+            .ConfigureServices(s => {
+                s.AddHostedService<BackgroundServiceStarter<ReplayParserService>>();
+                s.AddHostedService<BackgroundServiceStarter<AccountService>>();
+                s.AddHostedService<BackgroundServiceStarter<LeaderboardService>>();
+                s.AddHostedService<BackgroundServiceStarter<AnalyticsService>>();
             });
 }

ReplayBrowser/Services/AccountService.cs

@@ -195,56 +195,40 @@ public async Task UpdateAccount(Account? account)
 
     public async Task<AccountHistoryResponse?> GetAccountHistory(string username, int pageNumber)
     {
+        Account? account = null;
         // If the username is empty, we want to see logs for not logged in users.
         if (string.IsNullOrWhiteSpace(username))
         {
-            var systemAccount = GetSystemAccount();
-
-            systemAccount.History = systemAccount.History.OrderByDescending(h => h.Time).ToList();
-
-            AccountHistoryResponse response = new()
-            {
-                History = [],
-                Page = pageNumber,
-                TotalPages = 1
-            };
-
-            if (systemAccount.History.Count > 10)
-            {
-                response.TotalPages = systemAccount.History.Count / 10;
-            }
-
-            response.History = systemAccount.History.Skip(pageNumber * 10).Take(10).ToList();
-            return response;
-        } else {
+            account = GetSystemAccount();
+        }
+        else
+        {
             using var scope = _scopeFactory.CreateScope();
             var context = scope.ServiceProvider.GetRequiredService<ReplayDbContext>();
-            var account = context.Accounts
+            account = context.Accounts
                 .Include(a => a.History)
                 .FirstOrDefault(a => a.Username == username);
+        }
 
-            if (account == null)
-            {
-                return null;
-            }
+        if (account is null)
+            return null;
 
-            account.History = account.History.OrderByDescending(h => h.Time).ToList();
+        account.History = account.History.OrderByDescending(h => h.Time).ToList();
 
-            AccountHistoryResponse response = new()
-            {
-                History = [],
-                Page = pageNumber,
-                TotalPages = 1
-            };
-
-            if (account.History.Count > 10)
-            {
-                response.TotalPages = account.History.Count / 10;
-            }
+        AccountHistoryResponse response = new()
+        {
+            History = [],
+            Page = pageNumber,
+            TotalPages = 1
+        };
 
-            response.History = account.History.Skip(pageNumber * 10).Take(10).ToList();
-            return response;
+        if (account.History.Count > 10)
+        {
+            response.TotalPages = account.History.Count / 10;
         }
+
+        response.History = account.History.OrderByDescending(h => h.Time).Skip(pageNumber * 10).Take(10).ToList();
+        return response;
     }
 
     private Account GetSystemAccount()

ReplayBrowser/Services/ReplayParser/ReplayParserService.cs

@@ -7,9 +7,11 @@
 using ReplayBrowser.Data.Models;
 using ReplayBrowser.Helpers;
 using ReplayBrowser.Models;
+using ReplayBrowser.Models.Ingested;
 using ReplayBrowser.Services.ReplayParser.Providers;
 using Serilog;
 using YamlDotNet.Serialization;
+using YamlDotNet.Serialization.NamingConventions;
 
 namespace ReplayBrowser.Services.ReplayParser;
 
@@ -238,14 +240,16 @@ private Replay ParseReplay(Stream stream, string replayLink)
 
         var deserializer = new DeserializerBuilder()
             .IgnoreUnmatchedProperties()
+            .WithNamingConvention(CamelCaseNamingConvention.Instance)
             .Build();
-        var replay = deserializer.Deserialize<Replay>(reader);
-        if (replay.Map == null && replay.Maps == null)
+
+        var yamlReplay = deserializer.Deserialize<YamlReplay>(reader);
+        if (yamlReplay.Map == null && yamlReplay.Maps == null)
         {
             throw new Exception("Replay is not valid.");
         }
 
-        replay.Link = replayLink;
+        var replay = Replay.FromYaml(yamlReplay, replayLink);
 
         var replayUrls = _configuration.GetSection("ReplayUrls").Get<StorageUrl[]>()!;
         if (replay.ServerId == Constants.UnsetServerId)
@@ -258,19 +262,32 @@ private Replay ParseReplay(Stream stream, string replayLink)
             replay.ServerName = replayUrls.First(x => replay.Link!.Contains(x.Url)).FallBackServerName;
         }
 
+        if (yamlReplay.RoundEndPlayers == null)
+            return replay;
+
+        var jobs = GetDbContext().JobDepartments.ToList();
+
+        replay.RoundParticipants!.ForEach(
+            p => p.Players!
+                .Where(pl => pl.JobPrototypes.Count != 0)
+                .ToList()
+                .ForEach(
+                    pl => pl.EffectiveJobId = jobs.SingleOrDefault(j => j.Job == pl.JobPrototypes[0])?.Id
+                )
+        );
+
         // Check for GDPRed accounts
         var gdprGuids = GetDbContext().GdprRequests.Select(x => x.Guid).ToList();
-        if (replay.RoundEndPlayers != null)
+
+        foreach (var redact in gdprGuids)
         {
-            foreach (var player in replay.RoundEndPlayers)
-            {
-                if (gdprGuids.Contains(player.PlayerGuid))
-                {
-                    player.RedactInformation(true);
-                }
-            }
+            replay.RedactInformation(redact, true);
         }
 
+        var redacted = replay.RoundParticipants!.Where(p => p.PlayerGuid == Guid.Empty);
+        if (redacted.Any())
+            replay.RedactCleanup();
+
         return replay;
     }
 

ReplayBrowser/Startup.cs

@@ -47,50 +47,20 @@ public void ConfigureServices(IServiceCollection services)
                 Environment.Exit(1);
             }
 
-            options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"));
+            // FIXME: Timeout needs to bumped down to default after the big migration finishes
+            options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), o => o.CommandTimeout(600));
 
             options.EnableSensitiveDataLogging();
             options.UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole().AddSerilog()));
         });
 
-        // Run migrations on startup.
-        var replayContext = services.BuildServiceProvider().GetService<ReplayDbContext>();
-        replayContext.Database.Migrate();
-
-        // We need to create a "dummy" system Account to use for unauthenticated requests.
-        // This is because the Account system is used for logging and we need to have an account to log as.
-        // This is a bit of a hack but it works.
-        try
-        {
-            var systemAccount = new Account()
-            {
-                Guid = Guid.Empty,
-                Username = "[System] Unauthenticated user",
-            };
-
-            if (replayContext.Accounts.FirstOrDefault(a => a.Guid == Guid.Empty) == null) // Only add if it doesn't already exist.
-            {
-                replayContext.Accounts.Add(systemAccount);
-                replayContext.SaveChanges();
-            }
-        }
-        catch (Exception e)
-        {
-            Log.Error(e, "Failed to create system account.");
-        }
-
         services.AddSingleton<Ss14ApiHelper>();
         services.AddSingleton<AccountService>();
         services.AddSingleton<LeaderboardService>();
         services.AddSingleton<ReplayParserService>();
         services.AddSingleton<AnalyticsService>();
         services.AddSingleton<NoticeHelper>();
 
-        services.AddHostedService<BackgroundServiceStarter<ReplayParserService>>();
-        services.AddHostedService<BackgroundServiceStarter<AccountService>>();
-        services.AddHostedService<BackgroundServiceStarter<LeaderboardService>>();
-        services.AddHostedService<BackgroundServiceStarter<AnalyticsService>>();
-
         services.AddScoped<ReplayHelper>();
 
         services.AddCors(options =>
@@ -108,7 +78,7 @@ public void ConfigureServices(IServiceCollection services)
         {
             options.MessageTemplate = "Handled {RequestPath}";
         });
-        
+
         services.Configure<ForwardedHeadersOptions>(options =>
         {
             options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto;
@@ -122,7 +92,7 @@ public void ConfigureServices(IServiceCollection services)
             Log.Information("Proxy IP: {ProxyIP}", proxyIP);
             options.KnownProxies.Add(IPAddress.Parse(proxyIP));
         });
-        
+
         services.AddOpenTelemetry().WithMetrics(providerBuilder =>
         {
             providerBuilder.AddPrometheusExporter();
@@ -229,6 +199,35 @@ public void ConfigureServices(IServiceCollection services)
 
     public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
     {
+        // Run migrations on startup.
+        {
+            using var dbScope = app.ApplicationServices.CreateScope();
+            using var replayContext = dbScope.ServiceProvider.GetRequiredService<ReplayDbContext>();
+            replayContext!.Database.Migrate();
+
+            // We need to create a "dummy" system Account to use for unauthenticated requests.
+            // This is because the Account system is used for logging and we need to have an account to log as.
+            // This is a bit of a hack but it works.
+            try
+            {
+                var systemAccount = new Account()
+                {
+                    Guid = Guid.Empty,
+                    Username = "[System] Unauthenticated user",
+                };
+
+                if (replayContext.Accounts.FirstOrDefault(a => a.Guid == Guid.Empty) == null) // Only add if it doesn't already exist.
+                {
+                    replayContext.Accounts.Add(systemAccount);
+                    replayContext.SaveChanges();
+                }
+            }
+            catch (Exception e)
+            {
+                Log.Error(e, "Failed to create system account.");
+            }
+        }
+
         app.Use((context, next) =>
         {
             context.Request.Scheme = "https";