Merge remote-tracking branch 'origin/master' into EpicManifestParsing

Author: devedse

DeveLanCacheUI_Backend.Tests/DbContextTests/DbContextDateTimeTests.cs

@@ -0,0 +1,57 @@
+using DeveLanCacheUI_Backend.Db;
+using DeveLanCacheUI_Backend.Db.DbModels;
+using Microsoft.Data.Sqlite;
+using Microsoft.EntityFrameworkCore;
+
+namespace DeveLanCacheUI_Backend.Tests.DbContextTests
+{
+    [TestClass]
+    public class DbContextDateTimeTests
+    {
+        [TestMethod]
+        public async Task ConvertsDateTimeToUtcCorrectly()
+        {
+            // Create a new SQLite connection string for in-memory database
+            var connectionString = "Data Source=:memory:";
+            using var connection = new SqliteConnection(connectionString);
+            await connection.OpenAsync();
+
+            var options = new DbContextOptionsBuilder<DeveLanCacheUIDbContext>()
+                .UseSqlite(connection)
+                .Options;
+
+            var dateTimeInLocalTimeZone = new DateTime(2023, 10, 1, 12, 0, 0, DateTimeKind.Local);
+            var expectedUtcDateTime = dateTimeInLocalTimeZone.ToUniversalTime();
+
+            // First context: Create database and insert data
+            using (var context = new DeveLanCacheUIDbContext(options))
+            {
+                // Ensure the database is created
+                await context.Database.EnsureCreatedAsync();
+
+                var testDownloadEvent = new DbDownloadEvent
+                {
+                    Id = 1,
+                    CreatedAt = dateTimeInLocalTimeZone,
+                    ClientIp = "",
+                    CacheIdentifier = "",
+                    LastUpdatedAt = dateTimeInLocalTimeZone
+                };
+
+                context.DownloadEvents.Add(testDownloadEvent);
+                await context.SaveChangesAsync();
+            }
+
+            // Second context: Verify the data was persisted and converted correctly
+            using (var context = new DeveLanCacheUIDbContext(options))
+            {
+                var savedEvent = await context.DownloadEvents.FindAsync(1);
+                Assert.IsNotNull(savedEvent, "Saved event should not be null.");
+
+                Assert.AreEqual(DateTimeKind.Utc, savedEvent.CreatedAt.Kind, "CreatedAt should be in UTC.");
+                Assert.AreEqual(expectedUtcDateTime, savedEvent.CreatedAt, "CreatedAt should be converted to UTC.");
+                Assert.AreEqual(expectedUtcDateTime, savedEvent.LastUpdatedAt, "LastUpdatedAt should be converted to UTC.");
+            }
+        }
+    }
+}

DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogLineParserTests.cs

@@ -90,5 +90,29 @@ public void SuperSplit_ParseCorrectly3()
             Assert.AreEqual("ctldl.windowsupdate.com", splitted[13]);
             Assert.AreEqual("-", splitted[14]);
         }
+
+        [TestMethod]
+        public void SplittedToLanCacheLogEntryRaw_ParseCorrectly_AndAlsoCalculateFieldsWorks_ForDateTimeWithTimeZone()
+        {
+            var logEntry = LanCacheLogLineParser.LogLineToLanCacheLogEntryRaw("[steam] 10.88.10.1 / - - - [28/Jun/2023:20:14:49 +0800] \"GET /depot/434174/chunk/9437c354e87778aeafe94a65ee042432440d4037 HTTP/1.1\" 200 392304 \"-\" \"Valve/Steam HTTP Client 1.0\" \"HIT\" \"cache1-ams1.steamcontent.com\" \"-\"");
+
+            Assert.AreEqual("steam", logEntry.CacheIdentifier);
+            Assert.AreEqual("10.88.10.1", logEntry.RemoteAddress);
+            Assert.AreEqual("-", logEntry.ForwardedFor);
+            Assert.AreEqual("-", logEntry.RemoteUser);
+            Assert.AreEqual("28/Jun/2023:20:14:49 +0800", logEntry.TimeLocal);
+            Assert.AreEqual("GET /depot/434174/chunk/9437c354e87778aeafe94a65ee042432440d4037 HTTP/1.1", logEntry.Request);
+            Assert.AreEqual("200", logEntry.Status);
+            Assert.AreEqual("392304", logEntry.BodyBytesSent);
+            Assert.AreEqual("-", logEntry.Referer);
+            Assert.AreEqual("Valve/Steam HTTP Client 1.0", logEntry.UserAgent);
+            Assert.AreEqual("HIT", logEntry.UpstreamCacheStatus);
+            Assert.AreEqual("cache1-ams1.steamcontent.com", logEntry.Host);
+            Assert.AreEqual("-", logEntry.HttpRange);
+
+            logEntry.CalculateFields();
+
+            Assert.AreEqual(new DateTime(2023, 6, 28, 12, 14, 49, DateTimeKind.Utc), logEntry.DateTime.ToUniversalTime());
+        }
     }
 }

DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs

@@ -1,4 +1,6 @@
-namespace DeveLanCacheUI_Backend.Db
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+
+namespace DeveLanCacheUI_Backend.Db
 {
     public class DeveLanCacheUIDbContext : DbContext
     {
@@ -14,7 +16,7 @@ public DeveLanCacheUIDbContext(DbContextOptions options) : base(options)
         {
 
         }
-        
+
         protected override void OnModelCreating(ModelBuilder modelBuilder)
         {
             base.OnModelCreating(modelBuilder);
@@ -59,5 +61,23 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                             }
                         });
         }
+
+        protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder)
+        {
+            ArgumentNullException.ThrowIfNull(configurationBuilder);
+
+            configurationBuilder.Properties<DateTime>().HaveConversion<DateTimeAsUtcValueConverter>();
+            configurationBuilder.Properties<DateTime?>().HaveConversion<NullableDateTimeAsUtcValueConverter>();
+        }
     }
+
+    public class NullableDateTimeAsUtcValueConverter() : ValueConverter<DateTime?, DateTime?>(
+    v => !v.HasValue ? v : ToUtc(v.Value), v => v.HasValue ? DateTime.SpecifyKind(v.Value, DateTimeKind.Utc) : v)
+    {
+        private static DateTime? ToUtc(DateTime v) => v.Kind == DateTimeKind.Utc ? v : v.ToUniversalTime();
+    }
+
+    public class DateTimeAsUtcValueConverter() : ValueConverter<DateTime, DateTime>(
+        v => v.Kind == DateTimeKind.Utc ? v : v.ToUniversalTime(), v => DateTime.SpecifyKind(v, DateTimeKind.Utc));
+
 }

DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs

@@ -42,6 +42,7 @@ public class LanCacheLogEntryRaw
 
         public void CalculateFields()
         {
+            // Will automatically get converted to UTC when storing in Database
             DateTime = DateTime.ParseExact(TimeLocal, "dd/MMM/yyyy:HH:mm:ss zzz", CultureInfo.InvariantCulture);
             BodyBytesSentLong = long.Parse(BodyBytesSent);
 

DeveLanCacheUI_Backend/Program.cs

@@ -87,8 +87,8 @@ public static void Main(string[] args)
             }
             else
             {
-                builder.Services.AddHostedService<SteamDepotEnricherHostedService>();
                 builder.Services.AddHostedService<SteamDepotDownloaderHostedService>();
+                builder.Services.AddSingleton<SteamDepotEnricherHostedService>();
                 builder.Services.AddSingleton<ISteamAppObtainerService, OriginalSteamAppObtainerService>();
             }
 

DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamDepotDownloaderHostedService.cs

@@ -7,17 +7,20 @@ public class SteamDepotDownloaderHostedService : BackgroundService
         public IServiceProvider Services { get; }
 
         private readonly DeveLanCacheConfiguration _deveLanCacheConfiguration;
+        private readonly SteamDepotEnricherHostedService _steamDepotEnricherHostedService;
         private readonly ILogger<SteamDepotDownloaderHostedService> _logger;
         private readonly HttpClient _httpClient;
 
         private const string DeveLanCacheUISteamDepotFinderLatestUrl = "https://api.github.com/repos/devedse/DeveLanCacheUI_SteamDepotFinder_Runner/releases/latest";
 
         public SteamDepotDownloaderHostedService(IServiceProvider services,
             DeveLanCacheConfiguration deveLanCacheConfiguration,
+            SteamDepotEnricherHostedService steamDepotEnricherHostedService,
             ILogger<SteamDepotDownloaderHostedService> logger)
         {
             Services = services;
             _deveLanCacheConfiguration = deveLanCacheConfiguration;
+            _steamDepotEnricherHostedService = steamDepotEnricherHostedService;
             _logger = logger;
             _httpClient = new HttpClient();
             _httpClient.DefaultRequestHeaders.Add("User-Agent", "request");
@@ -40,39 +43,42 @@ private async Task GoRun(CancellationToken stoppingToken)
 
             var depotFileDirectory = Path.Combine(deveLanCacheUIDataDirectory, "depotdir");
 
-            if (!Directory.Exists(depotFileDirectory))
+            if (Directory.Exists(depotFileDirectory))
             {
-                Directory.CreateDirectory(depotFileDirectory);
+                // This is purely cleanup since we don't use the depot directory anymore. We just directly download and process.
+                Directory.Delete(depotFileDirectory, true);
             }
 
             while (!stoppingToken.IsCancellationRequested)
             {
                 var shouldDownload = await NewDepotFileAvailable();
 
+                _logger.LogInformation("New Depot File Available: {NewVersionAvailable}, LatestVersion: {LatestVersion}, DownloadUrl: {DownloadUrl}",
+                    shouldDownload.NewVersionAvailable, shouldDownload.LatestVersion, shouldDownload.DownloadUrl);
+
                 if (shouldDownload.NewVersionAvailable)
                 {
-                    var createdFileName = await GoDownload(depotFileDirectory, shouldDownload);
+                    if (shouldDownload.LatestVersion == null || string.IsNullOrWhiteSpace(shouldDownload.DownloadUrl))
+                    {
+                        throw new UnreachableException($"New version available, but LatestVersion or DownloadUrl is null. This should not happen. LatestVersion: {shouldDownload.LatestVersion}, DownloadUrl: {shouldDownload.DownloadUrl}");
+                    }
+                    var downloadedBytes = await GoDownload(depotFileDirectory, shouldDownload);
 
-                    //Remove all other csv files in DepotDir except this one
-                    var depotFiles = Directory.GetFiles(depotFileDirectory).Where(t => Path.GetExtension(t).Equals(".csv", StringComparison.OrdinalIgnoreCase) && !t.EndsWith(createdFileName, StringComparison.OrdinalIgnoreCase));
-                    foreach (var depotFile in depotFiles)
+                    if (downloadedBytes != null)
+                    {
+                        await _steamDepotEnricherHostedService.GoProcess(shouldDownload.LatestVersion, downloadedBytes, stoppingToken);
+                    }
+                    else
                     {
-                        try
-                        {
-                            File.Delete(depotFile);
-                        }
-                        catch (Exception ex)
-                        {
-                            _logger.LogWarning("Could not delete depot file: {DepotFile}, exception: {Exception}", depotFile, ex);
-                        }
+                        _logger.LogWarning("Downloaded depot file was null, not processing further.");
                     }
                 }
 
                 await Task.Delay(TimeSpan.FromHours(1));
             }
         }
 
-        private async Task<string?> GoDownload(string depotFileDirectory, (bool NewVersionAvailable, Version? LatestVersion, string? DownloadUrl) shouldDownload)
+        private async Task<byte[]?> GoDownload(string depotFileDirectory, (bool NewVersionAvailable, Version? LatestVersion, string? DownloadUrl) shouldDownload)
         {
             _logger.LogInformation("Detected that new version '{NewVersionAvailable}' of Depot File is available, so downloading: {DownloadUrl}...", shouldDownload.NewVersionAvailable, shouldDownload.DownloadUrl);
 
@@ -84,31 +90,7 @@ private async Task GoRun(CancellationToken stoppingToken)
             }
 
             var bytes = await downloadedCsv.Content.ReadAsByteArrayAsync();
-
-            var fileName = $"depot_{shouldDownload.LatestVersion}.csv";
-            var filePath = Path.Combine(depotFileDirectory, fileName);
-            File.WriteAllBytes(filePath, bytes);
-
-            await using (var scope = Services.CreateAsyncScope())
-            {
-                using var dbContext = scope.ServiceProvider.GetRequiredService<DeveLanCacheUIDbContext>();
-                var foundSetting = await dbContext.Settings.FirstOrDefaultAsync();
-                if (foundSetting == null || foundSetting.Value == null)
-                {
-                    foundSetting = new DbSetting()
-                    {
-                        Key = DbSetting.SettingKey_DepotVersion,
-                        Value = shouldDownload.LatestVersion?.ToString()
-                    };
-                    dbContext.Settings.Add(foundSetting);
-                }
-                else
-                {
-                    foundSetting.Value = shouldDownload.LatestVersion?.ToString();
-                }
-                await dbContext.SaveChangesAsync();
-            }
-            return fileName;
+            return bytes;
         }
 
         private async Task<(bool NewVersionAvailable, Version? LatestVersion, string? DownloadUrl)> NewDepotFileAvailable()
@@ -148,7 +130,7 @@ private async Task GoRun(CancellationToken stoppingToken)
             await using (var scope = Services.CreateAsyncScope())
             {
                 using var dbContext = scope.ServiceProvider.GetRequiredService<DeveLanCacheUIDbContext>();
-                var foundSetting = await dbContext.Settings.FirstOrDefaultAsync();
+                var foundSetting = await dbContext.Settings.FirstOrDefaultAsync(t => t.Key == DbSetting.SettingKey_DepotVersion);
                 if (foundSetting == null || foundSetting.Value == null)
                 {
                     _logger.LogInformation("Update of Depot File required because CurrentVersion could not be found");