Fixed bug where steam api is now suddenly gone

Author: devedse

DeveLanCacheUI_Backend/DeveLanCacheUI_Backend.csproj

@@ -31,8 +31,8 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="9.0.4" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.2" />
     <PackageReference Include="Polly" Version="8.5.2" />
-    <PackageReference Include="protobuf-net" Version="3.2.52" />
-    <PackageReference Include="SteamKit2" Version="3.2.0" />
+    <PackageReference Include="protobuf-net" Version="3.2.56" />
+    <PackageReference Include="SteamKit2" Version="3.3.1" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="8.1.1" />
   </ItemGroup>
 

DeveLanCacheUI_Backend/Helpers/RemoteFileDownloader.cs

@@ -0,0 +1,58 @@
+namespace DeveLanCacheUI_Backend.Helpers
+{
+    /// <summary>
+    /// Simple helper to download remote files. If the url ends with .gz it will stream-decompress and return the raw bytes.
+    /// Keeps implementation minimal and self-contained.
+    /// </summary>
+    public static class RemoteFileDownloader
+    {
+        public static async Task<byte[]> DownloadFileAsync(string url, CancellationToken cancellationToken = default)
+        {
+            using var httpClient = new HttpClient();
+            httpClient.DefaultRequestHeaders.Add("User-Agent", "request");
+
+            var rawBytes = await httpClient.GetByteArrayAsync(url, cancellationToken);
+            if (url.EndsWith(".gz", StringComparison.OrdinalIgnoreCase))
+            {
+                using var inputMs = new MemoryStream(rawBytes);
+                using var gzip = new GZipStream(inputMs, CompressionMode.Decompress);
+                using var outputMs = new MemoryStream();
+                await gzip.CopyToAsync(outputMs, cancellationToken);
+                return outputMs.ToArray();
+            }
+            return rawBytes;
+        }
+
+        /// <summary>
+        /// Downloads a specific asset from the latest GitHub release for the given owner/repo.
+        /// If the primary asset name is not found, optional alternate names will be tried in order.
+        /// Automatically decompresses .gz assets.
+        /// </summary>
+        public static async Task<byte[]> DownloadGithubLatestReleaseAssetAsync(
+            string owner,
+            string repo,
+            string assetName,
+            CancellationToken cancellationToken = default)
+        {
+            var releaseUrl = $"https://api.github.com/repos/{owner}/{repo}/releases/latest";
+            using var httpClient = new HttpClient();
+            httpClient.DefaultRequestHeaders.Add("User-Agent", "request");
+
+            var latestJson = await httpClient.GetStringAsync(releaseUrl, cancellationToken);
+            var release = JsonSerializer.Deserialize<DeveLanCacheUI_Backend.Services.OriginalDepotEnricher.Models.GithubLatestApiPoco>(latestJson);
+            if (release == null)
+            {
+                throw new InvalidOperationException($"Could not deserialize latest release for {owner}/{repo}");
+            }
+
+            var asset = release.assets.FirstOrDefault(a => string.Equals(a.name, assetName, StringComparison.OrdinalIgnoreCase));
+            if (asset == null)
+            {
+                throw new FileNotFoundException($"Asset '{assetName}' not found in latest release for {owner}/{repo}");
+            }
+
+            // Reuse existing download + auto .gz logic
+            return await DownloadFileAsync(asset.browser_download_url, cancellationToken);
+        }
+    }
+}

DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamApi.cs

@@ -10,17 +10,68 @@ public static class SteamApi
 
         private static SteamApiData LoadSteamApiData()
         {
-            using var c = new HttpClient();
-            var result = c.GetAsync("https://api.steampowered.com/ISteamApps/GetAppList/v2/").Result;
-            var resultString = result.Content.ReadAsStringAsync().Result;
+            // Pull steam-applist.json(.gz) from latest GitHub release (Steam GetAppList endpoint broken).
+            var bytes = RemoteFileDownloader.DownloadGithubLatestReleaseAssetAsync(
+                owner: "devedse",
+                repo: "DeveLanCacheUI_SteamDepotFinder_Runner",
+                assetName: "steam-applist.json.gz",
+                cancellationToken: CancellationToken.None
+            ).Result;
 
-            var retval = JsonSerializer.Deserialize<SteamApiData>(resultString);
+            var resultString = Encoding.UTF8.GetString(bytes);
+            
+            // Deserialize from GitHub format
+            var githubData = JsonSerializer.Deserialize<GitHubSteamApiData>(resultString);
+            
+            if (githubData?.response?.apps == null)
+            {
+                throw new FileNotFoundException("Could not download / parse / find the steam-applist.json.gz file from DeveLanCacheUI_SteamDepotFinder_Runner");
+            }
+            
+            // Convert to original SteamApiData format
+            var retval = new SteamApiData
+            {
+                applist = new Applist
+                {
+                    apps = githubData.response.apps
+                        .Select(a => new App { appid = a.appid, name = a.name })
+                        .DistinctBy(t => t.appid)
+                        .ToArray()
+                }
+            };
+            
+            return retval;
+
+            // OLD FALLBACK (commented out intentionally):
+            // using var c = new HttpClient();
+            // var result = c.GetAsync("https://api.steampowered.com/ISteamApps/GetAppList/v2/").Result;
+            // var resultString = result.Content.ReadAsStringAsync().Result;
+            // var retval = JsonSerializer.Deserialize<SteamApiData>(resultString) ?? new SteamApiData { applist = new Applist { apps = Array.Empty<App>() } };
+            // retval.applist.apps = retval.applist.apps.DistinctBy(t => t.appid).ToArray();
+            // return retval;
+        }
 
-            //There was one person with an issue where one appid was added as a duplicate???, no idea how but this seems to be a bug in the steam api.
-            //I'm just going to distinct on it
-            retval.applist.apps = retval.applist.apps.DistinctBy(t => t.appid).ToArray();
 
-            return retval;
+
+
+
+        // Classes for deserializing from GitHub release JSON format
+        private class GitHubSteamApiData
+        {
+            public GitHubSteamResponse response { get; set; }
+        }
+
+        private class GitHubSteamResponse
+        {
+            public GitHubSteamApp[] apps { get; set; }
+        }
+
+        private class GitHubSteamApp
+        {
+            public uint appid { get; set; }
+            public string name { get; set; }
+            public long last_modified { get; set; }
+            public long price_change_number { get; set; }
         }
     }
 }

DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamDepotDownloaderHostedService.cs

@@ -62,35 +62,16 @@ private async Task GoRun(CancellationToken stoppingToken)
                     {
                         throw new UnreachableException($"New version available, but LatestVersion or DownloadUrl is null. This should not happen. LatestVersion: {shouldDownload.LatestVersion}, DownloadUrl: {shouldDownload.DownloadUrl}");
                     }
-                    var downloadedBytes = await GoDownload(depotFileDirectory, shouldDownload);
 
-                    if (downloadedBytes != null)
-                    {
-                        await _steamDepotEnricherHostedService.GoProcess(shouldDownload.LatestVersion, downloadedBytes, stoppingToken);
-                    }
-                    else
-                    {
-                        _logger.LogWarning("Downloaded depot file was null, not processing further.");
-                    }
-                }
+                    _logger.LogInformation("Detected that new version '{LatestVersion}' of Depot File is available, downloading: {DownloadUrl}...", shouldDownload.LatestVersion, shouldDownload.DownloadUrl);
 
-                await Task.Delay(TimeSpan.FromHours(1));
-            }
-        }
+                    var downloadedBytes = await RemoteFileDownloader.DownloadFileAsync(shouldDownload.DownloadUrl, stoppingToken);
 
-        private async Task<byte[]?> GoDownload(string depotFileDirectory, (bool NewVersionAvailable, Version? LatestVersion, string? DownloadUrl) shouldDownload)
-        {
-            _logger.LogInformation("Detected that new version '{NewVersionAvailable}' of Depot File is available, so downloading: {DownloadUrl}...", shouldDownload.NewVersionAvailable, shouldDownload.DownloadUrl);
+                    await _steamDepotEnricherHostedService.GoProcess(shouldDownload.LatestVersion, downloadedBytes, stoppingToken);
+                }
 
-            var downloadedCsv = await _httpClient.GetAsync(shouldDownload.DownloadUrl);
-            if (!downloadedCsv.IsSuccessStatusCode)
-            {
-                _logger.LogWarning("Could not obtain {Url}: {StatusCode}, {ReasonPhrase}", DeveLanCacheUISteamDepotFinderLatestUrl, downloadedCsv.StatusCode, downloadedCsv.ReasonPhrase);
-                return null;
+                await Task.Delay(TimeSpan.FromHours(1));
             }
-
-            var bytes = await downloadedCsv.Content.ReadAsByteArrayAsync();
-            return bytes;
         }
 
         private async Task<(bool NewVersionAvailable, Version? LatestVersion, string? DownloadUrl)> NewDepotFileAvailable()
@@ -120,13 +101,17 @@ private async Task GoRun(CancellationToken stoppingToken)
                 return (false, null, null);
             }
 
-            var downloadUrl = dataParsed.assets.FirstOrDefault(t => t.browser_download_url.EndsWith(".csv", StringComparison.OrdinalIgnoreCase))?.browser_download_url;
-            if (downloadUrl == null)
+            var asset = dataParsed.assets?.FirstOrDefault(a => a.name == "app-depot-output-cleaned.csv.gz")
+                        ?? dataParsed.assets?.FirstOrDefault(a => a.name == "app-depot-output-cleaned.csv");
+
+            if (asset == null || string.IsNullOrWhiteSpace(asset.browser_download_url))
             {
-                _logger.LogWarning("Could not find download url in: {Data}", data);
+                _logger.LogWarning("Could not find app-depot-output-cleaned.csv.gz or .csv in release assets");
                 return (false, null, null);
             }
 
+            var downloadUrl = asset.browser_download_url;
+
             await using (var scope = Services.CreateAsyncScope())
             {
                 using var dbContext = scope.ServiceProvider.GetRequiredService<DeveLanCacheUIDbContext>();

DeveLanCacheUI_Backend/Steam/AppInfoHandler.cs

@@ -32,16 +32,15 @@ public async Task<List<App>> EnsureAppsAreLoaded()
         {
             _logger.LogInformation("Retrieving all known AppIds");
 
-            using var steamAppsApi = _steam3Session.Configuration.GetAsyncWebAPIInterface("ISteamApps");
-            var response = await steamAppsApi.CallAsync(HttpMethod.Get, "GetAppList", 2);
+            // Old live API call (broken) replaced by local cached SteamApi data from GitHub release.
+            //using var steamAppsApi = _steam3Session.Configuration.GetAsyncWebAPIInterface("ISteamApps");
+            //var response = await steamAppsApi.CallAsync(HttpMethod.Get, "GetAppList", 2);
 
-            var apiApps = response["apps"].Children.Select(app =>
-                new App()
-                {
-                    appid = app["appid"].AsUnsignedInteger(),
-                    name = app["name"].AsString() ?? "Unknown"
-                }
-                ).ToList();
+            var apiApps = SteamApi.SteamApiData.applist.apps.Select(app => new App
+            {
+                appid = app.appid,
+                name = string.IsNullOrWhiteSpace(app.name) ? "Unknown" : app.name
+            }).ToList();
 
             _cachedAppNames = apiApps.DistinctBy(t => t.appid).ToDictionary(t => t.appid, t => t);