Merge pull request #168 from Corona-Studio/download

Download

ProjBobcat/ProjBobcat/Class/Helper/Download/DownloadHelper.Multipart.cs

@@ -3,7 +3,6 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
@@ -19,20 +18,17 @@ namespace ProjBobcat.Class.Helper.Download;
 
 public static partial class DownloadHelper
 {
-    private const int DefaultPartialDownloadTimeoutMs = 3000;
+    private const int DefaultPartialDownloadTimeoutMs = 500;
 
     private static HttpClient Head => HttpClientHelper.HeadClient;
     private static HttpClient MultiPart => HttpClientHelper.MultiPartClient;
 
     record PreChunkInfo(
-        BufferBlock<PreChunkInfo> DownloadQueueBuffer,
-        FileStream? TempFileStream,
+        string DownloadUrl,
         DownloadRange Range,
         CancellationTokenSource Cts);
 
     record ChunkInfo(
-        BufferBlock<PreChunkInfo> DownloadQueueBuffer,
-        FileStream? TempFileStream,
         HttpResponseMessage Response,
         DownloadRange Range,
         CancellationTokenSource Cts);
@@ -48,7 +44,7 @@ private static async Task<double> ReceiveFromRemoteStreamAsync(
     {
         var startTime = Stopwatch.GetTimestamp();
 
-        using var buffer = MemoryPool<byte>.Shared.Rent(DefaultCopyBufferSize);
+        using var buffer = MemoryPool<byte>.Shared.Rent();
 
         while (true)
         {
@@ -139,7 +135,7 @@ private static IEnumerable<DownloadRange> CalculateDownloadRanges(
             {
                 Start = from + offset,
                 End = to + offset,
-                TempFileName = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName())
+                TempFileName = GetTempFilePath()
             };
         }
     }
@@ -237,7 +233,7 @@ public static async Task MultiPartDownloadTaskAsync(
                     downloadFile.Ranges = [];
 
                     foreach (var range in ranges)
-                        downloadFile.Ranges.TryAdd(range, range);
+                        downloadFile.Ranges.TryAdd(range, null);
                 }
 
                 if (!Directory.Exists(downloadFile.DownloadPath))
@@ -249,7 +245,7 @@ public static async Task MultiPartDownloadTaskAsync(
                     new TransformBlock<PreChunkInfo, ChunkInfo?>(
                         async preChunkInfo =>
                         {
-                            using var request = new HttpRequestMessage(HttpMethod.Get, downloadFile.GetDownloadUrl());
+                            using var request = new HttpRequestMessage(HttpMethod.Get, preChunkInfo.DownloadUrl);
 
                             if (downloadSettings.Authentication != null)
                                 request.Headers.Authorization = downloadSettings.Authentication;
@@ -265,20 +261,26 @@ public static async Task MultiPartDownloadTaskAsync(
                                     HttpCompletionOption.ResponseHeadersRead,
                                     preChunkInfo.Cts.Token);
 
+                                if (!downloadTask.IsSuccessStatusCode ||
+                                    !downloadTask.Content.Headers.ContentLength.HasValue ||
+                                    downloadTask.Content.Headers.ContentLength == 0)
+                                {
+                                    // Some mirror will return non-200 code during the high load
+                                    throw new HttpRequestException(
+                                        $"Failed to download part {preChunkInfo.Range.Start}-{preChunkInfo.Range.End}, status code: {downloadTask.StatusCode}");
+                                }
+
                                 return new ChunkInfo(
-                                    preChunkInfo.DownloadQueueBuffer,
-                                    preChunkInfo.TempFileStream,
                                     downloadTask,
                                     preChunkInfo.Range,
                                     preChunkInfo.Cts);
                             }
                             catch (HttpRequestException e)
                             {
-                                Console.WriteLine(e);
-                                await preChunkInfo.Cts.CancelAsync();
-                            }
+                                Debug.WriteLine(e);
 
-                            return null;
+                                throw;
+                            }
                         }, new ExecutionDataflowBlockOptions
                         {
                             BoundedCapacity = downloadSettings.DownloadParts,
@@ -299,8 +301,7 @@ public static async Task MultiPartDownloadTaskAsync(
                     using var res = chunkInfo.Response;
 
                     // We'll store the written file to a temp file stream, and keep its ref for the future use.
-                    var fileToWriteTo = chunkInfo.TempFileStream ?? File.Create(chunkInfo.Range.TempFileName);
-                    var currentPos = fileToWriteTo.Position;
+                    var fileToWriteTo = File.Create(chunkInfo.Range.TempFileName);
                     var elapsedTime = 0d;
 
                     try
@@ -313,6 +314,8 @@ public static async Task MultiPartDownloadTaskAsync(
                             fileToWriteTo,
                             chunkCts.Token);
 
+                        ArgumentOutOfRangeException.ThrowIfNotEqual(fileToWriteTo.Length, chunkInfo.Response.Content.Headers.ContentLength ?? 0);
+
                         downloadFile.FinishedRangeStreams.AddOrUpdate(chunkInfo.Range, fileToWriteTo, (_, oldStream) =>
                         {
                             try
@@ -328,43 +331,75 @@ public static async Task MultiPartDownloadTaskAsync(
                             return fileToWriteTo;
                         });
                     }
-                    catch (TaskCanceledException)
+                    catch (ArgumentOutOfRangeException)
                     {
-                        if (chunkInfo.Range.End - chunkInfo.Range.Start - fileToWriteTo.Position < 1024)
+                        // Size check failed, fast retry now
+                        // Dispose the file stream, and assign a new temp file path
+                        await fileToWriteTo.DisposeAsync();
+
+                        var regeneratedTempFileInfo = chunkInfo.Range with { TempFileName = GetTempFilePath() };
+
+                        ArgumentOutOfRangeException.ThrowIfEqual(downloadFile.Ranges.TryRemove(chunkInfo.Range, out _), false);
+                        ArgumentOutOfRangeException.ThrowIfEqual(downloadFile.Ranges.TryAdd(regeneratedTempFileInfo, null), false);
+
+                        throw;
+                    }
+                    catch (Exception)
+                    {
+                        // If we end up to here, which means either the chunk is too large or the download speed is too slow
+                        // So we need to further split the chunk into smaller parts
+                        var bytesTransmitted = fileToWriteTo.Length;
+
+                        if (bytesTransmitted == 0)
+                        {
+                            // If we haven't received any data, we need to retry
+                            await fileToWriteTo.DisposeAsync();
+                            throw;
+                        }
+
+                        // Remove the current range from the download queue
+                        ArgumentOutOfRangeException.ThrowIfEqual(downloadFile.Ranges.TryRemove(chunkInfo.Range, out _), false);
+
+                        var chunkLength = chunkInfo.Range.End - chunkInfo.Range.Start;
+                        var remaining = chunkLength - bytesTransmitted;
+                        var finishedRange = chunkInfo.Range with { End = chunkInfo.Range.End - remaining };
+
+                        // Add the finished parts
+                        ArgumentOutOfRangeException.ThrowIfEqual(downloadFile.Ranges.TryAdd(finishedRange, null), false);
+                        ArgumentOutOfRangeException.ThrowIfEqual(downloadFile.FinishedRangeStreams.TryAdd(finishedRange, fileToWriteTo), false);
+
+                        if (remaining < 1024)
                         {
                             // File is too small to be split, just retry
+                            var remainingRange = new DownloadRange
+                            {
+                                Start = finishedRange.End,
+                                End = chunkInfo.Range.End,
+                                TempFileName = GetTempFilePath()
+                            };
+
+                            if (remainingRange.Start >= remainingRange.End)
+                            {
+                                throw new ArgumentOutOfRangeException(
+                                    $"[{chunkLength}][{remaining}][{bytesTransmitted}][{chunkInfo.Range.End - remaining}]{remainingRange}[{e}]");
+                            }
+
+                            ArgumentOutOfRangeException.ThrowIfEqual(downloadFile.Ranges.TryAdd(remainingRange, null), false);
+
                             throw;
                         }
 
-                        // If we end up to here, which means either the chunk is too large or the download speed is too slow
-                        // So we need to further split the chunk into smaller parts
-                        var bytesReceived = fileToWriteTo.Position - currentPos;
-                        var chunkLength = chunkInfo.Range.End - chunkInfo.Range.Start + 1;
-                        var remaining = chunkLength - bytesReceived;
-                        var offset = chunkInfo.Range.Start + fileToWriteTo.Position - 1;
-                        var subChunkRanges = CalculateDownloadRanges(remaining, offset, downloadSettings);
-
-                        downloadFile.Ranges.TryRemove(chunkInfo.Range, out _);
+                        var subChunkRanges = CalculateDownloadRanges(remaining, finishedRange.End, downloadSettings);
 
                         // We add the sub chunks to the download queue
                         foreach (var range in subChunkRanges)
                         {
                             // Update the original range list
-                            downloadFile.Ranges.TryAdd(range, range);
+                            ArgumentOutOfRangeException.ThrowIfEqual(downloadFile.Ranges.TryAdd(range, null), false);
                         }
 
                         throw;
                     }
-                    catch (Exception)
-                    {
-                        downloadFile.Ranges.TryRemove(chunkInfo.Range, out _);
-
-                        var updatedRange = chunkInfo.Range with { Start = chunkInfo.Range.Start + fileToWriteTo.Position - 1};
-
-                        downloadFile.Ranges.TryAdd(updatedRange, updatedRange);
-
-                        throw;
-                    }
 
                     var addedAggregatedSpeedCount = Interlocked.Increment(ref aggregatedSpeedCount);
 
@@ -399,8 +434,11 @@ public static async Task MultiPartDownloadTaskAsync(
                 bufferBlock.LinkTo(requestCreationBlock, linkOptions);
                 requestCreationBlock.LinkTo(downloadActionBlock, linkOptions);
 
+                // Acquire the download URL
+                var downloadUrl = downloadFile.GetDownloadUrl();
+
                 foreach (var range in downloadFile.GetUndoneRanges())
-                    await bufferBlock.SendAsync(new PreChunkInfo(bufferBlock, null, range, cts), cts.Token);
+                    await bufferBlock.SendAsync(new PreChunkInfo(downloadUrl, range, cts), cts.Token);
 
                 bufferBlock.Complete();
                 await downloadActionBlock.Completion;
@@ -419,11 +457,10 @@ public static async Task MultiPartDownloadTaskAsync(
 
                 var fileStream = File.Create(filePath);
                 var hashStream = new CryptoStream(fileStream, hashProvider, CryptoStreamMode.Write);
-
+                
                 await using (Stream destStream = hashCheckFile ? hashStream : fileStream)
                 {
                     var index = 0;
-                    var xxx = downloadFile.Ranges.OrderBy(p => p.Key.Start).Select(p => p.Key).ToList();
 
                     foreach (var inputFileStream in downloadFile.GetFinishedStreamsInorder())
                     {
@@ -450,13 +487,15 @@ public static async Task MultiPartDownloadTaskAsync(
 
                 if (hashCheckFile)
                 {
-                    var checkSum = Convert.ToHexString(hashProvider.Hash.AsSpan());
+                    var checkSum = Convert.ToHexString(hashProvider.Hash.AsSpan()).ToLowerInvariant();
 
                     if (!checkSum.Equals(downloadFile.CheckSum!, StringComparison.OrdinalIgnoreCase))
                     {
                         downloadFile.RetryCount++;
-                        exceptions.Add(new HashMismatchException(filePath, checkSum, downloadFile.CheckSum!));
+                        exceptions.Add(new HashMismatchException(filePath, downloadFile.CheckSum!, checkSum, downloadFile));
 
+                        await RecycleDownloadFile(downloadFile);
+                        await fileStream.DisposeAsync();
                         FileHelper.DeleteFileWithRetry(filePath);
 
                         continue;

ProjBobcat/ProjBobcat/Class/Helper/Download/DownloadHelper.cs

@@ -18,6 +18,18 @@ public static partial class DownloadHelper
 
     private const int DefaultCopyBufferSize = 1024 * 8 * 10;
 
+    public static string GetTempDownloadPath()
+    {
+        var lxTempDir = Path.Combine(Path.GetTempPath(), "LauncherX");
+
+        return lxTempDir;
+    }
+
+    public static string GetTempFilePath()
+    {
+        return Path.Combine(GetTempDownloadPath(), Path.GetRandomFileName());
+    }
+
     private static async Task RecycleDownloadFile(AbstractDownloadBase download)
     {
         // Once we finished the download, we need to dispose the kept file stream
@@ -101,6 +113,11 @@ private static Task AdvancedDownloadFile(AbstractDownloadBase df, DownloadSettin
 
     private static (BufferBlock<AbstractDownloadBase> Input, ActionBlock<AbstractDownloadBase> Execution) BuildAdvancedDownloadTplBlock(DownloadSettings downloadSettings)
     {
+        var lxTempPath = GetTempDownloadPath();
+
+        if (!Directory.Exists(lxTempPath))
+            Directory.CreateDirectory(lxTempPath);
+
         var bufferBlock = new BufferBlock<AbstractDownloadBase>(new DataflowBlockOptions { EnsureOrdered = false });
         var actionBlock = new ActionBlock<AbstractDownloadBase>(
             d => AdvancedDownloadFile(d, downloadSettings),
@@ -125,6 +142,11 @@ public static async Task AdvancedDownloadListFile(
         IEnumerable<AbstractDownloadBase> fileEnumerable,
         DownloadSettings downloadSettings)
     {
+        var lxTempPath = GetTempDownloadPath();
+
+        if (!Directory.Exists(lxTempPath))
+            Directory.CreateDirectory(lxTempPath);
+
         var blocks = BuildAdvancedDownloadTplBlock(downloadSettings);
 
         foreach (var downloadFile in fileEnumerable)

ProjBobcat/ProjBobcat/Class/Model/Downloading/AbstractDownloadBase.cs

@@ -12,7 +12,7 @@ internal record UrlInfo(long FileLength, bool CanPartialDownload);
 
 public abstract class AbstractDownloadBase : IDownloadFile
 {
-    internal ConcurrentDictionary<DownloadRange, DownloadRange>? Ranges { get; set; }
+    internal ConcurrentDictionary<DownloadRange, object?>? Ranges { get; set; }
     internal UrlInfo? UrlInfo { get; set; }
     internal ConcurrentDictionary<DownloadRange, FileStream> FinishedRangeStreams { get; } = [];
 
@@ -22,9 +22,9 @@ internal IEnumerable<FileStream> GetFinishedStreamsInorder()
     {
         ArgumentNullException.ThrowIfNull(Ranges);
 
-        foreach (var downloadRange in Ranges.OrderBy(p => p.Key.Start))
+        foreach (var (downloadRange, _) in Ranges.OrderBy(p => p.Key.Start))
         {
-            if (!FinishedRangeStreams.TryGetValue(downloadRange.Key, out var stream))
+            if (!FinishedRangeStreams.TryGetValue(downloadRange, out var stream))
                 throw new InvalidOperationException("Stream not found.");
 
             yield return stream;
@@ -42,12 +42,12 @@ internal IEnumerable<DownloadRange> GetUndoneRanges()
     {
         ArgumentNullException.ThrowIfNull(Ranges);
 
-        foreach (var downloadRange in Ranges)
+        foreach (var (range, _) in Ranges)
         {
-            if (FinishedRangeStreams.ContainsKey(downloadRange.Key))
+            if (FinishedRangeStreams.ContainsKey(range))
                 continue;
 
-            yield return downloadRange.Key;
+            yield return range;
         }
     }
 

ProjBobcat/ProjBobcat/Class/Model/Downloading/DownloadRange.cs

@@ -25,6 +25,11 @@ namespace ProjBobcat.Class.Model.Downloading;
     /// </summary>
     public required string TempFileName { get; init; }
 
+    public override string ToString()
+    {
+        return $"[{Start}-{End}] {TempFileName}";
+    }
+
     public override int GetHashCode()
     {
         return HashCode.Combine(this.Start, this.End, this.TempFileName);

ProjBobcat/ProjBobcat/Exceptions/HashMismatchException.cs

@@ -1,26 +1,25 @@
 using System;
+using System.Linq;
+using ProjBobcat.Class.Model.Downloading;
 
 namespace ProjBobcat.Exceptions;
 
-public sealed class HashMismatchException : Exception
+public sealed class HashMismatchException(
+    string filePath,
+    string expectedHash,
+    string actualHash,
+    AbstractDownloadBase downloadBase)
+    : Exception(GetMessage(filePath, expectedHash, actualHash, downloadBase))
 {
-    public HashMismatchException(string filePath, string expectedHash, string actualHash) : base(GetMessage(filePath, expectedHash, actualHash))
-    {
-        FilePath = filePath;
-        ExpectedHash = expectedHash;
-        ActualHash = actualHash;
-    }
-
-    public string FilePath { get; }
-    public string ExpectedHash { get; }
-    public string ActualHash { get; }
-
-    static string GetMessage(string filePath, string expectedHash, string actualHash)
+    static string GetMessage(string filePath, string expectedHash, string actualHash, AbstractDownloadBase downloadBase)
     {
         return $"""
                 文件 {filePath} 的哈希值不匹配。
                 期望哈希值：{expectedHash}
                 实际哈希值：{actualHash}
+                [{downloadBase.FinishedRangeStreams.Count} files in total]
+                [{downloadBase.FinishedRangeStreams.Select(p => p.Value.Length).Sum()}/{downloadBase.UrlInfo?.FileLength ?? 0}]
+                {string.Join("\n", downloadBase.FinishedRangeStreams.OrderBy(p => p.Key.Start).Select(p => $"{p.Key} {p.Value.Length}"))}
                 """;
     }
 }