From 5a492092b48839ebb6e10ff64cb3414967098dea Mon Sep 17 00:00:00 2001 From: Ddggdd135 <1306334428@qq.com> Date: Sat, 17 Feb 2024 22:12:27 +0800 Subject: [PATCH] Update Downloader --- MinecraftLaunch.Test/Program.cs | 7 +- .../Classes/Interfaces/IDownloader.cs | 17 +- .../Components/Downloader/Downloader.cs | 474 ++++++++++++++++++ .../Extensions/DownloadExtension.cs | 4 +- 4 files changed, 494 insertions(+), 8 deletions(-) create mode 100644 MinecraftLaunch/Components/Downloader/Downloader.cs diff --git a/MinecraftLaunch.Test/Program.cs b/MinecraftLaunch.Test/Program.cs index 97d4fbe..c9294ca 100644 --- a/MinecraftLaunch.Test/Program.cs +++ b/MinecraftLaunch.Test/Program.cs @@ -10,9 +10,10 @@ MirrorDownloadManager.IsUseMirrorDownloadSource = true; var account = new OfflineAuthenticator("Yang114").Authenticate(); -var resolver = new GameResolver("C:\\Users\\w\\Desktop\\总整包\\MC\\mc启动器\\LauncherX\\.minecraft"); +Directory.CreateDirectory(".minecraft"); +var resolver = new GameResolver(".minecraft"); -var installer = new VanlliaInstaller(resolver, "1.19.4", MirrorDownloadManager.Bmcl); +var installer = new VanlliaInstaller(resolver, "1.12.2"); installer.ProgressChanged += (_, args) => { Console.WriteLine($"{args.ProgressStatus} - {args.Progress * 100:0.00}%"); }; @@ -26,7 +27,7 @@ var config = new LaunchConfig { Account = account, IsEnableIndependencyCore = true, - JvmConfig = new(@"C:\Program Files\Java\jdk1.8.0_301\bin\javaw.exe") { + JvmConfig = new(@"C:\Program Files\Java\jdk-17\bin\javaw.exe") { MaxMemory = 1024, } }; diff --git a/MinecraftLaunch/Classes/Interfaces/IDownloader.cs b/MinecraftLaunch/Classes/Interfaces/IDownloader.cs index f8b5619..18f3124 100644 --- a/MinecraftLaunch/Classes/Interfaces/IDownloader.cs +++ b/MinecraftLaunch/Classes/Interfaces/IDownloader.cs @@ -1,5 +1,16 @@ -namespace MinecraftLaunch.Classes.Interfaces; +using MinecraftLaunch.Classes.Models.Download; +using MinecraftLaunch.Classes.Models.Event; -public interface IDownloader { - ValueTask DownloadAsync(); +namespace MinecraftLaunch.Classes.Interfaces +{ + public interface IDownloader + { + event Action Completed; + event EventHandler ProgressChanged; + + void Cancel(); + ValueTask DownloadAsync(); + void Retry(); + void Setup(IEnumerable downloadItems); + } } \ No newline at end of file diff --git a/MinecraftLaunch/Components/Downloader/Downloader.cs b/MinecraftLaunch/Components/Downloader/Downloader.cs new file mode 100644 index 0000000..b4c0855 --- /dev/null +++ b/MinecraftLaunch/Components/Downloader/Downloader.cs @@ -0,0 +1,474 @@ +using Flurl.Http; +using MinecraftLaunch.Classes.Interfaces; +using MinecraftLaunch.Classes.Models.Download; +using System.Buffers; +using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.Net; +using System.Net.Sockets; +using System.Timers; +using static System.Runtime.InteropServices.JavaScript.JSType; +using DownloadProgressChangedEventArgs = MinecraftLaunch.Classes.Models.Event.DownloadProgressChangedEventArgs; +using Timer = System.Timers.Timer; + +namespace MinecraftLaunch.Components.Downloader; + +public sealed class Downloader : IDownloader, IDisposable +{ + public bool IsDisposed { get; private set; } = false; + public int MaxRetryCount { get; set; } = 4; + public int ChunkCount { get; set; } = 4; + public ThreadPoolImplement ThreadPool { get; private set; } + public HttpClient Client { get; set; } = new(); + public sealed class ThreadPoolImplement : IDisposable + { + private int _threadCount = 0; + public bool IsDisposed { get; private set; } = false; + public int ThreadCount + { + get + { + return _threadCount; + } + set + { + if (value < 0) throw new ArgumentException(); + if (value > _threadCount) + { + while (_threads.Count != value) + { + CancellationTokenSource cancellationTokenSource = new(); + CancellationToken token = cancellationTokenSource.Token; + Thread thread = null; + thread = new(() => + { + while (!token.IsCancellationRequested) + { + if (_firstTasks.IsEmpty && _tasks.IsEmpty) + { + Thread.Sleep((FreeThreadCount + ThreadCount / 2) / (ThreadCount + 1) * 64); + continue; + } + lock (_freeThreadCountLock) + FreeThreadCount--; + thread.Name = "ThreadPool Thread - Running"; + try + { + if (_firstTasks.TryDequeue(out Action action)) + { + action.Invoke(); + } + else if (_tasks.TryDequeue(out Action action2)) + { + action2.Invoke(); + } + } + catch { } + lock (_freeThreadCountLock) + FreeThreadCount++; + thread.Name = "ThreadPool Thread"; + } + }); + _threads[cancellationTokenSource] = thread; + lock (_freeThreadCountLock) + FreeThreadCount++; + thread.Name = "ThreadPool Thread"; + thread.IsBackground = true; + thread.Start(); + } + } + else if (value < _threadCount) + { + List toJoin = new(); + while (_threads.Count != ThreadCount) + { + KeyValuePair pair = _threads.TakeLast(1).First(); + pair.Key.Cancel(); + toJoin.Add(pair.Value); + lock (_freeThreadCountLock) + FreeThreadCount--; + } + foreach (Thread thread in toJoin) thread.Join(TimeSpan.FromSeconds(2)); + } + _threadCount = value; + } + } + private object _freeThreadCountLock = new(); + public int FreeThreadCount { get; private set; } + private ConcurrentDictionary _threads = new(); + private ConcurrentQueue _tasks = new(); + private ConcurrentQueue _firstTasks = new(); + ~ThreadPoolImplement() + { + Dispose(); + } + public ThreadPoolImplement(int maxThreadCount) + { + ThreadCount = maxThreadCount; + } + public void CancelAll() + { + _tasks.Clear(); + _firstTasks.Clear(); + } + public void WaitAll() + { + while (FreeThreadCount != ThreadCount || !_tasks.IsEmpty || !_firstTasks.IsEmpty) Thread.Sleep(16); + } + public void RunSync(Action action) + { + int _SpinLock = 1; + while (Interlocked.Exchange(ref _SpinLock, 1) != 0) + { + Thread.SpinWait(1); + } + _tasks.Enqueue(() => + { + action(); + Interlocked.Exchange(ref _SpinLock, 0); + }); + } + public void RunSync(Action action, T arg1) + { + int _SpinLock = 1; + while (Interlocked.Exchange(ref _SpinLock, 1) != 0) + { + Thread.SpinWait(1); + } + _tasks.Enqueue(() => + { + action(arg1); + Interlocked.Exchange(ref _SpinLock, 0); + }); + } + public void RunSync(Action action, T1 arg1, T2 arg2) + { + int _SpinLock = 1; + while (Interlocked.Exchange(ref _SpinLock, 1) != 0) + { + Thread.SpinWait(1); + } + _tasks.Enqueue(() => + { + action(arg1, arg2); + Interlocked.Exchange(ref _SpinLock, 0); + }); + } + public void RunSync(Action action, T1 arg1, T2 arg2, T3 arg3) + { + int _SpinLock = 1; + while (Interlocked.Exchange(ref _SpinLock, 1) != 0) + { + Thread.SpinWait(1); + } + _tasks.Enqueue(() => + { + action(arg1, arg2, arg3); + Interlocked.Exchange(ref _SpinLock, 0); + }); + } + public void RunSync(Action action, T1 arg1, T2 arg2, T3 arg3, T4 arg4) + { + int _SpinLock = 1; + while (Interlocked.Exchange(ref _SpinLock, 1) != 0) + { + Thread.SpinWait(1); + } + _tasks.Enqueue(() => + { + action(arg1, arg2, arg3, arg4); + Interlocked.Exchange(ref _SpinLock, 0); + }); + } + public void Run(Action action) + { + _tasks.Enqueue(() => + { + action(); + }); + } + public void Run(Action action, T arg1) + { + _tasks.Enqueue(() => + { + action(arg1); + }); + } + public void Run(Action action, T1 arg1, T2 arg2) + { + _tasks.Enqueue(() => + { + action(arg1, arg2); + }); + } + public void Run(Action action, T1 arg1, T2 arg2, T3 arg3) + { + _tasks.Enqueue(() => + { + action(arg1, arg2, arg3); + }); + } + public void Run(Action action, T1 arg1, T2 arg2, T3 arg3, T4 arg4) + { + _tasks.Enqueue(() => + { + action(arg1, arg2, arg3, arg4); + }); + } + public void RunFirst(Action action) + { + _firstTasks.Enqueue(() => + { + action(); + }); + } + public void Dispose() + { + if (!IsDisposed) + { + CancelAll(); + ThreadCount = 0; + GC.SuppressFinalize(this); + } + } + } + + public event Action Completed; + public event EventHandler ProgressChanged; + + private readonly Timer _timer; + private IEnumerable _downloadItems; + private ArrayPool _buffer { get; } = ArrayPool.Create(); + + private int _totalBytes; + private int _downloadedBytes; + private int _previousDownloadedBytes; + + private int _totalCount; + private int _completedCount; + private int _failedCount; + private int _chunkCount; + ~Downloader() + { + Dispose(); + } + public Downloader(int threadCount) + { + Client.Timeout = TimeSpan.FromSeconds(15); + Client.DefaultRequestHeaders.Connection.Add("keep-alive"); + ServicePointManager.DefaultConnectionLimit = 512; + _timer = new Timer + { + Interval = TimeSpan.FromSeconds(1).TotalMilliseconds + }; + + _timer.Elapsed += (sender, e) => UpdateDownloadProgress(); + ThreadPool = new(threadCount); + } + public Downloader() : this(128) { } + + public void Cancel() + { + _timer.Stop(); + ThreadPool.CancelAll(); + } + + public bool Download() + { + _timer.Start(); + int retryCount = 0; + while (retryCount < MaxRetryCount) + { + foreach (DownloadRequest downloadRequest in _downloadItems) + { + if (downloadRequest.IsCompleted) continue; + ThreadPool.Run(() => + { + if (!DownloadItem(downloadRequest)) + { + Console.WriteLine(downloadRequest.FileInfo.Name); + Interlocked.Increment(ref _failedCount); + } + else + { + Interlocked.Increment(ref _completedCount); + } + }); + } + ThreadPool.WaitAll(); + if (_failedCount > 0) + { + _downloadItems = _downloadItems.Where(x => !x.IsCompleted).ToList(); + _failedCount = 0; + retryCount++; + Thread.Sleep(16000); + } + else + { + break; + } + } + Completed?.Invoke(retryCount == 0); + _timer.Stop(); + return retryCount == 0; + } + + public async ValueTask DownloadAsync() + { + return await Task.Run(() => Download()); + } + + public bool DownloadItem(DownloadRequest downloadRequest, bool forceNotUsePartial = false) + { + downloadRequest.IsPartialContentSupported = GetIsPartialContentSupported(downloadRequest.Url); + if (downloadRequest.IsPartialContentSupported && downloadRequest.Size >= 1024 * 1024 && !forceNotUsePartial) + { + using ThreadPoolImplement pool = new(ChunkCount); + using MemoryStream result = new(downloadRequest.Size); + int chunkSize = (int)Math.Ceiling((double)downloadRequest.Size / ChunkCount); + bool failed = false; + for (int i = 0; i < (downloadRequest.Size / chunkSize); i++) + { + pool.Run(() => + { + failed |= !DownloadPart(downloadRequest, result, i, chunkSize); + }); + } + pool.WaitAll(); + if (!failed) + { + downloadRequest.FileInfo.Directory.Create(); + File.WriteAllBytes(downloadRequest.FileInfo.FullName, result.ToArray()); + downloadRequest.IsCompleted = true; + Interlocked.Add(ref _downloadedBytes, downloadRequest.Size); + downloadRequest.DownloadedBytes = downloadRequest.Size; + return true; + } + return DownloadItem(downloadRequest, true); + } + else + { + bool failed = !DownloadSingleFile(downloadRequest); + if (failed) + { + return false; + } + downloadRequest.IsCompleted = true; + Interlocked.Add(ref _downloadedBytes, downloadRequest.Size); + downloadRequest.DownloadedBytes = downloadRequest.Size; + return true; + } + } + + public void Retry() + { + Cancel(); + Setup(_downloadItems.Where(x => !x.IsCompleted)); + Download(); + } + + public void Setup(IEnumerable downloadItems) + { + // Initialize states + _downloadItems = downloadItems.ToImmutableList(); + _totalBytes = _downloadItems.Sum(item => item.Size); + _downloadedBytes = 0; + _previousDownloadedBytes = 0; + + _totalCount = _downloadItems.Count(); + _completedCount = 0; + _failedCount = 0; + } + + public void Dispose() + { + if (!IsDisposed) + { + _timer.Dispose(); + Client.Dispose(); + ThreadPool.Dispose(); + GC.SuppressFinalize(this); + } + } + private void UpdateDownloadProgress() + { + int diffBytes = _downloadedBytes - _previousDownloadedBytes; + _previousDownloadedBytes = _downloadedBytes; + + var progress = new DownloadProgressChangedEventArgs + { + TotalCount = _totalCount, + CompletedCount = _completedCount, + FailedCount = _failedCount, + TotalBytes = _totalBytes, + DownloadedBytes = _downloadedBytes, + Speed = diffBytes / 1, + }; + + ProgressChanged?.Invoke(this, progress); + } + private bool DownloadPart(DownloadRequest downloadRequest, Stream result, int i, int chunkSize) + { + try + { + using HttpRequestMessage message = new() + { + RequestUri = new(downloadRequest.Url) + }; + message.Headers.Range = new(i * chunkSize, Math.Min(((i + 1) * chunkSize) - 1, downloadRequest.Size)); + using HttpResponseMessage response = Client.Send(message); + if (!response.IsSuccessStatusCode) return false; + byte[] data = response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult(); + lock (result) + { + result.Position = i * chunkSize; + result.Write(data); + } + return true; + } + catch + { + return false; + } + } + private bool DownloadSingleFile(DownloadRequest downloadRequest) + { + try + { + DownloadExtensions.DownloadFileAsync(new FlurlRequest(downloadRequest.Url), downloadRequest.FileInfo.Directory.FullName, downloadRequest.FileInfo.Name).GetAwaiter().GetResult(); + //using HttpRequestMessage message = new() + //{ + // RequestUri = new(downloadRequest.Url) + //}; + //using HttpResponseMessage response = Client.Send(message); + //if (!response.IsSuccessStatusCode) return false; + //using Stream stream = response.Content.ReadAsStream(); + //int size = 1; + //downloadRequest.FileInfo.Directory.Create(); + //using FileStream fileStream = new(downloadRequest.FileInfo.FullName, FileMode.OpenOrCreate); + //byte[] buffer = _buffer.Rent(4096); + //while (size > 0) + //{ + // size = stream.Read(buffer); + // fileStream.Write(buffer, 0, size); + //} + //_buffer.Return(buffer); + //downloadRequest.IsCompleted = true; + return true; + } + catch { return false; } + } + private async Task GetIsPartialContentSupportedAsync(string url) + { + using var request = new HttpRequestMessage(HttpMethod.Head, url); + using var response = await new HttpClient().SendAsync(request); + return response.Headers.AcceptRanges.Contains("bytes"); + } + + private bool GetIsPartialContentSupported(string url) + { + using var request = new HttpRequestMessage(HttpMethod.Head, url); + using var response = new HttpClient().Send(request); + return response.Headers.AcceptRanges.Contains("bytes"); + } +} diff --git a/MinecraftLaunch/Extensions/DownloadExtension.cs b/MinecraftLaunch/Extensions/DownloadExtension.cs index 0a863cf..e63cc64 100644 --- a/MinecraftLaunch/Extensions/DownloadExtension.cs +++ b/MinecraftLaunch/Extensions/DownloadExtension.cs @@ -11,7 +11,7 @@ namespace MinecraftLaunch.Extensions; /// 下载扩展类 /// public static class DownloadExtension { - public static BatchDownloader DefaultDownloader { get; set; } = new(); + public static IDownloader DefaultDownloader { get; set; } = new Downloader(); public static IDownloadEntry OfMirrorSource(this IDownloadEntry entry, MirrorDownloadSource source) { @@ -37,7 +37,7 @@ public static ValueTask DownloadAsync(this DefaultDownloader.Setup(Enumerable.Repeat(request, 1)); DefaultDownloader.ProgressChanged += (sender, args) => { - action(args); + action?.Invoke(args); }; return DefaultDownloader.DownloadAsync();