求助：使用HttpClient批量下载图片为何出现间歇性停顿？

问题根源分析 #

1. UI卡顿：btnStartDownload_Click中同步调用Sites()方法，若Radar.PrepareLinks()耗时较长，会直接阻塞UI线程，导致程序卡顿。
2. 下载停顿：
   • HttpClient设置了ConnectionClose = true，每次请求后强制关闭连接，无法复用TCP连接，新请求需重新建立连接，大幅增加延迟。
   • 默认的ServicePointManager.DefaultConnectionLimit限制了同一服务器的并发连接数（默认值较低），当并行下载数超过限制时，后续请求会排队等待，出现间歇性停顿。
   • Parallel.ForEachAsync用于IO密集型任务时，线程调度易引发不必要的等待，不如异步任务并行处理高效。

修复方案 #

1. 异步化链接准备，避免UI阻塞

将Sites()改为异步方法，确保UI线程不被阻塞：

internal async Task<List<string>> SitesAsync()
{
    var list = new List<string>();
    Radar radar = new Radar(textBoxRadarPath.Text);
    // 异步执行PrepareLinks，避免阻塞UI线程
    await Task.Run(() => radar.PrepareLinks());

    foreach (var link in radar.links)
    {
        list.Add(link);
    }
    // [...] 添加更多URL
    return list;
}

修改按钮点击事件，异步获取链接：

private async void btnStartDownload_Click(object sender, EventArgs e)
{
    if (ResourceDownloader.IsBusy) return;
    var sites = await SitesAsync();

    List<ResourceDownloader.WebData>? downloads = null;

    using (cts = new CancellationTokenSource())
    {
        downloadProgress = new Progress<ResourceDownloader.ProgressReport>(Updater);
        downloads = await ResourceDownloader.Download(downloadProgress, sites, cts);
    }
}

2. 优化HttpClient配置，复用连接

调整HttpClient初始化逻辑，开启连接复用并提升并发连接限制：

private static readonly Lazy<HttpClient> client = new(() => {
    HttpClientHandler handler = CreateHandler(autoRedirect: true);

    // 提升同一服务器的并发连接数限制
    ServicePointManager.DefaultConnectionLimit = 16;
    handler.MaxConnectionsPerServer = 16;

    var client = new HttpClient(handler, true) { Timeout = TimeSpan.FromSeconds(60) };
    client.DefaultRequestHeaders.Add("User-Agent", @"Mozilla/5.0 (Windows NT 10; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0");
    client.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
    client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate");
    // 设置为false，复用TCP连接，避免重复建立连接的开销
    client.DefaultRequestHeaders.ConnectionClose = false;
    return client;
}, true);

3. 替换Parallel.ForEachAsync为异步任务并行处理

IO密集型任务更适合用Task.WhenAll处理，避免线程调度开销：

public static async Task<List<WebData>> Download(IProgress<ProgressReport> progress, IList<string> sites, CancellationTokenSource cts)
{
    IsBusy = true;
    processed = new ConcurrentBag<WebData>();
    progressCount = 0;
    totalCount = sites.Count;

    try
    {
        var downloadTasks = sites.Select(async site =>
        {
            try
            {
                var dataBytes = await client.Value.GetByteArrayAsync(site, cts.Token);
                ReportProgress(progress, dataBytes, site, null);
            }
            catch (Exception ex)
            {
                ReportProgress(progress, null, site, ex);
            }
        });

        await Task.WhenAll(downloadTasks);
    }
    catch (TaskCanceledException) { Debug.Print("The operation was canceled"); }
    finally { IsBusy = false; }
    return processed.ToList();
}

4. 优化进度计数，减少锁竞争

用Interlocked.Increment原子操作替代锁更新进度计数，提升并发效率：

private static void ReportProgress(IProgress<ProgressReport> progress, byte[]? data, string site, Exception? ex)
{
    var currentCount = Interlocked.Increment(ref progressCount);
    var percentage = currentCount * 100 / totalCount;
    WebData webData = new(site, data, ex is null, ex);
    processed.Add(webData);
    progress.Report(new ProgressReport(webData, percentage));
}

完整修改后代码 #

Form1修改后代码

using Ookii.Dialogs.WinForms;
using System.Text;

namespace Downloading_Files
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
            textBoxRadarPath.Text = Properties.Settings.Default.RadarFolder;
        }

        internal async Task<List<string>> SitesAsync()
        {
            var list = new List<string>();
            Radar radar = new Radar(textBoxRadarPath.Text);
            await Task.Run(() => radar.PrepareLinks());

            foreach (var link in radar.links)
            {
                list.Add(link);
            }
            // [...] 添加更多URL
            return list;
        }

        IProgress<ResourceDownloader.ProgressReport>? downloadProgress = null;
        CancellationTokenSource? cts = null;

        private void Updater(ResourceDownloader.ProgressReport progress)
        {
            StringBuilder log = new(1024);
            if (progress.Site.Completed)
            {
                log.Append($"Success! \t {progress.Site.Url}\r\n");
            }
            else
            {
                log.Append($"Failed! \t {progress.Site.Url}\r\n");
                log.Append($"\tReason: {progress.Site.Ex?.Message}\r\n");
            }
            logger.AppendText(log.ToString());
            progressBar.Value = progress.PercentageComplete;
        }

        private async void btnStartDownload_Click(object sender, EventArgs e)
        {
            if (ResourceDownloader.IsBusy) return;
            var sites = await SitesAsync();

            List<ResourceDownloader.WebData>? downloads = null;

            using (cts = new CancellationTokenSource())
            {
                downloadProgress = new Progress<ResourceDownloader.ProgressReport>(Updater);
                downloads = await ResourceDownloader.Download(downloadProgress, sites, cts);
            }
        }

        private void btnCancelDownload_Click(object sender, EventArgs e)
        {
            cts?.Cancel();
        }

        private void btnDownloadFolder_Click(object sender, EventArgs e)
        {
            RadarFolderDialog();
        }

        private void RadarFolderDialog()
        {
            VistaFolderBrowserDialog dlg = new VistaFolderBrowserDialog();
            dlg.ShowNewFolderButton = true;

            if (dlg.ShowDialog() == DialogResult.OK)
            {
                var targetPath = dlg.SelectedPath;
                if (!targetPath.EndsWith("Radar"))
                {
                    targetPath = Path.Combine(targetPath, "Radar");
                    if (!Directory.Exists(targetPath))
                    {
                        Directory.CreateDirectory(targetPath);
                    }
                }

                Properties.Settings.Default.RadarFolder = targetPath;
                textBoxRadarPath.Text = targetPath;
                Properties.Settings.Default.Save();
            }
        }
    }
}

ResourceDownloader修改后代码

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Threading.Tasks;

namespace Downloading_Files
{
    public class ResourceDownloader
    {
        private static readonly Lazy<HttpClient> client = new(() => {
            HttpClientHandler handler = CreateHandler(autoRedirect: true);

            ServicePointManager.DefaultConnectionLimit = 16;
            handler.MaxConnectionsPerServer = 16;

            var client = new HttpClient(handler, true) { Timeout = TimeSpan.FromSeconds(60) };
            client.DefaultRequestHeaders.Add("User-Agent", @"Mozilla/5.0 (Windows NT 10; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0");
            client.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
            client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate");
            client.DefaultRequestHeaders.ConnectionClose = false;
            return client;
        }, true);

        private static HttpClientHandler CreateHandler(bool autoRedirect)
        {
            return new HttpClientHandler()
            {
                AllowAutoRedirect = autoRedirect,
                CookieContainer = new CookieContainer(),
                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
            };
        }

        public record WebData(string Url, byte[]? Data, bool Completed = true, Exception? Ex = null);
        public record ProgressReport(WebData Site, int PercentageComplete);

        private static ConcurrentBag<WebData> processed = default!;
        private static int progressCount = 0;
        private static int totalCount = 0;

        public static bool IsBusy { get; internal set; } = false;

        public static async Task<List<WebData>> Download(IProgress<ProgressReport> progress, IList<string> sites, CancellationTokenSource cts)
        {
            IsBusy = true;
            processed = new ConcurrentBag<WebData>();
            progressCount = 0;
            totalCount = sites.Count;

            try
            {
                var downloadTasks = sites.Select(async site =>
                {
                    try
                    {
                        var dataBytes = await client.Value.GetByteArrayAsync(site, cts.Token);
                        ReportProgress(progress, dataBytes, site, null);
                    }
                    catch (Exception ex)
                    {
                        ReportProgress(progress, null, site, ex);
                    }
                });

                await Task.WhenAll(downloadTasks);
            }
            catch (TaskCanceledException) { Debug.Print("The operation was canceled"); }
            finally { IsBusy = false; }
            return processed.ToList();
        }

        private static void ReportProgress(IProgress<ProgressReport> progress, byte[]? data, string site, Exception? ex)
        {
            var currentCount = Interlocked.Increment(ref progressCount);
            var percentage = currentCount * 100 / totalCount;
            WebData webData = new(site, data, ex is null, ex);
            processed.Add(webData);
            progress.Report(new ProgressReport(webData, percentage));
        }
    }
}

内容的提问来源于stack exchange，提问作者Daniel Lip