使用SemaphoreSlim运行并行任务的性能困惑与优化咨询

我用HtmlAgilityPack和SemaphoreSlim实现异步批量发送HTTP请求并收集响应，遇到了性能瓶颈：

• 1000个任务、SemaphoreSlim初始计数15时，总耗时约40秒；
• 任务量翻倍到2000个时，耗时线性增长到约80秒；
• 但同时启动两个控制台应用，各处理1000个任务（Semaphore初始计数均为15），每个应用耗时约60秒，2000个任务总耗时仅60秒。

想知道怎么在单个应用内达到这种性能，以及如何进行性能扩容。

Main方法代码 #

using appParser.Services;
using HtmlAgilityPack;
using System.Diagnostics;

internal class Program
{
        static int j;
        static int k;
        static long sum=0;
        static HtmlWeb web = new HtmlWeb();

        static async Task Main(string[] args)
        {
            web.UserAgent = "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Mobile Safari/537.36";
            
            List<Task> tasks = new List<Task>();
            List<HtmlDocument> htmldocs = new();

            var stopwatch = new Stopwatch();

            Console.WriteLine("Put request number");
            k = Int32.Parse(Console.ReadLine());
            Console.WriteLine("Put initCount");
            j = Int32.Parse(Console.ReadLine());
            var PerformanceTest = new PerfrormanceTestSemaphore(j);
            Console.WriteLine("-------------------------------------------------------------");
            Console.WriteLine($"InitCount: {j}");
            stopwatch.Start();

            for (int i = 0; i < k; i++)
            {
                tasks.Add(PerformanceTest.LoadDocs("https://m.olx.pl/elektronika/gry-konsole/q-xbox/?search%5Border%5D=created_at:desc", web));
            }
            stopwatch.Stop();
            Console.WriteLine($"Tasks added in {stopwatch.ElapsedMilliseconds} ms");
            stopwatch.Restart();
                
            while (tasks.Count > 0)
            {
                stopwatch.Start();
                Task finished = await Task.WhenAny(tasks);
                htmldocs.Add(((Task<HtmlDocument>)finished).Result);
                tasks.Remove(finished);
                stopwatch.Stop();
                Console.WriteLine($"{tasks.Count} left, last finished in {stopwatch.ElapsedMilliseconds} ms");
                sum += stopwatch.ElapsedMilliseconds;
                stopwatch.Restart();
            }
            Console.WriteLine("-------------------------------------------------------------");
            Console.WriteLine($"Tasks finished in {sum} ms");
            Console.WriteLine("-------------------------------------------------------------");
            await File.WriteAllTextAsync("html.txt", htmldocs[k-1].Text.ToString());
            Console.ReadLine();
        }
}

PerfrormanceTestSemaphore类代码 #

using HtmlAgilityPack;

namespace appParser.Services
{
    internal class PerfrormanceTestSemaphore
    {
        public int SemaphoreNum { get; set; }
        private SemaphoreSlim _mutex;
        
        CancellationTokenSource cts = new();
        public PerfrormanceTestSemaphore(int semaphoreNum)
        {
            SemaphoreNum = semaphoreNum;
            _mutex = new SemaphoreSlim(semaphoreNum);
        }

        public async Task<HtmlDocument> LoadDocs(string url, HtmlWeb web)
        {
            await _mutex.WaitAsync(cts.Token);
            try
            {
                string loadUrl = url + "&view=list/full_page=True";
                return await web.LoadFromWebAsync(url);
            }
            finally
            {
                _mutex.Release();
            }
        }
    }
}

1. 替换HtmlWeb为复用的HttpClient #

HtmlWeb内部默认的HttpClient实例复用性差，每个请求可能创建新连接，导致连接池耗尽。改用单例HttpClient（或IHttpClientFactory）复用连接：

// 全局复用的HttpClient
private static readonly HttpClient _httpClient = new HttpClient(new HttpClientHandler
{
    MaxConnectionsPerServer = 30, // 匹配目标域名的最大并发连接数
    UseCookies = true,
    AllowAutoRedirect = true
})
{
    DefaultRequestHeaders =
    {
        {"User-Agent", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Mobile Safari/537.36"}
    }
};

// 替换原LoadDocs方法
public async Task<HtmlDocument> LoadDocs(string url)
{
    await _mutex.WaitAsync(cts.Token);
    try
    {
        string loadUrl = url + "&view=list/full_page=True";
        var response = await _httpClient.GetStringAsync(loadUrl);
        var doc = new HtmlDocument();
        doc.LoadHtml(response);
        return doc;
    }
    finally
    {
        _mutex.Release();
    }
}

2. 优化Task.WhenAny的低效循环 #

Task.WhenAny配合List<Task>.Remove是O(n)复杂度，任务越多越慢。改用Task.WhenAll批量等待所有任务完成：

// 替换原while循环
var allTasks = tasks.Cast<Task<HtmlDocument>>().ToArray();
await Task.WhenAll(allTasks);
htmldocs.AddRange(allTasks.Select(t => t.Result));

3. 匹配Semaphore并发数与连接池容量 #

单个应用内，Semaphore初始计数不要低于MaxConnectionsPerServer，否则会浪费连接池资源。比如将Semaphore初始计数设为30，和MaxConnectionsPerServer一致，同时可根据目标网站限流策略、本地CPU核心数调整。

4. 多分区并行处理（模拟多应用效果） #

拆分任务为多个独立的并发队列，每个队列用单独的Semaphore，相当于在单个应用内模拟多个独立请求池：

var semaphore1 = new SemaphoreSlim(15);
var semaphore2 = new SemaphoreSlim(15);

var tasks = new List<Task<HtmlDocument>>();
// 第一组任务
for (int i = 0; i < 1000; i++)
{
    tasks.Add(LoadDocsWithSemaphore(url, semaphore1));
}
// 第二组任务
for (int i = 0; i < 1000; i++)
{
    tasks.Add(LoadDocsWithSemaphore(url, semaphore2));
}

await Task.WhenAll(tasks);

5. 线程安全的结果收集 #

原代码中List<HtmlDocument>不是线程安全的，多任务同时添加会导致异常。改用ConcurrentBag<HtmlDocument>：

var htmldocs = new ConcurrentBag<HtmlDocument>();
// 在任务完成时添加结果
htmldocs.Add(await finishedTask);

6. 系统级连接配置（仅.NET Framework） #

如果使用.NET Framework，调整ServicePointManager参数提升连接复用效率：

ServicePointManager.DefaultConnectionLimit = 100;
ServicePointManager.Expect100Continue = false;

内容的提问来源于stack exchange，提问作者mrRemote