Yahoo财经INTC数据爬虫仅返回99条记录？是限制还是Bug？

核心原因 #

你的爬虫只能拿到99条数据，不是代码Bug，而是Yahoo财经历史数据页面的动态加载机制导致的：

• 用浏览器访问时，页面会先加载最近的约100条数据，之后随着页面滚动触发JS逻辑，逐步加载更早的历史记录。
• 而你的HttpClient直接请求返回的是初始静态HTML，仅包含默认加载的这部分数据，更早的记录并未包含在静态响应中，自然解析不到。
• 另外，HttpClient默认请求头与浏览器差异较大，Yahoo可能会对非浏览器请求返回精简内容，但这不是核心问题，核心还是动态加载。

解决方案 #

方案1：使用Yahoo Finance CSV下载端点（推荐）

Yahoo提供了直接导出历史数据的CSV接口，比爬HTML更稳定，不受页面结构变化影响，且能一次性获取指定时间范围的全部数据。

示例代码：

internal class Program
{
    static async Task Main(string[] args)
    {
        var startTime = new DateTime(2023, 1, 1);
        var endTime = new DateTime(2023, 7, 1);
        var client = new HttpClient();
        
        // 添加浏览器请求头，避免被拦截
        client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36");

        // CSV下载接口，参数与网页请求一致
        string uri = $"https://query1.finance.yahoo.com/v7/finance/download/INTC?period1={ToUnixTimeSeconds(startTime)}&period2={ToUnixTimeSeconds(endTime)}&interval=1d&events=history&includeAdjustedClose=true";
        
        var response = await client.GetAsync(uri);
        if (response.IsSuccessStatusCode)
        {
            string csvContent = await response.Content.ReadAsStringAsync();
            var historicalPrices = ParseCsv(csvContent);
            Console.WriteLine($"historical prices count {historicalPrices.Count}");
            foreach (var kvp in historicalPrices)
                Console.WriteLine($"{kvp.Key:yyyy-MM-dd} {kvp.Value}");
        }
        else
        {
            Console.WriteLine($"请求失败：{response.StatusCode}");
        }
    }

    private static ulong ToUnixTimeSeconds(DateTime date)
    {
        return Convert.ToUInt64((date - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds);
    }

    private static Dictionary<DateTime, float> ParseCsv(string csvContent)
    {
        var dictionary = new Dictionary<DateTime, float>();
        var lines = csvContent.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
        
        // 跳过表头行
        for (int i = 1; i < lines.Length; i++)
        {
            var parts = lines[i].Split(',');
            if (parts.Length >= 6 
                && DateTime.TryParse(parts[0], out DateTime date) 
                && float.TryParse(parts[5], out float adjClose))
            {
                dictionary[date] = adjClose;
            }
        }
        return dictionary;
    }
}

方案2：使用支持JS渲染的爬虫（PuppeteerSharp）

如果必须通过爬取网页的方式获取数据，可以用PuppeteerSharp模拟浏览器行为，等待JS加载全部历史记录。

步骤：

1. 先通过NuGet安装依赖：Install-Package PuppeteerSharp
2. 示例代码：

using PuppeteerSharp;

internal class Program
{
    static async Task Main(string[] args)
    {
        var startTime = new DateTime(2023, 1, 1);
        var endTime = new DateTime(2023, 7, 1);
        string uri = $"https://finance.yahoo.com/quote/INTC/history?period1={ToUnixTimeSeconds(startTime)}&period2={ToUnixTimeSeconds(endTime)}&interval=1d&filter=history&frequency=1d";

        // 首次运行需要下载Chromium，后续可注释
        await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultChromiumRevision);
        
        using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true });
        using var page = await browser.NewPageAsync();
        
        // 设置浏览器请求头
        await page.SetUserAgentAsync("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36");
        
        // 等待页面加载完成
        await page.GoToAsync(uri, new NavigationOptions { WaitUntil = WaitUntilNavigation.Networkidle2 });
        
        // 模拟滚动加载全部数据
        int previousHeight = 0;
        while (true)
        {
            int currentHeight = await page.EvaluateExpressionAsync<int>("document.body.scrollHeight");
            if (currentHeight == previousHeight) break;
            
            previousHeight = currentHeight;
            await page.EvaluateExpressionAsync("window.scrollTo(0, document.body.scrollHeight)");
            await Task.Delay(1000); // 等待数据加载
        }
        
        // 获取完整页面HTML并解析
        string html = await page.GetContentAsync();
        var historicalPrices = ParseHistoricalPrices(html);
        Console.WriteLine($"historical prices count {historicalPrices.Count}");
        foreach (var kvp in historicalPrices)
            Console.WriteLine($"{kvp.Key:yyyy-MM-dd} {kvp.Value}");
    }

    private static ulong ToUnixTimeSeconds(DateTime date)
    {
        return Convert.ToUInt64((date - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds);
    }

    // 复用你原有的HTML解析方法
    private static Dictionary<DateTime, float> ParseHistoricalPrices(string html)
    {
        Dictionary<DateTime, float> dictionary = new Dictionary<DateTime, float>();
        IElement matchingTable = null;
        AngleSharp.IBrowsingContext browsingContext = BrowsingContext.New(AngleSharp.Configuration.Default);
        AngleSharp.Dom.IDocument document = browsingContext.OpenAsync(virtualResponse => virtualResponse.Content(html)).Result;
        
        if (document.Title != null && document.Title.Contains("Historical Prices"))
        {
            int matchCount = 0;
            var tableCollection = document.GetElementsByTagName("table");
            foreach (var table in tableCollection)
            {
                var dataTestAttribute = table.GetAttribute("data-test");
                if (dataTestAttribute == "historical-prices")
                {
                    matchCount++;
                    matchingTable = table;
                }
            }
            
            if (matchCount == 1 && matchingTable != null)
            {
                var tbodyCollection = matchingTable.GetElementsByTagName("tbody");
                if (tbodyCollection.Length == 1)
                {
                    var trCollection = tbodyCollection[0].GetElementsByTagName("tr");
                    foreach (var trElement in trCollection)
                    {
                        var tdCollection = trElement.GetElementsByTagName("td");
                        if (tdCollection.Length == 7)
                        {
                            string dateString = tdCollection[0].TextContent.Trim();
                            string adjCloseString = tdCollection[5].TextContent.Trim().Replace(",", "");
                            DateTime date = DateTime.ParseExact(dateString, "MMM dd, yyyy", System.Globalization.CultureInfo.InvariantCulture);
                            float adjClose = (float)Convert.ToDouble(adjCloseString);
                            
                            if (dictionary.ContainsKey(date))
                                dictionary.Remove(date);
                            dictionary.Add(date, adjClose);
                        }
                    }
                }
            }
        }
        return dictionary;
    }
}

内容的提问来源于stack exchange，提问作者H2ONaCl