C#爬取谷歌搜索xGj8Mb内容失败，求解决方案（Java Jsoup可成功）

我需要按流派整理音乐文件，因此用C#编写代码，目标是获取谷歌搜索页面中class为xGj8Mb的div标签内容。但无论是用标准C# HttpClient方法，还是结合AngleSharp库，都无法获取该信息，保存下来的页面还缺失大量内容。而用Java结合Jsoup库重写相同逻辑，却能正常获取页面所有信息。

以下是我测试用的两段C#代码：

使用标准HttpClient的代码 #

public class Program {
private static async Task Main(string[] args) {
    const string folderPath = @"C:\Users\chris\Desktop\Attilio\ATTILIO MUSIC";
    if (!Directory.Exists(folderPath)) {
        Console.Error.WriteLine("DOES NOT EXIST");
        return;
    }

    var archive = Directory.GetFiles(folderPath);
    foreach (var entry in archive) {
        // 筛选mp3/wav文件
        var fileName = entry.Split(folderPath + @"\")[1];

        SearchOnInternet(fileName);
        return;
    }

    Console.WriteLine($"Number: {archive.Length}");
}

private static void SearchOnInternet(string fileName) {
    // 移除文件扩展名
    if (fileName.EndsWith(".mp3"))
        fileName = fileName.Split(".mp3")[0];
    else if (fileName.EndsWith(".wav"))
        fileName = fileName.Split(".wav")[0];

    // 转码为URL安全格式
    var fileNameAsUrl = Uri.EscapeDataString(fileName);
    Console.WriteLine(fileNameAsUrl);
    var url =
        $"https://www.google.com/search?q={fileNameAsUrl}";

    // 发起请求并保存页面
    using (var client = new HttpClient()) {
        var response = client.GetAsync(url);
        if (response.Result.IsSuccessStatusCode) {
            var responseBody = response.Result.Content.ReadAsStringAsync();

            var outputPath = @"C:\Users\chris\Documents\Universita\Programmi\C#\FileAura\FileAuraServer\Search";
            if (!Directory.Exists(outputPath))
                Directory.CreateDirectory(outputPath);
            File.WriteAllText(
                @$"{outputPath}\{fileName}.html",
                responseBody.Result);
        }
    }
}
}

使用AngleSharp的代码 #

private static async void SearchOnInternet(string fileName) {
    // 移除文件扩展名
    if (fileName.EndsWith(".mp3"))
        fileName = fileName.Split(".mp3")[0];
    else if (fileName.EndsWith(".wav"))
        fileName = fileName.Split(".wav")[0];

    // 转码为URL安全格式
    var fileNameAsUrl = Uri.EscapeDataString(fileName);
    var googleSearchUrl =
        $"https://www.google.com/search?q={fileNameAsUrl}";

    // 发起请求
    using var httpClient = new HttpClient();
    var response = await httpClient.GetAsync(googleSearchUrl);
    response.EnsureSuccessStatusCode();

    // 获取并保存页面内容
    var responseContent = await response.Content.ReadAsStringAsync();
    string filePath = "output.html";
    await File.WriteAllTextAsync(filePath, responseContent);

    Console.WriteLine($"The HTML content has been saved to {filePath}");

    // 解析HTML
    var context = BrowsingContext.New(Configuration.Default);
    var parser = context.GetService<IHtmlParser>();
    var document = await parser.ParseDocumentAsync(responseContent);

    // 提取页面标题和搜索结果示例
    var title = document.Title;
    Console.WriteLine("Title: " + title);

    var results = document.QuerySelectorAll("h3");
    foreach (var result in results)
    {
        Console.WriteLine(result.TextContent);
    }
}

期望获取的页面 #

代码返回的实际页面 #

内容的提问来源于stack exchange，提问作者Christian 30