使用.NET WebClient下载指定站点时遇403禁止错误求助

问题描述 #

使用.NET 4.8.1的C#代码，通过自定义CookieAwareWebClient3类依次访问以下页面：

• 主页：https://sos.ga.gov/
• 链接页：https://sos.ga.gov/licensing-division-license-lookup
• 目标搜索页：https://verify.sos.ga.gov/verification/Search.aspx

前两个页面可正常获取HTML内容，但请求目标搜索页时返回Forbidden错误。

原代码片段 #

主请求代码 #

string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0";
CookieAwareWebClient3 client = new CookieAwareWebClient3();
client.Headers.Add("Host", "sos.ga.gov");
client.Headers.Add("Sec-Fetch-Dest", "document");
client.Headers.Add("Sec-Fetch-Mode", "navigate");
client.Headers.Add("Sec-Fetch-Site", "none");
client.Headers.Add("Sec-Fetch-User", "?1");
client.Headers.Add("TE", "trailers");
client.Headers.Add("Upgrade-Insecure-Requests", "1");
client.Headers.Add("user-agent", UserAgent);
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;

string url = "https://sos.ga.gov/";
string beginOutput = "", beginOutput1 = "", beginOutput2 = "", error = "";
try
{
    byte[] beginResults = client.DownloadData(url);    
    beginOutput = Encoding.ASCII.GetString(beginResults);
}
catch(WebException ex)
{
    error = ex.Message;
}
client.TransferCookies();

// 访问链接页
client.Headers.Add("Host", "sos.ga.gov");
client.Headers.Add("Referer", "https://sos.ga.gov/");
client.Headers.Add("Sec-Fetch-Dest", "document");
client.Headers.Add("Sec-Fetch-Mode", "navigate");
client.Headers.Add("Sec-Fetch-Site", "same-origin");
client.Headers.Add("Sec-Fetch-User", "?1");
client.Headers.Add("Upgrade-Insecure-Requests", "1");
client.Headers.Add("user-agent", UserAgent);

url = "https://sos.ga.gov/licensing-division-license-lookup";
try
{
    byte[] beginResults1 = client.DownloadData(url);     
    beginOutput1 = Encoding.ASCII.GetString(beginResults1);
}
catch (WebException ex)
{
    error = ex.Message;
}
client.TransferCookies();

// 访问目标搜索页
client.Headers.Add("Host", "verify.sos.ga.gov");
client.Headers.Add("Referer", "https://sos.ga.gov/");
client.Headers.Add("Sec-Fetch-Dest", "document");
client.Headers.Add("Sec-Fetch-Mode", "navigate");
client.Headers.Add("Sec-Fetch-Site", "same-site");
client.Headers.Add("Sec-Fetch-User", "?1");
client.Headers.Add("Upgrade-Insecure-Requests", "1");
client.Headers.Add("user-agent", UserAgent);

url = "https://verify.sos.ga.gov/verification/Search.aspx";
try
{
    byte[] beginResults2 = client.DownloadData(url);    // 此处出现Forbidden错误
    beginOutput2 = Encoding.ASCII.GetString(beginResults2);
}
catch (WebException ex)
{
    error = ex.Message;
}

自定义WebClient类 #

public class CookieAwareWebClient3 : WebClient
{
    public CookieAwareWebClient3()
    {
        CookieContainer = new CookieContainer();
        ResponseCookies = new CookieCollection();
    }

    public CookieContainer CookieContainer { get; set; }
    public CookieCollection ResponseCookies { get; set; }

    public void TransferCookies()
    {
        CookieContainer.Add(ResponseCookies);
    }
    protected override WebRequest GetWebRequest(Uri address)
    {
        var request = (HttpWebRequest)base.GetWebRequest(address);
        request.KeepAlive = true;
        request.CookieContainer = CookieContainer;
        return request;

    }
    protected override WebResponse GetWebResponse(WebRequest request)
    {
        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12 | SecurityProtocolType.Ssl3;
        var response = (HttpWebResponse)base.GetWebResponse(request);

        this.ResponseCookies = response.Cookies;
        return response;
    }
}

问题排查与修复方案 #

1. 修复Cookie跨域传递逻辑 #

原CookieAwareWebClient3的Cookie处理存在缺陷：每次响应的Cookie会覆盖ResponseCookies，且TransferCookies方法未指定域名，导致子域名verify.sos.ga.gov无法获取主域名sos.ga.gov的Cookie。

修改后的CookieAwareWebClient3类：

public class CookieAwareWebClient3 : WebClient
{
    public CookieContainer CookieContainer { get; }

    public CookieAwareWebClient3()
    {
        CookieContainer = new CookieContainer();
    }

    protected override WebRequest GetWebRequest(Uri address)
    {
        var request = (HttpWebRequest)base.GetWebRequest(address);
        request.KeepAlive = true;
        request.CookieContainer = CookieContainer;
        return request;
    }

    protected override WebResponse GetWebResponse(WebRequest request)
    {
        var response = (HttpWebResponse)base.GetWebResponse(request);
        // 直接将响应Cookie添加到CookieContainer，自动处理域名关联
        CookieContainer.Add(response.Cookies);
        return response;
    }
}

移除冗余的ResponseCookies和TransferCookies方法，直接在GetWebResponse中将Cookie添加到CookieContainer，确保子域名能正确继承主域名的Cookie。

2. 避免重复添加请求头 #

原代码每次请求都重复添加相同Header，导致请求头重复，触发服务器反爬机制。每次请求前需清空或重新设置Headers：

主请求代码调整：

string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0";
// 初始化时设置安全协议，只需一次
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls13;

CookieAwareWebClient3 client = new CookieAwareWebClient3();
string beginOutput = "", beginOutput1 = "", beginOutput2 = "", error = "";

// 访问主页
client.Headers.Clear();
client.Headers.Add("Host", "sos.ga.gov");
client.Headers.Add("Sec-Fetch-Dest", "document");
client.Headers.Add("Sec-Fetch-Mode", "navigate");
client.Headers.Add("Sec-Fetch-Site", "none");
client.Headers.Add("Sec-Fetch-User", "?1");
client.Headers.Add("TE", "trailers");
client.Headers.Add("Upgrade-Insecure-Requests", "1");
client.Headers.Add("user-agent", UserAgent);

try
{
    byte[] beginResults = client.DownloadData("https://sos.ga.gov/");    
    beginOutput = Encoding.UTF8.GetString(beginResults); // 改用UTF8编码，避免乱码
}
catch(WebException ex)
{
    error = ex.Message;
}

// 访问链接页
client.Headers.Clear();
client.Headers.Add("Host", "sos.ga.gov");
client.Headers.Add("Referer", "https://sos.ga.gov/");
client.Headers.Add("Sec-Fetch-Dest", "document");
client.Headers.Add("Sec-Fetch-Mode", "navigate");
client.Headers.Add("Sec-Fetch-Site", "same-origin");
client.Headers.Add("Sec-Fetch-User", "?1");
client.Headers.Add("Upgrade-Insecure-Requests", "1");
client.Headers.Add("user-agent", UserAgent);

try
{
    byte[] beginResults1 = client.DownloadData("https://sos.ga.gov/licensing-division-license-lookup");     
    beginOutput1 = Encoding.UTF8.GetString(beginResults1);
}
catch (WebException ex)
{
    error = ex.Message;
}

// 访问目标搜索页
client.Headers.Clear();
client.Headers.Add("Host", "verify.sos.ga.gov");
client.Headers.Add("Referer", "https://sos.ga.gov/licensing-division-license-lookup"); // 修正Referer为链接页
client.Headers.Add("Sec-Fetch-Dest", "document");
client.Headers.Add("Sec-Fetch-Mode", "navigate");
client.Headers.Add("Sec-Fetch-Site", "same-site");
client.Headers.Add("Sec-Fetch-User", "?1");
client.Headers.Add("Upgrade-Insecure-Requests", "1");
client.Headers.Add("user-agent", UserAgent);

try
{
    byte[] beginResults2 = client.DownloadData("https://verify.sos.ga.gov/verification/Search.aspx");
    beginOutput2 = Encoding.UTF8.GetString(beginResults2);
}
catch (WebException ex)
{
    error = ex.Message;
}

3. 关键修正点 #

• 安全协议：初始化时一次设置支持TLS1.2和TLS1.3，避免重复设置。
• 编码：改用Encoding.UTF8解析响应，避免ASCII编码导致的乱码。
• Referer：访问目标页时，Referer设置为链接页（符合用户正常跳转路径），服务器会验证跳转来源的合法性。
• Header清理：每次请求前调用client.Headers.Clear()，避免重复添加相同Header。

内容的提问来源于stack exchange，提问作者sritt2024