Java(HttpsURLConnection/Jsoup)访问部分URL返回403错误求助

一、核心原因 #

Cloudflare会校验请求是否来自真实浏览器，缺失必要请求头、未维护会话Cookie或未执行页面JavaScript，都会触发403拦截。

二、修正HttpsURLConnection代码 #

通过模拟浏览器请求头、启用Cookie管理来绕过基础验证：

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.HttpURLConnection;

public class ACSFetcher {
    public static void main(String[] args) throws Exception {
        // 启用Cookie管理，保存会话状态
        CookieHandler.setDefault(new CookieManager());

        URL targetUrl = new URL("https://pubs.acs.org");
        HttpURLConnection conn = (HttpURLConnection) targetUrl.openConnection();

        // 模拟Chrome浏览器请求头
        conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
        conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8");
        conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
        conn.setRequestProperty("Referer", "https://pubs.acs.org");
        conn.setRequestProperty("Upgrade-Insecure-Requests", "1");
        conn.setRequestProperty("Connection", "keep-alive");

        conn.setInstanceFollowRedirects(true);
        conn.setDoInput(true);

        int responseCode = conn.getResponseCode();
        if (responseCode == HttpURLConnection.HTTP_OK) {
            BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
            StringBuilder content = new StringBuilder();
            String line;
            while ((line = reader.readLine()) != null) {
                content.append(line);
            }
            reader.close();
            System.out.println(content.toString());
        } else if (responseCode == HttpURLConnection.HTTP_MOVED_TEMP || responseCode == HttpURLConnection.HTTP_MOVED_PERM) {
            // 处理跳转，重新发起请求
            String redirectUrl = conn.getHeaderField("Location");
            conn = (HttpURLConnection) new URL(redirectUrl).openConnection();
            // 重复设置请求头
            conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
            conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8");
            
            BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
            StringBuilder content = new StringBuilder();
            String line;
            while ((line = reader.readLine()) != null) {
                content.append(line);
            }
            reader.close();
            System.out.println(content.toString());
        }
        conn.disconnect();
    }
}

三、修正Jsoup代码 #

Jsoup支持直接设置请求头和维护Cookie会话：

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.Connection;
import java.util.Map;

public class ACSJsoupFetcher {
    public static void main(String[] args) throws Exception {
        // 首次请求获取会话Cookie和页面状态
        Connection.Response initialResp = Jsoup.connect("https://pubs.acs.org")
                .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
                .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
                .header("Accept-Language", "en-US,en;q=0.5")
                .header("Referer", "https://pubs.acs.org")
                .header("Upgrade-Insecure-Requests", "1")
                .header("Connection", "keep-alive")
                .execute();

        // 获取会话Cookie
        Map<String, String> cookies = initialResp.cookies();

        // 二次请求带上Cookie和完整请求头
        Document doc = Jsoup.connect("https://pubs.acs.org")
                .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
                .headers(initialResp.headers())
                .cookies(cookies)
                .get();

        System.out.println(doc.html());
    }
}

四、进阶方案：使用Selenium（应对JS渲染验证） #

如果上述方法仍失效，说明Cloudflare需要执行JavaScript验证，此时用Selenium模拟真实浏览器：

import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;

public class ACSSeleniumFetcher {
    public static void main(String[] args) {
        // 设置ChromeDriver路径（根据本地实际路径调整）
        System.setProperty("webdriver.chrome.driver", "path/to/chromedriver");

        ChromeOptions options = new ChromeOptions();
        // 模拟正常浏览器，绕过自动化检测
        options.addArguments("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
        options.addArguments("--disable-blink-features=AutomationControlled");
        options.addArguments("--start-maximized");
        // 可选：启用无头模式
        // options.addArguments("--headless=new");

        WebDriver driver = new ChromeDriver(options);
        try {
            driver.get("https://pubs.acs.org");
            // 等待页面加载完成（可替换为显式等待提升稳定性）
            Thread.sleep(3000);
            String pageSource = driver.getPageSource();
            System.out.println(pageSource);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            driver.quit();
        }
    }
}

注意事项 #

• 定期更新User-Agent为当前主流浏览器的标识，避免使用过时值。
• 控制请求频率，避免触发Cloudflare的频率限制。
• 使用Selenium时，需保证ChromeDriver版本与本地Chrome浏览器版本一致。

内容的提问来源于stack exchange，提问作者MrSalesi