爬虫前如何获取新会话Cookie？自动获取返回403问题排查

手动从Firefox复制Cookie到代码中时，爬虫程序可正常运行，但使用Selenium自动获取Cookie后发送请求却返回403响应，相关代码如下：

import requests
import pandas as pd
from selenium import webdriver
import time
from selenium.webdriver.chrome.options import Options

#Function to grab new cookies
def update_cookies(session):
    driver_path = 'C:/Program Files/chromedriver.exe'
    opts = Options()
    opts.add_argument("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0")
    driver = webdriver.Chrome(executable_path=driver_path,chrome_options=opts)
    driver.get('https://www.reuter.de/mariner-duschsystem-mit-eden-edge-thermostat-inkl-grundkoerper-kopfbrause-slim-300-mm-und-metall-brauseset-schwarz-matt-a1161798.php')
    time.sleep(10)
    cookies_new = driver.get_cookies()
    driver.quit()
    
    for c in cookies_new:
        print(str(c))
        if c['name'] == '__cf_bm':
            print('Found __cf_bm')
            session.cookies['__cf_bm'] = c['value']
        elif c['name'] == 'cf_clearance':
            print('Found cf_clearance')
            session.cookies['cf_clearance'] = c['value']
        elif c['name'] == 'XTCsid':
            print('Found XTCsid')
            session.cookies['XTCsid'] = c['value']

#Start a session
session = requests.Session()

#Old cookies to be updated (not necessary when update_cookies() is working)
cookies = {
    "user_locale_selection": "https://www.reuter.de",
    "cookie_test": "please_accept_for_session",
    "__cf_bm": "Oayhkm3Ps7J.onL1Km1NO3S2xXMlxDjPEbL3H4myz7Y-1691765714-0-ARw0RHJXQNMM7RN3aw1OTIbQNpRk4AQIY63pQdj3IRDeYzpAQm4gGmNKgsM2qVDw4X7i13zSLIvGV9eir0cw54BNxlWuVRKdkxWyaSEfH4fs",
    "cf_clearance": "2XeYVrW.r5zZAddHaHJTBglFsWJx8wRe140z1kLZJOw-1691765716-0-1-ee7419d2.733d08ea.a668f614-0.2.1691765716",
    "feedback": "1",
    "XTCsid": "s:a118e3fbb3a07fdd44c51ec9e44b4ad2.JMFQIZZCu/8CZRQgPoSYzdqva80TX2qycnxngYQ5BxY"
}

#Assign old cookies to session
session.cookies.update(cookies)

#Define headers
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0",
    "Accept": "application/json",
    "Accept-Language": "de,en-US;q=0.7,en;q=0.3",
    "Accept-Encoding": "utf-8",
    "DNT": "1",
}

#Set request params and url
base_url = "https://www.reuter.de/services/products/"
params = {
    "$language": "de",
    "$select": "id,name,price,category,manufacturer,series,url"
}

#Empty List
all_ids = []

#Update cookies
update_cookies(session)


#Start scraping
while True:

    try:
        response = session.get(base_url, params=params, headers=headers)
        
        r = response.json()
        
        #Add r to list
        ##all_ids.extend(item["id"] for item in data.get("items", []))
        all_ids.extend(r['data'])

        #End condition
        if r["limit"] + r["skip"] >= r["total"]:
            break
        
        #Set next request
        params["$skip"] = r["limit"] + r["skip"]
        print(counter)
        print(r)
        counter = counter+1

    except Exception as e:
        print('Error:', e)
        print(response.status_code)
        break

#Info to df -> csv
df = pd.json_normalize(all_ids)
df.to_csv('all_perfect.csv',sep=';',encoding='utf-8',decimal=',')

1. User-Agent配置错误 #

你在Selenium的Chrome Options中直接添加UA字符串是无效的，正确方式需要加上--user-agent=前缀。另外，使用Chrome驱动却设置Firefox的UA，容易触发网站反爬检测。

2. Cookie复制不完整且属性处理缺失 #

• 手动Cookie包含user_locale_selection、cookie_test、feedback等字段，你只复制了3个Cookie，网站可能需要这些字段才能通过验证。
• Selenium获取的Cookie包含domain、path等属性，直接给requests session赋值时需要保留这些属性，否则Cookie可能无法正确匹配请求的域名和路径。

3. Cloudflare自动化检测 #

网站使用Cloudflare防护，Chrome驱动默认会暴露navigator.webdriver等自动化特征，导致生成的cf_clearance Cookie无效——Cloudflare会验证浏览器环境是否为真实用户。

修复后的代码 #

import requests
import pandas as pd
from selenium import webdriver
import time
from selenium.webdriver.chrome.options import Options

#Function to grab new cookies
def update_cookies(session):
    driver_path = 'C:/Program Files/chromedriver.exe'
    opts = Options()
    # 正确设置UA，匹配requests里的Firefox UA
    opts.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0')
    # 隐藏Chrome自动化特征
    opts.add_argument('--disable-blink-features=AutomationControlled')
    opts.add_experimental_option('excludeSwitches', ['enable-automation'])
    opts.add_experimental_option('useAutomationExtension', False)
    
    driver = webdriver.Chrome(executable_path=driver_path, options=opts)
    # 执行脚本隐藏webdriver属性
    driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
    
    driver.get('https://www.reuter.de/mariner-duschsystem-mit-eden-edge-thermostat-inkl-grundkoerper-kopfbrause-slim-300-mm-und-metall-brauseset-schwarz-matt-a1161798.php')
    time.sleep(10)
    cookies_new = driver.get_cookies()
    driver.quit()
    
    # 完整复制所有Cookie，保留domain、path等属性
    for c in cookies_new:
        print(str(c))
        session.cookies.set(
            name=c['name'],
            value=c['value'],
            domain=c.get('domain'),
            path=c.get('path'),
            secure=c.get('secure'),
            expires=c.get('expiry')
        )

#Start a session
session = requests.Session()

#Define headers
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0",
    "Accept": "application/json",
    "Accept-Language": "de,en-US;q=0.7,en;q=0.3",
    "Accept-Encoding": "gzip, deflate, br",  # 修正编码格式，requests默认支持对应解析
    "DNT": "1",
}

#Set request params and url
base_url = "https://www.reuter.de/services/products/"
params = {
    "$language": "de",
    "$select": "id,name,price,category,manufacturer,series,url"
}

#Empty List
all_ids = []
counter = 0  # 初始化counter变量，原代码未定义

#Update cookies
update_cookies(session)


#Start scraping
while True:

    try:
        response = session.get(base_url, params=params, headers=headers)
        response.raise_for_status()  # 主动抛出HTTP错误，提前发现问题
        
        r = response.json()
        
        #Add r to list
        all_ids.extend(r['data'])

        #End condition
        if r["limit"] + r["skip"] >= r["total"]:
            break
        
        #Set next request
        params["$skip"] = r["limit"] + r["skip"]
        print(counter)
        print(r)
        counter = counter+1

    except Exception as e:
        print('Error:', e)
        if 'response' in locals():
            print(response.status_code)
            print(response.text)  # 打印响应内容，方便排查具体原因
        break

#Info to df -> csv
df = pd.json_normalize(all_ids)
df.to_csv('all_perfect.csv',sep=';',encoding='utf-8',decimal=',')

额外说明 #

• 修复了counter未初始化的问题，避免运行报错。
• 修改了Accept-Encoding为标准格式，requests会自动处理编码解析，无需指定utf-8。
• 添加了response.raise_for_status()，可以更早发现HTTP请求错误；异常块中新增打印响应内容，方便排查具体问题。

内容的提问来源于stack exchange，提问作者hm2330