配合selenium使用Python requests下载PDF得到无后缀空文件

• 文件名未添加.pdf后缀，导致Mac系统无法识别文件类型，默认标记为通用Document文件
• requests请求未携带Selenium会话中的Cookies、用户代理等身份凭证，网站校验身份不通过，返回空内容或非PDF内容，即使response.ok返回True也不代表拿到了真实PDF数据
• 代码逻辑存在多处错误：
  • 需要登录的分支下，未完成登录操作就提前发起PDF请求，拿到的是登录前的无效页面内容
  • for循环内定义的title_article变量未传入check_need_to_sign_in函数，存在变量作用域问题，文件名读取错误
  • 计数器写法错误，i = + 1实际等价于i = 1，规范写法为i += 1

import time
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import requests as req

# 全局初始化requests会话，同步Selenium的Cookies
req_session = req.Session()
# 禁用SSL告警
req_session.verify = False

issues_numb = driver.find_elements(By.XPATH, "//section[@class='article-list-item-content-block ']")
parent_tab = driver.current_window_handle
timeout = 30

def check_need_to_sign_in(file_name):
    new_tab = driver.window_handles
    driver.switch_to.window(str(new_tab[-1]))
    i = 0
    try:
        WebDriverWait(driver, 20).until(EC.element_to_be_clickable(
            (By.XPATH, "//input[@class='form-control ltr_override input ext-input text-box ext-text-box']")))
        print("Sign-in necessary")
        sign_in()
        # 登录完成后再获取最终URL、同步Cookies
        url = driver.current_url
        # 同步Selenium的Cookies到requests会话
        for cookie in driver.get_cookies():
            req_session.cookies.set(cookie['name'], cookie['value'])
        # 携带身份信息发起请求
        response = req_session.get(url, headers={"User-Agent": driver.execute_script("return navigator.userAgent")})
        
        while i < 1:
            print(response.ok, len(response.content))
            if response.ok and len(response.content) > 100: # 加长度校验避免空内容
                # 文件名加.pdf后缀
                with open(f"{file_name}.pdf", 'wb') as f:
                    f.write(response.content)
                i += 1
    except TimeoutException:
        print("No need to sign-in")
        time.sleep(5) # 等待重定向完成
        url = driver.current_url
        # 同步Cookies和UA
        for cookie in driver.get_cookies():
            req_session.cookies.set(cookie['name'], cookie['value'])
        response = req_session.get(url, headers={"User-Agent": driver.execute_script("return navigator.userAgent")})
        while i < 1:
            print(response.ok, len(response.content))
            if response.ok and len(response.content) > 100:
                with open(f"{file_name}.pdf", 'wb') as f:
                    f.write(response.content)
                i += 1
    # 处理完成关闭当前PDF标签页
    driver.close()
    driver.switch_to.window(parent_tab)

# 主循环
for elem in issues_numb:
    title_article = elem.get_attribute("aria-label")
    file_name = title_article[13:]
    print(file_name)
    try:
        pdf_icon = elem.find_element(By.XPATH,".//span[@class='icon fal fa-file-pdf']")
        print("pdf object found for", str(elem))
        ActionChains(driver).move_to_element(pdf_icon).click(pdf_icon).perform()
        WebDriverWait(driver, timeout).until(lambda d: len(d.window_handles) > len([parent_tab]))
        # 把文件名传入处理函数
        check_need_to_sign_in(file_name)
    except Exception as e:
        print(f"处理{file_name}出错：{e}")
        # 出错后切回主标签页避免后续流程异常
        for handle in driver.window_handles:
            if handle != parent_tab:
                driver.switch_to.window(handle)
                driver.close()
        driver.switch_to.window(parent_tab)

运行代码后查看控制台打印的len(response.content)数值，正常PDF文件大小至少为几十KB（即数值大于10240），如果数值过小说明请求到的仍不是真实PDF内容，可打印response.text查看返回的具体内容排查网站反爬规则。

内容的提问来源于stack exchange，提问作者double_wizz