Python爬虫写入CSV遭覆盖仅存2页数据，寻求解决办法

核心问题分析 #

你的代码里，每页循环都用'w'模式打开product.csv，这个模式会清空文件原有内容再写入，所以爬完第2页时，第1页的数据会被完全覆盖，最终只保留最后一页的内容。同时每次都写入表头，也会导致表头重复出现。

方案1：使用追加模式写入，仅在第一次写入表头 #

把文件初始化（写表头）移到循环外，后续每页用追加模式写入数据：

from selenium import webdriver
import time
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.wait import WebDriverWait
from csv import writer

options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-gpu")
options.add_argument("--window-size=1920x1080")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
wait = WebDriverWait(driver, 20) 

# 先创建文件并写入表头，仅执行一次
with open('product.csv', 'w', newline='', encoding='utf-8') as csvfile:
    thewriter = writer(csvfile)
    header = ['name','url','website_link','company','rating']
    thewriter.writerow(header)

for page in range(1,3):             
    URL = f'https://www.askgamblers.com/online-casinos/countries/ca/{page}'
    driver.get(URL)
    time.sleep(2)

    urls= []
    page_links = driver.find_elements(By.XPATH, "//div[@class='card__desc']//a[starts-with(@href, '/online')]")
    for link in page_links:
        href = link.get_attribute("href")
        urls.append(href)    
    
    # 用追加模式打开文件，写入当前页数据
    with open('product.csv', 'a', newline='', encoding='utf-8') as csvfile:
        thewriter = writer(csvfile)
        
        for url in urls:
            driver.get(url)
            time.sleep(1)
            
            # 给字段设置默认值，避免异常后变量未定义
            name = ''
            company = ''
            link = ''
            rate = ''
            
            try:
                name = driver.find_element(By.CSS_SELECTOR,"h1.review-intro__title").text   
            except:
                pass
            
            try:
                company = driver.find_element(By.XPATH,"//p[span[contains(.,'Company')]]/following-sibling::div").text   
            except:
                pass
            try:
                link = driver.find_element(By.XPATH,"//p[span[contains(.,'Website')]]/following-sibling::div").text   
            except:
                pass
            
            try:
                rate = driver.find_element(By.CSS_SELECTOR,"span.rating-ring__number").text
            except:
                pass
            
            jobinfo = [name, url, link, company, rate]
            thewriter.writerow(jobinfo)

driver.quit()

方案2：先收集所有数据，最后一次性写入CSV #

这种方式减少文件IO操作，更高效，也从根源避免覆盖问题：

from selenium import webdriver
import time
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.wait import WebDriverWait
from csv import writer

options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-gpu")
options.add_argument("--window-size=1920x1080")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
wait = WebDriverWait(driver, 20) 

# 初始化存储所有数据的列表，先加入表头
all_data = [['name','url','website_link','company','rating']]

for page in range(1,3):             
    URL = f'https://www.askgamblers.com/online-casinos/countries/ca/{page}'
    driver.get(URL)
    time.sleep(2)

    urls= []
    page_links = driver.find_elements(By.XPATH, "//div[@class='card__desc']//a[starts-with(@href, '/online')]")
    for link in page_links:
        href = link.get_attribute("href")
        urls.append(href)    
    
    for url in urls:
        driver.get(url)
        time.sleep(1)
        
        name = ''
        company = ''
        link = ''
        rate = ''
        
        try:
            name = driver.find_element(By.CSS_SELECTOR,"h1.review-intro__title").text   
        except:
            pass
        
        try:
            company = driver.find_element(By.XPATH,"//p[span[contains(.,'Company')]]/following-sibling::div").text   
        except:
            pass
        try:
            link = driver.find_element(By.XPATH,"//p[span[contains(.,'Website')]]/following-sibling::div").text   
        except:
            pass
        
        try:
            rate = driver.find_element(By.CSS_SELECTOR,"span.rating-ring__number").text
        except:
            pass
        
        all_data.append([name, url, link, company, rate])

# 所有数据收集完成后，一次性写入文件
with open('product.csv', 'w', newline='', encoding='utf-8') as csvfile:
    thewriter = writer(csvfile)
    thewriter.writerows(all_data)

driver.quit()

额外优化说明 #

• 移除了重复导入的模块（如两次导入Service和By）
• 给每个字段设置默认空字符串，避免异常后变量未定义导致报错
• 添加driver.quit()，避免浏览器进程残留
• 用f-string替代format()，代码更简洁

内容的提问来源于stack exchange，提问作者Amen Aziz