Python多线程爬虫向列表追加数据失败，仅单个列表正常

核心问题排查 #

• 未定义线程锁lock：代码中使用with lock但未初始化锁对象，触发NameError导致锁块内的数据追加逻辑完全不执行，这是大部分列表为空的直接原因。
• 全局列表未初始化：names、price等存储用列表未提前定义为空列表，多线程环境下会引发变量未定义错误，导致数据无法正常追加。
• 变量名拼写错误：存储条件判断中，循环变量为things，但赋值时写成stor = thing.text，触发NameError导致storage列表无法写入数据。
• 元素查找的空值处理错误：直接调用soup.find(...).text != None的写法错误，若find返回None，调用.text会直接抛出异常，中断价格等字段的追加逻辑。
• 未捕获多线程任务异常：使用executor.submit提交任务时未捕获异常，单个任务出错会默默失败，导致对应数据缺失。

修正后的代码 #

import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from bs4 import BeautifulSoup
import pandas as pd

# 初始化全局存储列表与线程锁
names = []
price = []
pack_type = []
packaging = []
salt = []
storage = []
manufacturer = []
image_list = []
lock = threading.Lock()

def init_driver():
    PROXY = 'http://p.webshare.io:9999'
    chrome_options = Options()
    chrome_options.add_argument('log-level=3')
    chrome_options.add_argument('--proxy-server=%s' % PROXY)
    chrome_options.add_argument("--headless")
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("--no-sandbox")
    driver_service = Service(executable_path='chromedriver.exe')
    driver = webdriver.Chrome(service=driver_service, options=chrome_options)
    return driver


def Get_info(url2):
    driver = None
    try:
        driver = init_driver()
        driver.get(url2)
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        
        # 提取药品名称
        title = soup.find('h1', class_='DrugHeader__title-content___2ZaPo').text
        
        # 提取图片链接并处理
        images = soup.find_all('img', alt=title)
        pop = []
        for image in images:
            blah = image['src']
            final = blah.replace('l_watermark_346,w_480,h_480/a_ignore,w_480,h_480,c_fit,q_auto,f_auto/', '')
            pop.append(final)
        img_str = ', '.join(pop)
        
        # 提取存储条件
        stor = 'N/A'
        store_elements = soup.find_all(class_='saltInfo DrugHeader__meta-value___vqYM0')
        for thing in store_elements:
            if 'store' in thing.text.lower():
                stor = thing.text.strip()
                break
        
        # 提取厂商、成分
        meta_values = soup.find_all(class_='DrugHeader__meta-value___vqYM0')
        manu = meta_values[0].text.strip() if len(meta_values) >=1 else 'N/A'
        salt_comp = meta_values[1].text.strip() if len(meta_values) >=2 else 'N/A'
        
        # 提取价格（安全判断元素是否存在）
        price_val = 'N/A'
        price_elem = soup.find(class_='DrugPriceBox__best-price___32JXw')
        if price_elem:
            price_val = price_elem.text.strip()
        else:
            price_elem = soup.find(class_='PriceBoxPlanOption__offer-price___3v9x8 PriceBoxPlanOption__offer-price-cp___2QPU_')
            if price_elem:
                price_val = price_elem.text.strip()
            else:
                price_elem = soup.find(class_='DrugPriceBox__price___dj2lv')
                if price_elem:
                    price_val = price_elem.text.strip()
        
        # 提取包装信息
        pack_val = 'N/A'
        pack_elem = soup.find(class_='DrugPriceBox__quantity___2LGBX')
        if pack_elem:
            pack_val = pack_elem.text.strip()
        
        # 判断包装类型
        pack_type_val = 'N/A'
        pack_lower = pack_val.lower()
        if 'vial' in pack_lower:
            pack_type_val = 'vial'
        elif 'strip' in pack_lower:
            pack_type_val = 'strip'
        elif 'bottle' in pack_lower:
            pack_type_val = 'bottle'
        elif 'tube' in pack_lower:
            pack_type_val = 'tube'
        elif 'packet' in pack_lower:
            pack_type_val = 'packet'
        elif 'box' in pack_lower:
            pack_type_val = 'box'
        elif 'cartridge' in pack_lower:
            pack_type_val = 'cartridge'
        elif 'ampoule' in pack_lower:
            pack_type_val = 'ampoule'
        elif 'syringe' in pack_lower:
            pack_type_val = 'syringe'
        
        # 线程安全追加数据
        with lock:
            names.append(title)
            image_list.append(img_str)
            storage.append(stor)
            manufacturer.append(manu)
            salt.append(salt_comp)
            price.append(price_val)
            pack_type.append(pack_type_val)
            packaging.append(pack_val)
            
    except Exception as e:
        print(f"处理URL {url2} 时出错: {str(e)}")
    finally:
        if driver:
            driver.quit()


# 执行多线程任务
if __name__ == "__main__":
    url_list = []  # 替换为你的实际URL列表
    
    print('Start!')
    print(f"待处理URL数量: {len(url_list)}")
    
    with ThreadPoolExecutor(max_workers=10) as executor:
        futures = [executor.submit(Get_info, url) for url in url_list]
        for future in as_completed(futures):
            try:
                future.result()
            except Exception as e:
                print(f"任务执行异常: {str(e)}")
    
    # 验证数据长度
    print(f'Drug Name数量: {len(names)}')
    print(f'Price数量: {len(price)}')
    print(f'Packaging数量: {len(packaging)}')
    print(f'Composition数量: {len(salt)}')
    print(f'Storage数量: {len(storage)}')
    print(f'Manufacturer数量: {len(manufacturer)}')
    print(f'Images数量: {len(image_list)}')
    
    # 导出到Excel
    listing_dict = {
        'Drug Name': names,
        'Price': price,
        'Packaging type': pack_type,
        'Packaging': packaging,
        'Composition': salt,
        'Storage': storage,
        'Manufacturer': manufacturer,
        'Images': image_list
    }
    
    print("______________________Job Finished!______________________")
    df = pd.DataFrame(listing_dict)
    df.to_excel("Test.xlsx", index=False)

关键优化说明 #

• 新增线程锁初始化，确保多线程下数据追加的安全性
• 提前定义所有存储用空列表，避免变量未定义错误
• 修复变量名拼写错误，统一循环变量命名
• 优化元素查找逻辑，先判断元素存在再提取文本，避免空值异常
• 添加全局异常捕获，方便排查单个URL的处理错误
• 降低线程并发数至10，避免触发网站反爬或本地资源耗尽
• 为所有字段设置默认值（如'N/A'），保证数据完整性

内容的提问来源于stack exchange，提问作者Tzeboys