Python开发Naver新闻多关键词爬虫仅返回最后一个结果如何解决

• 可变参数共享冲突：你定义的params是全局字典对象，Python中字典属于可变对象，多线程同时修改/读取同一个字典时会出现值覆盖问题，多关键词场景下后提交的任务会把前一个任务设置的query等参数覆盖，最终所有请求都带上最后一个关键词的参数。
• 请求参数被意外销毁：你处理下一页时直接把params设为None，且没有在每个关键词/每个日期区间的循环开头重置初始的link和params，导致第一个关键词第一次翻页后，后续所有请求都没有正确的查询参数，自然只能拿到最后一个处理成功的关键词结果。

import requests
import concurrent.futures
from bs4 import BeautifulSoup
from urllib.parse import urljoin

year_list_start = ['2013.01.01','2014.01.02']
year_list_upto = ['2014.01.01','2015.01.01']
base = 'https://search.naver.com/search.naver'

def fetch_content(s, keyword):
    # 每个关键词独立使用初始链接
    base_link = 'https://search.naver.com/search.naver'
    for start_date, date_upto in zip(year_list_start, year_list_upto):
        # 每个日期区间生成独立参数，不共用全局对象
        current_params = {
            'where': 'news',
            'sm': 'tab_pge',
            'query': keyword,
            'sort': '1',
            'photo': '0',
            'field': '0',
            'pd': '',
            'ds': '',
            'de': '',
            'cluster_rank': '',
            'mynews': '0',
            'office_type': '0',
            'office_section_code': '0',
            'news_office_checked': '',
            'nso': '',
            'start': '',
        }
        current_link = base_link
        ds = start_date.replace(".", "")
        de = date_upto.replace(".", "")
        current_params['ds'] = ds
        current_params['de'] = de
        current_params['nso'] = f'so:r,p:from{ds}to{de},a:all'
        current_params['start'] = 1

        while True:
            res = s.get(current_link, params=current_params)
            print(f"当前关键词：{keyword}，响应状态：{res.status_code}")
            print(res.url)
            soup = BeautifulSoup(res.text, "lxml")
            if not soup.select_one("ul.list_news .news_area .info_group > a.press"):
                break
            for item in soup.select("ul.list_news .news_area"):
                newspaper_name = item.select_one(".info_group > a.press").get_text(strip=True).lstrip("=")
                # 输出带关键词前缀，方便区分不同关键词的结果
                print(f"[{keyword}] {newspaper_name}")

            if soup.select_one("a.btn_next[aria-disabled='true']"):
                break
            next_page = soup.select_one("a.btn_next").get("href")
            current_link = urljoin(base, next_page)
            # 下一页地址已经带全参数，不需要额外传params
            current_params = None


if __name__ == '__main__':
    with requests.Session() as s:
        s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'
        
        # 替换为你需要的多关键词列表
        keywords = ['CMG제약','DB하이텍','ES큐브','EV첨단소재']

        with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
            future_to_keyword = {executor.submit(fetch_content, s, keyword): keyword for keyword in keywords}
            # 遍历执行结果，捕获异常避免程序提前退出
            for future in concurrent.futures.as_completed(future_to_keyword):
                kw = future_to_keyword[future]
                try:
                    future.result()
                except Exception as e:
                    print(f"关键词{kw}处理出错：{str(e)}")

内容的提问来源于stack exchange，提问作者SMTH