Biopython Entrez模块EFetch调用抛出HTTP 400错误：Bad Request

问题背景 #

使用Biopython的Entrez模块时，ESearch能正常获取PubMed ID列表，但遍历调用EFetch提取文献属性时频繁触发HTTP Error 400: Bad Request，即使添加5秒延时也无法解决。

原代码 #

Entrez.email = "myemail@example.com"
Entrez.api_key = "api_key_generated_from_NCBI"

def fetch_pub_data(pmid):
    # Fetch PubMed record in XML format
    handle = Entrez.efetch(db="pubmed", id=pmid, retmode="xml", api_key="api_key_generated_from_NCBI")
    record = Entrez.read(handle, validate=False)
    handle.close()
    try:
        # Extract publication date
        pub_date = record['PubmedArticle'][0]['MedlineCitation']['Article']['Journal']['JournalIssue']['PubDate']
    except (KeyError, IndexError):
        publication_date = "No Publication date available"
    try:
        abstract_text = record['PubmedArticle'][0]['MedlineCitation']['Article']['Abstract']['AbstractText']
    except (KeyError, IndexError):
        abstract_text = "No abstract available"
    try:
        title = record['PubmedArticle'][0]['MedlineCitation']['Article']['ArticleTitle']
    except (KeyError, IndexError):
        title = "No Title available"
    return publication_date, abstract_text, title

def search_pubmed(query, retmax=10):
    handle = Entrez.esearch(db="pubmed", term=query, sort='relevance', retmax=retmax, api_key="api_key_generated_from_NCBI")
    search_results = Entrez.read(handle)
    pubmed_ids = search_results["IdList"]

    # Create an empty DataFrame to store the results
    df = pd.DataFrame(columns=['PubMed_ID', 'Publication_Date', 'Title', 'Abstract'])

    # Fetch and store information for the retrieved PubMed IDs in the DataFrame
    for pmid in pubmed_ids:
        pub_date, abstract, title = fetch_pub_data(pmid)
        new_row_data = {'PubMed_ID': pmid, 'Publication_Date': pub_date, 'Title': title, 'Abstract': abstract}
        num_rows = len(df)
        df.loc[num_rows] = new_row_data
        time.sleep(5)
    return df

# search query
search_term = ('((2024/1/1:2024/1/31[pdat]))')

# Search PubMed and fetch PubMed IDs with their respective publication dates and abstracts
results= search_pubmed(search_term, retmax=10) 
print(results)

错误信息 #

<ipython-input-1-f55c5b745c60> in fetch_pub_data(pmid)
---&gt; 12     handle = Entrez.efetch(db="pubmed", id=pmid, retmode="xml", api_key="#################")
     13     record = Entrez.read(handle, validate=False)

~/.local/lib/python3.8/site-packages/Bio/Entrez/__init__.py in efetch(db, **keywords)
    194     variables.update(keywords)
    195     request = _build_request(cgi, variables)
--&gt; 196     return _open(request)
    197 
    198 

~/.local/lib/python3.8/site-packages/Bio/Entrez/__init__.py in _open(request)
    592     for i in range(max_tries):
    593         try:
--&gt; 594             handle = urlopen(request)
    595         except HTTPError as exception:
    596             # Reraise if the final try fails

/mmm/jupyterhub/anaconda3/lib/python3.8/urllib/request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    220     else:
    221         opener = _opener
--&gt; 222     return opener.open(url, data, timeout)
    223 
    224 def install_opener(opener):

/mmm/jupyterhub/anaconda3/lib/python3.8/urllib/request.py in open(self, fullurl, data, timeout)
    529         for processor in self.process_response.get(protocol, []):
    530             meth = getattr(processor, meth_name)
--&gt; 531             response = meth(req, response)
    532 
    533         return response

/mmm/jupyterhub/anaconda3/lib/python3.8/urllib/request.py in http_response(self, request, response)
    638         # request was successfully received, understood, and accepted.
    639         if not (200 &lt;= code &lt; 300):
--&gt; 640             response = self.parent.error(
    641                 'http', request, response, code, msg, hdrs)
    642 

/mmm/jupyterhub/anaconda3/lib/python3.8/urllib/request.py in error(self, proto, *args)
    567         if http_err:
    568             args = (dict, 'default', 'http_error_default') + orig_args
--&gt; 569             return self._call_chain(*args)
    570 
    571 # XXX probably also want an abstract factory that knows when it makes

/mmm/jupyterhub/anaconda3/lib/python3.8/urllib/request.py in _call_chain(self, chain, kind, meth_name, *args)
    500         for handler in handlers:
    501             func = getattr(handler, meth_name)
--&gt; 502             result = func(*args)
    503             if result is not None:
    504                 return result

/mmm/jupyterhub/anaconda3/lib/python3.8/urllib/request.py in http_error_default(self, req, fp, code, msg, hdrs)
    647 class HTTPDefaultErrorHandler(BaseHandler):
    648     def http_error_default(self, req, fp, code, msg, hdrs):
--&gt; 649         raise HTTPError(req.full_url, code, msg, hdrs, fp)
    650 
    651 class HTTPRedirectHandler(BaseHandler):

HTTPError: HTTP Error 400: Bad Request

修复方案 #

核心问题分析 #

1. 重复传入API Key：全局已设置Entrez.api_key，每次调用EFetch/ESearch时重复传参可能导致参数解析冲突，触发400错误。
2. 单条请求效率低：逐个请求PMID违反NCBI API最佳实践，容易触发限流或请求格式错误。
3. 变量名不匹配：fetch_pub_data中捕获异常时赋值publication_date，但返回未定义变量，导致逻辑错误。
4. 默认重试机制不足：Biopython默认重试次数少，遇到临时网络波动或API限流时无法自动恢复。

修改后的代码 #

import time
import pandas as pd
from Bio import Entrez

# 全局配置，只需设置一次
Entrez.email = "myemail@example.com"
Entrez.api_key = "api_key_generated_from_NCBI"
# 增加重试次数与间隔，应对临时错误
Entrez.max_tries = 5
Entrez.sleep_between_tries = 3

def fetch_pub_data(pmid_list):
    # 批量请求多个PMID，减少请求次数
    handle = Entrez.efetch(db="pubmed", id=",".join(pmid_list), retmode="xml")
    records = Entrez.read(handle, validate=False)
    handle.close()
    
    results = []
    for record in records['PubmedArticle']:
        # 提取并格式化发表日期
        try:
            pub_date = record['MedlineCitation']['Article']['Journal']['JournalIssue']['PubDate']
            pub_date_str = f"{pub_date.get('Year', '')} {pub_date.get('Month', '')} {pub_date.get('Day', '')}".strip()
            pub_date_str = pub_date_str if pub_date_str else "No Publication date available"
        except (KeyError, IndexError):
            pub_date_str = "No Publication date available"
        
        # 提取并合并多段摘要
        try:
            abstract_text = record['MedlineCitation']['Article']['Abstract']['AbstractText']
            abstract_str = " ".join([str(part) for part in abstract_text]) if isinstance(abstract_text, list) else str(abstract_text)
        except (KeyError, IndexError):
            abstract_str = "No abstract available"
        
        # 提取标题
        try:
            title = record['MedlineCitation']['Article']['ArticleTitle']
        except (KeyError, IndexError):
            title = "No Title available"
        
        # 获取PMID
        pmid = record['MedlineCitation']['PMID']
        results.append({
            'PubMed_ID': pmid,
            'Publication_Date': pub_date_str,
            'Title': title,
            'Abstract': abstract_str
        })
    return results

def search_pubmed(query, retmax=10):
    handle = Entrez.esearch(db="pubmed", term=query, sort='relevance', retmax=retmax)
    search_results = Entrez.read(handle)
    pubmed_ids = search_results["IdList"]
    
    # 批量获取数据并生成DataFrame
    data = fetch_pub_data(pubmed_ids)
    df = pd.DataFrame(data)
    return df

# 搜索查询
search_term = '2024/1/1:2024/1/31[pdat]'

# 执行搜索并获取结果
results = search_pubmed(search_term, retmax=10)
print(results)

关键优化点 #

• 移除所有EFetch/ESearch调用中的重复api_key参数，使用全局配置。
• 改为批量EFetch请求，将多个PMID用逗号拼接成一个请求，符合NCBI API规范，减少请求次数。
• 修复变量名不匹配问题，确保返回值正确。
• 增加全局重试配置，提升临时错误的容错能力。
• 优化日期和摘要的格式化，提升结果可读性。
• 直接用列表生成DataFrame，替代逐行添加的低效操作。

内容的提问来源于stack exchange，提问作者Hari