使用Python访问网站遭遇HTTP 500内部服务器错误求助

我需要用Python访问指定网站的HTML并提取法国邮政编码，浏览器能正常访问该网站，但通过Python代码请求时却持续出现HTTP 500内部服务器错误。请问是网站拦截了请求还是我的程序存在问题？

我的代码 #

import requests
import urllib
import re

link = "https://annuaire.118712.fr/magasin/mcdonald-s_1"
f = urllib.request.urlopen(link)
myfile = f.read()
for result in re.findall(r"postalCode\": \"(\d{5})", str(myfile)):
    print(result)

错误信息 #

---------------------------------------------------------------------------
HTTPError                                 Traceback (most recent call last)
<ipython-input-2-76fbd165889e> in <module>
      5 
      6 link = "https://annuaire.118712.fr/magasin/mcdonald-s_1"
----> 7 f = urllib.request.urlopen(link)
      8 myfile = f.read()
      9 for result in re.findall(r"postalCode\": \"(\d{5})", str(myfile)):

~\Anaconda3\lib\urllib\request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    220     else:
    221         opener = _opener
---> 222     return opener.open(url, data, timeout)
    223 
    224 def install_opener(opener):

~\Anaconda3\lib\urllib\request.py in open(self, fullurl, data, timeout)
    529         for processor in self.process_response.get(protocol, []):
    530             meth = getattr(processor, meth_name)
---> 531             response = meth(req, response)
    532 
    533         return response

~\Anaconda3\lib\urllib\request.py in http_response(self, request, response)
    638         # request was successfully received, understood, and accepted.
    639         if not (200 <= code < 300):
---> 640             response = self.parent.error(
    641                 'http', request, response, code, msg, hdrs)
    642 

~\Anaconda3\lib\urllib\request.py in error(self, proto, *args)
    561             http_err = 0
    562         args = (dict, proto, meth_name) + args
---> 563         result = self._call_chain(*args)
    564         if result:
    565             return result

~\Anaconda3\lib\urllib\request.py in _call_chain(self, chain, kind, meth_name, *args)
    500         for handler in handlers:
    501             func = getattr(handler, meth_name)
---> 502             result = func(*args)
    503             if result is not None:
    504                 return result

~\Anaconda3\lib\urllib\request.py in http_error_302(self, req, fp, code, msg, headers)
    753         fp.close()
    754 
---> 755         return self.parent.open(new, timeout=req.timeout)
    756 
    757     http_error_301 = http_error_303 = http_error_307 = http_error_302

~\Anaconda3\lib\urllib\request.py in open(self, fullurl, data, timeout)
    529         for processor in self.process_response.get(protocol, []):
    530             meth = getattr(processor, meth_name)
---> 531             response = meth(req, response)
    532 
    533         return response

~\Anaconda3\lib\urllib\request.py in http_response(self, request, response)
    638         # request was successfully received, understood, and accepted.
    639         if not (200 <= code < 300):
---> 640             response = self.parent.error(
    641                 'http', request, response, code, msg, hdrs)
    642 

~\Anaconda3\lib\urllib\request.py in error(self, proto, *args)
    567         if http_err:
    568             args = (dict, 'default', 'http_error_default') + orig_args
---> 569             return self._call_chain(*args)
    570 
    571 # XXX probably also want an abstract factory that knows when it makes

~\Anaconda3\lib\urllib\request.py in _call_chain(self, chain, kind, meth_name, *args)
    500         for handler in handlers:
    501             func = getattr(handler, meth_name)
---> 502             result = func(*args)
    503             if result is not None:
    504                 return result

~\Anaconda3\lib\urllib\request.py in http_error_default(self, req, fp, code, msg, hdrs)
    647 class HTTPDefaultErrorHandler(BaseHandler):
    648     def http_error_default(self, req, fp, code, msg, hdrs):
---> 649         raise HTTPError(req.full_url, code, msg, hdrs, fp)
    650 
    651 class HTTPRedirectHandler(BaseHandler):

HTTPError: HTTP Error 500: Internal Server Error

原因与解决方案 #

HTTP 500是服务器内部错误，但浏览器能正常访问，说明你的请求与浏览器请求存在差异，核心问题是请求头缺失关键字段，导致服务器无法正确处理请求，抛出内部错误。

修复步骤 #

给请求添加模拟浏览器的请求头，让服务器识别为正常访问，同时可以用你已导入的requests库简化请求流程：

修改后的代码：

import requests
import re

link = "https://annuaire.118712.fr/magasin/mcdonald-s_1"
# 模拟浏览器请求头
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
}

response = requests.get(link, headers=headers)
response.raise_for_status()  # 检查请求是否成功

# 提取邮政编码
for result in re.findall(r'postalCode": "(\d{5})', response.text):
    print(result)

说明 #

• User-Agent字段告诉服务器你是正常浏览器，避免被识别为非人工请求；
• Accept字段指定客户端能接受的内容类型，匹配浏览器的默认设置；
• requests.get比urllib更简洁，默认处理重定向等问题更友好。

内容的提问来源于stack exchange，提问作者user19725009