为何requests.get在Databricks中运行超时，本地却正常？

我尝试抓取美国运输统计局网站内容，本地VSCode中编写的代码可正常获取数据并通过Beautiful Soup解析，但将代码迁移至Databricks notebook作为数据流水线一部分时，持续出现TimeoutError。

本地正常运行的代码 #

# 导入必要库
import requests
import pandas as pd
from bs4 import BeautifulSoup
import ssl

ssl._create_default_https_context = ssl._create_unverified_context

# 下载汇总网页内容
base_url = 'https://www.transtats.bts.gov/HomeDrillChart.asp'
years_data = requests.get(base_url, verify=False).text

Databricks中报错信息 #

ConnectionError: HTTPSConnectionPool(host='www.transtats.bts.gov', port=443): Max retries exceeded with url: /HomeDrillChart.asp (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fce182a0d90>: Failed to establish a new connection: [Errno 110] Connection timed out'))

尝试的无效解决方案 #

我尝试使用公开代理修改网络设置，但仍出现相同的TimeoutError：

base_url = 'https://www.transtats.bts.gov/HomeDrillChart.asp'
proxies = {
    'https': 'https://75.72.55.108:8118',
    'http': 'http://198.49.68.80:80'
}
try:
    r = requests.get(url=base_url, verify=False, proxies=proxies)
    print(r.status_code)
    print(r.content)
except requests.exceptions.RequestException as e:
    print(e)

此外，我还尝试传递包含User-Agent和Cookie信息的自定义请求头（包括移除Cookie过期时间），但均无效果。

完整错误栈 #

---------------------------------------------------------------------------
TimeoutError                              Traceback (most recent call last)
/databricks/python/lib/python3.8/site-packages/urllib3/connection.py in _new_conn(self)
    158         try:
---> 159             conn = connection.create_connection(
    160                 (self._dns_host, self.port), self.timeout, **extra_kw

/databricks/python/lib/python3.8/site-packages/urllib3/util/connection.py in create_connection(address, timeout, source_address, socket_options)
     83     if err is not None:
---> 84         raise err
     85 

/databricks/python/lib/python3.8/site-packages/urllib3/util/connection.py in create_connection(address, timeout, source_address, socket_options)
     73                 sock.bind(source_address)
---> 74             sock.connect(sa)
     75             return sock

TimeoutError: [Errno 110] Connection timed out

During handling of the above exception, another exception occurred:

NewConnectionError                        Traceback (most recent call last)
/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    669             # Make the request on the httplib connection object.
---> 670             httplib_response = self._make_request(
    671                 conn,

/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    380         try:
---> 381             self._validate_conn(conn)
    382         except (SocketTimeout, BaseSSLError) as e:

/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py in _validate_conn(self, conn)
    977         if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`
---> 978             conn.connect()
    979 

/databricks/python/lib/python3.8/site-packages/urllib3/connection.py in connect(self)
    308         # Add certificate verification
---> 309         conn = self._new_conn()
    310         hostname = self.host

/databricks/python/lib/python3.8/site-packages/urllib3/connection.py in _new_conn(self)
    170         except SocketError as e:
---> 171             raise NewConnectionError(
    172                 self, "Failed to establish a new connection: %s" % e

NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7fce182a0d90>: Failed to establish a new connection: [Errno 110] Connection timed out

During handling of the above exception, another exception occurred:

MaxRetryError                             Traceback (most recent call last)
/databricks/python/lib/python3.8/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    438             if not chunked:
---> 439                 resp = conn.urlopen(
    440                     method=request.method,

/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    725 
---> 726             retries = retries.increment(
    727                 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]

/databricks/python/lib/python3.8/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
    445         if new_retry.is_exhausted():
---> 446             raise MaxRetryError(_pool, url, error or ResponseError(cause))
    447 

MaxRetryError: HTTPSConnectionPool(host='www.transtats.bts.gov', port=443): Max retries exceeded with url: /HomeDrillChart.asp (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fce182a0d90>: Failed to establish a new connection: [Errno 110] Connection timed out'))

During handling of the above exception, another exception occurred:

ConnectionError                           Traceback (most recent call last)
<command-4350135934483206> in <module>
      1 # Downloading contents of the summary web page
      2 base_url = 'https://www.transtats.bts.gov/HomeDrillChart.asp'
----> 3 years_data = requests.get(base_url, verify=False).text
      4 
      5 # Creating BeautifulSoup object, getting a dict of years and links to each year page

/databricks/python/lib/python3.8/site-packages/requests/api.py in get(url, params, **kwargs)
     74 
     75     kwargs.setdefault('allow_redirects', True)
---> 76     return request('get', url, params=params, **kwargs)
     77 
     78 

/databricks/python/lib/python3.8/site-packages/requests/api.py in request(method, url, **kwargs)
     59     # cases, and look like a memory leak in others.
     60     with sessions.Session() as session:
---> 61         return session.request(method=method, url=url, **kwargs)
     62 
     63 

/databricks/python/lib/python3.8/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    540         }
    541         send_kwargs.update(settings)
---> 542         resp = self.send(prep, **send_kwargs)
    543 
    544         return resp

/databricks/python/lib/python3.8/site-packages/requests/sessions.py in send(self, request, **kwargs)
    653 
    654         # Send the request
---> 655         r = adapter.send(request, **kwargs)
    656 
    657         # Total elapsed time of the request (approximately)

/databricks/python/lib/python3.8/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    514                 raise SSLError(e, request=request)
    515 
---> 516             raise ConnectionError(e, request=request)
    517 
    518         except ClosedPoolError as e:

ConnectionError: HTTPSConnectionPool(host='www.transtats.bts.gov', port=443): Max retries exceeded with url: /HomeDrillChart.asp (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fce182a0d90>: Failed to establish a new connection: [Errno 110] Connection timed out'))

内容的提问来源于stack exchange，提问作者Helaman Sanchez