在AWS Lambda中使用Selenium/BeautifulSoup获取数据结果与VS Code不一致

本地VS Code中运行Selenium+BeautifulSoup代码访问http://www.cell2gps.com/，点击g-recaptcha按钮后能成功获取目标节点body > center > center > script:nth-child(12)的内容；但将代码部署到AWS Lambda后，点击该按钮后获取的目标内容为None，期望Lambda环境下能得到和本地一致的结果。

本地（VS Code）代码 #

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
from bs4 import BeautifulSoup


options = webdriver.ChromeOptions()
options.add_argument("headless")
options.add_argument('--no-sandbox')
options.add_argument('--single-process')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--blink-settings=imagesEnabled=false')
driver = webdriver.Chrome('[path]/chromedriver.exe',options=options)
#driver = webdriver.Chrome('path',options=options)

driver.implicitly_wait(0.5)
driver.get('http://www.cell2gps.com/')

html=driver.page_source
soup=BeautifulSoup(html,'html.parser')
t_list2=soup.select_one('body > center > center > script:nth-child(12)')
data=str(t_list2)
print(data)
print('--------------------------')

#driver.find_element_by_name('mcc').clear()
#driver.find_element_by_name('mcc').send_keys('@@')
#driver.find_element_by_name('mnc').clear()
#driver.find_element_by_name('mnc').send_keys('@@')
#driver.find_element_by_name('lac').clear()
#driver.find_element_by_name('lac').send_keys('@@')
#driver.find_element_by_name('cellid').clear()
#driver.find_element_by_name('cellid').send_keys('@@')
driver.find_element_by_class_name('g-recaptcha').click()
time.sleep(10)

html=driver.page_source
soup=BeautifulSoup(html,'html.parser')
t_list2=soup.select_one('body > center > center > script:nth-child(12)')
data=str(t_list2)
print(data)

driver.close()
driver.quit()

本地运行结果 #

<script>
var point = new mapboxgl.LngLat(-122.078643,37.422161);
mapboxgl.accessToken = 'pk.eyJ1Ijoid2FuZ3dkIiwiYSI6ImNrNjJvdTBjcjBncm4za3FuaHhod3I5Ym8ifQ.BXm0oZg7ZfoFqlsT9IlAAQ';
map = new mapboxgl.Map({
container: 'map',
center: point,
zoom: 4,
style: 'mapbox://styles/mapbox/streets-v11'
});

map.on('load', function() {

});
</script>
--------------------------
<script>
var point = new mapboxgl.LngLat(-122.065285,37.397690);
mapboxgl.accessToken = 'pk.eyJ1Ijoid2FuZ3dkIiwiYSI6ImNrNjJvdTBjcjBncm4za3FuaHhod3I5Ym8ifQ.BXm0oZg7ZfoFqlsT9IlAAQ';
map = new mapboxgl.Map({
container: 'map',
center: point,
zoom: 14,
style: 'mapbox://styles/mapbox/streets-v11'
});
...
map.on('load', function() {
createPoint(1,37.397690,-122.065285,700,"GSM CellTower 310-260-328-29021 location is (37.397690,-122.065285) Accuracy:700 m",0,0,18);
});
</script>

AWS Lambda代码 #

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
from bs4 import BeautifulSoup

def lambda_handler(event, context):

    options = webdriver.ChromeOptions()
    options.add_argument('--headless')
    options.add_argument('--no-sandbox')
    options.add_argument('--single-process')
    options.add_argument('--disable-dev-shm-usage')
    options.binary_location = "/opt/headless/python/bin/headless-chromium"
    driver= webdriver.Chrome("/opt/headless/python/bin/chromedriver",chrome_options=options)
    driver.implicitly_wait(0.5)

    driver.get('http://www.cell2gps.com/')

    html=driver.page_source
    soup=BeautifulSoup(html,'html.parser')
    t_list2=soup.select_one('body > center > center > script:nth-child(12)')
    data=str(t_list2)
    print(data)
    print('--------------------------')

    #driver.find_element_by_name('mcc').clear()
    #driver.find_element_by_name('mcc').send_keys('@@')
    #driver.find_element_by_name('mnc').clear()
    #driver.find_element_by_name('mnc').send_keys('@@')
    #driver.find_element_by_name('lac').clear()
    #driver.find_element_by_name('lac').send_keys('@@')
    #driver.find_element_by_name('cellid').clear()
    #driver.find_element_by_name('cellid').send_keys('@@')
    driver.find_element_by_class_name('g-recaptcha').click()
    time.sleep(6)  #second
    
    html=driver.page_source
    soup=BeautifulSoup(html,'html.parser')
    t_list2=soup.select_one('body > center > center > script:nth-child(12)')
    data=str(t_list2)
    print(data)

    driver.close()
    driver.quit()
    return 0

AWS Lambda运行结果 #

<script>
var point = new mapboxgl.LngLat(-122.078643,37.422161);
mapboxgl.accessToken = 'pk.eyJ1Ijoid2FuZ3dkIiwiYSI6ImNrNjJvdTBjcjBncm4za3FuaHhod3I5Ym8ifQ.BXm0oZg7ZfoFqlsT9IlAAQ';
map = new mapboxgl.Map({
container: 'map',
center: point,
zoom: 4,
style: 'mapbox://styles/mapbox/streets-v11'
});
map.on('load', function() {
});
</script>
--------------------------
None

解决建议 #

• 延长等待时间：Lambda环境网络速度通常慢于本地，当前设置的6秒等待比本地的10秒短，建议调整为10秒或更长，确保页面完全加载。
• 替换固定等待为显式等待：使用Selenium的WebDriverWait等待目标节点出现，避免固定等待的不确定性，示例代码：

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

# 点击按钮后等待目标节点加载
WebDriverWait(driver, 15).until(
    EC.presence_of_element_located((By.CSS_SELECTOR, 'body > center > center > script:nth-child(12)'))
)
html = driver.page_source

• 模拟真实浏览器环境：Lambda的headless Chrome可能被网站识别为爬虫，添加参数模拟正常浏览器：

options.add_argument('--window-size=1920,1080')
options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36')
options.add_argument('--enable-javascript')

• 检查reCAPTCHA验证状态：Lambda环境中可能未通过验证码验证，可添加截图功能查看页面状态：

driver.save_screenshot('/tmp/screenshot.png')
# 可将截图上传至S3查看验证是否成功

内容的提问来源于stack exchange，提问作者user20588340