Python Selenium爬虫无法跳转至指定页面问题求助

使用Python结合Selenium编写爬虫，目标访问www.apartments.com，输入指定地点（如Santa Monica, CA）后调用.click()点击搜索按钮，但页面始终跳转到默认的Chicago, IL页面。相关代码如下：

from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.keys import Keys

chrome_options = Options()
chrome_options.headless = False
chrome_options.add_argument("start-maximized")
# options.add_experimental_option("detach", True)
chrome_options.add_argument("--no-sandbox")
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('excludeSwitches', ['enable-logging'])
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.add_argument('--disable-blink-features=AutomationControlled')

driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=chrome_options)
driver.get('https://www.apartments.com/')

driver.implicitly_wait(10)

var_inp='Santa Monica, CA'
#search for rentals
#driver.execute_script("arguments[0].setAttribute('value',arguments[1])",element, value)
#WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "search-box-input"))).send_keys(var_inp+Keys.RETURN)

def foo():
    element=WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "quickSearchLookup")))
    driver.execute_script("arguments[0].setAttribute('value','Santa Monica, CA')", element)
    #element.send_keys(var_inp)
    button=WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[@class='go']")))
    button.click()

foo()
##find it
##//div[contains(@class,"StyledPropertyCardDataArea")]/ul
#driver.execute_script("arguments[0].setAttribute('value',arguments[1])",driver.find_element(By.CSS_SELECTOR, "input#search-box-input"), 'New York, NY')

#driver.implicitly_wait(30)
prices=WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//p[@class='property-pricing']")))
beds=WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//p[@class='property-beds']")))
specials=WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//p[@class='property-specials']")))
lst_beds=[]
for bed in beds:
    lst_beds.append(bed.text)
lst_prices=[]
for price in prices:
    lst_prices.append(price.text)
lst_specials=[]
for special in specials:
    lst_specials.append(special.text)

print(lst_specials,lst_prices,lst_beds)

driver.quit()

核心问题 #

直接通过execute_script修改输入框的value属性，不会触发网页的输入事件（如input、change），网站的搜索逻辑无法识别到输入内容，因此会使用默认地点执行搜索。

解决步骤 #

1. 模拟真实用户输入：放弃JS修改属性，改用send_keys输入内容，同时先清空输入框避免默认值干扰，这样能触发页面事件监听，确保网站识别到输入内容。
2. 优先用回车触发搜索：输入完成后按回车键，比点击按钮更稳定，减少因按钮状态未就绪导致的交互失败。
3. 检查弹窗干扰：如果仍有问题，先处理页面的cookie授权弹窗，这类弹窗可能会阻断元素交互。

修改后的核心代码 #

替换原代码中的foo函数：

def foo():
    # 等待搜索框可交互
    element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "quickSearchLookup")))
    # 清空输入框原有内容
    element.clear()
    # 输入目标地点
    element.send_keys(var_inp)
    # 按回车键触发搜索
    element.send_keys(Keys.RETURN)

内容的提问来源于stack exchange，提问作者xlmaster