Python爬取Just Dial时部分网页无法加载的问题求助

问题场景 #

我有一个包含Just Dial网站URL的数据集，想要提取商家名称等信息。示例数据如下：

dict_test  = {"Id" : [1000, 1001, 1002],
             "Online_url" : ['https://www.justdial.com/Mumbai/Sunrise-Info-Solutions-Pvt-Ltd-Near-Airtel-Gallery/022PXX22-XX22-220719102528-J5Q2_BZDET?xid=TXVtYmFpIE1vYmlsZSBEZWFsZXJz',
                            'https://www.justdial.com/Mumbai/Riddhi-Siddhi-Mobile-Gallery-Electronic-Opposite-Jain-Plaza-Ambernath/022PXX22-XX22-210519191020-K2U6_BZDET?xid=TXVtYmFpIE1vYmlsZSBEZWFsZXJz',
                            'https://www.justdial.com/Mumbai/Bharat-Communication-Opposite-Vibgyor-School-Goregaon-West/022PXX22-XX22-130103150323-S4V9_BZDET?xid=TXVtYmFpIE1vYmlsZSBEZWFsZXJz']}
df_test = pd.DataFrame(dict_test)

使用以下Selenium脚本提取信息时，第一个URL能正常提取，但第二个URL加载后显示about:blank，无法继续处理后续URL：

options = webdriver.ChromeOptions() 
options.add_experimental_option("excludeSwitches", ['enable-automation'])
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument("--disable-notifications")
options.add_argument( "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
options.add_argument("--remote-debugging-port=9222")
driver = webdriver.Chrome(executable_path=r'C:\Users\admin\Downloads\chromedriver_lates\chromedriver.exe', options = options)
driver.maximize_window()
driver.implicitly_wait(10)
driver.get('https://www.justdial.com/')
time.sleep(2)

def webpage_extract(min_count, max_count, df_test, folder, file_name):
    for i in range(min_count,max_count):
        try:
            driver.set_page_load_timeout(5)
            driver.switch_to.window(driver.window_handles[0])
            driver.execute_script("window.open('');")
            # Switch to the new window and open new URL
            driver.switch_to.window(driver.window_handles[1])
            driver.get(df_test['Online_url'].iloc[i])
            time.sleep(5)
        except TimeoutException as ex:
            isrunning = 0
            print("Exception has been thrown.")

        try:
            myElem = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.CLASS_NAME, 'fn')))
        except TimeoutException:
            print("Loading took too much time!")
        
### Below command is used to close the Pop-up
        try:
            driver.find_element("xpath", '//*[@id="best_deal_detail_div"]/section/span').click()
            driver.find_element("xpath", '//*[@id="best_deal_detail_div"]/section/span').click()
        except:
            "Pop_Up"
            
        try:
            seller_info=driver.find_element("xpath", "/html/body/div[2]/div[1]/div/div[1]/div[2]/div/div/h1/span/span").text
            print("Seller_Name: ", seller_info)
        except:
            seller_info="Extraction_Error"
            
        print("Iteration {} : Information Extracted for Seller {}".format(i, seller_info))
        driver.delete_all_cookies()
        # Closing the tab
        driver.close()
        time.sleep(2)
    return None


%time webpage_extract(0, len(df_test), df_test, folder, file_name)

解决方案 #

1. 核心问题分析 #

出现about:blank的主要原因是窗口切换逻辑错误：关闭新标签页后，driver没有切回主窗口，导致下一次打开新窗口时上下文异常，无法正确加载URL。同时缺少对加载失败的判断，导致后续代码仍会执行无效操作。

2. 修改后的代码 #

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import time
import pandas as pd

# 初始化浏览器
options = webdriver.ChromeOptions() 
options.add_experimental_option("excludeSwitches", ['enable-automation'])
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument("--disable-notifications")
options.add_argument( "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
options.add_argument("--remote-debugging-port=9222")
driver = webdriver.Chrome(executable_path=r'C:\Users\admin\Downloads\chromedriver_lates\chromedriver.exe', options = options)
driver.maximize_window()
driver.implicitly_wait(10)
driver.get('https://www.justdial.com/')
time.sleep(2)

def webpage_extract(min_count, max_count, df_test, folder, file_name):
    for i in range(min_count, max_count):
        seller_info = "Extraction_Error"
        main_window = driver.window_handles[0]  # 缓存主窗口句柄
        
        try:
            driver.set_page_load_timeout(10)  # 适当延长超时时间
            driver.switch_to.window(main_window)
            # 打开新标签页
            driver.execute_script("window.open('');")
            new_window = driver.window_handles[-1]  # 获取最新打开的窗口
            driver.switch_to.window(new_window)
            
            # 加载目标URL
            target_url = df_test['Online_url'].iloc[i]
            driver.get(target_url)
            time.sleep(3)  # 缩短固定等待，优先用显式等待
            
            # 检查是否加载失败（about:blank）
            if driver.current_url == "about:blank":
                print(f"Iteration {i}: URL加载失败，跳过该链接")
                continue
            
            # 等待关键元素加载
            WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.CLASS_NAME, 'fn')))
            
            # 关闭弹窗
            try:
                close_btn = WebDriverWait(driver, 3).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="best_deal_detail_div"]/section/span')))
                close_btn.click()
            except:
                pass  # 无弹窗则跳过
            
            # 提取商家名称
            seller_info = driver.find_element(By.XPATH, "/html/body/div[2]/div[1]/div/div[1]/div[2]/div/div/h1/span/span").text
            print(f"Seller_Name: {seller_info}")
            
        except TimeoutException:
            print(f"Iteration {i}: 页面加载超时")
        except Exception as e:
            print(f"Iteration {i}: 发生错误 - {str(e)}")
        finally:
            # 确保关闭当前标签页并切回主窗口
            try:
                if driver.window_handles[-1] != main_window:
                    driver.close()
                driver.switch_to.window(main_window)
                driver.delete_all_cookies()
            except:
                pass
            
        print(f"Iteration {i} : 提取结果 - {seller_info}")
        time.sleep(1)
    return None

# 执行函数（注意folder和file_name需要提前定义）
%time webpage_extract(0, len(df_test), df_test, folder, file_name)

3. 关键优化点 #

• 修复窗口切换逻辑：每次操作前缓存主窗口句柄，关闭新标签页后强制切回主窗口，避免上下文混乱。
• 增加加载失败判断：检查driver.current_url是否为about:blank，若是则直接跳过后续步骤，处理下一个URL。
• 优化等待策略：减少固定time.sleep的使用，优先用显式等待（WebDriverWait），同时适当延长页面加载超时时间。
• 完善异常处理：用finally块确保无论是否出错，都会关闭当前标签页并切回主窗口，保证循环能持续执行。

内容的提问来源于stack exchange，提问作者Data-7scientist