求助：完善NFRC网站_doPostBack下公司链接爬取Python代码（无编程基础）

以下是完善后的完整代码，可自动遍历所有分页，提取每个公司对应的隐藏详情链接，并将结果保存到Excel文件中：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import pandas as pd
import time

# Setup Chrome options
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--start-maximized")  # Maximize the browser window

# Initialize Selenium WebDriver using ChromeDriverManager
driver = webdriver.Chrome(service=webdriver.chrome.service.Service(ChromeDriverManager().install()), options=chrome_options)

# Initialize an empty list to store company data
company_data = []

# Function to wait and click with retry
def wait_and_click(driver, by, value, retries=3):
    for _ in range(retries):
        try:
            element = WebDriverWait(driver,20).until(EC.element_to_be_clickable((by, value)))
            driver.execute_script("arguments[0].scrollIntoView(true);", element)
            element.click()
            return True
        except Exception as e:
            print(f"Error clicking element {value}: {e}")
            time.sleep(2)  # Wait before retrying
    return False

# Open the NFRC website
driver.get("https://www.nfrc.co.uk/search-members")

# Click on "Search for a roofing contractor now"
wait_and_click(driver, By.LINK_TEXT, "Search for a roofing contractor now")

# Click to open the dropdown for contractor type selection
dropdown_locator = (By.CLASS_NAME, "sfDropdownList")
wait_and_click(driver, By.CLASS_NAME, "sfDropdownList")

# Select "For a Domestic Property" from the dropdown
wait_and_click(driver, By.XPATH, '//*[@id="MainContent_C001_contractorType"]/option[@value="CONTRACTOR_ADVANCED_DOMESTIC_PROPERTY_VIEW"]')

# Select "Pitched Roof"
wait_and_click(driver, By.XPATH, '//*[@id="MainContent_C001_rblRoofType"]/li[1]/label')

# Proceed to the next step
wait_and_click(driver, By.ID, "MainContent_C001_btnSearch")

# Add a delay to ensure the page loads completely after clicking search button
time.sleep(5)  # Adjust the time delay as needed

# 定义处理单页数据的函数
def process_page(driver):
    # 等待列表加载完成
    WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.CLASS_NAME, "contractor-item")))
    # 获取页面源码解析
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    # 遍历所有公司项
    for item in soup.find_all(class_="contractor-item"):
        company_name = item.find(class_="contractor-name").get_text(strip=True)
        # 提取_doPostBack链接中的目标地址
        postback_script = item.find('a')['href']
        # 解析_doPostBack参数，提取会员详情页链接
        control_id = postback_script.split("'")[1]
        detail_link = f"https://www.nfrc.co.uk/search-members?{control_id}=x"
        # 添加到数据列表
        company_data.append({
            "公司名称": company_name,
            "详情链接": detail_link
        })
    print(f"当前页已提取 {len(soup.find_all(class_='contractor-item'))} 条数据")

# 开始遍历所有分页
while True:
    # 处理当前页
    process_page(driver)
    
    # 检查是否有下一页按钮，且可点击
    try:
        next_button = WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.XPATH, "//a[@title='Next Page']"))
        )
        # 判断按钮是否被禁用
        if 'disabled' in next_button.get_attribute('class'):
            print("已到达最后一页，停止遍历")
            break
        # 点击下一页
        wait_and_click(driver, By.XPATH, "//a[@title='Next Page']")
        # 等待页面加载
        time.sleep(3)
    except Exception as e:
        print(f"无更多分页或分页出错: {e}")
        break

# 将数据保存到Excel
df = pd.DataFrame(company_data)
df.to_excel("NFRC会员公司列表.xlsx", index=False)
print(f"数据已保存到 NFRC会员公司列表.xlsx，共 {len(company_data)} 条记录")

# 关闭浏览器
driver.quit()

关键逻辑说明 #

• 提取隐藏链接：公司的点击事件是__doPostBack脚本，通过拆分脚本字符串提取控件ID，构造可直接访问的详情页链接（NFRC网站支持通过该参数跳转至会员详情页）
• 分页遍历：循环检查“Next Page”按钮状态，若按钮未被禁用则点击进入下一页，直到遍历完所有分页
• 数据持久化：使用Pandas将爬取的公司名称和链接保存为Excel文件，方便后续查看和使用

内容的提问来源于stack exchange，提问作者Maira Zafar