网页爬虫需求：整合多页链接抓取与联系人信息提取代码

实现步骤 #

1. 遍历目标网站分页，抓取所有详情页链接并保存至本地文本文件
2. 读取保存的链接，逐个访问详情页提取联系人、传真信息并汇总保存

完整代码实现 #

import requests
from bs4 import BeautifulSoup
import time

def crawl_links(base_url, total_pages):
    links = []
    for page in range(1, total_pages + 1):
        page_url = f"{base_url}?page={page}"
        try:
            # 模拟浏览器请求，避免被反爬拦截
            headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"}
            response = requests.get(page_url, headers=headers)
            response.raise_for_status()  # 捕获HTTP请求错误
            soup = BeautifulSoup(response.text, "html.parser")
            
            # 定位详情页链接（需根据目标网站实际HTML结构调整选择器）
            link_elements = soup.select("div.contractor-list-item h3 a")
            for elem in link_elements:
                full_link = elem.get("href")
                # 处理相对链接（如果网站返回的是相对路径）
                if not full_link.startswith("http"):
                    full_link = f"https://www.usaopps.com{full_link}"
                if full_link not in links:
                    links.append(full_link)
            
            print(f"完成第{page}页抓取，累计获取{len(links)}个链接")
            time.sleep(1)  # 控制请求频率，防反爬
        except Exception as e:
            print(f"第{page}页抓取失败: {str(e)}")
            continue
    
    # 保存链接到本地文件
    with open("contractor_links.txt", "w", encoding="utf-8") as f:
        f.write("\n".join(links))
    print(f"所有链接已保存至contractor_links.txt，共{len(links)}条")

def extract_info_from_links(link_file):
    # 读取本地链接文件
    with open(link_file, "r", encoding="utf-8") as f:
        links = [line.strip() for line in f if line.strip()]
    
    collected_info = []
    for idx, link in enumerate(links, 1):
        try:
            headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"}
            response = requests.get(link, headers=headers)
            response.raise_for_status()
            soup = BeautifulSoup(response.text, "html.parser")
            
            # 提取联系人和传真信息（需根据目标网站实际HTML结构调整选择器）
            contact_name = soup.select_one("div.contact-info span.name").get_text(strip=True) if soup.select_one("div.contact-info span.name") else "无联系人信息"
            fax_number = soup.select_one("div.contact-info span.fax").get_text(strip=True) if soup.select_one("div.contact-info span.fax") else "无传真信息"
            
            # 整理信息格式
            info_item = f"""序号: {idx}
详情页链接: {link}
联系人: {contact_name}
传真: {fax_number}
------------------------"""
            collected_info.append(info_item)
            print(f"完成第{idx}条信息提取")
            time.sleep(1)
        except Exception as e:
            print(f"第{idx}条链接提取失败: {str(e)}")
            continue
    
    # 保存提取的信息
    with open("contractor_info.txt", "w", encoding="utf-8") as f:
        f.write("\n".join(collected_info))
    print(f"所有信息已保存至contractor_info.txt，共{len(collected_info)}条有效数据")

if __name__ == "__main__":
    # 目标网站基础URL
    target_base_url = "https://www.usaopps.com/government_contractors/naics-111110-Soybean-Farming.htm"
    # 替换为网站实际总页数（可手动查看分页栏获取）
    total_page_count = 5
    
    # 第一步：抓取多页链接
    crawl_links(target_base_url, total_page_count)
    
    # 第二步：批量提取详情页信息
    extract_info_from_links("contractor_links.txt")

关键注意事项 #

• 调整CSS选择器：代码中用于定位链接、联系人、传真的选择器（如div.contractor-list-item h3 a），需要你用浏览器开发者工具查看目标网站的HTML结构后修改，确保能精准定位元素
• 防反爬设置：代码中添加了请求延迟和浏览器标识，若网站反爬严格，可适当延长time.sleep()的时间，或添加代理IP
• 异常处理：代码捕获了请求和解析过程中的异常，避免程序崩溃，同时输出错误信息方便排查问题

内容的提问来源于stack exchange，提问作者Life Unexplained