Python新手：使用BeautifulSoup爬取数据遇两类问题求助

1. 提取列表形式的CQC Service Type和CQC Specialism #

目标页面的CQC分类数据（服务类型、专长）都包裹在dl.ai-group-fields标签内，每个分类对应一个dt标题和包含li列表的dd容器。我们可以遍历这些dl元素，匹配标题后提取列表项并转为字符串格式：

# 在loop_excel_sheet函数内处理div_CQC
cqc_service_type_str = ''
cqc_specialism_str = ''
for dl in div_CQC:
    dt_text = dl.find('dt').text.strip()
    dd_container = dl.find('dd')
    if dt_text == 'CQC Service Type' and dd_container:
        # 提取所有列表项并拼接为逗号分隔的字符串
        items = [li.text.strip() for li in dd_container.find_all('li')]
        cqc_service_type_str = ', '.join(items)
    elif dt_text == 'CQC Specialism' and dd_container:
        items = [li.text.strip() for li in dd_container.find_all('li')]
        cqc_specialism_str = ', '.join(items)

2. 提取JavaScript嵌入的CQC overall rating和Date of inspection #

静态HTML无法直接获取JS加载的数据，但目标页面的CQC评分数据会以JSON形式嵌入在script标签中（标记为window.cqcWidgetData）。我们可以用正则提取该JSON字符串，再转为Python字典获取所需字段：

import re
import json

# 在获取soup之后添加
cqc_rating = ''
inspection_date = ''
# 遍历所有script标签查找目标数据
script_tags = soup.find_all('script')
for script in script_tags:
    script_content = script.string
    if script_content and 'window.cqcWidgetData' in script_content:
        # 匹配JSON部分
        match = re.search(r'window\.cqcWidgetData = (\{.*?\});', script_content, re.DOTALL)
        if match:
            try:
                cqc_data = json.loads(match.group(1))
                cqc_rating = cqc_data.get('overallRating', '')
                inspection_date = cqc_data.get('lastReportedDate', '')
            except json.JSONDecodeError:
                pass
            break

修改后的完整代码 #

将上述逻辑整合到原有代码中，最终实现所有数据的爬取：

import pandas as pd  
import requests  
from bs4 import BeautifulSoup 
import http.client
import re
import json

http.client._MAXHEADERS = 1000

file="/Users/dhesis/PycharmProjects/pythonProject/CH_ID (2).xlsx"
Carehome_list = pd.read_excel(file).iloc[:, 0].to_list()
print(Carehome_list)

def loop_excel_sheet():
    for CH in Carehome_list[:2]:
        url = CH
        headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36'}
        r = requests.get(url, headers=headers)
        soup = BeautifulSoup(r.content, 'html.parser')
        divs = soup.find_all('div', class_ ="container-fluid bem-js-service-details")
        div_CQC = soup.find_all('dl', class_="ai-group-fields read-data bem-description-list bem-description-list--ai-group")
        
        # 提取CQC Service Type和Specialism
        cqc_service_type_str = ''
        cqc_specialism_str = ''
        for dl in div_CQC:
            dt_text = dl.find('dt').text.strip()
            dd_container = dl.find('dd')
            if dt_text == 'CQC Service Type' and dd_container:
                items = [li.text.strip() for li in dd_container.find_all('li')]
                cqc_service_type_str = ', '.join(items)
            elif dt_text == 'CQC Specialism' and dd_container:
                items = [li.text.strip() for li in dd_container.find_all('li')]
                cqc_specialism_str = ', '.join(items)
        
        # 提取CQC评分和检查日期
        cqc_rating = ''
        inspection_date = ''
        script_tags = soup.find_all('script')
        for script in script_tags:
            script_content = script.string
            if script_content and 'window.cqcWidgetData' in script_content:
                match = re.search(r'window\.cqcWidgetData = (\{.*?\});', script_content, re.DOTALL)
                if match:
                    try:
                        cqc_data = json.loads(match.group(1))
                        cqc_rating = cqc_data.get('overallRating', '')
                        inspection_date = cqc_data.get('lastReportedDate', '')
                    except json.JSONDecodeError:
                        pass
                    break
        
        for item in divs:
            name = item.find('dd', itemprop = 'branchOf').text
            address = item.find('span', itemprop = 'location').text
            try:
                website = item.find('a', itemprop = 'url').text
            except:
                website = ''
            last_updated = item.find('p').text.strip()
            try:
                telephone = item.find('dd', itemprop='telephone').text
            except:
                telephone = ''

            info = {
                'Name': name,
                'Address': address,
                'Website': website,
                'Telephone': telephone,
                'Website Last Updated': last_updated,
                'CQC Service Type': cqc_service_type_str,
                'CQC Specialism': cqc_specialism_str,
                'CQC Overall Rating': cqc_rating,
                'Date of Inspection': inspection_date
            }
            chlist.append(info)


chlist = []
loop_excel_sheet()
df= pd.DataFrame(chlist)
df.to_csv('CH Tester.csv')

额外提示 #

• 处理90个站点时，建议添加time.sleep(1)之类的请求延迟，避免触发目标网站的反爬限制
• 若目标页面的JSON结构或HTML标签有变动，需根据实际页面源码调整匹配的键名和标签选择器

内容的提问来源于stack exchange，提问作者Simran Dhesi