使用BeautifulSoup爬取Goodreads书籍genres与language信息失败求助

问题分析 #

你的代码无法获取genres和language，核心原因是Goodreads页面结构已更新，原选择器失效：

• 体裁（genres）：旧版的div.left和a.bookPageGenreLink选择器不再匹配当前页面的体裁区域
• 语言（language）：TruncatedContent类对应的是其他截断内容，和语言信息无关

修正后的完整代码 #

import re
from bs4 import BeautifulSoup
from argparse import Namespace
from urllib.request import urlopen
import pandas as pd

def get_id(book_id_str):
    # 提取书籍纯数字ID（兼容带标题的book_id格式）
    match = re.match(r'^\d+', book_id_str)
    return match.group() if match else book_id_str

def get_genres(soup):
    genres = []
    # 定位当前页面的体裁区域
    genre_section = soup.find('div', class_='BookPageMetadataSection__genres')
    if not genre_section:
        return genres
    
    # 遍历所有体裁分组
    genre_groups = genre_section.find_all('div', class_='BookPageMetadataSection__genreButton')
    for group in genre_groups:
        # 获取主体裁
        main_genre = group.find('span', class_='BookPageMetadataSection__genreName').text.strip()
        # 获取子体裁列表
        sub_genres = [g.text.strip() for g in group.find_all('a', class_='Button Button--tag-inline Button--small')]
        
        if sub_genres:
            genres.append(f"{main_genre} > {' > '.join(sub_genres)}")
        else:
            genres.append(main_genre)
    return genres

def scrape_book(book_id: str, args: Namespace):
    url = "https://www.goodreads.com/book/show/" + book_id
    source = urlopen(url)
    soup = BeautifulSoup(source, "html.parser")

    # 初始化默认值，避免元素不存在导致报错
    book_title = soup.find("h1", {"class": "Text Text__title1", "data-testid":"bookTitle"}).text.strip() if soup.find("h1", {"class": "Text Text__title1", "data-testid":"bookTitle"}) else "Unknown"
    book_description = soup.find("span", {"class": "Formatted"}).text.strip() if soup.find("span", {"class": "Formatted"}) else "Unknown"
    book_pages = soup.find("p", {"data-testid": "pagesFormat"}).text.strip() if soup.find("p", {"data-testid": "pagesFormat"}) else "Unknown"
    book_published_date = soup.find("p", {"data-testid": "publicationInfo"}).text.strip() if soup.find("p", {"data-testid": "publicationInfo"}) else "Unknown"
    book_author = soup.find("span", {"class": "ContributorLink__name"}).text.strip() if soup.find("span", {"class": "ContributorLink__name"}) else "Unknown"
    book_author_url = soup.find("a", {"class": "ContributorLink"})['href'] if soup.find("a", {"class": "ContributorLink"}) else "Unknown"
    
    # 获取语言信息
    book_language = "Unknown"
    language_elem = soup.find('div', data_testid='language')
    if language_elem:
        book_language = language_elem.text.strip()
    else:
        # 备选方案：通过详情区域的dt/dd配对查找
        details_section = soup.find('div', class_='BookDetailsSection')
        if details_section:
            language_dt = details_section.find('dt', string=lambda x: x and 'Language' in x)
            if language_dt:
                book_language = language_dt.find_next_sibling('dd').text.strip()

    book = {
        "book_id_title": book_id,
        "book_id": get_id(book_id),
        "book_title": book_title,
        "book_description": book_description,
        "book_url": url,
        "Book_work_pages": book_pages,
        "Book_work_published_date": book_published_date,
        "book_author": book_author,
        "book_author_url": f"https://www.goodreads.com{book_author_url}" if book_author_url != "Unknown" else "Unknown",
        "book_language": book_language,
        "genres": get_genres(soup)
    }

    return book

# 测试调用
result = scrape_book(book_id='343', args=Namespace())
print(result)

关键修改说明 #

1. 体裁获取：
   • 改用BookPageMetadataSection__genres定位体裁区域，适配当前Goodreads页面结构
   • 区分主体裁和子体裁，按层级拼接成主体裁 > 子体裁的格式
2. 语言获取：
   • 优先通过data-testid='language'直接定位语言元素（最可靠）
   • 增加备选方案：在详情区域查找包含"Language"的标签，再取对应的值，避免单一选择器失效
3. 容错处理：
   • 为所有字段增加默认值，避免页面元素缺失导致代码崩溃
   • 修复book_author_url获取逻辑，拼接完整的Goodreads链接

内容的提问来源于stack exchange，提问作者Mohammed Almulla