使用Python提取YouTube评论回复用户名失败的技术求助

我编写的Python代码能够获取YouTube视频主评论的用户名，也可以点击“查看回复”按钮展开评论的回复内容，但始终无法提取到回复用户的用户名，多次尝试后结果中仍不显示该类用户名。以下是我的代码：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

def extract_usernames(video_url):
    # Initialize the Chrome driver
    driver = webdriver.Chrome()

    # Navigate to the video URL
    driver.get(video_url)

    # Wait for the page to load completely
    time.sleep(10)  # Increase this value if your internet connection is slow

    # Scroll to the bottom of the page to load all comments
    last_height = driver.execute_script("return document.documentElement.scrollHeight")
    while True:
        driver.execute_script("window.scrollTo(0, document.documentElement.scrollHeight);")
        time.sleep(2)  # Slowly scroll down to load all comments
        new_height = driver.execute_script("return document.documentElement.scrollHeight")
        if new_height == last_height:
            break
        last_height = new_height

    # Find all the comment elements
    comment_elements = driver.find_elements(By.CSS_SELECTOR, "ytd-comment-thread-renderer")

    # Extract the usernames from the comment elements (including replies)
    usernames = []
    for comment_element in comment_elements:
        # Click on the "View replies" button if it exists
        try:
            view_replies_button = comment_element.find_element(By.CSS_SELECTOR, "#more-replies.more-button.style-scope.ytd-comment-replies-renderer")
            view_replies_button.click()
            time.sleep(2)  # Wait for replies to load after clicking
        except:
            pass

        extract_usernames_recursive(comment_element, usernames)

    # Close the browser
    driver.quit()

    return usernames

def extract_usernames_recursive(comment_element, usernames):
    # Extract the username from the top-level comment
    username_element = comment_element.find_element(By.CSS_SELECTOR, "h3.style-scope.ytd-comment-view-model > a.yt-simple-endpoint.style-scope.ytd-comment-view-model")
    username_link = username_element.get_attribute('href')
    if username_link.startswith('https://www.youtube.com/@'):
        username = username_link.split('@')[1]
        usernames.append(username)

    # Check for replies and extract usernames recursively
    reply_elements = comment_element.find_elements(By.CSS_SELECTOR, "ytd-comment-thread-renderer")
    for reply_element in reply_elements:
        extract_usernames_recursive(reply_element, usernames)

# Example usage
video_url = "https://www.youtube.com/watch?v=A1III_DQU4I"  # Replace with the actual video URL
usernames = extract_usernames(video_url)
print(usernames)

原代码无法提取回复用户名的核心问题在于DOM结构识别错误以及选择器使用不当：

1. 回复评论并非独立的ytd-comment-thread-renderer元素，而是嵌套在ytd-comment-replies-renderer下的ytd-comment-view-model中
2. CSS选择器中的>是HTML转义字符，实际应直接使用>
3. 原选择器依赖不稳定的层级类名，容易因页面更新失效

修改后的代码如下：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

def extract_usernames(video_url):
    driver = webdriver.Chrome()
    driver.get(video_url)
    
    # 等待评论区加载（用显式等待替代固定sleep）
    try:
        WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, "ytd-comment-thread-renderer"))
        )
    except:
        print("评论区加载超时")
        driver.quit()
        return []

    # 滚动加载所有评论
    last_height = driver.execute_script("return document.documentElement.scrollHeight")
    while True:
        driver.execute_script("window.scrollTo(0, document.documentElement.scrollHeight);")
        time.sleep(3)
        new_height = driver.execute_script("return document.documentElement.scrollHeight")
        if new_height == last_height:
            break
        last_height = new_height

    comment_threads = driver.find_elements(By.CSS_SELECTOR, "ytd-comment-thread-renderer")
    usernames = []

    for thread in comment_threads:
        # 点击展开回复
        try:
            view_replies_btn = thread.find_element(By.CSS_SELECTOR, "ytd-comment-replies-renderer #more-replies")
            if view_replies_btn.is_displayed():
                view_replies_btn.click()
                time.sleep(2)
                # 等待回复加载完成
                WebDriverWait(thread, 10).until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, "ytd-comment-replies-renderer ytd-comment-view-model"))
                )
        except:
            pass

        # 提取主评论用户名
        extract_single_comment_username(thread, usernames)
        # 提取回复用户名
        extract_reply_usernames(thread, usernames)

    driver.quit()
    return usernames

def extract_single_comment_username(comment_container, usernames):
    """提取单个评论（主评论或回复）的用户名"""
    try:
        # 两种方式提取：从href解析或直接取文本
        username_link = comment_container.find_element(By.CSS_SELECTOR, "#author-text a").get_attribute('href')
        if username_link:
            username = username_link.split('@')[-1]
            usernames.append(username)
        else:
            username_text = comment_container.find_element(By.CSS_SELECTOR, "#author-text span").text.strip()
            usernames.append(username_text)
    except:
        # 忽略提取失败的情况
        pass

def extract_reply_usernames(thread, usernames):
    """提取当前评论线程下的所有回复用户名"""
    try:
        reply_comments = thread.find_elements(By.CSS_SELECTOR, "ytd-comment-replies-renderer ytd-comment-view-model")
        for reply in reply_comments:
            extract_single_comment_username(reply, usernames)
    except:
        pass

# 示例使用
video_url = "https://www.youtube.com/watch?v=A1III_DQU4I"
usernames = extract_usernames(video_url)
print("提取到的用户名：", usernames)

关键修改说明： #

• 分离提取逻辑：单独编写函数处理单个评论和回复的用户名提取，结构更清晰
• 显式等待替代固定sleep：提高代码稳定性，避免因页面加载慢导致的元素查找失败
• 稳定选择器：使用#author-text相关元素定位用户名，这类ID元素不易随页面更新失效
• 正确识别回复DOM结构：从ytd-comment-replies-renderer下直接定位回复评论元素

内容的提问来源于stack exchange，提问作者tamer_mz