Webscraping代码无法采集ASOS网站全部商品URL问题

问题背景 #

尝试采集ASOS女装连衣裙分类页面的全部商品URL，逻辑为：先获取商品总数（约18000个）和每页商品数（72个），计算总页数后遍历每页采集URL，预期得到包含约18000条URL的列表。

实际异常 #

仅前2页能采集到72个URL，后续页面采集数量为0，测试5页仅得到144个URL。

原代码 #

# Upload google drive files
from google.colab import drive
drive.mount('/content/drive')

# Import libraries
import urllib
import urllib.request
from bs4 import BeautifulSoup
import re
import requests
import matplotlib.pyplot as plt
from io import BytesIO 

# Beautiful soup function
user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'

headers={'User-Agent':user_agent,} 

def make_soup(url):
    request= urllib.request.Request(url, None,headers) 
    thepage = urllib.request.urlopen(request)
    soupdata = BeautifulSoup(thepage, "html.parser")
    return soupdata

# Get total number of pages 

site = 'https://www.asos.com/us/women/dresses/cat/?cid=8799'
soup = make_soup(site)
element = soup.find('p', class_='label_Ph1fi')
element = element.text

numbers = re.findall(r'\d{1,3}(?:,\d{3})*', element)

if len(numbers) >= 2:
    offset = int(numbers[0].replace(',', ''))
    num_images = int(numbers[1].replace(',', ''))
    num_pages = int(num_images / offset)
    print(f"Images Per Page: {offset}")
    print(f"Total Images: {num_images}")
    print(f"Total Pages:{num_pages}")
else:
    print("Numbers not found")

# Get all product urls 

product_urls = []
for i in range(5): # testing with 5 instead of num_pages 
    site = 'https://www.asos.com/us/women/dresses/cat/?cid=8799&page='
    site = site + str(i)
    soup = make_soup(site)
    items = soup.find_all('a',class_='productLink_E9Lfb',href=True)
    url_cnt = 0
    for link in items:
        href = link.get('href')
        if href:
            product_urls.append(href)
            url_cnt += 1
    
    print(f'Page {i} done. URLs collected: {url_cnt}')

print(f'Total number of product urls: {len(product_urls)}')

问题原因 #

1. 页码参数错误：ASOS分页参数page从1开始计数，但代码中用range(5)生成0-4的页码，第0页实际跳转到第1页，第3、4页为无效参数，返回页面无商品数据。
2. User-Agent过旧：使用的Firefox 3.0.7版本UA过于老旧，易被网站反爬机制识别并限制，导致后续页面返回空内容。
3. 总页数计算缺陷：直接用int(num_images / offset)会丢失余数部分的页面，导致最后一页商品被遗漏。

修复方案 #

修改要点 #

• 调整页码起始值：循环从1开始，匹配网站分页规则
• 更新UA为现代浏览器标识，降低被反爬拦截概率
• 改用requests库发送请求，简化代码同时支持会话保持
• 增加请求延迟，避免请求频率过高触发反爬
• 修正总页数计算逻辑，采用向上取整确保不遗漏页面

修复后代码 #

# 挂载Google Drive（无需可注释）
from google.colab import drive
drive.mount('/content/drive')

# 导入依赖库
import re
import time
import requests
from bs4 import BeautifulSoup

# 配置现代浏览器请求头
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36'
}

def make_soup(url):
    # 增加1秒请求延迟，规避反爬
    time.sleep(1)
    response = requests.get(url, headers=headers)
    response.raise_for_status()  # 捕获HTTP请求错误
    return BeautifulSoup(response.text, "html.parser")

# 获取分页信息
site = 'https://www.asos.com/us/women/dresses/cat/?cid=8799'
soup = make_soup(site)
element = soup.find('p', class_='label_Ph1fi')

if element:
    element_text = element.text
    numbers = re.findall(r'\d{1,3}(?:,\d{3})*', element_text)
    if len(numbers) >= 2:
        per_page = int(numbers[0].replace(',', ''))
        total_items = int(numbers[1].replace(',', ''))
        # 向上取整计算总页数，避免遗漏最后一页
        total_pages = (total_items + per_page - 1) // per_page
        print(f"每页商品数: {per_page}")
        print(f"总商品数: {total_items}")
        print(f"总页数: {total_pages}")
    else:
        print("未提取到有效分页数字")
else:
    print("未找到分页信息元素")

# 采集商品URL
product_urls = []
# 测试前5页，实际使用可改为range(1, total_pages + 1)
for page_num in range(1, 6):
    page_url = f'https://www.asos.com/us/women/dresses/cat/?cid=8799&page={page_num}'
    soup = make_soup(page_url)
    items = soup.find_all('a', class_='productLink_E9Lfb', href=True)
    url_count = len(items)
    # 批量添加有效URL
    product_urls.extend([link['href'] for link in items if link['href']])
    print(f'第{page_num}页采集完成，获取URL数量: {url_count}')

print(f'总采集URL数量: {len(product_urls)}')

内容的提问来源于stack exchange，提问作者Danny_webb