如何从不一致的网页爬取数据创建DataFrame并处理长度不匹配

问题描述 #

爬取房产列表信息并导出CSV时，遇到错误：

'All arrays must be of the same length.'

原因是部分房产缺失部分字段，导致各数据列表长度不一致，需要为缺失信息填充空值。

问题代码 #

# 安装必要库（未安装时执行）
!pip install requests
!pip install beautifulsoup4
!pip install pandas

# 导入库
import requests
from bs4 import BeautifulSoup
import pandas as pd
from lxml import etree

link = 'https://listado.mercadolibre.cl/inmuebles/departamentos/venta/propiedades-usadas/_NoIndex_True#applied_filter_id%3DOPERATION_SUBTYPE%26applied_filter_name%3DModalidad%26applied_filter_order%3D5%26applied_value_id%3D244562%26applied_value_name%3DPropiedades+usadas%26applied_value_order%3D1%26applied_value_results%3D68139%26is_custom%3Dfalse'

r = requests.get(link)

r.status_code

soup = BeautifulSoup(r.content, 'html.parser')

dom=etree.HTML(str(soup))

titulos = soup.find_all('h2', attrs={"class":"ui-search-item__title"})
titulos = [i.text for i in titulos]

urls = soup.find_all('div',attrs={"class":"ui-search-item__group__element ui-search-item__title-grid"})
urls = [item.find('a')['href'] for item in urls]

moneda = soup.find_all('span', attrs={"class":"andes-money-amount__currency-symbol"})
moneda = [i.text for i in moneda]

precio = soup.find_all('span', attrs={"class":"andes-money-amount__fraction"})
precio = [i.text for i in precio]

dormitorio = dom.xpath('//div[@class="ui-search-item__attributes-container-grid"]/ul/li[contains(normalize-space(.), "dormitorios")]')
dormitorio = [i.text for i in dormitorio]

baño = dom.xpath('//div[@class="ui-search-item__attributes-container-grid"]/ul/li[contains(normalize-space(.), "baño")]')
baño = [i.text for i in baño]

m2 = dom.xpath('//div[@class="ui-search-item__attributes-container-grid"]/ul/li[contains(normalize-space(.), "m²")]')
m2 = [i.text for i in m2]

locacion = soup.find_all('span',attrs={"class":"ui-search-item__location-label"})
locacion = [i.text for i in locacion]

vendedor = soup.find_all('div',attrs={"class":"ui-search-item__official-store-grid"})
vendedor = [i.text for i in vendedor]
vendedor

# df = pd.DataFrame({"titulos" :titulos, "urs" :urls})

# df.to_csv('Ejemplo')

siguiente = dom.xpath('//div[@class="ui-search-pagination"]/nav/ul/li[contains(@class,"--next")]/a')[0].get('href')

ini = soup.find('span',attrs={"class":"andes-pagination__link"}).text
ini = int(ini)

can = soup.find('li',attrs={"class":"andes-pagination__page-count"})
can = int(can.text.split(" ")[1])

lista_titulos = []
lista_urls = []
lista_dormitorio = []
lista_baño = []
lista_m2 = []
lista_moneda = []
lista_precio = []
lista_vendedor = []
lista_locacion = []

siguiente = link
while True:
    r = requests.get(siguiente)
    if r.status_code == 200:
        soup = BeautifulSoup(r.content, 'html.parser')

        #titulos
        titulos = soup.find_all('h2', attrs={"class":"ui-search-item__title"})
        titulos = [i.text for i in titulos]
        lista_titulos.extend(titulos)

        #urls
        urls = soup.find_all('div',attrs={"class":"ui-search-item__group__element ui-search-item__title-grid"})
        urls = [item.find('a')['href'] for item in urls]
        lista_urls.extend(urls)

        #moneda
        moneda = soup.find_all('span', attrs={"class":"andes-money-amount__currency-symbol"})
        moneda = [i.text for i in moneda]
        lista_moneda.extend(moneda)

        #precio
        precio = soup.find_all('span', attrs={"class":"andes-money-amount__fraction"})
        precio = [i.text for i in precio]
        lista_precio.extend(precio)

        #dormitorios
        dormitorio = dom.xpath('//div[@class="ui-search-item__attributes-container-grid"]/ul/li[contains(normalize-space(.), "dormitorios")]')
        dormitorio = [i.text for i in dormitorio]
        lista_dormitorio.extend(dormitorio)

        #baños
        baño = dom.xpath('//div[@class="ui-search-item__attributes-container-grid"]/ul/li[contains(normalize-space(.), "baño")]')
        baño = [i.text for i in baño]
        lista_baño.extend(baño)

        #m2
        m2 = dom.xpath('//div[@class="ui-search-item__attributes-container-grid"]/ul/li[contains(normalize-space(.), "m²")]')
        m2 = [i.text for i in m2]
        lista_m2.extend(m2)

        #vendedor
        vendedor = soup.find_all('div',attrs={"class":"ui-search-item__official-store-grid"})
        vendedor = [i.text for i in vendedor]
        lista_vendedor.extend(vendedor)


        #locacion
        locacion = soup.find_all('span',attrs={"class":"ui-search-item__location-label"})
        locacion = [i.text for i in locacion]
        lista_locacion.extend(locacion)

        #inicial
        ini = soup.find('span',attrs={"class":"andes-pagination__link"}).text
        ini = int(ini)

        #cantidad
        dom=etree.HTML(str(soup))
        can = soup.find('li',attrs={"class":"andes-pagination__page-count"})
        can = int(can.text.split(" ")[1])

    else:
          break
    print(ini,can)
    if ini==can:
          break
    siguiente = dom.xpath('//div[@class="ui-search-pagination"]/nav/ul/li[contains(@class,"--next")]/a')[0].get('href')

print(len(lista_titulos))
print(len(lista_urls))
print(len(lista_dormitorio))
print(len(lista_baño))
print(len(lista_m2))
print(len(lista_locacion))
print(len(lista_vendedor))

df = pd.DataFrame({"titulos" :lista_titulos, "url" :lista_urls, "moneda" :lista_moneda, "precio" :lista_precio, "dormitorios" :lista_dormitorio, "baños" :lista_baño, "m2" :lista_m2})
df.to_csv('Venta_Departamentos_Usados_MasRelevantes')

解决方案 #

核心思路是遍历每个房产条目，单独提取对应字段，缺失字段直接填充空值，确保每个数据列表长度一致。

修改后的代码 #

# 安装必要库（未安装时执行）
!pip install requests
!pip install beautifulsoup4
!pip install pandas

# 导入库
import requests
from bs4 import BeautifulSoup
import pandas as pd
from lxml import etree

link = 'https://listado.mercadolibre.cl/inmuebles/departamentos/venta/propiedades-usadas/_NoIndex_True#applied_filter_id%3DOPERATION_SUBTYPE%26applied_filter_name%3DModalidad%26applied_filter_order%3D5%26applied_value_id%3D244562%26applied_value_name%3DPropiedades+usadas%26applied_value_order%3D1%26applied_value_results%3D68139%26is_custom%3Dfalse'

# 初始化存储所有数据的列表
data = []

siguiente = link
while True:
    r = requests.get(siguiente)
    if r.status_code != 200:
        break
        
    soup = BeautifulSoup(r.content, 'html.parser')
    dom = etree.HTML(str(soup))
    
    # 获取当前页面所有房产条目
    items = soup.find_all('div', class_='ui-search-result__content-wrapper')
    
    for item in items:
        # 提取每个条目的字段，缺失则为空字符串
        item_data = {}
        
        # 标题
        title_tag = item.find('h2', class_='ui-search-item__title')
        item_data['titulos'] = title_tag.text.strip() if title_tag else ''
        
        # URL
        url_container = item.find('div', class_='ui-search-item__group__element ui-search-item__title-grid')
        url_tag = url_container.find('a') if url_container else None
        item_data['url'] = url_tag['href'] if url_tag else ''
        
        # 货币和价格
        currency_tag = item.find('span', class_='andes-money-amount__currency-symbol')
        item_data['moneda'] = currency_tag.text.strip() if currency_tag else ''
        
        price_tag = item.find('span', class_='andes-money-amount__fraction')
        item_data['precio'] = price_tag.text.strip() if price_tag else ''
        
        # 卧室数量
        dorm_tag = item.find('li', string=lambda text: text and 'dormitorios' in text)
        item_data['dormitorios'] = dorm_tag.text.strip() if dorm_tag else ''
        
        # 卫生间数量
        bath_tag = item.find('li', string=lambda text: text and 'baño' in text)
        item_data['baños'] = bath_tag.text.strip() if bath_tag else ''
        
        # 面积
        m2_tag = item.find('li', string=lambda text: text and 'm²' in text)
        item_data['m2'] = m2_tag.text.strip() if m2_tag else ''
        
        # 位置
        loc_tag = item.find('span', class_='ui-search-item__location-label')
        item_data['locacion'] = loc_tag.text.strip() if loc_tag else ''
        
        # 卖家
        seller_tag = item.find('div', class_='ui-search-item__official-store-grid')
        item_data['vendedor'] = seller_tag.text.strip() if seller_tag else ''
        
        data.append(item_data)
    
    # 处理分页
    ini_tag = soup.find('span', class_='andes-pagination__link')
    ini = int(ini_tag.text.strip()) if ini_tag else 1
    
    can_tag = soup.find('li', class_='andes-pagination__page-count')
    can = int(can_tag.text.split(" ")[1]) if can_tag else 1
    
    print(f"当前页面: {ini}/{can}")
    
    if ini == can:
        break
        
    next_link_tag = dom.xpath('//div[@class="ui-search-pagination"]/nav/ul/li[contains(@class,"--next")]/a')
    siguiente = next_link_tag[0].get('href') if next_link_tag else ''

# 转换为DataFrame并导出CSV
df = pd.DataFrame(data)
df.to_csv('Venta_Departamentos_Usados_MasRelevantes.csv', index=False, encoding='utf-8')

print(f"爬取完成，共获取 {len(df)} 条数据")

关键修改点 #

• 不再单独提取每个字段的全局列表，而是逐个处理房产条目，确保每个条目都有所有字段的对应值（空值或实际内容）
• 使用find方法结合条件判断提取字段，避免因部分条目缺失字段导致列表长度不一致
• 最后直接将字典列表转换为DataFrame，所有字段长度天然一致，解决CSV导出错误

内容的提问来源于stack exchange，提问作者JTB