Excel VBA爬取idealista页面手机号失败，寻求技术帮助

需要爬取URL https://www.idealista.com/inmueble/96599356/ 中的手机号607176984，目前已成功获取页面其他数据，但无法提取目标手机号。

页面相关HTML代码（Chrome获取） #

<div class="ide-box-contact module-contact-gray contact-data-container ">
<h2 class="txt-big txt-bold mb-small">
Anunciante
</h2>
<div id="contact-phones-container" class="hidden-contact-phones no-form">
<a role="button" aria-label="contact phone" class="icon-phone hidden-contact-phones_link" href="">
<span class="hidden-contact-phones_text">Ver teléfono</span>
<span class="animation_circle">
<span class="circle"></span>
<span class="circle"></span>
<span class="circle"></span>
</span>
</a>
</div>
<div class="professional-name">
<div class="name">
Particular
</div>
<span class="particular">
<span class="icon-block"></span>

已尝试的Excel VBA代码 #

Sub detalles_de_cada_trastero()
ActiveSheet.Select

hoja_actual = ActiveSheet.Name
   
Application.ScreenUpdating = False
    
Dim XMLPage As New MSXML2.XMLHTTP60
Dim htmldoc As New MSHTML.HTMLDocument
Dim htmlim As MSHTML.IHTMLElement
Dim htmlims As MSHTML.IHTMLElementCollection
Dim url As String
    
url = "https://www.idealista.com/inmueble/96599356/"   
    
XMLPage.Open "Get", url_corregida, False
XMLPage.setRequestHeader "Content-Type", "text/xml"
On Error Resume Next
XMLPage.send
htmldoc.body.innerHTML = XMLPage.responseText

' 第一种尝试（无效）
Set htmlims = htmldoc.getElementsByClassName("hidden-contact-phones_text")
For Each htmlim In htmlims
    Sheets("hoja_actual").Cells(Fila, 12).Value = htmlim.innerText        
    texto = htmlim.innerText   ' 仅获取到"Ver teléfono"        
Next htmlim
'====================================================
   
' 第二种尝试
Set htmlims = htmldoc.getElementsByClassName("icon-phone hidden-contact-phones_formatted-phone _mobilePhone")
For Each htmlim In htmlims ' 未进入循环，直接跳过              
    texto = htmlim.getAttribute("href")        
    'Sheets("hoja_actual").Cells(Fila, 12).Value = htmlim.innerText        
Next htmlim
'===============================================================
' 第三种尝试
texto = htmldoc.getElementsByClassName("icon-phone hidden-contact-phones_formatted-phone _mobilePhone")  ' 返回"nothing"
texto = htmldoc.getElementsByClassName("icon-phone hidden-contact-phones_formatted-phone _mobilePhone").Item
texto = htmldoc.getElementsByClassName("icon-phone hidden-contact-phones_formatted-phone _mobilePhone").toString 
'===============================================================
' 第四种尝试
texto = htmldoc.getElementById("contact-phones-container")
Set htmlims = htmldoc.getElementById("contact-phones-container")    ' 报错！
For Each htmlim In htmlims            
    texto_1 = htmlim.Children(0).innerText  ' 输出"profesonal"
    texto = htmlim.getAttribute("href")  
    texto_2 = htmlim.Children(1).innerText         
Next

解决方案 #

问题根源 #

目标手机号是动态加载的：页面初始HTML仅显示"Ver teléfono"按钮，点击后才会通过JavaScript请求加载真实手机号。你用MSXML2.XMLHTTP60获取的是初始静态HTML，自然拿不到动态渲染的手机号元素。

两种解决方法 #

方法1：使用IE控件模拟交互

利用InternetExplorer对象打开页面，等待JavaScript加载完成后再获取元素：

Sub GetIdealistaPhone()
    Dim ie As Object
    Dim doc As Object
    Dim phoneElem As Object
    Dim url As String
    
    url = "https://www.idealista.com/inmueble/96599356/"
    
    ' 创建IE对象
    Set ie = CreateObject("InternetExplorer.Application")
    ie.Visible = True ' 调试时设为True，发布后可改为False
    
    ' 打开页面并等待加载完成
    ie.navigate url
    Do While ie.Busy Or ie.readyState <> 4
        DoEvents
    Loop
    
    ' 等待动态内容加载（可根据实际情况调整等待时长）
    Application.Wait Now + TimeValue("00:00:03")
    
    ' 尝试直接获取手机号元素
    On Error Resume Next
    Set phoneElem = ie.document.querySelector(".hidden-contact-phones_formatted-phone._mobilePhone")
    On Error GoTo 0
    
    If Not phoneElem Is Nothing Then
        ' 去掉href里的tel:前缀，提取纯手机号
        ActiveSheet.Cells(Fila, 12).Value = Replace(phoneElem.getAttribute("href"), "tel:", "")
    Else
        ' 点击"Ver teléfono"按钮后再次尝试获取
        ie.document.querySelector(".hidden-contact-phones_link").Click
        Application.Wait Now + TimeValue("00:00:02")
        Set phoneElem = ie.document.querySelector(".hidden-contact-phones_formatted-phone._mobilePhone")
        If Not phoneElem Is Nothing Then
            ActiveSheet.Cells(Fila, 12).Value = Replace(phoneElem.getAttribute("href"), "tel:", "")
        Else
            ActiveSheet.Cells(Fila, 12).Value = "未获取到手机号"
        End If
    End If
    
    ' 关闭IE并释放对象
    ie.Quit
    Set ie = Nothing
End Sub

方法2：直接调用API接口（更高效）

抓包查看点击"Ver teléfono"时的请求，idealista会调用专属接口返回手机号。直接请求该接口无需模拟浏览器：

1. 用Chrome开发者工具抓包，找到包含房源ID（如96599356）的手机号接口URL
2. 复制浏览器请求头中的User-Agent、Cookie等必要参数
3. 解析接口返回的JSON数据提取手机号

示例代码（需添加Microsoft Scripting Runtime引用处理JSON，或使用VBA-JSON库）：

Sub GetPhoneViaAPI()
    Dim XMLPage As New MSXML2.XMLHTTP60
    Dim jsonText As String
    Dim jsonObj As Object
    Dim url As String
    
    ' 替换为抓包得到的真实接口URL
    url = "https://www.idealista.com/api/v1/contact/phone?propertyId=96599356"
    
    XMLPage.Open "GET", url, False
    ' 添加从浏览器复制的请求头
    XMLPage.setRequestHeader "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
    XMLPage.setRequestHeader "Cookie", "替换为你的浏览器Cookie"
    XMLPage.send
    
    If XMLPage.Status = 200 Then
        jsonText = XMLPage.responseText
        Set jsonObj = CreateObject("Scripting.Dictionary")
        ' 解析JSON（需JSON解析工具，示例假设手机号字段为"phone"）
        ActiveSheet.Cells(Fila, 12).Value = jsonObj("phone")
    Else
        ActiveSheet.Cells(Fila, 12).Value = "接口请求失败"
    End If
    
    Set XMLPage = Nothing
    Set jsonObj = Nothing
End Sub

注意事项 #

• 方法1中的Fila变量需确保已正确定义并赋值
• 方法2的接口URL和请求头可能随网站更新变化，需定期抓包验证
• 频繁爬取可能触发反爬机制，建议添加请求间隔

内容的提问来源于stack exchange，提问作者FrancisoRo