无页码URL的多页面数据爬取问题求助

针对静态URL分页的场景，提供两种优先级不同的可行方案：

方案1：抓取AJAX接口（高效稳定） #

这类网站翻页通常依赖AJAX异步加载数据，而非刷新页面。你可以通过浏览器开发者工具定位真实数据接口：

1. 打开目标页面按F12进入开发者工具，切换到Network标签
2. 点击下一页按钮，观察XHR/Fetch请求，找到返回分页数据的接口（通常包含page、offset等参数）
3. 复制接口URL和请求参数，直接在Power Query中调用并遍历所有页码

示例代码（假设找到的接口为https://apps2.xxxxx.net/api/campaign/inquiries，每页25条）：

let
    // 定义获取单页数据的函数
    GetPage = (page as number) =>
        let
            Source = Json.Document(Web.Contents("https://apps2.xxxxx.net/api/campaign/inquiries", [
                Headers = [
                    #"Content-Type" = "application/json",
                    // 替换为浏览器中实际的Cookie、User-Agent
                    #"Cookie" = "your_cookie_here",
                    #"User-Agent" = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36..."
                ],
                Query = [
                    tag = "olb_inquiries",
                    page = Text.From(page),
                    pageSize = "25"
                ]
            ])),
            // 解析JSON为表格
            DataTable = Table.FromRecords(Source[data]),
            // 调整列类型（匹配实际返回字段）
            ChangedType = Table.TransformColumnTypes(DataTable,{{"ID", type text}, {"Request Type", type text}, {"Customers", type text}, {"Accounts", Int64.Type}, {"Agent", type text}, {"Queued Since", type datetime}})
        in
            ChangedType,

    // 获取总页数（假设接口返回total字段）
    TotalPages = let
        FirstPage = Json.Document(Web.Contents("https://apps2.xxxxx.net/api/campaign/inquiries", [
            Headers = [
                #"Cookie" = "your_cookie_here",
                #"User-Agent" = "your_user_agent_here"
            ],
            Query = [tag = "olb_inquiries", page = "1", pageSize = "25"]
        ]))
    in
        Number.RoundUp(FirstPage[total] / 25),

    // 生成页码列表并批量抓取
    PageList = List.Generate(() => 1, each _ <= TotalPages, each _ + 1),
    AllPages = List.Transform(PageList, GetPage),
    // 合并所有页面数据
    CombinedTables = Table.Combine(AllPages)
in
    CombinedTables

方案2：模拟浏览器翻页（适合无法找到API的场景） #

如果无法定位AJAX接口，可通过Power Query模拟点击下一页按钮，循环抓取所有页面：

let
    // 定义递归抓取函数
    ScrapePages = (currentPage as table, browser as any) =>
        let
            // 复用原有逻辑提取当前页数据
            CurrentData = let
                Source = browser,
                #"Extracted Table From Html" = Html.Table(Source, {{"Column1", "DIV.gridless > TABLE > * > TR > :nth-child(1)"}, {"Column2", "DIV.gridless > TABLE > * > TR > :nth-child(2)"}, {"Column3", "DIV.gridless > TABLE > * > TR > :nth-child(3)"}, {"Column4", "DIV.gridless > TABLE > * > TR > :nth-child(4)"}, {"Column5", "DIV.gridless > TABLE > * > TR > :nth-child(5)"}, {"Column6", "DIV.gridless > TABLE > * > TR > :nth-child(6)"}, {"Column7", "DIV.gridless > TABLE > * > TR > :nth-child(7)"}}, [RowSelector="DIV.gridless > TABLE > * > TR"]),
                #"Promoted Headers" = Table.PromoteHeaders(#"Extracted Table From Html", [PromoteAllScalars=true]),
                #"Changed Type" = Table.TransformColumnTypes(#"Promoted Headers",{{"", type text}, {"ID", Int64.Type}, {"Request Type", type text}, {"Customers", type text}, {"Accounts", Int64.Type}, {"Agent", type text}, {"Queued Since", type datetime}}),
                #"Removed Columns" = Table.RemoveColumns(#"Changed Type",{""}),
                #"Changed Type1" = Table.TransformColumnTypes(#"Removed Columns",{{"ID", type text}})
            in
                ChangedType1,

            // 检查下一页按钮是否存在（替换为实际按钮的CSS选择器）
            NextButtonExists = try Html.SelectSingleNode(browser, "button.next-page") <> null otherwise false,

            // 递归抓取下一页
            NextPage = if NextButtonExists then
                let
                    ClickNext = Web.BrowserContents("https://apps2.xxxxx.net/customerservice/campaign/?tag=olb_inquiries", [
                        WaitFor = 2000,
                        NavigationMethod = "Click",
                        ElementSelector = "button.next-page"
                    ]),
                    NextData = ScrapePages(CurrentData, ClickNext)
                in
                    Table.Combine({CurrentData, NextData})
            else
                CurrentData
        in
            NextPage,

    // 初始化抓取第一页
    FirstPage = Web.BrowserContents("https://apps2.xxxxx.net/customerservice/campaign/?tag=olb_inquiries", [WaitFor = 3000]),
    AllData = ScrapePages(Table.FromRows({}), FirstPage)
in
    AllData

注意事项 #

• 方案1需替换Cookie和User-Agent为浏览器中的实际值（可从开发者工具请求头复制）
• 方案2需修改ElementSelector为网站下一页按钮的真实CSS选择器（右键检查元素获取）
• 若网站有反爬机制，可调整WaitFor参数增加延迟

内容的提问来源于stack exchange，提问作者Soumya