Expedia POST请求处理及酒店评论全量抓取技术问询

1. Hotels.com全量抓取评论实现 #

原代码仅返回50条评论，核心原因是GraphQL查询的分页机制：单请求默认返回单页数据，要获取全部评论，需利用接口返回的分页信息循环发起请求。

核心逻辑说明 #

原请求的variables参数中包含paginationURL，初始为空；接口返回的data.reviews.body.reviewContent.pagination里有三个关键字段：

• currentPage：当前页码
• nextURL：下一页的分页令牌（非完整URL）
• totalPages：总页数

只需在每次请求后，将nextURL赋值给下一次请求的paginationURL，直到nextURL为空，即可完成全量抓取。

修改后的代码 #

import trio
import httpx
import pandas as pd


async def main():
    async with httpx.AsyncClient(timeout=None) as client:
        all_reviews = []
        # 初始请求参数，paginationURL为空
        data = {
            "operationName": "reviewsQuery",
            "query": "query reviewsQuery($hotelId: String!, $reviewType: String, $reviewOrder: String, $tripTypeFilter: String, $paginationURL: String) {\n  reviews(\n    hotelId: $hotelId\n    reviewType: $reviewType\n    reviewOrder: $reviewOrder\n    tripTypeFilter: $tripTypeFilter\n    paginationURL: $paginationURL\n  ) {\n    body {\n      reviewContent {\n        filters {\n          type\n          name\n          count\n          url\n          __typename\n        }\n        overall {\n          selectedFilterType\n          rating\n          badgeText\n          total\n          scores {\n            score\n            count\n            url\n            __typename\n          }\n          ratingAspects {\n            cleanliness\n            service\n            comfort\n            condition\n            neighbourhood\n            __typename\n          }\n          whatGuestsSay {\n            type\n            text\n            __typename\n          }\n          topRated {\n            category\n            explanation\n            __typename\n          }\n          __typename\n        }\n        sort {\n          url\n          options {\n            value\n            label\n            __typename\n          }\n          __typename\n        }\n        reviews {\n          hermes {\n            groups {\n              separatorText\n              items {\n                itineraryId\n                brand\n                googleTranslateEnabled\n                reviewDbDate\n                ...GuestReviewsFragment\n                __typename\n              }\n              __typename\n            }\n            __typename\n          }\n          __typename\n        }\n        pagination {\n          currentPage\n          nextURL\n          totalPages\n          __typename\n        }\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n}\n\nfragment GuestReviewsFragment on ReviewsItem {\n  genuineMsg\n  tripType\n  tripTypeText\n  reviewDate\n  reviewSubmitDate\n  rating\n  reviewer {\n    name\n    locality\n    locale\n    __typename\n  }\n  badge\n  summary\n  description\n  __typename\n}\n",
            "variables": {
                "hotelId": "344560",
                "reviewOrder": "date_newest_first",
                "reviewType": "brand",
                "tripTypeFilter": "all",
                "paginationURL": ""
            }
        }

        while True:
            r = await client.post('https://fr.hotels.com/kes/graphql', json=data)
            response_data = r.json()['data']['reviews']['body']['reviewContent']
            
            # 提取当前页评论
            for x in response_data['reviews']['hermes']['groups']:
                all_reviews.extend(x['items'])
            
            # 获取分页信息
            pagination = response_data['pagination']
            next_url = pagination.get('nextURL')
            print(f"已抓取第{pagination['currentPage']}/{pagination['totalPages']}页")
            
            # 无下一页则终止循环
            if not next_url:
                break
            
            # 更新分页令牌，准备下一次请求
            data['variables']['paginationURL'] = next_url

        # 转换为DataFrame输出
        df = pd.DataFrame(all_reviews)
        print(df)


if __name__ == "__main__":
    trio.run(main)

2. Expedia.com适配问题解决 #

错误1：gaierror: [Errno -2] Name or service not known #

该错误是因为请求的域名fr.expedia.com/kes/graphql不存在，Expedia的GraphQL端点与Hotels.com完全不同，不能直接复用路径。需通过浏览器抓包工具（如Chrome开发者工具），获取Expedia真实的评论查询端点和请求结构。

错误2：429 Client Error: Too Many Requests #

这是触发了Expedia的频率限制反爬机制，解决方法包括：

• 添加真实的浏览器User-Agent
• 增加请求间隔（如每次请求后休眠2-3秒）
• 必要时使用代理IP
• 携带从浏览器复制的有效Cookie

适配后的参考代码 #

（注：Expedia的GraphQL结构可能随时更新，建议自行抓包验证最新请求参数）

import requests
import pandas as pd
import time

def main():
    headers = {
        "Content-Type": "application/json",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
        # 可添加从浏览器复制的Cookie，提升请求成功率
        # "Cookie": "your_cookie_here"
    }

    all_reviews = []
    start_index = 0
    page_size = 10  # Expedia单页默认返回10条，可根据抓包调整

    while True:
        data = [
            {
                "operationName": "PropertyFilteredReviewsQuery",
                "variables": {
                    "context": {
                        "siteId": 1,
                        "locale": "en_US",
                        "eapid": 0,
                        "currency": "USD",
                        "device": {"type": "DESKTOP"},
                        "identity": {
                            "duaid": "-1",
                            "expUserId": "-1",
                            "tuid": "-1",
                            "authState": "ANONYMOUS"
                        },
                        "privacyTrackingState": "CAN_TRACK"
                    },
                    "propertyId": "24625",
                    "searchCriteria": {
                        "primary": {
                            "dateRange": None,
                            "rooms": [{"adults": 2}],
                            "destination": {"regionId": "178305"}
                        },
                        "secondary": {
                            "booleans": [
                                {"id": "includeRecentReviews", "value": True},
                                {"id": "includeRatingsOnlyReviews", "value": True}
                            ],
                            "counts": [
                                {"id": "startIndex", "value": start_index},
                                {"id": "size", "value": page_size}
                            ],
                            "selections": [
                                {"id": "sortBy", "value": "NEWEST_TO_OLDEST_BY_LANGUAGE"},
                                {"id": "searchTerm", "value": ""}
                            ]
                        }
                    }
                },
                "query": "query PropertyFilteredReviewsQuery($context: ContextInput!, $propertyId: String!, $searchCriteria: PropertySearchCriteriaInput!) {\n  propertyReviewSummaries(\n    context: $context\n    propertyIds: [$propertyId]\n    searchCriteria: $searchCriteria\n  ) {\n    ...__PropertyReviewSummaryFragment\n    __typename\n  }\n  propertyInfo(context: $context, propertyId: $propertyId) {\n    id\n    reviewInfo(searchCriteria: $searchCriteria) {\n      ...__PropertyReviewsListFragment\n      sortAndFilter {\n        ...TravelerTypeFragment\n        ...SortTypeFragment\n        ...SearchTextFragment\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n}\n\nfragment __PropertyReviewSummaryFragment on PropertyReviewSummary {\n  id\n  reviewCount\n  averageOverallRating\n  ratingBreakdown {\n    rating\n    count\n    __typename\n  }\n  __typename\n}\n\nfragment __PropertyReviewsListFragment on PropertyReviewsList {\n  reviews {\n    content {\n      reviews {\n        id\n        rating\n        title\n        text\n        travelerType\n        stayDate\n        reviewer {\n          name\n          location\n          __typename\n        }\n        __typename\n      }\n      __typename\n    }\n    hasNextPage\n    __typename\n  }\n  __typename\n}\n\nfragment TravelerTypeFragment on SortAndFilterSection {\n  id\n  options {\n    id\n    value\n    label\n    count\n    __typename\n  }\n  __typename\n}\n\nfragment SortTypeFragment on SortAndFilterSection {\n  id\n  options {\n    id\n    value\n    label\n    __typename\n  }\n  __typename\n}\n\nfragment SearchTextFragment on SortAndFilterSection {\n  id\n  placeholder\n  __typename\n}"
            }
        ]

        try:
            response = requests.post("https://www.expedia.com/graphql", json=data, headers=headers)
            response.raise_for_status()

            reviews_data = response.json()[0]['data']['propertyInfo']['reviewInfo']['reviews']
            current_reviews = reviews_data['content']['reviews']
            all_reviews.extend(current_reviews)

            # 检查是否还有下一页
            if not reviews_data['hasNextPage']:
                break

            start_index += page_size
            time.sleep(2)  # 添加请求间隔，避免触发反爬

        except Exception as e:
            print(f"请求出错: {e}")
            break

    df = pd.DataFrame(all_reviews)
    print(df)

if __name__ == "__main__":
    main()

内容的提问来源于stack exchange，提问作者Dev Burman