如何修改沃尔玛商品抓取代码获取单个商品详情页的JSON响应

核心修改逻辑 #

• 原有搜索页抓取逻辑保留，新增收集商品详情页URL的步骤
• 无需额外构造复杂的GraphQL请求：沃尔玛详情页HTML中内置了全量商品预加载JSON数据，直接提取即可获得商品完整描述、全量高清缩略图、规格参数、售后政策等全部拓展信息
• 新增请求间隔逻辑，避免触发反爬限制

前置依赖安装 #

需要安装HTML解析库，执行命令：

pip install beautifulsoup4

修改后完整代码 #

import requests
import json
import time
from bs4 import BeautifulSoup

# 原有搜索请求参数保持不变
data= {
   
   "query":"query Browse( $query:String $page:Int $prg:Prg! $facet:String $sort:Sort $catId:String! $max_price:String $min_price:String $module_search:String $affinityOverride:AffinityOverride $ps:Int $ptss:String $beShelfId:String $fitmentFieldParams:JSON ={}$fitmentSearchParams:JSON ={}$rawFacet:String $seoPath:String $trsp:String $fetchMarquee:Boolean! $fetchSkyline:Boolean! $additionalQueryParams:JSON ={}){search( query:$query page:$page prg:$prg facet:$facet sort:$sort cat_id:$catId max_price:$max_price min_price:$min_price module_search:$module_search affinityOverride:$affinityOverride additionalQueryParams:$additionalQueryParams ps:$ps ptss:$ptss trsp:$trsp _be_shelf_id:$beShelfId ){query searchResult{...BrowseResultFragment}}contentLayout( channel:\"WWW\" pageType:\"BrowsePage\" tenant:\"WM_GLASS\" version:\"v1\" searchArgs:{query:$query cat_id:$catId _be_shelf_id:$beShelfId prg:$prg}){modules{...ModuleFragment configs{...on EnricherModuleConfigsV1{zoneV1}__typename...on _TempoWM_GLASSWWWSearchSortFilterModuleConfigs{facetsV1{...FacetFragment}}...on TempoWM_GLASSWWWPillsModuleConfigs{moduleSource pillsV2{...PillsModuleFragment}}...on TempoWM_GLASSWWWSearchFitmentModuleConfigs{fitments( fitmentSearchParams:$fitmentSearchParams fitmentFieldParams:$fitmentFieldParams ){...FitmentFragment sisFitmentResponse{...BrowseResultFragment}}}...on TempoWM_GLASSWWWStoreSelectionHeaderConfigs{fulfillmentMethodLabel storeDislayName}...on TempoWM_GLASSWWWBreadcrumbConfigs{_rawConfigs}...on TempoWM_GLASSWWWSponsoredProductCarouselConfigs{_rawConfigs}...PopularInModuleFragment...CopyBlockModuleFragment...BannerModuleFragment...HeroPOVModuleFragment...InlineSearchModuleFragment...MarqueeDisplayAdConfigsFragment @include(if:$fetchMarquee)...SkylineDisplayAdConfigsFragment @include(if:$fetchSkyline)...HorizontalChipModuleConfigsFragment}}...LayoutFragment pageMetadata{location{postalCode stateOrProvinceCode city storeId}pageContext}}seoBrowseMetaData( id:$catId facets:$rawFacet path:$seoPath facet_query_param:$facet _be_shelf_id:$beShelfId ){metaTitle metaDesc metaCanon h1}}fragment BrowseResultFragment on SearchInterface{title aggregatedCount...BreadCrumbFragment...DebugFragment...ItemStacksFragment...PageMetaDataFragment...PaginationFragment...RequestContextFragment...ErrorResponse modules{facetsV1{...FacetFragment}pills{...PillsModuleFragment}}}fragment ModuleFragment on TempoModule{name version type moduleId schedule{priority}matchedTrigger{zone}}fragment LayoutFragment on ContentLayout{layouts{id layout}}fragment BreadCrumbFragment on SearchInterface{breadCrumb{id name url}}fragment DebugFragment on SearchInterface{debug{sisUrl}}fragment ItemStacksFragment on SearchInterface{itemStacks{displayMessage meta{adsBeacon{adUuid moduleInfo max_ads}query stackId stackType title layoutEnum totalItemCount totalItemCountDisplay viewAllParams{query cat_id sort facet affinityOverride recall_set min_price max_price}}itemsV2{...ItemFragment...InGridMarqueeAdFragment}}}fragment ItemFragment on Product{__typename id usItemId fitmentLabel name checkStoreAvailabilityATC seeShippingEligibility brand type shortDescription imageInfo{...ProductImageInfoFragment}canonicalUrl externalInfo{url}category{path{name url}}badges{flags{...on BaseBadge{key text type id}}tags{...on BaseBadge{key text type}}}classType averageRating numberOfReviews esrb mediaRating salesUnitType sellerId sellerName hasSellerBadge availabilityStatusV2{display value}productLocation{displayValue aisle{zone aisle}}badge{type dynamicDisplayName}fulfillmentSpeed offerId preOrder{...PreorderFragment}priceInfo{...ProductPriceInfoFragment}variantCriteria{...VariantCriteriaFragment}fulfillmentBadge fulfillmentTitle fulfillmentType brand manufacturerName showAtc sponsoredProduct{spQs clickBeacon spTags}showOptions}fragment ProductImageInfoFragment on ProductImageInfo{thumbnailUrl}fragment ProductPriceInfoFragment on ProductPriceInfo{priceRange{minPrice maxPrice}currentPrice{...ProductPriceFragment}wasPrice{...ProductPriceFragment}unitPrice{...ProductPriceFragment}listPrice{...ProductPriceFragment}shipPrice{...ProductPriceFragment}subscriptionPrice{priceString subscriptionString}priceDisplayCodes{priceDisplayCondition finalCostByWeight}}fragment PreorderFragment on PreOrder{isPreOrder preOrderMessage preOrderStreetDateMessage}fragment ProductPriceFragment on ProductPrice{price priceString}fragment VariantCriteriaFragment on VariantCriterion{name type id isVariantTypeSwatch variantList{id images name rank swatchImageUrl availabilityStatus products selectedProduct{canonicalUrl usItemId}}}fragment InGridMarqueeAdFragment on MarqueePlaceholder{__typename type moduleLocation lazy}fragment PageMetaDataFragment on SearchInterface{pageMetadata{storeSelectionHeader{fulfillmentMethodLabel storeDislayName}title canonical description location{addressId}}}fragment PaginationFragment on SearchInterface{paginationV2{maxPage pageProperties}}fragment RequestContextFragment on SearchInterface{requestContext{vertical isFitmentFilterQueryApplied searchMatchType categories{id name}}}fragment ErrorResponse on SearchInterface{errorResponse{correlationId source errors{errorType statusCode statusMsg source}}}fragment PillsModuleFragment on PillsSearchInterface{title url image:imageV1{src alt}baseSeoURL}fragment BannerModuleFragment on TempoWM_GLASSWWWSearchBannerConfigs{moduleType viewConfig{title image imageAlt displayName description url urlAlt appStoreLink appStoreLinkAlt playStoreLink playStoreLinkAlt}}fragment PopularInModuleFragment on TempoWM_GLASSWWWPopularInBrowseConfigs{seoBrowseRelmData(id:$catId){relm{id name url}}}fragment CopyBlockModuleFragment on TempoWM_GLASSWWWCopyBlockConfigs{copyBlock(id:$catId){cwc}}fragment FacetFragment on Facet{name type layout min max selectedMin selectedMax unboundedMax stepSize values{id name description type itemCount isSelected baseSeoURL}}fragment FitmentFragment on Fitments{partTypeIDs result{status formId position quantityTitle extendedAttributes{...FitmentFieldFragment}labels{...LabelFragment}resultSubTitle}labels{...LabelFragment}savedVehicle{vehicleYear{...VehicleFieldFragment}vehicleMake{...VehicleFieldFragment}vehicleModel{...VehicleFieldFragment}additionalAttributes{...VehicleFieldFragment}}fitmentFields{...VehicleFieldFragment}fitmentForms{id fields{...FitmentFieldFragment}title labels{...LabelFragment}}}fragment LabelFragment on FitmentLabels{ctas{...FitmentLabelEntityFragment}messages{...FitmentLabelEntityFragment}links{...FitmentLabelEntityFragment}images{...FitmentLabelEntityFragment}}fragment FitmentLabelEntityFragment on FitmentLabelEntity{id label}fragment VehicleFieldFragment on FitmentVehicleField{id label value}fragment FitmentFieldFragment on FitmentField{id displayName value extended data{value label}dependsOn}fragment HeroPOVModuleFragment on TempoWM_GLASSWWWHeroPovConfigsV1{povCards{card{povStyle image{mobileImage{...TempoCommonImageFragment}desktopImage{...TempoCommonImageFragment}}heading{text textColor textSize}subheading{text textColor}detailsView{backgroundColor isTransparent}ctaButton{button{linkText clickThrough{value}}}logo{...TempoCommonImageFragment}links{link{linkText}}}}}fragment TempoCommonImageFragment on TempoCommonImage{src alt assetId uid clickThrough{value}}fragment InlineSearchModuleFragment on TempoWM_GLASSWWWInlineSearchConfigs{headingText placeholderText}fragment MarqueeDisplayAdConfigsFragment on TempoWM_GLASSWWWMarqueeDisplayAdConfigs{_rawConfigs ad{...DisplayAdFragment}}fragment DisplayAdFragment on Ad{...AdFragment adContent{type data{__typename...AdDataDisplayAdFragment}}}fragment AdFragment on Ad{status moduleType platform pageId pageType storeId stateCode zipCode pageContext moduleConfigs adsContext adRequestComposite}fragment AdDataDisplayAdFragment on AdData{...on DisplayAd{json status}}fragment SkylineDisplayAdConfigsFragment on TempoWM_GLASSWWWSkylineDisplayAdConfigs{_rawConfigs ad{...SkylineDisplayAdFragment}}fragment SkylineDisplayAdFragment on Ad{...SkylineAdFragment adContent{type data{__typename...SkylineAdDataDisplayAdFragment}}}fragment SkylineAdFragment on Ad{status moduleType platform pageId pageType storeId stateCode zipCode pageContext moduleConfigs adsContext adRequestComposite}fragment SkylineAdDataDisplayAdFragment on AdData{...on DisplayAd{json status}}fragment HorizontalChipModuleConfigsFragment on TempoWM_GLASSWWWHorizontalChipModuleConfigs{chipModuleSource:moduleSource chipModule{title url{linkText title clickThrough{type value}}}chipModuleWithImages{title url{linkText title clickThrough{type value}}image{alt clickThrough{type value}height src title width}}}",
   "variables":{
      "id":"",
      "affinityOverride":"default",
      "dealsId":"",
      "query":"",
      "page":1,
      "prg":"desktop",
      "catId":"3920",
      "facet":"",
      "sort":"best_seller",
      "rawFacet":"",
      "seoPath":"",
      "ps":40,
      "ptss":"",
      "trsp":"",
      "beShelfId":"",
      "recall_set":"",
      "module_search":"",
      "min_price":"",
      "max_price":"",
      "storeSlotBooked":"",
      "additionalQueryParams":None,
      "fitmentFieldParams":None,
      "fitmentSearchParams":{
         "id":"",
         "affinityOverride":"default",
         "dealsId":"",
         "query":"",
         "page":1,
         "prg":"desktop",
         "catId":"3920",
         "facet":"",
         "sort":"best_seller",
         "rawFacet":"",
         "seoPath":"",
         "ps":40,
         "ptss":"",
         "trsp":"",
         "beShelfId":"",
         "recall_set":"",
         "module_search":"",
         "min_price":"",
         "max_price":"",
         "storeSlotBooked":"",
         "additionalQueryParams":None,
         "cat_id":"3920",
         "_be_shelf_id":""
      },
      "fetchMarquee":True,
      "fetchSkyline":True,
      "fetchSbaTop":False
   }

}
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
    'content-type':'application/json',
    'wm_mp': 'true',
    'wm_page_url': 'https://www.walmart.com/browse/books/3920?sort=best_seller&affinityOverride=default',
    'wm_qos.correlation_id': 'FWpup9KEKUrLFOY68gppqfprABL16K6qE76g',
    'x-apollo-operation-name': 'Browse',
    'x-enable-server-timing': '1',
    'x-latency-trace': '1',
    'x-o-ccm': 'server',
    'x-o-correlation-id': 'FWpup9KEKUrLFOY68gppqfprABL16K6qE76g',
    'x-o-gql-query': 'query Browse',
    'x-o-market': 'us',
    'x-o-platform': 'rweb',
    'x-o-platform-version': 'main-176-e8acb5',
    'x-o-segment': 'oaoh'
    }
params= {
  "affinityOverride": "default",
    "page": "1",
    "prg": "desktop",
    "catId": "3920",
    "sort": "best_seller",
    "ps": "40",
    "fetchMarquee": "true",
    "fetchSkyline": "true",
    "fetchSbaTop": "false"}

# 存储所有商品详情页URL
product_urls = []

# 第一步：抓取搜索结果，收集商品URL
for i in range(1,25,1):
    params['maxPage']=i
    api_url='https://www.walmart.com/orchestra/home/graphql/browse'
    try:
        resp = requests.post(api_url, data=json.dumps(data), headers=headers,params=params, timeout=10)
        r=resp.json()
        items = r['data']['search']['searchResult']['itemStacks'][0]['itemsV2']
        for item in items:
            if 'canonicalUrl' in item:
                # 拼接完整URL，去掉无用查询参数
                full_url = f"https://www.walmart.com{item['canonicalUrl'].split('?')[0]}"
                if full_url not in product_urls:
                    product_urls.append(full_url)
        print(f"第{i}页抓取完成，累计获取商品数：{len(product_urls)}")
        time.sleep(1)
    except Exception as e:
        print(f"第{i}页抓取失败：{e}")
        time.sleep(2)
        continue

# 第二步：批量抓取商品详情
detail_headers = headers.copy()
for idx, p_url in enumerate(product_urls):
    print(f"\n正在抓取第{idx+1}/{len(product_urls)}个商品：{p_url}")
    try:
        # 更新请求头当前页参数，降低反爬识别概率
        detail_headers['wm_page_url'] = p_url
        resp = requests.get(p_url, headers=detail_headers, timeout=10)
        if resp.status_code != 200:
            print(f"请求失败，状态码：{resp.status_code}")
            time.sleep(3)
            continue
        # 提取页面内嵌的全量商品JSON数据
        soup = BeautifulSoup(resp.text, 'html.parser')
        next_data_tag = soup.find('script', id='__NEXT_DATA__')
        if not next_data_tag:
            print("未找到商品数据")
            time.sleep(3)
            continue
        all_data = json.loads(next_data_tag.string)
        # 提取全量商品信息，可按需取对应字段
        full_product_info = all_data['props']['pageProps']['initialData']['data']['product']
        # 示例字段提取
        product_name = full_product_info.get('name', '')
        product_desc = full_product_info.get('shortDescription', '')
        all_images = full_product_info.get('imageInfo', {}).get('allImages', [])
        print(f"商品名称：{product_name}")
        print(f"商品描述长度：{len(product_desc)} 字符")
        print(f"商品图片数量：{len(all_images)} 张")
        # 此处可添加数据存储逻辑（存本地文件/数据库等）
        time.sleep(2)
    except Exception as e:
        print(f"商品抓取失败：{e}")
        time.sleep(3)
        continue

注意事项 #

• 可根据自身需求调整time.sleep()的间隔时长，降低IP被封禁的概率
• 可随机生成wm_qos.correlation_id和x-o-correlation_id参数替换固定值，进一步降低反爬识别概率
• 全量商品字段均可从full_product_info中提取，包括长描述、规格参数、变体信息、售后政策等

内容的提问来源于stack exchange，提问作者e.iluf