Python网页爬取遇401错误：需配置HTTP请求头的技术求助

API 探路者

2026-8-13

问题概述

我是编程新手，完全摸不着头绪，就像一只学编程的猫。我在Ubuntu 22.04.1 LTS系统上用PyCharm写Python代码，用Insomnia调试。代码是合并了两篇教程的内容，加了页面滚动的JS代码，后来换成httpx并添加了Insomnia里的请求头，但现在搞不清楚请求头的正确顺序，尤其是Cookie的位置，不知道怎么让请求不被识别成爬虫，陷入了困境。

我的代码

# sad_scrape_code_attempt.py

import time
import httpx
from playwright.sync_api import sync_playwright

HEADERS = {
    "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0",
    "Accept": "*/*",
    "Accept-Language": "en-US,en;q=0.5",
    "Accept-Encoding": "gzip, deflate, br",
    "Referer": "https://shop.metro.bg/shop/cart",
    "CallTreeId": "||BTOC-1BF47A0C-CCDD-47BB-A9DA-592009B5FB38",
    "Content-Type": "application/json; charset=UTF-8",
    "x-timeout-ms": "5000",
    "DNT": "1",
    "Connection": "keep-alive",

    "Sec-Fetch-Dest": "empty",
    "Sec-Fetch-Mode": "cors",
    "Sec-Fetch-Site": "same-origin"
}

def get_cookie_playwright():
    with sync_playwright() as p:
        browser = p.firefox.launch(headless=False, slow_mo=50)
        context = browser.new_context()
        page = context.new_page()
        page.goto('https://shop.metro.bg/shop/cart')
        page.fill('input#user_id', 'the_sad_cat_username')
        page.fill('input#password', 'the_sad_cat_password')
        page.click('button[type=submit]')
        page.click('button.btn-primary.accept-btn.field-accept-button-name')
        page.evaluate(
            """
            var intervalID = setInterval(function () {
                var scrollingElement = (document.scrollingElement || document.body);
                scrollingElement.scrollTop = scrollingElement.scrollHeight;
            }, 200);

            """
        )
        prev_height = None
        while True:
            curr_height = page.evaluate('(window.innerHeight + window.scrollY)')
            if not prev_height:
                prev_height = curr_height
                time.sleep(1)
            elif prev_height == curr_height:
                page.evaluate('clearInterval(intervalID)')
                break
            else:
                prev_height = curr_height
                time.sleep(1)

# print(context.cookies())
        cookie_for_requests = context.cookies()[11]['value']
        browser.close()
    return cookie_for_requests

def req_with_cookie(cookie_for_requests):
    cookies = dict(
        Cookie=f'BIGipServerbetty.metrosystems.net-80={cookie_for_requests};')
    r = httpx.get('https://shop.metro.bg/ordercapture.customercart.v1/carts/alias/current', cookies=cookies)
    return r.text

if __name__ == '__main__':
    data = req_with_cookie(get_cookie_playwright())
    print(data)

# Used packages

#Playwright
#PyTest
#PyTest-Playwirght
#JavaScript
#TypeScript
#httpx

参考的代码（Insomnia生成）

import requests

url = "https://shop.metro.bg/ordercapture.customercart.v1/carts/alias/current"

querystring = {"customerId":"1001100022726355","cardholderNumber":"1","storeId":"00022","country":"BG","locale":"bg-BG","fsdAddressId":"1001100022726355994-AD0532EI","__t":"1668082324830"}

headers = {
    "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0",
    "Accept": "*/*",
    "Accept-Language": "en-US,en;q=0.5",
    "Accept-Encoding": "gzip, deflate, br",
    "Referer": "https://shop.metro.bg/shop/cart",
    "CallTreeId": "||BTOC-1BF47A0C-CCDD-47BB-A9DA-592009B5FB38",
    "Content-Type": "application/json; charset=UTF-8",
    "x-timeout-ms": "5000",
    "DNT": "1",
    "Connection": "keep-alive",
    "Cookie": "selectedLocale_BG=bg-BG; BIGipServerbetty.metrosystems.net-80=!DHrH53oKfz3YHEsEdKzHuTxiWd+ak6uA3C+dv7oHRDuEk+ScE0MCf7DPAzLTCmE+GApsIOFM2GKufYk=; anonymousUserId=24EE2F84-55B5-4F94-861E-33C4EB770DC6; idamUserIdToken=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IktfYWE1NTAxNWEtMjA2YS0xMWVkLTk4ZDUtZTJjYzEyYjBkYzUwIn0.eyJleHAiOjE2NjgwODQxMjIsImlhdCI6MTY2ODA4MjMyMiwiYXVkIjoiQlRFWCIsInNlc3Npb25fc3RhdGUiOiJPTnJweFVhOG12WHRJeDR0c3pIZ09GR296WHUyeHZVVzVvNnc3eW1lLUdZLnJMRU1EWGFGIiwiaXNzIjoiaHR0cHM6Ly9pZGFtLm1ldHJvLmJnIiwiZW1haWwiOiJvZmZpY2VAdGVydmlvbi5iZyIsIm5vbmNlIjoiZjg3ZDMyYzEyYTRkNDY1ZGEzYjQwMTQ3OTlkYzc4NzMiLCJjX2hhc2giOiIiLCJzdWIiOiJVX2Y0MjBhY2E4LWY2OTMtNGMxNS1iOTIzLTc1NWY5NTc3ZTIwMCIsImF0X2hhc2giOiJlbkFGRFNJdUdmV0wzNnZ0UnJEQ253IiwicmVhbG0iOiJTU09fQ1VTVF9CRyIsImF1dGhfdGltZSI6MTY2ODA4MjMyMiwiYW1yIjpbIlVTRVJfQ1JFREVOVElBTFMiXX0.AC9vccz5PBe0d2uD6tHV5KdQ8_zbZvdARGUqo5s8KpJ0bGw97vm3xadF5TTHBUwkXX3oyJsbygC1tKvQInycU-zE0sqycIDtjP_hAGf6tUG-VV5xvtRsxBkacTBMy8OmbNHi5oncko7-dZ_tSOzQwSclLZKgKaqBcCqPBQVF0ug4pvbbqyZcw6D-MH6_T5prF7ppyqY11w9Ps_c7pFCciFR965gsO3Q-zr8CjKq1qGJeEpBFMKF0vfwinrc4wDpC5zd0Vgyf4ophzo6JkzA8TiWOGou5Z0khIpl435qUzxzt-WPFwPsPefhg_X9fYHma_OqQIpNjnV2tQwHqBD1qMTGXijtfOFQ; USER_TYPE=CUST; compressedJWT=eNpVUtlyozAQ/CJvcdrhEZtLGGGbQ4BeUlwGiTMhMeCvX5Fkt3YfVKrqme6eaalc7Tozc3Ihth8+Ae8SW/lVrvaYi3AD7Vx0eRwD4pzsuohuG3YsIm/EkUyxDybQjVzqgz2gqnBrj0ZpthNEtzUNqjWl3uqb4xkxA8Z/FhHY+ATHHld+cdFnYbZcZqIPpsflK9PpsBbw4LvfVFYcsh6LzdLJfGbOE+hR8B9ObOmG4FTqLgz4InCs+hhw81Q0BnQsHIQGmBLe3TR/7nzC7fHqmBh6uuIDMpMCuVwm2u2Xf2NbngbWDc9NQ85MpcYnhvcfOejtB5s1B3TMQefyueg9sgit8QlM8cnmc1P+rlF9hpq+QE2dIQUipMnTDRiPLBuvzjtvyISlwbF9KSKe5WH/8Izvnt5rE6FGuYDWsFMmjOa/+zMfLmWegYkEHC0/PO+P9qPYcuzbb5ztwvqVr1061LHzTHX8yDu33XbCnTHlQsgydcesK5iPO2JBvmbk3xpmH6RtNt00YnNQXXBpNV+0UIYU8lCD2ztKOdODQSJcNFVyg2aF60zS2GVvjvQk9lpAh8WliQS1aoVPwPJQn/fbr0vdxRiDJLh7d8pJhzVeNIW+75QK7H0zFVp9Z3BeGmZlA17s5LAcHDgjmc8vO/QiqorcSOenYVEx0/HJATQIqDJxAS7qsKnGQqrrXf5qNaf9GyRl3emruki8vxg0It5IhsxSfI8lGkvl+72qsoNMjhUp75xzR7NRq83w0Pp6oRqg74eq65zPaD/H9TX6GIyDfmFccfA8/fVtkPe7y5AUosA+fpZWBO0l9QzSZIfuoeG2n8aJNKG0WMfoap2XOcVJKT0ex9ep0m9vZv0gJwkqKue+Xb0TZ0Bjz+HMqi9W6Z81h+8PCaRZTJtoFYOun46FkQiPyFmGF65/VX33RdKl+ZYcXDvs7/Nv6PdLkg==; SES2_customerAdr_1001100022726355={%22addressId%22:%221001100022726355994-AD0532EI%22%2C%22addressHash%22:%221001100022726355994-AD0532EI%22%2C%22storeId%22:%2200022%22}; SES2_customerAdr_={%22addressId%22:null%2C%22addressHash%22:null%2C%22storeId%22:%2200022%22}; UserSettings=SelectedStore=1b1fc6ac-2ad6-4243-806e-a4a28c96dff4&SelectedAddress=1001100022726355994-ad0532ei",
    "Sec-Fetch-Dest": "empty",
    "Sec-Fetch-Mode": "cors",
    "Sec-Fetch-Site": "same-origin"
}

response = requests.request("GET", url, headers=headers, params=querystring)

print(response.text)

遇到的困境

代码合并了两篇教程，加了页面滚动的JS逻辑，换成httpx后添加了Insomnia的HEADERS，但不确定请求头的顺序是否正确
听说浏览器的请求头顺序是反爬虫的识别点，很多HTTP客户端会自动排序请求头，导致被识别
不知道怎么确定正确的请求头顺序，尤其是Cookie的位置，猜测是第11或第3个，但没依据

解决方案建议

不要手动提取单个Cookie，直接复用Playwright的上下文Cookie
Playwright的context.cookies()返回的是所有Cookie的列表，你只需要把这些Cookie直接传给httpx，不需要单独提取某一个。修改req_with_cookie函数：

def req_with_cookie(context_cookies):
    # 把Playwright的Cookie格式转换成httpx需要的格式
    cookies = {cookie['name']: cookie['value'] for cookie in context_cookies}
    # 传入所有请求头
    r = httpx.get(
        'https://shop.metro.bg/ordercapture.customercart.v1/carts/alias/current',
        cookies=cookies,
        headers=HEADERS
    )
    return r.text

同时修改get_cookie_playwright，返回所有Cookie而不是单个：

def get_cookie_playwright():
    with sync_playwright() as p:
        browser = p.firefox.launch(headless=False, slow_mo=50)
        context = browser.new_context()
        page = context.new_page()
        # ... 登录和滚动逻辑不变 ...
        all_cookies = context.cookies()
        browser.close()
    return all_cookies

这样能确保所有必要的Cookie都被带上，不会遗漏。

用Playwright直接发起API请求，跳过httpx
既然已经用Playwright打开了浏览器，完全可以直接用Playwright的page.request.get()来发起请求，这样请求会自动带上浏览器的所有Cookie和请求头，和真实浏览器的请求完全一致，不需要手动处理请求头顺序：

def get_cart_data():
    with sync_playwright() as p:
        browser = p.firefox.launch(headless=False, slow_mo=50)
        context = browser.new_context()
        page = context.new_page()
        # ... 登录和滚动逻辑不变 ...
        # 直接用context的request发起请求
        response = context.request.get(
            'https://shop.metro.bg/ordercapture.customercart.v1/carts/alias/current'
        )
        browser.close()
    return response.text

然后主函数直接调用这个方法就行，这是最省心的方式，完全模拟真实浏览器的请求。

如果一定要用httpx，确保请求头顺序和浏览器一致
打开浏览器的开发者工具（F12），切换到网络标签，找到对应的API请求，查看请求头的顺序。然后用httpx的OrderedDict来定义HEADERS，这样就能保持顺序：
```
from collections import OrderedDict

HEADERS = OrderedDict([
    ("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0"),
    ("Accept", "*/*"),
    ("Accept-Language", "en-US,en;q=0.5"),
    # 按照浏览器里的顺序依次添加所有请求头
    # ...
    ("Cookie", "..."),  # 这里的Cookie要带上所有值，不是单个
])
```
不过这种方式比较麻烦，因为每次Cookie更新都要手动同步，不如直接用Playwright的请求方便。
添加必要的查询参数
从Insomnia的代码里可以看到，API请求需要querystring里的参数，比如customerId、storeId等，你的httpx请求里没加这些参数，这可能也是请求失败的原因之一。不管用哪种方式发起请求，都要带上这些参数。

内容的提问来源于stack exchange，提问作者Peter