Python urllib请求Unreal Marketplace API遇403 Forbidden错误求助

手动在Firefox、Edge、Opera浏览器访问目标URL正常，但Python脚本运行时出现urllib.error.HTTPError: HTTP Error 403: Forbidden错误。已尝试添加User-Agent请求头，问题仍未解决。

403错误通常是服务器检测到请求并非来自真实浏览器，仅User-Agent不足以通过校验，需补充更多浏览器特征的请求头，或优化请求方式，以下是可行修复方案：

方案1：完善urllib请求头并添加请求延迟 #

服务器会校验多个请求头字段，补充Accept、Accept-Language、Referer等常见字段，同时添加请求延迟避免触发反爬：

import json
import os
import urllib.request
import time

# 模拟真实浏览器的请求头组合
opener = urllib.request.build_opener()
opener.addheaders = [
    ('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'),
    ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'),
    ('Accept-Language', 'en-US,en;q=0.5'),
    ('Referer', 'https://www.unrealengine.com/marketplace'),
    # 可选：从浏览器开发者工具复制真实Cookie添加
    # ('Cookie', 'your_cookie_content_here')
]
urllib.request.install_opener(opener)

MP_URL = "https://www.unrealengine.com/marketplace"
JSON_DIR = "./data"
OUT_FILE = "mp_vault_list.md"


def main():
    mp_jsons = []
    cost = 0.
    disc = 0.
    currency = ""
    count = 0
    for f in os.listdir(JSON_DIR):
        if not f.endswith(".json"):
            continue
        with open(os.path.join(JSON_DIR, f), encoding="utf8") as fd:
            j = json.load(fd)
            count += len(j['data']['elements'])
            mp_jsons.append(j)
    with open(OUT_FILE, "w+", encoding="utf8") as fd:
        fd.write(f"# Vault content\n\n")
        i = 0
        for j in mp_jsons:
            for el in j['data']['elements']:
                i += 1
                print(f"\rProcessing Vault {i}/{count}", end="")
                # 添加1秒延迟，避免频繁请求触发反爬
                time.sleep(1)
                try:
                    with urllib.request.urlopen(f"{MP_URL}/api/assets/asset/{el['id']}") as f:
                        jj = json.load(f)
                        jjd = jj['data']['data']
                        fd.write(f"* [{jjd['title']}]({MP_URL}/en-US/product/{jjd['urlSlug']})\n")
                        cost += jjd['priceValue']
                        currency = jjd['currencyCode']
                        try:
                            disc += jjd['discountPriceValue']
                        except KeyError:
                            pass
                except urllib.error.HTTPError as e:
                    print(f"\nError accessing {el['id']}: {e}")

        fd.write(f"\n# Vault stats\n\n")
        fd.write(f"* Value: {cost / 100}{currency}\n")
        fd.write(f"* Current cost: {disc / 100}{currency}\n")
        fd.write(f"* Size: {count}\n")


if __name__ == '__main__':
    main()

方案2：改用requests库（更简便） #

requests库对请求头和会话的处理更友好，能轻松维持浏览器式的请求状态：

import json
import os
import requests
import time

MP_URL = "https://www.unrealengine.com/marketplace"
JSON_DIR = "./data"
OUT_FILE = "mp_vault_list.md"

# 构建会话，统一设置请求头
session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
    'Referer': 'https://www.unrealengine.com/marketplace',
    # 可选：添加浏览器Cookie
    # 'Cookie': 'your_cookie_content_here'
})


def main():
    mp_jsons = []
    cost = 0.
    disc = 0.
    currency = ""
    count = 0
    for f in os.listdir(JSON_DIR):
        if not f.endswith(".json"):
            continue
        with open(os.path.join(JSON_DIR, f), encoding="utf8") as fd:
            j = json.load(fd)
            count += len(j['data']['elements'])
            mp_jsons.append(j)
    with open(OUT_FILE, "w+", encoding="utf8") as fd:
        fd.write(f"# Vault content\n\n")
        i = 0
        for j in mp_jsons:
            for el in j['data']['elements']:
                i += 1
                print(f"\rProcessing Vault {i}/{count}", end="")
                time.sleep(1)
                try:
                    response = session.get(f"{MP_URL}/api/assets/asset/{el['id']}")
                    response.raise_for_status()  # 主动触发HTTP错误
                    jj = response.json()
                    jjd = jj['data']['data']
                    fd.write(f"* [{jjd['title']}]({MP_URL}/en-US/product/{jjd['urlSlug']})\n")
                    cost += jjd['priceValue']
                    currency = jjd['currencyCode']
                    try:
                        disc += jjd['discountPriceValue']
                    except KeyError:
                        pass
                except requests.exceptions.RequestException as e:
                    print(f"\nError accessing {el['id']}: {e}")

        fd.write(f"\n# Vault stats\n\n")
        fd.write(f"* Value: {cost / 100}{currency}\n")
        fd.write(f"* Current cost: {disc / 100}{currency}\n")
        fd.write(f"* Size: {count}\n")


if __name__ == '__main__':
    main()

关键修复说明 #

• 补充请求头：服务器会校验Accept、Referer等字段，模拟真实浏览器的请求头组合能降低被拦截概率
• 添加请求延迟：time.sleep(1)避免短时间内大量请求触发反爬机制
• 会话维持：使用会话对象能保持请求状态，更接近浏览器的访问行为
• Cookie（可选）：若服务器需要验证会话，可从浏览器开发者工具的Network面板复制Cookie添加到请求头

内容的提问来源于stack exchange，提问作者Franklyn W. R. Tigier