Python Requests流传输问题：目标端接收数据非JSON格式

使用Python的requests库通过GET请求从源API获取流式响应事件，筛选特定payload后POST到另一API。代码无报错，但目标端收到的是字节串格式的表单数据，而非预期的JSON。搭建Flask测试服务器验证，结果一致。

原代码 #

import json
import requests
import get_token
from requests.auth import HTTPBasicAuth

source_streaming_url = 'https://livestream/api/stream/stream1'
destination_streaming_url = 'https://destinationstream/v1/streams/stream1'

def source_api():
    headers = {
        'x-api-key': <api_key>,
        'Authorization': 'Bearer ' + get_token.get_token().get('result').get('access_token'),
    }
    response = requests.get(source_streaming_url, headers=headers, stream=True)

    for line in response.iter_lines():
        if line:
            yield line


def main():
    for line in source_api():
        payload = json.loads(line.decode('utf-8'))
        evars_main = payload.get('variable1')
        if evars_main and evars_main.get('variable2'):
            if evars_main.get('variable2').get('myvar'):
            
                secrets =  <get_secret>

                destination_response = requests.post(
                        destination_streaming_url,
                        headers={"Content-Type": "application/json"},
                        data=payload,
                        auth=HTTPBasicAuth(secrets.get('user'), secrets.get('secret')),
                        stream=True
                        )
                print(destination_response.content)

目标端收到的内容（表单格式） #

reportSuite=stream1&exclude=id&exclude=value&exclude=description&pageEvent=id&pageEvent=type&pageEvent=description&pageEvent=linkURL&pageEvent=linkName&timeGMT=1682996441&receivedTimeGMT=1682996441&hitIdHigh=3614207338102325248&hitIdLow=46195734853736065634&mcVisIdHigh=2951833792067086269&mcVisIdLow=367663990508238318&visIdHigh=41978984813332068260&visIdLow=8004941667833356853&visIdType=5&props=prop29&props=prop40&props=prop73&props=prop30&props=prop33&props=prop21&props=site_section&props=prop48&evars=evars&events=event21&events=event89&geoCountry=jpn&geoRegion=13&geoCity=honan&geoZip=168-0072&geoDMA=30001&geoLatitude=35.63&geoLongitude=139.66&connectionType=LAN%2FWifi&topLevelDomain=.net&languageAbbrev=ja&language=Japanese&searchEngine=Microsoft+Bing&bot=%3CNot+a+bot%3E&operatingSystem=Windows+10&browserType=&browser=&javascriptVersion=1.6&monitorWidth=1600&monitorHeight=1600&monitorColorDepth=16+million+%2824-bit%29&carrier=&mobileDeviceType=&mobileDeviceName=&mobileManufacturer=&mobileScreenWidth=0
&mobileScreenHeight=0&mobileCookieSupport=False&mobileColorDepth=&mobileAudioSupport=&mobileVideoSupport=&pageURL=&pageName=&usesPersistentCookie=1&homePage=0&browserHeight=842&browserWidth=1479&javaEnabled=False&ip=<ip>&isErrorPage=False&purchaseId=&referrer=https%3A%2F%2Fwww.dr.com%2F&state=&userAgent=Mozilla%2F5.0+%28Windows+NT+10.0%3B+Win64%3B+x64%29+AppleWebKit%2F537.36+%28KHTML%2C+like+Gecko%29+Chrome%2F112.0.0.0+Safari%2F537.36+Edg%2F&plugins=&currency=USD&hitSource=1&transactionId=&truncated=False&zip=&mcAudiences=values&mcAudiences=delim

原Flask测试服务器代码 #

from flask import Flask, jsonify, request, Response, stream_with_context
import io
import json

app = Flask(__name__)

@app.route('/stream', methods=['POST'])
def stream():
    def generate():

        stream = io.BytesIO(request.get_data())
        while chunk := stream.read(1024):
            yield chunk
            print(chunk.decode()) # Display the received data to stdout
    return app.response_class(stream_with_context(generate()), mimetype='application/octet-stream')

if __name__ == "__main__":
    app.run(debug=True)

问题出在requests.post的参数使用逻辑上：

• 当给data参数传入Python字典时，requests会自动将其编码为**application/x-www-form-urlencoded格式的表单数据**，哪怕手动设置了Content-Type: application/json，这个自动编码逻辑会导致实际发送的内容格式与声明的不一致。
• 目标端收到表单格式的数据后，自然解析为键值对的字节串，而非预期的JSON结构。

要发送JSON格式的数据，有两种可靠方式：

1. 使用json参数替代data参数：requests会自动将字典序列化为JSON字符串，并正确设置Content-Type: application/json（即使不手动设置headers也生效，保留headers更明确）。
2. 手动将字典序列化为JSON字符串，再传给data参数，同时确保headers里的Content-Type正确。

推荐第一种方式，简洁且不易出错。

修改后的主函数代码 #

def main():
    for line in source_api():
        payload = json.loads(line.decode('utf-8'))
        evars_main = payload.get('variable1')
        if evars_main and evars_main.get('variable2'):
            if evars_main.get('variable2').get('myvar'):
            
                secrets =  <get_secret>

                destination_response = requests.post(
                        destination_streaming_url,
                        headers={"Content-Type": "application/json"},
                        json=payload,  # 将data替换为json参数
                        auth=HTTPBasicAuth(secrets.get('user'), secrets.get('secret')),
                        stream=True
                        )
                print(destination_response.content)

修改后的Flask测试服务器代码（用于验证JSON接收） #

若要在测试服务器中正确接收并查看JSON数据，需调整代码以解析JSON：

from flask import Flask, request, jsonify

app = Flask(__name__)

@app.route('/stream', methods=['POST'])
def stream():
    # 直接解析请求体为JSON
    received_json = request.get_json()
    print("Received JSON:", received_json)
    # 返回确认响应
    return jsonify({"status": "success", "received": received_json})

if __name__ == "__main__":
    app.run(debug=True)

内容的提问来源于stack exchange，提问作者user2993021