如何向Gemini发送大体积JSON数据分析并解决500内部错误？

原Python代码在处理小体积JSON文件时可正常调用Gemini-Pro进行分析，但处理大体积JSON文件时触发InternalServerError（500内部服务器错误），错误栈如下：

ERROR:tornado.access:500 POST /v1beta/models/gemini-pro:generateContent?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 3329.36ms
---------------------------------------------------------------------------
InternalServerError                       Traceback (most recent call last)
<ipython-input-22-8b84417dca32> in <cell line: 40>()
     38 chat = model.start_chat(history=[])
     39 
---> 40 response = chat.send_message(json.dumps(data))
     41 
     42 

8 frames
/usr/local/lib/python3.10/dist-packages/google/ai/generativelanguage_v1beta/services/generative_service/transports/rest.py in __call__(self, request, retry, timeout, metadata)
    854             # subclass.
    855             if response.status_code >= 400:
---> 856                 raise core_exceptions.from_http_response(response)
    857 
    858             # Return the response

InternalServerError: 500 POST https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:generateContent?%24alt=json%3Benum-encoding%3Dint: An internal error has occurred. Please retry or report in https://developers.generativeai.google/guide/troubleshooting

原代码：

import pathlib
import json
import textwrap

import google.generativeai as genai

from IPython.display import display
from IPython.display import Markdown


def to_markdown(text):
  text = text.replace('•', '  *')
  return Markdown(textwrap.indent(text, '> ', predicate=lambda _: True))

from google.colab import userdata

GOOGLE_API_KEY='xxxxxxx'

genai.configure(api_key=GOOGLE_API_KEY)

json_file_path = pathlib.Path("/content/sample_data/BigData.json")
with open(json_file_path, "r") as f:
    json_data = json.load(f)

prompt = "can you report analysis of this json data?"

data = {
    "prompt": prompt,
    "data": json_data
}

print("data= ", data)

model = genai.GenerativeModel('gemini-pro')

chat = model.start_chat(history=[])

response = chat.send_message(json.dumps(data))


to_markdown(response.text)

核心原因是大体积JSON序列化后超出了Gemini-Pro的上下文令牌限制，或者请求体过大导致服务器处理失败。Gemini-Pro存在上下文窗口长度限制，直接发送完整大JSON会触发服务器内部错误。

通过将大JSON数据分批次发送给模型，逐步完成分析后汇总结果。修改后的代码如下：

import pathlib
import json
import textwrap

import google.generativeai as genai

from IPython.display import display
from IPython.display import Markdown


def to_markdown(text):
    text = text.replace('•', '  *')
    return Markdown(textwrap.indent(text, '> ', predicate=lambda _: True))

from google.colab import userdata

GOOGLE_API_KEY='xxxxxxx'
genai.configure(api_key=GOOGLE_API_KEY)

# 读取大JSON文件
json_file_path = pathlib.Path("/content/sample_data/BigData.json")
with open(json_file_path, "r") as f:
    json_data = json.load(f)

# 初始化模型和对话
model = genai.GenerativeModel('gemini-pro')
chat = model.start_chat(history=[])

# 告知模型将分块发送数据
initial_prompt = """我将分块发送一份大JSON数据，请你逐步分析每一块的内容并记录关键信息。所有块发送完成后，请汇总所有分析结果，生成完整的数据分析报告。"""
chat.send_message(initial_prompt)

# 分块处理逻辑（可根据JSON结构调整分块方式）
chunk_size = 50  # 每块包含的条目数，可根据数据复杂度调整
if isinstance(json_data, list):
    total_chunks = (len(json_data) + chunk_size - 1) // chunk_size
    for i in range(total_chunks):
        start_idx = i * chunk_size
        end_idx = min((i+1)*chunk_size, len(json_data))
        chunk = json_data[start_idx:end_idx]
        chunk_prompt = f"这是第{i+1}/{total_chunks}块数据，请分析该块内容并记录关键信息：\n{json.dumps(chunk)}"
        print(f"正在发送第{i+1}/{total_chunks}块数据...")
        chat.send_message(chunk_prompt)
else:
    # 若JSON为对象类型，按键值对分组分块
    items = list(json_data.items())
    total_chunks = (len(items) + chunk_size - 1) // chunk_size
    for i in range(total_chunks):
        start_idx = i * chunk_size
        end_idx = min((i+1)*chunk_size, len(items))
        chunk_dict = dict(items[start_idx:end_idx])
        chunk_prompt = f"这是第{i+1}/{total_chunks}块数据，请分析该块内容并记录关键信息：\n{json.dumps(chunk_dict)}"
        print(f"正在发送第{i+1}/{total_chunks}块数据...")
        chat.send_message(chunk_prompt)

# 请求汇总分析结果
final_prompt = """所有数据块已发送完毕，请汇总所有分析结果，生成一份详细的数据分析报告，包含数据整体概况、关键发现、趋势总结等内容。"""
response = chat.send_message(final_prompt)

# 展示结果
display(to_markdown(response.text))

• 调整分块大小：根据JSON数据的字段复杂度调整chunk_size，确保每块序列化后的令牌数不超过Gemini-Pro的32768令牌上限。
• 过滤冗余字段：提前过滤JSON中不需要分析的冗余字段，减少发送的数据量。
• 添加重试机制：针对偶尔出现的500错误，可引入重试逻辑，示例代码如下：

from tenacity import retry, stop_after_attempt, wait_exponential

@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
def send_with_retry(chat, message):
    return chat.send_message(message)

# 使用send_with_retry替代chat.send_message
send_with_retry(chat, chunk_prompt)

内容的提问来源于stack exchange，提问作者user17063944