如何反序列化包含超大Base64值的JSON成员？

针对API返回的JSON中包含GB级Base64字段的场景，直接反序列化为byte[]会导致内存溢出。核心解决方案是利用Newtonsoft.Json的JsonTextReader逐令牌读取能力，在解析目标字段时，实时读取Base64字符块并解码写入Stream，而非加载整个字符串到内存。

步骤1：定义目标实体类 #

将data字段声明为Stream类型，并指定自定义转换器：

public class ApiResponse
{
    public string Name0 { get; set; }
    public string Name1 { get; set; }
    [JsonConverter(typeof(LargeBase64ToStreamConverter))]
    public Stream Data { get; set; }
    public string Name2 { get; set; }
}

步骤2：实现自定义JsonConverter #

该转换器会跳过常规的字符串读取逻辑，逐块读取Base64字符并实时解码到目标流：

public class LargeBase64ToStreamConverter : JsonConverter
{
    private const int BufferSize = 4096; // 可根据系统IO性能调整

    public override bool CanConvert(Type objectType)
    {
        return objectType == typeof(Stream);
    }

    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
    {
        if (reader.TokenType != JsonToken.PropertyName || (string)reader.Value != "data")
        {
            throw new JsonSerializationException("Unexpected token or property name.");
        }

        // 移动到data字段的字符串值起始位置
        if (!reader.Read() || reader.TokenType != JsonToken.String)
        {
            throw new JsonSerializationException("Expected string value for 'data' property.");
        }

        // 建议替换为FileStream等持久化流，避免内存占用
        var targetStream = new MemoryStream();
        var decoder = new Base64ChunkDecoder(targetStream);
        char[] buffer = new char[BufferSize];
        int charsRead;

        // 逐块读取Base64字符，不加载完整字符串到内存
        while ((charsRead = reader.ReadAsStringBuffer(buffer, 0, buffer.Length)) > 0)
        {
            decoder.Write(buffer, 0, charsRead);
        }

        decoder.FinalizeDecoding();
        targetStream.Position = 0; // 重置流指针以便后续读取
        return targetStream;
    }

    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
    {
        throw new NotImplementedException("此转换器仅支持反序列化");
    }
}

// 辅助类：处理Base64字符块的分段解码
public class Base64ChunkDecoder
{
    private readonly Stream _output;
    private char[] _remainingChars = Array.Empty<char>();

    public Base64ChunkDecoder(Stream outputStream)
    {
        _output = outputStream ?? throw new ArgumentNullException(nameof(outputStream));
    }

    public void Write(char[] buffer, int index, int count)
    {
        // 合并上一次剩余的字符和当前读取的字符
        char[] combined = new char[_remainingChars.Length + count];
        Array.Copy(_remainingChars, combined, _remainingChars.Length);
        Array.Copy(buffer, index, combined, _remainingChars.Length, count);

        // 只处理能被4整除的字符段（Base64编码规则：4字符对应3字节）
        int processLength = combined.Length - (combined.Length % 4);
        if (processLength > 0)
        {
            byte[] decodedBytes = Convert.FromBase64CharArray(combined, 0, processLength);
            _output.Write(decodedBytes, 0, decodedBytes.Length);
        }

        // 保存剩余不足4个的字符，用于下一次解码
        int remainingCount = combined.Length - processLength;
        _remainingChars = new char[remainingCount];
        Array.Copy(combined, processLength, _remainingChars, 0, remainingCount);
    }

    public void FinalizeDecoding()
    {
        if (_remainingChars.Length == 0)
        {
            _output.Flush();
            return;
        }

        // 处理剩余字符，补全Base64填充字符后解码
        string paddedChars = new string(_remainingChars).PadRight(4, '=');
        byte[] decodedBytes = Convert.FromBase64String(paddedChars);

        // 根据剩余字符数量计算实际有效字节数（去除填充带来的冗余）
        int validLength = _remainingChars.Length switch
        {
            1 => 0, // 单个Base64字符无法解码，视为无效数据
            2 => 1,
            3 => 2,
            _ => decodedBytes.Length
        };

        if (validLength > 0)
        {
            _output.Write(decodedBytes, 0, validLength);
        }

        _remainingChars = Array.Empty<char>();
        _output.Flush();
    }
}

步骤3：流式反序列化API响应 #

使用JsonTextReader配合响应流进行反序列化，全程不加载完整JSON到内存：

using (var httpResponse = await httpClient.GetAsync("your-api-url", HttpCompletionOption.ResponseHeadersRead))
{
    httpResponse.EnsureSuccessStatusCode();
    using (var responseStream = await httpResponse.Content.ReadAsStreamAsync())
    using (var streamReader = new StreamReader(responseStream))
    using (var jsonReader = new JsonTextReader(streamReader))
    {
        var serializer = new JsonSerializer();
        var response = serializer.Deserialize<ApiResponse>(jsonReader);

        // 示例：将Data流写入文件
        using (var fileStream = new FileStream("output.data", FileMode.Create, FileAccess.Write))
        {
            await response.Data.CopyToAsync(fileStream);
        }

        // 务必释放Data流
        response.Data.Dispose();
    }
}

关键注意事项 #

• 避免内存流：处理GB级数据时，不要用MemoryStream，直接用FileStream写入磁盘，彻底规避内存压力
• 缓冲区大小：BufferSize可根据服务器IO性能调整，建议设置为4096、8192或16384
• 资源释放：所有Stream和IDisposable对象必须通过using语句或手动Dispose释放
• 错误处理：可根据业务需求添加Base64无效字符、字段缺失等异常处理逻辑

内容的提问来源于stack exchange，提问作者Sean Killian