.NET环境下如何仅用内存创建2-5GB的超大Stream

核心思路 #

MemoryStream的2GB限制根源在于它依赖单一int索引的数组（.NET中数组最大长度为int.MaxValue，对应约2GB）。要绕过这个限制，我们可以实现一个基于多内存块数组的自定义Stream——用多个固定大小的内存块（比如100MB/块）拼接成逻辑上的大Stream，通过计算偏移量定位到对应块和块内位置，完全在内存中操作，无需创建文件。

自定义LargeMemoryStream实现 #

以下是可直接使用的实现，支持2-5GB甚至更大的内存Stream，兼容StreamReader等标准Stream消费者：

using System;
using System.Collections.Generic;
using System.IO;

public class LargeMemoryStream : Stream
{
    private const int DefaultBlockSize = 1024 * 1024 * 100; // 100MB/块，可按需调整
    private readonly List<byte[]> _blocks = new List<byte[]>();
    private long _position;
    private long _length;
    private readonly int _blockSize;

    public LargeMemoryStream() : this(DefaultBlockSize) { }
    public LargeMemoryStream(int blockSize)
    {
        _blockSize = blockSize > 0 ? blockSize : DefaultBlockSize;
    }

    public override bool CanRead => true;
    public override bool CanSeek => true;
    public override bool CanWrite => true;
    public override long Length => _length;
    public override long Position
    {
        get => _position;
        set
        {
            if (value < 0 || value > _length)
                throw new ArgumentOutOfRangeException(nameof(value));
            _position = value;
        }
    }

    public override void Flush() { }

    public override int Read(byte[] buffer, int offset, int count)
    {
        if (buffer == null) throw new ArgumentNullException(nameof(buffer));
        if (offset < 0 || count < 0 || offset + count > buffer.Length)
            throw new ArgumentOutOfRangeException();

        int totalRead = 0;
        while (count > 0 && _position < _length)
        {
            var (blockIndex, blockOffset) = GetBlockInfo(_position);
            var block = _blocks[blockIndex];
            int readSize = Math.Min(count, _blockSize - blockOffset);
            // 处理最后一块可能不满的情况
            if (blockIndex == _blocks.Count - 1)
                readSize = Math.Min(readSize, (int)(_length - _position));

            Buffer.BlockCopy(block, blockOffset, buffer, offset, readSize);
            totalRead += readSize;
            offset += readSize;
            count -= readSize;
            _position += readSize;
        }
        return totalRead;
    }

    public override long Seek(long offset, SeekOrigin origin)
    {
        long newPosition = origin switch
        {
            SeekOrigin.Begin => offset,
            SeekOrigin.Current => _position + offset,
            SeekOrigin.End => _length + offset,
            _ => throw new ArgumentOutOfRangeException(nameof(origin))
        };
        Position = newPosition;
        return _position;
    }

    public override void SetLength(long value)
    {
        if (value < 0) throw new ArgumentOutOfRangeException(nameof(value));
        if (value <= _length)
        {
            _length = value;
            if (_position > _length) _position = _length;
            return;
        }

        // 扩展内存块到指定长度
        while (_length < value)
        {
            int addSize = (int)Math.Min(_blockSize, value - _length);
            _blocks.Add(new byte[addSize]);
            _length += addSize;
        }
    }

    public override void Write(byte[] buffer, int offset, int count)
    {
        if (buffer == null) throw new ArgumentNullException(nameof(buffer));
        if (offset < 0 || count < 0 || offset + count > buffer.Length)
            throw new ArgumentOutOfRangeException();

        while (count > 0)
        {
            var (blockIndex, blockOffset) = GetBlockInfo(_position);
            // 确保当前块存在，不存在则创建
            while (blockIndex >= _blocks.Count)
            {
                _blocks.Add(new byte[_blockSize]);
                _length += _blockSize;
            }

            var block = _blocks[blockIndex];
            int writeSize = Math.Min(count, _blockSize - blockOffset);
            Buffer.BlockCopy(buffer, offset, block, blockOffset, writeSize);
            offset += writeSize;
            count -= writeSize;
            _position += writeSize;
            // 更新实际长度（如果写入超过原长度）
            if (_position > _length) _length = _position;
        }
    }

    protected override void Dispose(bool disposing)
    {
        if (disposing)
        {
            _blocks.Clear();
            _length = 0;
            _position = 0;
        }
        base.Dispose(disposing);
    }

    // 计算当前偏移量对应的块索引和块内偏移
    private (int BlockIndex, int BlockOffset) GetBlockInfo(long position)
    {
        int blockIndex = (int)(position / _blockSize);
        int blockOffset = (int)(position % _blockSize);
        return (blockIndex, blockOffset);
    }
}

使用示例 #

写入大数据并通过StreamReader读取 #

using (var largeStream = new LargeMemoryStream())
{
    // 模拟写入3GB数据（实际替换为业务数据写入逻辑即可）
    byte[] chunk = new byte[1024 * 1024]; // 1MB写入块
    for (int i = 0; i < 3072; i++)
    {
        largeStream.Write(chunk, 0, chunk.Length);
    }

    // 重置流位置，供读取使用
    largeStream.Seek(0, SeekOrigin.Begin);

    // 用StreamReader读取内容
    using (var reader = new StreamReader(largeStream))
    {
        string line;
        while ((line = reader.ReadLine()) != null)
        {
            // 处理读取到的内容
        }
    }
}

关键注意事项 #

• 块大小调整：DefaultBlockSize可根据内存情况调整，内存充足时设为200MB/块减少块数量；内存紧张时设小，但避免过小导致频繁块操作。
• 内存占用：全程在内存中操作，需确保系统有足够空闲内存（比如5GB数据至少需要5GB以上可用内存）。
• 兼容性：该实现完全遵循Stream抽象类规范，可直接替换MemoryStream，兼容所有依赖Stream的组件（包括StreamReader、BinaryReader等）。

内容的提问来源于stack exchange，提问作者Who_Said_Code