使用Utf8JsonReader迭代读取JSON令牌报错CS4013的原因与解决

问题场景 #

想要使用Utf8JsonReader从流中提取所有JSON令牌，参考官方文档的GetMoreBytesFromStream方法编写了迭代器函数，但编译时标注的两行触发错误：

var buffer = new byte[defaultBufferSize];
if (stream.Read(buffer) == 0)
{
    yield break;
}
var reader = new Utf8JsonReader(buffer, isFinalBlock: false, state: default);
while (true)
{
    if (reader.Read()) // <- 此处报错
    {
        var tokenType = (TokenType)reader.TokenType;
        ... // 处理后返回令牌
    }
    else if (reader.IsFinalBlock) // <- 此处报错
    {
        yield break;
    }
    else
    {
        GetMoreBytesFromStream(stream, ref buffer, ref reader, maxBufferSize);
    }
}

编译错误信息：

Error CS4013 Instance of type 'Utf8JsonReader' cannot be used inside a nested function, query expression, iterator block or async method

错误原因 #

Utf8JsonReader是ref struct类型，这类类型的核心约束是必须始终存储在栈上，绝对不能被分配到托管堆中。而使用yield关键字的迭代器块，会被C#编译器自动转换为包含状态字段的状态机类，这个类的实例会被分配到托管堆。如果允许在迭代器中使用Utf8JsonReader，就意味着要把这个ref struct存储到状态机类的字段里，直接违反了ref struct的栈约束，因此编译器直接禁止这种用法。

正确实现方式 #

方案1：收集所有令牌后一次性返回 #

放弃迭代器模式，将提取到的所有令牌收集到集合中，处理完成后统一返回：

public List<Token> ExtractAllTokens(Stream stream, int defaultBufferSize, int maxBufferSize)
{
    var tokens = new List<Token>();
    var buffer = new byte[defaultBufferSize];
    int bytesRead = stream.Read(buffer);
    if (bytesRead == 0)
        return tokens;

    var reader = new Utf8JsonReader(buffer.AsSpan(0, bytesRead), isFinalBlock: false, state: default);
    while (true)
    {
        if (reader.Read())
        {
            var tokenType = (TokenType)reader.TokenType;
            // 处理令牌并添加到集合
            tokens.Add(new Token { Type = tokenType /* 补充其他字段 */ });
        }
        else if (reader.IsFinalBlock)
        {
            break;
        }
        else
        {
            GetMoreBytesFromStream(stream, ref buffer, ref reader, maxBufferSize);
        }
    }
    return tokens;
}

// 适配Span的GetMoreBytesFromStream实现示例
private static void GetMoreBytesFromStream(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader, int maxBufferSize)
{
    int bytesRead = stream.Read(buffer);
    if (bytesRead == 0)
    {
        reader = new Utf8JsonReader(ReadOnlySpan<byte>.Empty, isFinalBlock: true, reader.CurrentState);
        return;
    }
    reader = new Utf8JsonReader(buffer.AsSpan(0, bytesRead), isFinalBlock: stream.Position == stream.Length, reader.CurrentState);
}

方案2：使用回调函数处理令牌 #

如果不想一次性加载所有令牌到内存，可以通过回调函数逐个处理提取到的令牌：

public void ProcessTokens(Stream stream, int defaultBufferSize, int maxBufferSize, Action<Token> tokenHandler)
{
    var buffer = new byte[defaultBufferSize];
    int bytesRead = stream.Read(buffer);
    if (bytesRead == 0)
        return;

    var reader = new Utf8JsonReader(buffer.AsSpan(0, bytesRead), isFinalBlock: false, state: default);
    while (true)
    {
        if (reader.Read())
        {
            var tokenType = (TokenType)reader.TokenType;
            // 构造Token并传递给回调
            tokenHandler(new Token { Type = tokenType /* 补充其他字段 */ });
        }
        else if (reader.IsFinalBlock)
        {
            break;
        }
        else
        {
            GetMoreBytesFromStream(stream, ref buffer, ref reader, maxBufferSize);
        }
    }
}

// 使用示例
ProcessTokens(stream, 4096, 16384, token => {
    // 处理单个令牌
    Console.WriteLine($"Token type: {token.Type}");
});

方案3：手动实现枚举器（接近原迭代器需求） #

如果一定要实现延迟返回的迭代器效果，可以手动实现IEnumerator<Token>，但要确保Utf8JsonReader始终在栈上，不被存储到类字段中。这种方式代码量较大，适合内存敏感场景：

public class JsonTokenEnumerator : IEnumerator<Token>
{
    private readonly Stream _stream;
    private readonly int _defaultBufferSize;
    private readonly int _maxBufferSize;
    private byte[] _buffer;
    private Utf8JsonReader _reader;
    private Token _current;
    private bool _disposed;

    public JsonTokenEnumerator(Stream stream, int defaultBufferSize, int maxBufferSize)
    {
        _stream = stream ?? throw new ArgumentNullException(nameof(stream));
        _defaultBufferSize = defaultBufferSize;
        _maxBufferSize = maxBufferSize;
        _buffer = new byte[defaultBufferSize];
        int bytesRead = _stream.Read(_buffer);
        _reader = new Utf8JsonReader(_buffer.AsSpan(0, bytesRead), isFinalBlock: false, state: default);
    }

    public Token Current => _current;

    object IEnumerator.Current => Current;

    public bool MoveNext()
    {
        while (true)
        {
            if (_reader.Read())
            {
                _current = new Token { Type = (TokenType)_reader.TokenType /* 补充其他字段 */ };
                return true;
            }
            else if (_reader.IsFinalBlock)
            {
                return false;
            }
            else
            {
                GetMoreBytesFromStream(_stream, ref _buffer, ref _reader, _maxBufferSize);
            }
        }
    }

    public void Reset()
    {
        throw new NotSupportedException();
    }

    public void Dispose()
    {
        if (!_disposed)
        {
            _disposed = true;
            // 清理相关资源
        }
    }

    private static void GetMoreBytesFromStream(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader, int maxBufferSize)
    {
        int bytesRead = stream.Read(buffer);
        if (bytesRead == 0)
        {
            reader = new Utf8JsonReader(ReadOnlySpan<byte>.Empty, isFinalBlock: true, reader.CurrentState);
            return;
        }
        reader = new Utf8JsonReader(buffer.AsSpan(0, bytesRead), isFinalBlock: stream.Position == stream.Length, reader.CurrentState);
    }
}

// 使用示例
using var enumerator = new JsonTokenEnumerator(stream, 4096, 16384);
while (enumerator.MoveNext())
{
    var token = enumerator.Current;
    // 处理令牌
}

内容的提问来源于stack exchange，提问作者Weier