C# Socket大文件传输数据接收异常，致反序列化失败求助

开发类似FileZilla的远程文件浏览器（C# TCP Socket实现），服务端与客户端共用SocketHelper处理通信，小数据传输正常，但大文件下载时频繁出现数据不完整/冗余，导致JSON反序列化失败，无规律出错。

核心通信类代码：

public class SocketHelper
{
    public static async Task SendAsync(Socket socket, object message)
    {
        // Size
        var sendBuffer = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(message));
        await socket.SendAsync(BitConverter.GetBytes(sendBuffer.Length));

        // Payload
        var sentBytes = 0;

        while (sentBytes < sendBuffer.Length)
        {
            sentBytes += await socket.SendAsync(sendBuffer);
        }
    }

    public static async Task<string> ReceiveAsync(Socket socket)
    {
        // Size
        var preflightBuffer = new byte[32];
        await socket.ReceiveAsync(preflightBuffer);

        // Payload
        var receiveBuffer = new byte[BitConverter.ToInt32(preflightBuffer)];
        var receivedBytes = 0;

        while (receivedBytes < receiveBuffer.Length)
        {
            receivedBytes += await socket.ReceiveAsync(receiveBuffer);
        }

        return Encoding.UTF8.GetString(receiveBuffer);
    }
}

小数据传输示例（正常）：

public class FolderListingRequest
{
    public string Path { get; set; }
}

public class FolderListingResponse : Response
{
    public IList<FileData> Files { get; set; } = [];
}

服务端下载处理代码：

public async Task DownloadFiles(Socket socket, DownloadFilesRequest request)
{
    var payload = new Payload
    {
        Type = PayloadTypeEnum.DownloadFiles,
        Data = request,
    };
    var fileCountToReceive = request.Paths.Count;
    var fileCountReceived = 0;
    await SocketHelper.SendAsync(socket, payload);

    while (fileCountReceived < fileCountToReceive)
    {
        Console.WriteLine("Receiving file");
        var receivedData = await SocketHelper.ReceiveAsync(socket);
        var fileData = JsonConvert.DeserializeObject<FileData>(receivedData);

        if (fileData == null)
        {
            Console.WriteLine("Could not deserialize downloaded file info");
        }

        localFileService.StoreFile(fileData);
        fileCountReceived++;
    }
}

客户端发送文件代码：

public async Task SendFiles(Socket socket, DownloadFilesRequest request)
{
    foreach (var path in request.Paths)
    {
        var response = new FileData()
        {
            Path = path,
            Contents = localFileService.GetFileContents(path),
        };

        await SocketHelper.SendAsync(socket, response);
    }
}

疑问：是否是客户端发送量超出服务端接收能力的同步问题？字节计数为何失效？是否需要消息标记或大文件分块？

1. 核心问题：SocketHelper的字节计数逻辑存在致命错误 #

字节计数方案本身可行，但SocketHelper的实现有多个bug，导致数据传输错位：

错误点1：长度前缀接收不完整 #

ReceiveAsync用32字节缓冲区接收长度前缀，但BitConverter.ToInt32仅需前4字节，且socket.ReceiveAsync不一定一次性返回4字节。若只收到1-3字节就解析，会得到错误长度，后续payload接收必然出错。

错误点2：发送payload时未指定偏移量 #

循环发送sendBuffer时，每次都发送整个数组而非从sentBytes位置开始的剩余字节，导致前面字节重复发送、后面字节可能未发完，最终payload内容混乱。

错误点3：接收payload时未指定偏移量 #

接收payload时每次都写入缓冲区起始位置，覆盖之前接收的内容，导致数据丢失、payload不完整。

修复后的SocketHelper代码 #

public class SocketHelper
{
    private const int Int32Bytes = 4;

    public static async Task SendAsync(Socket socket, object message)
    {
        var json = JsonConvert.SerializeObject(message);
        var sendBuffer = Encoding.UTF8.GetBytes(json);
        // 发送固定4字节的长度前缀
        var lengthBytes = BitConverter.GetBytes(sendBuffer.Length);
        await SendAllAsync(socket, lengthBytes, 0, lengthBytes.Length);
        // 发送完整payload
        await SendAllAsync(socket, sendBuffer, 0, sendBuffer.Length);
    }

    public static async Task<string> ReceiveAsync(Socket socket)
    {
        // 确保收满4字节长度前缀
        var lengthBytes = new byte[Int32Bytes];
        await ReceiveAllAsync(socket, lengthBytes, 0, Int32Bytes);
        var payloadLength = BitConverter.ToInt32(lengthBytes, 0);
        // 接收指定长度的payload
        var payloadBuffer = new byte[payloadLength];
        await ReceiveAllAsync(socket, payloadBuffer, 0, payloadLength);
        return Encoding.UTF8.GetString(payloadBuffer);
    }

    // 确保发送指定长度字节的辅助方法
    private static async Task SendAllAsync(Socket socket, byte[] buffer, int offset, int count)
    {
        var sent = 0;
        while (sent < count)
        {
            var bytesSent = await socket.SendAsync(buffer.AsMemory(offset + sent, count - sent), SocketFlags.None);
            sent += bytesSent;
        }
    }

    // 确保接收指定长度字节的辅助方法
    private static async Task ReceiveAllAsync(Socket socket, byte[] buffer, int offset, int count)
    {
        var received = 0;
        while (received < count)
        {
            var bytesReceived = await socket.ReceiveAsync(buffer.AsMemory(offset + received, count - received), SocketFlags.None);
            if (bytesReceived == 0)
            {
                throw new IOException("Socket connection closed");
            }
            received += bytesReceived;
        }
    }
}

2. 大文件传输优化：避免直接序列化整个文件到JSON #

即使修复SocketHelper，将大文件内容直接序列化到JSON仍会引发两个问题：

• 内存占用过高：大文件全部加载到内存易引发OOM
• JSON反序列化耗时久，出错会导致整个文件传输失败

优化方案：分块传输文件 #

重新设计协议，分步骤发送：

1. 客户端先发送文件元数据（路径、文件名、总大小），用修复后的SocketHelper发送
2. 服务端接收元数据后回复确认（ACK）
3. 客户端分块发送文件内容：每块固定大小（如4KB/64KB），每块前加4字节块长度（0表示传输结束）
4. 服务端按块接收并写入本地文件，直到收到长度为0的块

客户端分块发送示例 #

public async Task SendFileChunked(Socket socket, string filePath)
{
    var fileInfo = new FileInfo(filePath);
    // 发送文件元数据
    var metadata = new FileMetadata
    {
        Path = filePath,
        FileName = fileInfo.Name,
        TotalSize = fileInfo.Length
    };
    await SocketHelper.SendAsync(socket, metadata);
    // 等待服务端ACK
    var ack = await SocketHelper.ReceiveAsync(socket);
    if (!ack.Equals("ACK"))
    {
        throw new Exception("Server rejected file metadata");
    }
    // 分块发送内容
    const int chunkSize = 4096;
    using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read);
    var buffer = new byte[chunkSize];
    int bytesRead;
    while ((bytesRead = await fs.ReadAsync(buffer, 0, chunkSize)) > 0)
    {
        await SendAllAsync(socket, BitConverter.GetBytes(bytesRead), 0, Int32Bytes);
        await SendAllAsync(socket, buffer, 0, bytesRead);
        // 等待块ACK（增强可靠性）
        await SocketHelper.ReceiveAsync(socket);
    }
    // 发送结束标记
    await SendAllAsync(socket, BitConverter.GetBytes(0), 0, Int32Bytes);
}

服务端分块接收示例 #

public async Task ReceiveFileChunked(Socket socket, string saveDir)
{
    // 接收文件元数据
    var metadataJson = await SocketHelper.ReceiveAsync(socket);
    var metadata = JsonConvert.DeserializeObject<FileMetadata>(metadataJson);
    // 发送ACK
    await SocketHelper.SendAsync(socket, "ACK");
    // 分块接收内容
    var savePath = Path.Combine(saveDir, metadata.FileName);
    using var fs = new FileStream(savePath, FileMode.Create, FileAccess.Write);
    while (true)
    {
        // 接收块长度
        var chunkLengthBytes = new byte[Int32Bytes];
        await ReceiveAllAsync(socket, chunkLengthBytes, 0, Int32Bytes);
        var chunkLength = BitConverter.ToInt32(chunkLengthBytes, 0);
        if (chunkLength == 0)
        {
            break; // 传输结束
        }
        // 接收块内容并写入文件
        var chunkBuffer = new byte[chunkLength];
        await ReceiveAllAsync(socket, chunkBuffer, 0, chunkLength);
        await fs.WriteAsync(chunkBuffer, 0, chunkLength);
        // 发送块ACK
        await SocketHelper.SendAsync(socket, "ACK");
    }
}

3. 额外可靠性增强建议 #

• 增加校验和：每个消息/块附带MD5/SHA1校验值，服务端接收后验证数据完整性
• 错误重传机制：校验失败或未收到ACK时，客户端重传对应消息/块
• 限速控制：客户端根据服务端处理能力调整发送速率，避免网络拥塞

内容的提问来源于stack exchange，提问作者xIcarus