桌面捕获图像序列编码为视频流并通过网络传输的实现方案问询

问题描述 #

我已经通过自定义的DesktopCapture类获取了大量桌面捕获图像，希望将这些图像序列通过视频编码器编码为视频流，再通过网络发送至另一接收端应用，由该应用接收数据并播放。请问如何实现图像编码为视频并进行网络传输？

现有桌面捕获类代码 #

// 简易桌面捕获类
public class DesktopCapture
{
    public readonly Bitmap Frame;
    public readonly Graphics FrameGraphics;
    private readonly Rectangle CaptureArea;

    public DesktopCapture(Rectangle captureArea)
    {
        CaptureArea = captureArea;
        Frame = new Bitmap(captureArea.Width, captureArea.Height);
        FrameGraphics = Graphics.FromImage(Frame);
    }
    
    public void Capture()
    {
        FrameGraphics.CopyFromScreen(CaptureArea.Location, Point.Empty, CaptureArea.Size);
    }
}

期望实现的代码示例 #

TcpClient client = new TcpClient();
client.Connect(new IPEndPoint(IPAddress.Loopback, 5555));

DesktopCapture desktopCap = new DesktopCapture(Screen.PrimaryScreen.Bounds);

NetworkStream clientStream = client.GetStream();
VideoStreamWriter videoStream = new VideoStreamWriter(Encoder.SomeEncoder, clientStream);
videoStreamWriter.StartEncoding();
while (true)
{
    desktopCap.Capture();
    videoStreamWriter.EncodeFrame(desktopCap.Frame);
}

实现方案 #

推荐使用FFmpeg.AutoGen（FFmpeg的C#绑定库）完成视频编码，结合TCP协议实现网络传输，以下是完整可运行的代码实现：

步骤1：安装依赖包

通过NuGet安装FFmpeg.AutoGen：

Install-Package FFmpeg.AutoGen

步骤2：实现视频编码写入类（VideoStreamWriter）

这个类封装FFmpeg的H.264编码逻辑，将Bitmap帧转换为视频流并写入NetworkStream：

using System;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using FFmpeg.AutoGen;

public unsafe class VideoStreamWriter : IDisposable
{
    private readonly AVCodecContext* _codecContext;
    private readonly AVFormatContext* _formatContext;
    private readonly AVStream* _videoStream;
    private readonly NetworkStream _networkStream;
    private long _pts;

    public VideoStreamWriter(int width, int height, int fps, NetworkStream stream)
    {
        _networkStream = stream;
        
        // 初始化FFmpeg组件
        ffmpeg.av_register_all();
        ffmpeg.avcodec_register_all();
        ffmpeg.avformat_network_init();

        // 查找H.264编码器
        var codec = ffmpeg.avcodec_find_encoder(AVCodecID.AV_CODEC_ID_H264);
        if (codec == null) throw new InvalidOperationException("找不到H.264编码器，请确保FFmpeg环境正确");

        // 创建输出格式上下文
        ffmpeg.avformat_alloc_output_context2(&_formatContext, null, "h264", null);
        if (_formatContext == null) throw new InvalidOperationException("无法创建视频格式上下文");

        // 创建视频流
        _videoStream = ffmpeg.avformat_new_stream(_formatContext, codec);
        if (_videoStream == null) throw new InvalidOperationException("无法创建视频流");

        // 配置编码器参数
        _codecContext = _videoStream->codec;
        _codecContext->codec_id = AVCodecID.AV_CODEC_ID_H264;
        _codecContext->codec_type = AVMediaType.AVMEDIA_TYPE_VIDEO;
        _codecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_YUV420P;
        _codecContext->width = width;
        _codecContext->height = height;
        _codecContext->time_base = new AVRational { num = 1, den = fps };
        _codecContext->framerate = new AVRational { num = fps, den = 1 };
        _codecContext->gop_size = 10; // 每10帧生成一个关键帧
        _codecContext->max_b_frames = 1;
        _codecContext->bit_rate = 5000000; // 5Mbps码率，可根据需求调整

        // 打开编码器
        if (ffmpeg.avcodec_open2(_codecContext, codec, null) < 0)
            throw new InvalidOperationException("无法打开H.264编码器");

        // 写入流头部（包含SPS/PPS等关键信息）
        var headerBuffer = new byte[1024];
        fixed (byte* bufPtr = headerBuffer)
        {
            int ret = ffmpeg.avformat_write_header(_formatContext, null);
            if (ret < 0) throw new InvalidOperationException("无法写入视频流头部");
            
            // 将头部数据写入网络流
            while (ret > 0)
            {
                int writeSize = Math.Min(ret, headerBuffer.Length);
                _networkStream.Write(headerBuffer, 0, writeSize);
                ret -= writeSize;
            }
        }
    }

    public void EncodeFrame(Bitmap frame)
    {
        // 分配YUV格式的AVFrame
        var yuvFrame = ffmpeg.av_frame_alloc();
        yuvFrame->format = (int)AVPixelFormat.AV_PIX_FMT_YUV420P;
        yuvFrame->width = _codecContext->width;
        yuvFrame->height = _codecContext->height;
        ffmpeg.av_frame_get_buffer(yuvFrame, 32);

        // 将Bitmap的RGB格式转换为YUV420P
        using (var bitmapData = frame.LockBits(new Rectangle(0, 0, frame.Width, frame.Height), ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb))
        {
            var swsContext = ffmpeg.sws_getContext(frame.Width, frame.Height, AVPixelFormat.AV_PIX_FMT_BGR24,
                _codecContext->width, _codecContext->height, AVPixelFormat.AV_PIX_FMT_YUV420P,
                ffmpeg.SWS_BILINEAR, null, null, null);
            
            var srcData = new byte_ptrArray4 { [0] = (byte*)bitmapData.Scan0 };
            var srcLinesize = new int_array4 { [0] = bitmapData.Stride };
            ffmpeg.sws_scale(swsContext, srcData, srcLinesize, 0, frame.Height, yuvFrame->data, yuvFrame->linesize);
            ffmpeg.sws_freeContext(swsContext);
        }

        // 设置帧的显示时间戳（PTS）
        yuvFrame->pts = _pts++;

        // 编码帧
        var pkt = ffmpeg.av_packet_alloc();
        ffmpeg.av_init_packet(pkt);
        pkt->data = null;
        pkt->size = 0;

        int ret = ffmpeg.avcodec_send_frame(_codecContext, yuvFrame);
        if (ret < 0) throw new InvalidOperationException("发送帧到编码器失败");

        while (ret >= 0)
        {
            ret = ffmpeg.avcodec_receive_packet(_codecContext, pkt);
            if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN) || ret == ffmpeg.AVERROR_EOF) break;
            if (ret < 0) throw new InvalidOperationException("接收编码后的数据包失败");

            // 调整时间戳适配流格式
            ffmpeg.av_packet_rescale_ts(pkt, _codecContext->time_base, _videoStream->time_base);
            pkt->stream_index = _videoStream->index;

            // 将编码后的数据包写入网络流
            _networkStream.Write(new Span<byte>(pkt->data, pkt->size));
            
            ffmpeg.av_packet_unref(pkt);
        }

        // 释放资源
        ffmpeg.av_frame_free(&yuvFrame);
        ffmpeg.av_packet_free(&pkt);
    }

    public void Dispose()
    {
        // 写入流尾部
        ffmpeg.av_write_trailer(_formatContext);
        
        // 释放FFmpeg资源和网络流
        ffmpeg.avcodec_close(_codecContext);
        ffmpeg.avformat_free_context(_formatContext);
        _networkStream?.Dispose();
    }
}

步骤3：编码端主程序

结合你的DesktopCapture类，实现桌面捕获、编码、传输的完整流程：

using System;
using System.Drawing;
using System.Net;
using System.Net.Sockets;

class Program
{
    static void Main(string[] args)
    {
        try
        {
            using var tcpClient = new TcpClient();
            tcpClient.Connect(new IPEndPoint(IPAddress.Loopback, 5555));
            
            var screenBounds = Screen.PrimaryScreen.Bounds;
            using var desktopCapture = new DesktopCapture(screenBounds);
            using var networkStream = tcpClient.GetStream();
            // 以30fps的帧率编码
            using var videoWriter = new VideoStreamWriter(screenBounds.Width, screenBounds.Height, 30, networkStream);

            Console.WriteLine("开始桌面捕获与传输，按任意键停止...");
            while (!Console.KeyAvailable)
            {
                desktopCapture.Capture();
                videoWriter.EncodeFrame(desktopCapture.Frame);
                // 控制帧率，避免CPU占用过高
                System.Threading.Thread.Sleep(1000 / 30);
            }
        }
        catch (Exception ex)
        {
            Console.WriteLine($"发生错误：{ex.Message}");
        }
    }
}

步骤4：接收端实现思路

接收端通过TcpListener监听指定端口，接收数据流后用FFmpeg解码，再将帧显示到UI控件（如WinForm的PictureBox）。核心逻辑如下：

using System;
using System.Net;
using System.Net.Sockets;
using FFmpeg.AutoGen;

public unsafe class VideoStreamReceiver : IDisposable
{
    private readonly TcpListener _listener;
    private AVCodecContext* _decoderContext;

    public VideoStreamReceiver(int port)
    {
        _listener = new TcpListener(IPAddress.Loopback, port);
        _listener.Start();
        
        // 初始化FFmpeg
        ffmpeg.av_register_all();
        ffmpeg.avcodec_register_all();

        // 初始化H.264解码器
        var decoder = ffmpeg.avcodec_find_decoder(AVCodecID.AV_CODEC_ID_H264);
        _decoderContext = ffmpeg.avcodec_alloc_context3(decoder);
        if (ffmpeg.avcodec_open2(_decoderContext, decoder, null) < 0)
            throw new InvalidOperationException("无法打开H.264解码器");
    }

    public void StartReceiving()
    {
        using var client = _listener.AcceptTcpClient();
        using var stream = client.GetStream();

        var buffer = new byte[4096];
        var pkt = ffmpeg.av_packet_alloc();
        
        while (stream.Read(buffer, 0, buffer.Length) > 0)
        {
            fixed (byte* bufPtr = buffer)
            {
                pkt->data = bufPtr;
                pkt->size = buffer.Length;
                
                // 发送数据包到解码器
                int ret = ffmpeg.avcodec_send_packet(_decoderContext, pkt);
                if (ret >= 0)
                {
                    var frame = ffmpeg.av_frame_alloc();
                    ret = ffmpeg.avcodec_receive_frame(_decoderContext, frame);
                    if (ret >= 0)
                    {
                        // 将解码后的YUV帧转换为Bitmap并显示
                        DisplayDecodedFrame(frame);
                    }
                    ffmpeg.av_frame_free(&frame);
                }
            }
        }
        
        ffmpeg.av_packet_free(&pkt);
    }

    private void DisplayDecodedFrame(AVFrame* frame)
    {
        // 实现YUV到Bitmap的转换，更新UI控件
        // 可参考编码端的格式转换逻辑反向实现，将YUV420P转为RGB格式
    }

    public void Dispose()
    {
        ffmpeg.avcodec_close(_decoderContext);
        _listener.Stop();
    }
}

内容的提问来源于stack exchange，提问作者SlimeNull