在Avalonia的.NET应用中，如何为gstreamer-sharp-netcore Pipeline添加gdkpixbufoverlay

你遇到的问题根源在于直接使用playbin时，无法在其内部的视频解码链路和AppSink之间插入滤镜元素。要解决这个问题，需要手动构建视频处理链路，替代playbin的默认流程，这样就能在解码后的视频流中插入gdkpixbufoverlay。

解决方案步骤 #

1. 拆解playbin链路：使用uridecodebin替代playbin，手动处理视频流的解码、滤镜叠加和输出。
2. 插入gdkpixbufoverlay：在解码后的视频流路径中添加gdkpixbufoverlay元素，并配置叠加图片的路径和位置。
3. 动态链接元素：处理uridecodebin的pad-added信号，动态将解码后的视频pad链接到后续的滤镜链和AppSink。

完整实现代码 #

Pipeline构建代码

private Pipeline _pipeline;
private Element _uriDecodeBin;
private Element _videoConvert1;
private Element _overlay;
private Element _videoConvert2;
private AppSink _videoSink;

public void InitializePipeline()
{
    _pipeline = new Pipeline("playback");
    
    // 创建uridecodebin替代playbin
    _uriDecodeBin = Gst.ElementFactory.Make("uridecodebin", "uridecodebin");
    _uriDecodeBin["uri"] = AppController.Configuration["keying:videoStream"];
    _pipeline.Add(_uriDecodeBin);

    // 创建视频处理元素链：videoconvert -> gdkpixbufoverlay -> videoconvert
    _videoConvert1 = Gst.ElementFactory.Make("videoconvert", "videoconvert1");
    _overlay = Gst.ElementFactory.Make("gdkpixbufoverlay", "overlay");
    // 配置叠加图片路径和位置（按需调整x/y坐标，示例为左上角）
    _overlay["location"] = "/path/to/your/overlay.png";
    _overlay["x"] = 10;
    _overlay["y"] = 10;
    _videoConvert2 = Gst.ElementFactory.Make("videoconvert", "videoconvert2");
    
    _pipeline.Add(_videoConvert1);
    _pipeline.Add(_overlay);
    _pipeline.Add(_videoConvert2);

    // 链接滤镜链内部元素
    if (!_videoConvert1.Link(_overlay))
        throw new Exception("Failed to link videoconvert1 to overlay");
    if (!_overlay.Link(_videoConvert2))
        throw new Exception("Failed to link overlay to videoconvert2");

    // 创建并配置AppSink
    _videoSink = new AppSink("videoSink");
    _videoSink["caps"] = Caps.FromString("video/x-raw,format=RGBA");
    _videoSink.Drop = true;
    _videoSink.Sync = true;
    _videoSink.MaxLateness = (1000 / 30) * 1000000;
    _videoSink.MaxBuffers = 1;
    _videoSink.Qos = true;
    _videoSink.EnableLastSample = false;
    
    _pipeline.Add(_videoSink);
    // 链接最后一个videoconvert到AppSink
    if (!_videoConvert2.Link(_videoSink))
        throw new Exception("Failed to link videoconvert2 to videosink");

    // 处理uridecodebin的pad-added信号，动态链接视频输出
    _uriDecodeBin.PadAdded += OnUriDecodeBinPadAdded;

    // 启动Pipeline
    _pipeline.SetState(Gst.State.Playing);
}

private void OnUriDecodeBinPadAdded(object? sender, PadAddedArgs args)
{
    var srcPad = args.Pad;
    // 只处理视频流
    var caps = srcPad.CurrentCaps;
    if (caps == null || !caps.ToString().StartsWith("video/x-raw"))
        return;

    // 获取videoconvert1的sink pad
    var sinkPad = _videoConvert1.GetStaticPad("sink");
    if (sinkPad.IsLinked)
        return;

    // 动态链接uridecodebin的视频pad到videoconvert1
    if (!srcPad.Link(sinkPad))
        throw new Exception("Failed to link uridecodebin video pad to videoconvert1");
}

渲染代码（保持原有逻辑不变）

private int renderLock = 0;

private void OnTick(TimeSpan time)
{
    if (System.Threading.Interlocked.CompareExchange(ref renderLock, 1, 0) == 0)
    {
        try
        {
            PullVideoSample();
        }
        finally
        {
            System.Threading.Interlocked.Decrement(ref renderLock);
        }
    }
}

private void PullVideoSample()
{
    AppSink? sink = _videoSink;

    if (sink == null)
        return;

    using Sample sample = sink.TryPullSample(0);
    if (sample == null)
        return;

    using Caps caps = sample.Caps;
    Structure cap = caps[0];

    string format = cap.GetString("format");
    cap.GetInt("width", out int width);
    cap.GetInt("height", out int height);

    using Gst.Buffer buffer = sample.Buffer;
    if (format == "RGBA" && buffer.Map(out MapInfo map, MapFlags.Read))
    {
        FrameImage.UpdateImage(ref map, width, height);
        buffer.Unmap(map);
    }
}

关键说明 #

• 使用uridecodebin替代playbin，可以更灵活地控制解码后的视频流路径，方便插入滤镜元素。
• videoconvert元素用于转换视频格式，确保gdkpixbufoverlay能处理输入的视频格式，同时最终输出RGBA格式给AppSink。
• 必须处理uridecodebin的pad-added信号，因为它的视频输出pad是动态创建的，需要在pad可用时手动链接到后续链路。

内容的提问来源于stack exchange，提问作者Jiří Darmovzal