如何用OpenCV获取摄像头帧并通过FFmpeg编码后UDP发送？

Got it, let's tweak your existing code to stream H.264 over UDP instead of saving to a file. The main changes revolve around reconfiguring FFmpeg's output context for network streaming rather than file writing—most of your OpenCV capture and encoding logic stays intact. Here's a complete, adjusted version of your code with explanations:

Key Modifications Explained #

• UDP Output URL: We use a URL like udp://127.0.0.1:1234 instead of a file path. For cross-network streaming, replace 127.0.0.1 with the receiver's IP address.
• MPEG-TS Format: We choose MPEG Transport Stream (mpegts) as the output format—it’s ideal for UDP because it handles packet fragmentation and synchronization, preventing playback glitches from lost packets.
• Streaming Flags: We enable AVFMT_FLAG_FLUSH_PACKETS to ensure encoded frames are sent immediately (critical for real-time streaming) and adjust buffer settings for low latency.

Full Working Code #

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libavutil/imgutils.h>
#include <libavutil/mathematics.h>
#include <libswscale/swscale.h>
#include <libswresample/swresample.h>
}
#include <opencv2/opencv.hpp>

using namespace std;
using namespace cv;

#define WIDTH 640
#define HEIGHT 480
#define CODEC_ID AV_CODEC_ID_H264
#define STREAM_PIX_FMT AV_PIX_FMT_YUV420P
#define FRAME_RATE 30

int main() {
    // Initialize FFmpeg components
    av_register_all();
    avformat_network_init();

    // Open OpenCV camera
    VideoCapture cap(0);
    if (!cap.isOpened()) {
        fprintf(stderr, "Could not open camera\n");
        return -1;
    }
    cap.set(CAP_PROP_FRAME_WIDTH, WIDTH);
    cap.set(CAP_PROP_FRAME_HEIGHT, HEIGHT);
    cap.set(CAP_PROP_FPS, FRAME_RATE);

    // FFmpeg output context setup for UDP
    AVFormatContext* fmt_ctx = NULL;
    const char* out_url = "udp://127.0.0.1:1234"; // Replace with receiver IP:port
    AVOutputFormat* fmt = av_guess_format("mpegts", NULL, NULL);
    if (!fmt) {
        fprintf(stderr, "Could not guess output format\n");
        return -1;
    }
    if (avformat_alloc_output_context2(&fmt_ctx, fmt, NULL, out_url) < 0) {
        fprintf(stderr, "Could not create output context\n");
        return -1;
    }

    // Add video stream
    AVStream* stream = avformat_new_stream(fmt_ctx, NULL);
    if (!stream) {
        fprintf(stderr, "Could not create stream\n");
        return -1;
    }
    AVCodecContext* codec_ctx = stream->codec;

    // Configure codec parameters
    codec_ctx->codec_id = CODEC_ID;
    codec_ctx->codec_type = AVMEDIA_TYPE_VIDEO;
    codec_ctx->width = WIDTH;
    codec_ctx->height = HEIGHT;
    codec_ctx->pix_fmt = STREAM_PIX_FMT;
    codec_ctx->bit_rate = 1000000; // 1 Mbps adjust based on your needs
    codec_ctx->time_base = av_make_q(1, FRAME_RATE);
    codec_ctx->framerate = av_make_q(FRAME_RATE, 1);
    codec_ctx->gop_size = 10; // Keyframe every 10 frames
    codec_ctx->max_b_frames = 1;

    // Set codec options (for H.264, use libx264)
    if (fmt_ctx->oformat->flags & AVFMT_GLOBALHEADER) {
        codec_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    }
    av_opt_set(codec_ctx->priv_data, "preset", "ultrafast", 0); // Low latency preset
    av_opt_set(codec_ctx->priv_data, "tune", "zerolatency", 0);

    // Open codec
    AVCodec* codec = avcodec_find_encoder(codec_ctx->codec_id);
    if (!codec) {
        fprintf(stderr, "Could not find encoder\n");
        return -1;
    }
    if (avcodec_open2(codec_ctx, codec, NULL) < 0) {
        fprintf(stderr, "Could not open codec\n");
        return -1;
    }

    // Initialize pixel format converter (OpenCV BGR to FFmpeg YUV420P)
    struct SwsContext* sws_ctx = sws_getContext(
        WIDTH, HEIGHT, AV_PIX_FMT_BGR24,
        WIDTH, HEIGHT, STREAM_PIX_FMT,
        SWS_BILINEAR, NULL, NULL, NULL
    );
    if (!sws_ctx) {
        fprintf(stderr, "Could not create sws context\n");
        return -1;
    }

    // Allocate frame buffers
    AVFrame* frame = av_frame_alloc();
    frame->format = STREAM_PIX_FMT;
    frame->width = WIDTH;
    frame->height = HEIGHT;
    if (av_frame_get_buffer(frame, 32) < 0) {
        fprintf(stderr, "Could not allocate frame buffer\n");
        return -1;
    }

    // Prepare output headers
    av_dump_format(fmt_ctx, 0, out_url, 1);
    if (!(fmt_ctx->oformat->flags & AVFMT_NOFILE)) {
        if (avio_open(&fmt_ctx->pb, out_url, AVIO_FLAG_WRITE) < 0) {
            fprintf(stderr, "Could not open UDP output\n");
            return -1;
        }
    }
    if (avformat_write_header(fmt_ctx, NULL) < 0) {
        fprintf(stderr, "Could not write header\n");
        return -1;
    }

    // Stream loop
    AVPacket pkt;
    av_init_packet(&pkt);
    pkt.data = NULL;
    pkt.size = 0;
    int frame_idx = 0;
    Mat cv_frame;

    while (cap.read(cv_frame)) {
        // Convert OpenCV BGR frame to FFmpeg YUV420P
        const uint8_t* src_data[1] = {cv_frame.data};
        int src_linesize[1] = {cv_frame.step};
        sws_scale(sws_ctx, src_data, src_linesize, 0, HEIGHT, frame->data, frame->linesize);

        // Set frame timestamp
        frame->pts = frame_idx++;

        // Encode frame
        int got_packet = 0;
        if (avcodec_encode_video2(codec_ctx, &pkt, frame, &got_packet) < 0) {
            fprintf(stderr, "Encoding failed\n");
            break;
        }

        if (got_packet) {
            // Adjust packet timestamp for stream
            av_packet_rescale_ts(&pkt, codec_ctx->time_base, stream->time_base);
            pkt.stream_index = stream->index;

            // Send packet via UDP
            if (av_interleaved_write_frame(fmt_ctx, &pkt) < 0) {
                fprintf(stderr, "Failed to send packet\n");
                break;
            }
            av_packet_unref(&pkt);
        }
    }

    // Flush remaining frames
    int got_packet = 0;
    do {
        avcodec_encode_video2(codec_ctx, &pkt, NULL, &got_packet);
        if (got_packet) {
            av_packet_rescale_ts(&pkt, codec_ctx->time_base, stream->time_base);
            pkt.stream_index = stream->index;
            av_interleaved_write_frame(fmt_ctx, &pkt);
            av_packet_unref(&pkt);
        }
    } while (got_packet);

    // Cleanup
    av_write_trailer(fmt_ctx);
    avcodec_close(codec_ctx);
    av_frame_free(&frame);
    sws_freeContext(sws_ctx);
    if (!(fmt_ctx->oformat->flags & AVFMT_NOFILE)) {
        avio_close(fmt_ctx->pb);
    }
    avformat_free_context(fmt_ctx);
    cap.release();

    return 0;
}

Testing the Stream #

To receive and play the stream on the same machine (or another machine with the correct IP), use FFplay:

ffplay udp://127.0.0.1:1234

Notes #

• Latency: The ultrafast preset and zerolatency tune are critical for minimizing delay—adjust the bit rate if you see artifacts or need better quality.
• Network Firewall: Ensure the UDP port (1234 in this example) is open on both sender and receiver machines.
• Cross-Network Streaming: Replace 127.0.0.1 in the output URL with the receiver's public or local IP address, depending on your network setup.

内容的提问来源于stack exchange，提问作者Rendres