如何基于WebRTC在视频处理后仅返回文本结果

我的任务是获取客户端摄像头的视频流，对其进行分析后将结果以字符串形式返回。参考aiortc文档及GitHub示例实现了服务端代码，但当前方案仍需返回视频帧，无法满足仅返回文本结果的需求，请问该如何调整代码？

现有服务端代码 #

async def offer(request):
    params = await request.json()
    offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])

    pc = RTCPeerConnection()
    pcs.add(pc)

    await server(pc, offer)

    return web.Response(
        content_type="application/json",
        text=json.dumps(
            {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
        ),
    )

pcs = set()

async def server(pc, offer):
    @pc.on("connectionstatechange")
    async def on_connectionstatechange():
        print("Connection state is %s" % pc.connectionState)
        if pc.connectionState == "failed":
            await pc.close()
            pcs.discard(pc)

    @pc.on("track")
    def on_track(track):
        print("======= received track: ", track)
        if track.kind == "video":
            global new_video_track
            new_video_track = Sign(track)
            pc.addTrack(new_video_track)

    @pc.on("datachannel")
    def on_datachannel(channel):
        global new_video_track
        new_video_track.channel = channel
        print("mounted channel")

        @channel.on("message")
        async def on_message(message):
            if isinstance(message, str):
                data = message.encode("utf-8")
            else:
                data = message
            print("receive data: ", data)

    await pc.setRemoteDescription(offer)
    answer = await pc.createAnswer()
    await pc.setLocalDescription(answer)


async def on_shutdown(app):
    # close peer connections
    coros = [pc.close() for pc in pcs]
    await asyncio.gather(*coros)
    pcs.clear()

class Sign(VideoStreamTrack):
    kind = "video"

    def __init__(self, track):
        super().__init__()
        self.track = track
        self.channel = None

    async def recv(self):
        frame = await self.track.recv()
        
        # getting the result I need

        if self.channel and self.sentence:
                self.channel.send(json.dumps({'word': self.sentence}))
        return frame

现有客户端代码 #

var pc = null;
var localVideo = document.querySelector("video#localVideo");
var serverVideo = document.querySelector("video#serverVideo");

navigator.mediaDevices.getUserMedia({
  video: {
    height: 1080,
    width: 1920,
    frameRate: {
      max: 10
    }   
}
}).then(stream => {
  localVideo.srcObject = stream;
  localVideo.addEventListener('loadedmetadata', () => {
    localVideo.play();
  });
});

function negotiate () {
  return pc.createOffer().then(function (offer) {
    return pc.setLocalDescription(offer);
  }).then(function () {
    // wait for ICE gathering to complete
    return new Promise(function (resolve) {
      if (pc.iceGatheringState === 'complete') {
        resolve();
      } else {
        function checkState () {
          if (pc.iceGatheringState === 'complete') {
            pc.removeEventListener('icegatheringstatechange', checkState);
            resolve();
          }
        }
        pc.addEventListener('icegatheringstatechange', checkState);
      }
    });
  }).then(function () {
    var offer = pc.localDescription;
    return fetch('/offer', {
      body: JSON.stringify({
        sdp: offer.sdp,
        type: offer.type,
      }),
      headers: {
        'Content-Type': 'application/json'
      },
      method: 'POST'
    });
  }).then(function (response) {
    return response.json();
  }).then(function (answer) {
    return pc.setRemoteDescription(answer);
  }).catch(function (e) {
    alert(e);
  });
}

function start () {
  var config = {
    sdpSemantics: 'unified-plan',
    iceServers: [{ urls: ['stun:stun.l.google.com:19302'] }]
  };

  pc = new RTCPeerConnection(config);

  localVideo.srcObject.getVideoTracks().forEach(track => {
    pc.addTrack(track);
  });
  pc.addEventListener('track', function (evt) {
    console.log("receive server video");
    if (evt.track.kind == 'video') {
      serverVideo.srcObject = evt.streams[0];
    }
  });
  ch = pc.createDataChannel("chat", {
    ordered: false,
    maxRetransmits: 0,
  });
  ch.addEventListener("message", function (evt) {
    console.log(Date.now() - JSON.parse(evt.data).now);
  });

  document.getElementById('start').style.display = 'none';
  negotiate();
  document.getElementById('stop').style.display = 'inline-block';
}

function stop () {
  document.getElementById('stop').style.display = 'none';
  setTimeout(function () {
    pc.close();
  }, 500);
}

服务端调整 #

核心思路是移除视频回传逻辑，仅保留视频流接收与分析，通过DataChannel传输文本结果：

1. 删除Sign类，无需继承VideoStreamTrack
2. 修改on_track回调，循环接收视频帧并分析，直接通过DataChannel发送结果
3. 不再调用pc.addTrack添加回传视频轨道

调整后的服务端关键代码：

async def server(pc, offer):
    data_channel = None

    @pc.on("connectionstatechange")
    async def on_connectionstatechange():
        print("Connection state is %s" % pc.connectionState)
        if pc.connectionState == "failed":
            await pc.close()
            pcs.discard(pc)

    @pc.on("track")
    async def on_track(track):
        print("======= received track: ", track)
        if track.kind == "video":
            while True:
                try:
                    frame = await track.recv()
                    # 执行你的视频分析逻辑，替换成实际的结果生成代码
                    sentence = "识别到的文本结果"
                    # 通过DataChannel发送分析结果
                    if data_channel and sentence:
                        data_channel.send(json.dumps({'word': sentence}))
                except Exception as e:
                    print(f"Track ended or error: {e}")
                    break

    @pc.on("datachannel")
    def on_datachannel(channel):
        nonlocal data_channel
        data_channel = channel
        print("mounted channel")

        @channel.on("message")
        async def on_message(message):
            if isinstance(message, str):
                data = message.encode("utf-8")
            else:
                data = message
            print("receive data: ", data)

    await pc.setRemoteDescription(offer)
    answer = await pc.createAnswer()
    await pc.setLocalDescription(answer)

客户端调整 #

核心思路是移除服务端视频流接收逻辑，新增文本结果展示：

1. 删除pc.addEventListener('track')的回调代码，无需处理服务端回传的视频
2. 修改DataChannel的message回调，将接收到的文本结果展示到页面（需提前在HTML中添加结果展示元素，比如<div id="result"></div>）

调整后的客户端关键代码：

// 新增结果展示元素
var resultDiv = document.querySelector("div#result");

function start () {
  var config = {
    sdpSemantics: 'unified-plan',
    iceServers: [{ urls: ['stun:stun.l.google.com:19302'] }]
  };

  pc = new RTCPeerConnection(config);

  localVideo.srcObject.getVideoTracks().forEach(track => {
    pc.addTrack(track);
  });
  // 移除服务端视频接收逻辑
  // pc.addEventListener('track', function (evt) { ... });

  ch = pc.createDataChannel("chat", {
    ordered: false,
    maxRetransmits: 0,
  });
  ch.addEventListener("message", function (evt) {
    var data = JSON.parse(evt.data);
    // 在页面展示分析结果
    resultDiv.textContent = "分析结果：" + data.word;
  });

  document.getElementById('start').style.display = 'none';
  negotiate();
  document.getElementById('stop').style.display = 'inline-block';
}

内容的提问来源于stack exchange，提问作者Rasul Makhmudov