如何使用Expo GL与Camera录制带人脸滤镜的视频？

核心思路 #

你已经实现了人脸检测和滤镜，现在需要把GL View渲染的带滤镜画面录制成视频，核心是将相机帧传入GL View渲染滤镜效果，再持续捕获GL渲染帧并编码为视频。以下是具体实现步骤：

1. 安装依赖 #

需要用到Expo的GL渲染、相机、人脸检测和音视频录制工具：

npx expo install expo-gl expo-camera expo-av expo-face-detector

2. 配置权限 #

在app.json中添加必要权限：

{
  "expo": {
    "ios": {
      "infoPlist": {
        "NSCameraUsageDescription": "需要访问相机拍摄视频",
        "NSFaceIDUsageDescription": "需要人脸检测来应用滤镜"
      }
    },
    "android": {
      "permissions": ["CAMERA", "RECORD_AUDIO", "WRITE_EXTERNAL_STORAGE"]
    }
  }
}

3. 完整代码实现 #

这个示例会渲染前置相机画面，检测人脸并给人脸区域添加灰度滤镜，同时支持录制带滤镜的视频：

import React, { useRef, useEffect, useState } from 'react';
import { View, Button, Text } from 'react-native';
import { GLView } from 'expo-gl';
import { Camera } from 'expo-camera';
import { FaceDetector } from 'expo-face-detector';
import { AV } from 'expo-av';

export default function GLFilterVideoRecorder() {
  const cameraRef = useRef(null);
  const glViewRef = useRef(null);
  const [hasPermission, setHasPermission] = useState(null);
  const [isRecording, setIsRecording] = useState(false);
  const recorderRef = useRef(null);

  // 请求权限
  useEffect(() => {
    (async () => {
      const cameraStatus = await Camera.requestCameraPermissionsAsync();
      await FaceDetector.requestPermissionsAsync();
      setHasPermission(cameraStatus.status === 'granted');
    })();
  }, []);

  // 初始化视频录制器
  const initRecorder = async () => {
    const recorder = new AV.Recorder();
    await recorder.prepareToRecordAsync({
      videoCodec: AV.VideoCodec.H264,
      audioCodec: AV.AudioCodec.AAC,
      videoBitrate: 5_000_000, // 5Mbps，可根据需求调整
    });
    recorderRef.current = recorder;
  };

  // 开始录制
  const startRecording = async () => {
    if (!recorderRef.current) await initRecorder();
    await recorderRef.current.startAsync();
    setIsRecording(true);
  };

  // 停止录制并获取视频URI
  const stopRecording = async () => {
    if (!recorderRef.current) return;
    await recorderRef.current.stopAndUnloadAsync();
    const videoUri = recorderRef.current.getURI();
    console.log('录制完成，视频路径:', videoUri);
    // 可在此处添加保存到相册的逻辑（需安装expo-media-library）
    setIsRecording(false);
  };

  // GL上下文初始化和渲染逻辑
  const onGLContextCreate = async (gl) => {
    // --------------------------
    // 1. 编译着色器（实现人脸滤镜）
    // --------------------------
    const vertexShader = gl.createShader(gl.VERTEX_SHADER);
    gl.shaderSource(vertexShader, `
      attribute vec2 position;
      attribute vec2 texCoord;
      varying vec2 vTexCoord;
      void main() {
        gl_Position = vec4(position, 0.0, 1.0);
        vTexCoord = texCoord;
      }
    `);
    gl.compileShader(vertexShader);

    const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER);
    gl.shaderSource(fragmentShader, `
      precision mediump float;
      varying vec2 vTexCoord;
      uniform sampler2D cameraTexture;
      uniform vec4 faceRect; // 人脸UV坐标：x, y, width, height

      void main() {
        vec2 uv = vTexCoord;
        uv.y = 1.0 - uv.y; // 翻转Y轴，适配相机纹理方向

        // 判断当前像素是否在人脸区域
        bool isInFace = uv.x > faceRect.x && uv.x < faceRect.x + faceRect.z &&
                        uv.y > faceRect.y && uv.y < faceRect.y + faceRect.w;

        vec4 color = texture2D(cameraTexture, uv);
        // 人脸区域应用灰度滤镜，其他区域保持原色彩
        if (isInFace) {
          float gray = dot(color.rgb, vec3(0.299, 0.587, 0.114));
          gl_FragColor = vec4(gray, gray, gray, color.a);
        } else {
          gl_FragColor = color;
        }
      }
    `);
    gl.compileShader(fragmentShader);

    const program = gl.createProgram();
    gl.attachShader(program, vertexShader);
    gl.attachShader(program, fragmentShader);
    gl.linkProgram(program);
    gl.useProgram(program);

    // --------------------------
    // 2. 设置顶点和纹理坐标数据
    // --------------------------
    // 全屏顶点数据
    const vertices = new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]);
    const vertexBuffer = gl.createBuffer();
    gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
    gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW);
    const positionLoc = gl.getAttribLocation(program, 'position');
    gl.enableVertexAttribArray(positionLoc);
    gl.vertexAttribPointer(positionLoc, 2, gl.FLOAT, false, 0, 0);

    // 纹理坐标数据
    const texCoords = new Float32Array([0, 0, 1, 0, 0, 1, 1, 1]);
    const texCoordBuffer = gl.createBuffer();
    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
    gl.bufferData(gl.ARRAY_BUFFER, texCoords, gl.STATIC_DRAW);
    const texCoordLoc = gl.getAttribLocation(program, 'texCoord');
    gl.enableVertexAttribArray(texCoordLoc);
    gl.vertexAttribPointer(texCoordLoc, 2, gl.FLOAT, false, 0, 0);

    // --------------------------
    // 3. 初始化相机纹理
    // --------------------------
    const cameraTexture = gl.createTexture();
    gl.bindTexture(gl.TEXTURE_2D, cameraTexture);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
    const cameraTexLoc = gl.getUniformLocation(program, 'cameraTexture');
    gl.uniform1i(cameraTexLoc, 0);

    // 人脸矩形uniform变量
    const faceRectLoc = gl.getUniformLocation(program, 'faceRect');
    gl.uniform4f(faceRectLoc, 0, 0, 0, 0);

    // --------------------------
    // 4. 相机帧监听和人脸检测
    // --------------------------
    let currentFaceRect = [0, 0, 0, 0];
    if (cameraRef.current) {
      await cameraRef.current.setFacingAsync(Camera.Constants.Type.front);
      cameraRef.current.setOnCameraFrameReceivedAsync(async (frame) => {
        // 更新相机纹理内容
        gl.bindTexture(gl.TEXTURE_2D, cameraTexture);
        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);

        // 检测人脸
        const faces = await FaceDetector.detectFacesAsync(frame, {
          mode: FaceDetector.Constants.Mode.fast,
          detectLandmarks: FaceDetector.Constants.Landmarks.none,
        });

        if (faces.length > 0) {
          // 将人脸坐标转换为UV坐标（0-1范围）
          const face = faces[0];
          const x = face.bounds.origin.x / frame.width;
          const y = face.bounds.origin.y / frame.height;
          const width = face.bounds.size.width / frame.width;
          const height = face.bounds.size.height / frame.height;
          currentFaceRect = [x, y, width, height];
        } else {
          currentFaceRect = [0, 0, 0, 0];
        }
      });
    }

    // --------------------------
    // 5. 渲染循环+录制逻辑
    // --------------------------
    const renderLoop = () => {
      gl.clearColor(0, 0, 0, 1);
      gl.clear(gl.COLOR_BUFFER_BIT);

      // 更新人脸矩形数据
      gl.uniform4f(faceRectLoc, ...currentFaceRect);

      // 绘制全屏纹理
      gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
      gl.endFrameEXP();

      // 录制时捕获GL帧
      if (isRecording && recorderRef.current) {
        const pixels = new Uint8Array(gl.drawingBufferWidth * gl.drawingBufferHeight * 4);
        gl.readPixels(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
        // 将RGBA像素数据传入录制器
        recorderRef.current.appendFrameRaw(pixels, {
          width: gl.drawingBufferWidth,
          height: gl.drawingBufferHeight,
          format: AV.VideoFrameFormat.RGBA,
        });
      }

      requestAnimationFrame(renderLoop);
    };

    renderLoop();
  };

  // 权限状态处理
  if (hasPermission === null) return <View />;
  if (hasPermission === false) return <View style={{ flex: 1, justifyContent: 'center', alignItems: 'center' }}><Text>无相机权限，请在设置中开启</Text></View>;

  return (
    <View style={{ flex: 1 }}>
      {/* 隐藏原生相机视图，仅用GL渲染 */}
      <Camera ref={cameraRef} style={{ position: 'absolute', width: 0, height: 0 }} />
      {/* GL渲染视图 */}
      <GLView ref={glViewRef} style={{ flex: 1 }} onContextCreate={onGLContextCreate} />
      {/* 录制控制按钮 */}
      <View style={{ position: 'absolute', bottom: 30, alignSelf: 'center' }}>
        <Button title={isRecording ? '停止录制' : '开始录制'} onPress={isRecording ? stopRecording : startRecording} />
      </View>
    </View>
  );
}

关键注意事项 #

• 相机视图隐藏：原生Camera组件设置为0宽高，避免重复渲染，我们只需要它的帧数据
• 坐标转换：人脸检测的像素坐标需要转换为GL纹理的UV坐标（0-1范围），才能正确定位人脸区域
• 滤镜自定义：修改片段着色器的代码可以实现各种人脸滤镜，比如美颜、贴纸、色彩调整等
• 视频格式：appendFrameRaw传入的像素格式要和录制器配置匹配，若出现颜色异常，可尝试转换为YUV420格式
• 性能优化：人脸检测使用fast模式，避免过度消耗性能；录制时可适当降低分辨率或码率

内容的提问来源于stack exchange，提问作者helyz247