如何将ImageReader图像传入MediaRecorder？Camera2多输出兼容方案

要解决Camera2无法同时挂载四个输出Surface的问题，同时实现帧处理与视频录制的管线合并，你可以通过OpenGL手动将YUV_420_888格式的Image数据转换为纹理，再渲染到MediaRecorder的输入Surface，无需依赖API28+的HardwareBuffer。以下是具体实现步骤和代码：

1. 核心思路 #

通过OpenGL创建三个纹理分别对应YUV_420_888的Y、U、V平面，使用着色器将YUV数据转换为RGB（或直接适配MediaRecorder的输入格式），最终渲染到MediaRecorder的Surface上。同时在帧处理回调中完成自定义逻辑。

2. OpenGL纹理渲染工具类 #

这个类负责初始化EGL环境、创建YUV纹理、编写着色器以及将Image数据渲染到输出Surface：

class GLRenderer(private val outputSurface: Surface) {
    private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY
    private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT
    private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE
    private var yTextureId = 0
    private var uTextureId = 0
    private var vTextureId = 0
    private var program = 0

    fun init() {
        // 初始化EGL显示
        eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY)
        val version = IntArray(2)
        EGL14.eglInitialize(eglDisplay, version, 0, version, 1)

        // 选择EGL配置
        val configAttribs = intArrayOf(
            EGL14.EGL_RED_SIZE, 8,
            EGL14.EGL_GREEN_SIZE, 8,
            EGL14.EGL_BLUE_SIZE, 8,
            EGL14.EGL_ALPHA_SIZE, 8,
            EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
            EGL14.EGL_SURFACE_TYPE, EGL14.EGL_WINDOW_BIT,
            EGL14.EGL_NONE
        )
        val configs = arrayOfNulls<EGLConfig>(1)
        val numConfigs = IntArray(1)
        EGL14.eglChooseConfig(eglDisplay, configAttribs, 0, configs, 0, 1, numConfigs, 0)
        val config = configs[0] ?: throw RuntimeException("No valid EGL config found")

        // 创建EGL上下文
        val contextAttribs = intArrayOf(
            EGL14.EGL_CONTEXT_CLIENT_VERSION, 2,
            EGL14.EGL_NONE
        )
        eglContext = EGL14.eglCreateContext(eglDisplay, config, EGL14.EGL_NO_CONTEXT, contextAttribs, 0)

        // 创建输出Surface
        eglSurface = EGL14.eglCreateWindowSurface(eglDisplay, config, outputSurface, null, 0)
        EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)

        // 初始化着色器和纹理
        initShaders()
        initTextures()
    }

    private fun initShaders() {
        // 顶点着色器：处理顶点坐标映射
        val vertexShader = """
            attribute vec4 aPosition;
            attribute vec2 aTexCoord;
            varying vec2 vTexCoord;
            void main() {
                gl_Position = aPosition;
                vTexCoord = aTexCoord;
            }
        """.trimIndent()

        // 片段着色器：YUV转RGB
        val fragmentShader = """
            precision mediump float;
            varying vec2 vTexCoord;
            uniform sampler2D yTexture;
            uniform sampler2D uTexture;
            uniform sampler2D vTexture;
            
            void main() {
                float y = texture2D(yTexture, vTexCoord).r;
                float u = texture2D(uTexture, vTexCoord).r - 0.5;
                float v = texture2D(vTexture, vTexCoord).r - 0.5;
                
                float r = y + 1.402 * v;
                float g = y - 0.344136 * u - 0.714136 * v;
                float b = y + 1.772 * u;
                
                gl_FragColor = vec4(r, g, b, 1.0);
            }
        """.trimIndent()

        program = createProgram(vertexShader, fragmentShader)
    }

    private fun createProgram(vertexSource: String, fragmentSource: String): Int {
        val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource)
        val fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource)
        val program = GLES20.glCreateProgram()
        GLES20.glAttachShader(program, vertexShader)
        GLES20.glAttachShader(program, fragmentShader)
        GLES20.glLinkProgram(program)
        return program
    }

    private fun loadShader(type: Int, source: String): Int {
        val shader = GLES20.glCreateShader(type)
        GLES20.glShaderSource(shader, source)
        GLES20.glCompileShader(shader)
        return shader
    }

    private fun initTextures() {
        val textures = IntArray(3)
        GLES20.glGenTextures(3, textures, 0)
        yTextureId = textures[0]
        uTextureId = textures[1]
        vTextureId = textures[2]

        // 配置Y纹理（单通道亮度）
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yTextureId)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)

        // 配置U纹理（单通道色度）
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, uTextureId)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)

        // 配置V纹理（单通道色度）
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, vTextureId)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR)
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)
    }

    fun renderImage(image: Image) {
        EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)
        val planes = image.planes

        // 上传Y平面数据（处理rowStride对齐问题）
        val yPlane = planes[0]
        val yRowStride = yPlane.rowStride
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yTextureId)
        for (y in 0 until image.height) {
            yPlane.buffer.position(y * yRowStride)
            val rowBuffer = yPlane.buffer.slice()
            GLES20.glTexSubImage2D(
                GLES20.GL_TEXTURE_2D, 0,
                0, y, image.width, 1,
                GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE,
                rowBuffer
            )
        }

        // 上传U平面数据（宽高为原图像的1/2）
        val uPlane = planes[1]
        val uWidth = image.width / 2
        val uHeight = image.height / 2
        val uRowStride = uPlane.rowStride
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, uTextureId)
        for (y in 0 until uHeight) {
            uPlane.buffer.position(y * uRowStride)
            val rowBuffer = uPlane.buffer.slice()
            GLES20.glTexSubImage2D(
                GLES20.GL_TEXTURE_2D, 0,
                0, y, uWidth, 1,
                GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE,
                rowBuffer
            )
        }

        // 上传V平面数据
        val vPlane = planes[2]
        val vRowStride = vPlane.rowStride
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, vTextureId)
        for (y in 0 until uHeight) {
            vPlane.buffer.position(y * vRowStride)
            val rowBuffer = vPlane.buffer.slice()
            GLES20.glTexSubImage2D(
                GLES20.GL_TEXTURE_2D, 0,
                0, y, uWidth, 1,
                GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE,
                rowBuffer
            )
        }

        // 执行渲染
        GLES20.glUseProgram(program)

        val positionHandle = GLES20.glGetAttribLocation(program, "aPosition")
        val texCoordHandle = GLES20.glGetAttribLocation(program, "aTexCoord")
        val yTextureHandle = GLES20.glGetUniformLocation(program, "yTexture")
        val uTextureHandle = GLES20.glGetUniformLocation(program, "uTexture")
        val vTextureHandle = GLES20.glGetUniformLocation(program, "vTexture")

        // 顶点坐标和纹理坐标
        val vertices = floatArrayOf(-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f)
        val texCoords = floatArrayOf(0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f)

        val vertexBuffer = ByteBuffer.allocateDirect(vertices.size * 4)
            .order(ByteOrder.nativeOrder())
            .asFloatBuffer()
            .put(vertices)
            .position(0)

        val texCoordBuffer = ByteBuffer.allocateDirect(texCoords.size * 4)
            .order(ByteOrder.nativeOrder())
            .asFloatBuffer()
            .put(texCoords)
            .position(0)

        GLES20.glEnableVertexAttribArray(positionHandle)
        GLES20.glVertexAttribPointer(positionHandle, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer)

        GLES20.glEnableVertexAttribArray(texCoordHandle)
        GLES20.glVertexAttribPointer(texCoordHandle, 2, GLES20.GL_FLOAT, false, 0, texCoordBuffer)

        // 绑定纹理到着色器
        GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yTextureId)
        GLES20.glUniform1i(yTextureHandle, 0)

        GLES20.glActiveTexture(GLES20.GL_TEXTURE1)
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, uTextureId)
        GLES20.glUniform1i(uTextureHandle, 1)

        GLES20.glActiveTexture(GLES20.GL_TEXTURE2)
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, vTextureId)
        GLES20.glUniform1i(vTextureHandle, 2)

        // 绘制画面
        GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)

        // 交换缓冲区，输出到MediaRecorder
        EGL14.eglSwapBuffers(eglDisplay, eglSurface)

        // 必须释放Image，避免Camera2阻塞
        image.close()
    }

    fun release() {
        // 清理EGL资源
        EGL14.eglMakeCurrent(eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_CONTEXT)
        EGL14.eglDestroySurface(eglDisplay, eglSurface)
        EGL14.eglDestroyContext(eglDisplay, eglContext)
        EGL14.eglTerminate(eglDisplay)

        // 清理OpenGL资源
        GLES20.glDeleteTextures(3, intArrayOf(yTextureId, uTextureId, vTextureId), 0)
        GLES20.glDeleteProgram(program)
    }
}

3. 整合Camera2与MediaRecorder流程 #

将GLRenderer与你的Camera2、MediaRecorder逻辑整合，只需要挂载三个Surface到CaptureSession：

// 初始化MediaRecorder
val recorderSurface = MediaCodec.createPersistentInputSurface()
val mediaRecorder = MediaRecorder(context).apply {
    setAudioSource(MediaRecorder.AudioSource.MIC)
    setVideoSource(MediaRecorder.VideoSource.SURFACE)
    setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
    setVideoEncoder(MediaRecorder.VideoEncoder.H264)
    setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
    setVideoSize(3840, 2160) // 4K分辨率
    setVideoFrameRate(30)
    setInputSurface(recorderSurface)
    setOutputFile(/* 你的视频输出路径 */)
    prepare()
}

// 初始化GLRenderer（必须在GL线程执行，这里用主线程）
val glRenderer = GLRenderer(recorderSurface)
Handler(Looper.getMainLooper()).post {
    glRenderer.init()
}

// 初始化帧处理用的ImageReader
val frameReader = ImageReader.newInstance(4000, 2256, ImageFormat.YUV_420_888, 3)
frameReader.setOnImageAvailableListener({ reader ->
    val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener
    // 在GL线程执行渲染和帧处理
    Handler(Looper.getMainLooper()).post {
        // 先执行自定义帧处理逻辑
        callback.onVideoFrameCaptured(image)
        // 再渲染到MediaRecorder的Surface
        glRenderer.renderImage(image)
    }
}, handler)

// 创建CameraCaptureSession，只挂载三个Surface：预览、照片、帧处理
val surfaces = listOf(
    surfaceView.holder.surface,
    photoImageReader.surface,
    frameReader.surface
)
cameraDevice.createCaptureSession(surfaces, object : CameraCaptureSession.StateCallback() {
    override fun onConfigured(session: CameraCaptureSession) {
        // 配置预览请求、拍照请求等
        val previewRequest = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW).apply {
            addTarget(surfaceView.holder.surface)
            addTarget(frameReader.surface)
        }.build()
        session.setRepeatingRequest(previewRequest, null, handler)
    }

    override fun onConfigureFailed(session: CameraCaptureSession) {
        // 处理配置失败逻辑
    }
}, handler)

4. 关键注意事项 #

• 线程安全：所有OpenGL操作必须在同一个GL线程执行（示例用主线程，也可以创建专门的GL线程）。
• Image释放：必须调用