MediaMuxer合成MP4丢失帧问题及合规替代方案问询

问题背景 #

此前我已创建并分享过将多帧Bitmap转换为MP4文件的示例，基于某开源项目实现，代码已发布至代码托管平台。单帧转换正常，示例代码如下：

@WorkerThread
private fun testImage() {
    Log.d("AppLog", "testImage")
    val startTime = System.currentTimeMillis()
    Log.d("AppLog", "start")
    val videoFile = File(ContextCompat.getExternalFilesDirs(this, null)[0], "image.mp4")
    if (videoFile.exists())
        videoFile.delete()
    videoFile.parentFile!!.mkdirs()
    val timeLapseEncoder = TimeLapseEncoder()
    val bitmap = BitmapFactory.decodeResource(resources, R.drawable.test)
    val width = bitmap.width
    val height = bitmap.height
    timeLapseEncoder.prepareForEncoding(videoFile.absolutePath, width, height)
    val frameDurationInMs = 1000
    timeLapseEncoder.encodeFrame(bitmap, frameDurationInMs)
    timeLapseEncoder.finishEncoding()
    val endTime = System.currentTimeMillis()
    Log.d("AppLog", "it took ${endTime - startTime} ms to convert a single image ($width x $height) to mp4")
}

问题现象 #

处理多帧（甚至仅2帧）时，偶尔会出现帧丢失，导致视频时长缩短。例如，2帧各设置5秒时长，输出视频仅5秒且丢失第二帧：

@WorkerThread
private fun testImages() {
    Log.d("AppLog", "testImages")
    val startTime = System.currentTimeMillis()
    Log.d("AppLog", "start")
    val videoFile = File(ContextCompat.getExternalFilesDirs(this, null)[0], "images.mp4")
    if (videoFile.exists())
        videoFile.delete()
    videoFile.parentFile!!.mkdirs()
//        Log.d("AppLog", "success creating parent?${videoFile.parentFile.exists()}")
    val timeLapseEncoder = TimeLapseEncoder()
    val bitmap = BitmapFactory.decodeResource(resources, R.drawable.frame1)
    val width = bitmap.width
    val height = bitmap.height
    timeLapseEncoder.prepareForEncoding(videoFile.absolutePath, width, height)
    val delay = 5000
    timeLapseEncoder.encodeFrame(bitmap, delay)
    val   bitmap2 = BitmapFactory.decodeResource(resources, R.drawable.frame2)
    timeLapseEncoder.encodeFrame(bitmap2, delay)
    timeLapseEncoder.finishEncoding()
    val endTime = System.currentTimeMillis()
    Log.d("AppLog", "it took ${endTime - startTime} ms to convert a single image ($width x $height) to ${videoFile.absolutePath} ${videoFile.exists()} ${videoFile.length()}")
}

已尝试操作 #

排查并调试代码未发现问题，奇怪的是修改时长或增加帧数时有时正常（如两帧各6秒可生成12秒视频）。尝试用循环生成等效时长的多帧：

for (i in 0 until 500)
    timeLapseEncoder.encodeFrame(bitmap, 10)
val bitmap2 = BitmapFactory.decodeResource(resources, R.drawable.frame2)
for (i in 0 until 500)
    timeLapseEncoder.encodeFrame(bitmap2, 10)

但仍无法生成各5秒的视频。猜测是帧率问题，但代码已设为合理的30fps。随后重写了BitmapToVideoEncoder实现，仍存在相同问题（偶尔时长为0）：

class BitmapToVideoEncoder(outputPath: String?, width: Int, height: Int, bitRate: Int, frameRate: Int) {
    private var encoder: MediaCodec?
    private val inputSurface: Surface
    private var mediaMuxer: MediaMuxer?
    private var videoTrackIndex = 0
    private var isMuxerStarted: Boolean
    private var presentationTimeUs: Long

    init {
        val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height)
        format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
        format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate)
        format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate)
        format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
        encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)
        encoder!!.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
        inputSurface = encoder!!.createInputSurface()
        encoder!!.start()
        mediaMuxer = MediaMuxer(outputPath!!, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
        isMuxerStarted = false
        presentationTimeUs = 0
    }

    @Throws(IOException::class)
    fun encodeFrame(bitmap: Bitmap, durationInMs: Long) {
        val frameDurationUs = durationInMs * 1000
        drawBitmapToSurface(bitmap)
        drainEncoder(false)
        presentationTimeUs += frameDurationUs
    }

    @Throws(IOException::class)
    fun finishEncoding() {
        drainEncoder(true)
        release()
    }

    private fun drawBitmapToSurface(bitmap: Bitmap) {
        val canvas = inputSurface.lockCanvas(null)
        canvas.drawBitmap(bitmap, 0f, 0f, null)
        inputSurface.unlockCanvasAndPost(canvas)
    }

    @Throws(IOException::class)
    private fun drainEncoder(endOfStream: Boolean) {
        if (endOfStream) {
          //Sending end of stream signal to encoder
            encoder!!.signalEndOfInputStream()
        }

        val bufferInfo = MediaCodec.BufferInfo()
        while (true) {
            val encoderStatus = encoder!!.dequeueOutputBuffer(bufferInfo, 10000)
            @Suppress("DEPRECATION")
            when {
                encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER -> {
                    if (!endOfStream) {
                        break
                    }
                }
                encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED -> {
                    //Output buffers changed
                }
                encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
                    if (isMuxerStarted) {
                        throw RuntimeException("format changed twice")
                    }
                    val newFormat = encoder!!.outputFormat
                    videoTrackIndex = mediaMuxer!!.addTrack(newFormat)
                    mediaMuxer!!.start()
                    isMuxerStarted = true
                }
                encoderStatus < 0 -> {
        //                Unexpected result from encoder
                }
                else -> {
                    val encodedData = encoder!!.getOutputBuffer(encoderStatus)
                        ?: throw RuntimeException("encoderOutputBuffer $encoderStatus was null")
                    if (bufferInfo.size != 0) {
                        if (!isMuxerStarted) {
                            throw RuntimeException("muxer hasn't started")
                        }
                        // Adjust the bufferInfo to have the correct presentation time
                        bufferInfo.presentationTimeUs = presentationTimeUs
                        encodedData.position(bufferInfo.offset)
                        encodedData.limit(bufferInfo.offset + bufferInfo.size)
                        mediaMuxer!!.writeSampleData(videoTrackIndex, encodedData, bufferInfo)
                    }
                    encoder!!.releaseOutputBuffer(encoderStatus, false)
                    if ((bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                        //End of stream reached
                        break
                    }
                }
            }
        }
    }

    private fun release() {
        if (encoder != null) {
            encoder!!.stop()
            encoder!!.release()
            encoder = null
        }
        if (mediaMuxer != null) {
            mediaMuxer!!.stop()
            mediaMuxer!!.release()
            mediaMuxer = null
        }
    }

}

该实现仅支持同分辨率、非透明Bitmap，且需匹配设备编码支持的分辨率，可通过以下代码检测支持的分辨率：

MediaCodec codec = MediaCodec.createEncoderByType(mimeType);
MediaCodecInfo codecInfo = codec.getCodecInfo();
MediaCodecInfo.CodecCapabilities capabilities = codecInfo.getCapabilitiesForType(mimeType);
MediaCodecInfo.VideoCapabilities videoCapabilities = capabilities.getVideoCapabilities();
codec.release();

技术问询 #

• 如何基于现有API正确创建支持不同帧时长、分辨率及透明帧的MP4视频？
• 是否存在轻量、协议合规的Bitmap转视频方案，支持自定义单帧时长，无需使用FFmpeg？

内容的提问来源于stack exchange，提问作者android developer