如何修改TensorFlow Lite代码实现视频帧实时超分辨率处理与显示

一、先修复输出图像与原图差异极小的问题 #

输出与原图差异小，核心原因是输入输出尺寸硬编码不匹配模型真实要求，或预处理逻辑冗余，修复步骤如下：

1. 从模型动态获取输入输出Shape
   在SRModel类中添加成员变量存储模型输入输出维度，避免硬编码：

   private int[] inputShape;
   private int[] outputShape;
   
   private void init() throws IOException {
       // ... 原有初始化代码
       interpreter = new Interpreter(loadModelFile(), options);
       // 从模型读取真实输入输出Shape（格式为[batch, height, width, channel]）
       inputShape = interpreter.getInputTensor(0).shape();
       outputShape = interpreter.getOutputTensor(0).shape();
   }
   

2. 修正预处理逻辑
   原代码的ResizeOp是将图像缩放到与原图一致，无实际意义，改为缩放到模型要求的输入尺寸：

   public TensorImage prepareInputTensor(Bitmap bitmap_lr) {
       TensorImage inputImage = TensorImage.fromBitmap(bitmap_lr);
       // 取模型要求的输入高宽
       int inputHeight = inputShape[1];
       int inputWidth = inputShape[2];
   
       ImageProcessor imageProcessor = new ImageProcessor.Builder()
               .add(new ResizeOp(inputHeight, inputWidth, ResizeOp.ResizeMethod.BILINEAR)) // 双线性插值更适合超分输入
               .add(new NormalizeOp(0.0f, 255.0f))
               .build();
       inputImage = imageProcessor.process(inputImage);
   
       return inputImage;
   }
   

3. 修正输出Tensor初始化逻辑
   用模型真实输出Shape初始化输出Tensor，替代硬编码：

   public TensorImage prepareOutputTensor() {
       TensorImage srImage = new TensorImage(DataType.FLOAT32);
       // 取输出的高宽通道维度
       int[] srShape = new int[]{outputShape[1], outputShape[2], outputShape[3]};
       srImage.load(TensorBuffer.createFixedSize(srShape, DataType.FLOAT32));
   
       return srImage;
   }
   

4. 补充资源释放方法
   在SRModel类中添加资源关闭方法，避免内存泄漏：

   public void close() {
       if (interpreter != null) {
           interpreter.close();
       }
       if (gpuDelegate != null) {
           gpuDelegate.close();
       }
   }
   

二、实现实时视频逐帧处理 #

基于Android的MediaCodec和MediaExtractor实现本地视频逐帧解码、超分处理、实时显示，步骤如下：

1. 添加权限到AndroidManifest.xml #

<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" android:maxSdkVersion="32" />

2. 创建视频处理布局activity_video_sr.xml #

<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:layout_width="match_parent"
    android:layout_height="match_parent">

    <SurfaceView
        android:id="@+id/surface_view"
        android:layout_width="match_parent"
        android:layout_height="match_parent" />

    <Button
        android:id="@+id/btn_control"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignParentBottom="true"
        android:layout_centerHorizontal="true"
        android:text="开始处理视频" />

</RelativeLayout>

3. 编写视频处理核心代码VideoSRActivity.java #

package com.example.mobedsr;

import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.os.Bundle;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.View;
import android.widget.Button;
import androidx.appcompat.app.AppCompatActivity;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Matrix;
import android.os.Handler;
import android.os.Looper;
import java.io.IOException;
import java.nio.ByteBuffer;

public class VideoSRActivity extends AppCompatActivity implements SurfaceHolder.Callback {

    private SurfaceView surfaceView;
    private SurfaceHolder surfaceHolder;
    private Button btnControl;
    private SRModel srModel;
    private boolean isProcessing = false;
    private final Handler mainHandler = new Handler(Looper.getMainLooper());

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_video_sr);

        surfaceView = findViewById(R.id.surface_view);
        btnControl = findViewById(R.id.btn_control);
        surfaceHolder = surfaceView.getHolder();
        surfaceHolder.addCallback(this);

        // 初始化超分模型（开启GPU加速）
        try {
            srModel = new SRModel(getAssets(), true);
        } catch (IOException e) {
            e.printStackTrace();
        }

        btnControl.setOnClickListener(v -> {
            if (!isProcessing) {
                startVideoProcessing();
                btnControl.setText("停止处理");
            } else {
                isProcessing = false;
                btnControl.setText("开始处理视频");
            }
        });
    }

    private void startVideoProcessing() {
        isProcessing = true;
        new Thread(() -> {
            MediaExtractor extractor = new MediaExtractor();
            MediaCodec decoder = null;
            try {
                // 替换为你的本地视频路径（可改为相册选择逻辑）
                String videoPath = "/storage/emulated/0/DCIM/Camera/test_video.mp4";
                extractor.setDataSource(videoPath);

                // 定位视频轨道
                int videoTrackIndex = -1;
                for (int i = 0; i < extractor.getTrackCount(); i++) {
                    MediaFormat format = extractor.getTrackFormat(i);
                    String mime = format.getString(MediaFormat.KEY_MIME);
                    if (mime.startsWith("video/")) {
                        videoTrackIndex = i;
                        break;
                    }
                }
                if (videoTrackIndex == -1) return;

                extractor.selectTrack(videoTrackIndex);
                MediaFormat videoFormat = extractor.getTrackFormat(videoTrackIndex);

                // 初始化解码器
                decoder = MediaCodec.createDecoderByType(videoFormat.getString(MediaFormat.KEY_MIME));
                decoder.configure(videoFormat, null, null, 0);
                decoder.start();

                MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();

                while (isProcessing) {
                    // 填充解码器输入数据
                    int inputBufferId = decoder.dequeueInputBuffer(10000);
                    if (inputBufferId >= 0) {
                        ByteBuffer inputBuffer = decoder.getInputBuffer(inputBufferId);
                        int sampleSize = extractor.readSampleData(inputBuffer, 0);
                        if (sampleSize < 0) {
                            decoder.queueInputBuffer(inputBufferId, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
                            break;
                        } else {
                            long presentationTimeUs = extractor.getSampleTime();
                            decoder.queueInputBuffer(inputBufferId, 0, sampleSize, presentationTimeUs, 0);
                            extractor.advance();
                        }
                    }

                    // 获取解码后的帧并处理
                    int outputBufferId = decoder.dequeueOutputBuffer(bufferInfo, 10000);
                    if (outputBufferId >= 0) {
                        Bitmap frameBitmap = decoder.getOutputBuffer(outputBufferId).asBitmap();
                        if (frameBitmap != null) {
                            // 调用超分模型处理帧
                            TensorImage inputTensor = srModel.prepareInputTensor(frameBitmap);
                            TensorImage outputTensor = srModel.prepareOutputTensor();
                            srModel.run(inputTensor.getBuffer(), outputTensor.getBuffer());
                            Bitmap srBitmap = srModel.tensorToImage(outputTensor);

                            // 实时绘制处理后的帧到SurfaceView
                            drawBitmapToSurface(srBitmap);
                        }

                        decoder.releaseOutputBuffer(outputBufferId, false);
                        if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) break;
                    }
                }

                decoder.stop();
                decoder.release();
                extractor.release();

            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                mainHandler.post(() -> {
                    isProcessing = false;
                    btnControl.setText("开始处理视频");
                });
            }
        }).start();
    }

    private void drawBitmapToSurface(Bitmap bitmap) {
        mainHandler.post(() -> {
            if (surfaceHolder.getSurface().isValid()) {
                Canvas canvas = surfaceHolder.lockCanvas();
                if (canvas != null) {
                    // 缩放Bitmap适配SurfaceView尺寸
                    Matrix matrix = new Matrix();
                    float scaleX = (float) canvas.getWidth() / bitmap.getWidth();
                    float scaleY = (float) canvas.getHeight() / bitmap.getHeight();
                    float scale = Math.max(scaleX, scaleY);
                    matrix.postScale(scale, scale);
                    matrix.postTranslate(
                            (canvas.getWidth() - bitmap.getWidth() * scale) / 2,
                            (canvas.getHeight() - bitmap.getHeight() * scale) / 2
                    );
                    canvas.drawBitmap(bitmap, matrix, null);
                    surfaceHolder.unlockCanvasAndPost(canvas);
                }
            }
        });
    }

    @Override
    public void surfaceCreated(SurfaceHolder holder) {
        surfaceHolder = holder;
    }

    @Override
    public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {}

    @Override
    public void surfaceDestroyed(SurfaceHolder holder) {
        isProcessing = false;
    }

    @Override
    protected void onDestroy() {
        super.onDestroy();
        if (srModel != null) srModel.close();
    }
}

三、实时处理性能优化建议 #

• 强制开启GPU加速：GPU delegate能将推理速度提升3-10倍，是实时处理的核心保障
• 帧跳过策略：若处理速度跟不上视频帧率，可每隔1-2帧跳过处理，优先保证流畅度
• Bitmap复用：避免频繁创建Bitmap对象，复用已有的Bitmap减少内存开销
• 模型量化：将浮点型模型转换为INT8量化模型，进一步降低推理延迟
• 线程优先级：给视频处理线程设置较高优先级，避免被系统调度打断

内容的提问来源于stack exchange，提问作者angel_30