使用PyTorch Mobile(Kotlin)在Mat对象上计算BoundingBox的问题

开发者阿泽

2026-7-16

问题

使用OpenCV结合PyTorch Mobile，基于2类YOLOv5模型做视频帧目标检测时，遇到两个核心问题：

检测出的BoundingBox无法正确映射到原始Mat图像上
频繁触发IndexOutOfBound异常

已将JNI传来的ByteBuffer转成OpenCV Mat，从Tensor中获取FloatArray后，通过自定义的outputsToNMSPredictionsMat处理检测结果，但问题未解决。相关代码如下：

MainActivity核心代码

override fun onImageProcessingMain(data: ByteBuffer): ByteArray {

    val color = Scalar(0.0, 0.0, 255.0) // BGR
    val thickness = 2

    val inputBytes = ByteArray(data.remaining())
    data.get(inputBytes)

    val imgRgb = Imgcodecs.imdecode(MatOfByte(*inputBytes), Imgcodecs.IMREAD_COLOR)
    val imgBgr = Mat()
    val img = Mat(720, 1280, CvType.CV_8UC3, data)
    // Convert RGB To BGR
    cvtColor(img, imgBgr, Imgproc.COLOR_RGB2BGR)

    // Convert BGR To RGB
    cvtColor(imgBgr, imgRgb, Imgproc.COLOR_BGR2RGB)

    val resizedImg = Mat()
    Imgproc.resize(imgRgb, resizedImg, Size(640.0, 640.0))


    val inputTensor = matToFloat32TensorFun(
        resizedImg
    )
    val outputTuple = mModule?.forward(IValue.from(inputTensor))?.toTuple()

    val outputTensor = outputTuple?.get(0)?.toTensor()
    val outputs = outputTensor?.dataAsFloatArray

    mResults = PrePostProcessor.outputsToNMSPredictions(outputs!!)

         for (result in mResults) {
             val pttl = Point(result.rect.left.toDouble(), result.rect.top.toDouble())
             val ptbr = Point(
                 (result.rect.left + TEXT_WIDTH).toDouble(),  // result.rect.height(),
                 (result.rect.top +  TEXT_HEIGHT).toDouble()  //result.rect.width()
             )
             val rec = Rect(result.rect.left,result.rect.top,
                 result.rect.left+result.rect.width() ,result.rect.top+result.rect.height()
             )


             Imgproc.rectangle(imgRgb, rec, color, thickness)
             //Imgproc.rectangle()
        

             Imgproc.putText(
                 imgRgb,
                 String.format("%s %.2f", PrePostProcessor.mClasses[result.classIndex], result.score),
                 pttl,
                 Core.COVAR_NORMAL,
                 2.0,
                 Scalar(255.0, 255.0, 255.0),
                 1
             )
         }
    cvtColor(imgRgb, imgBgr, Imgproc.COLOR_BGR2RGB)
    val outputBytes = ByteArray(imgBgr.total().toInt() * imgBgr.channels())
    imgBgr.get(0, 0, outputBytes)
    val mBitmap = Bitmap.createBitmap(imgRgb.cols(), imgRgb.rows(), Bitmap.Config.ARGB_8888)
    Utils.matToBitmap(imgRgb, mBitmap)
    runOnUiThread{
        mImageView?.setImageBitmap(mBitmap)
    }


    return outputBytes

}

检测结果处理函数

fun outputsToNMSPredictionsMat(outputs: FloatArray): ArrayList<Result> {
    val results = ArrayList<Result>()
    val imgWidth = 640
    val imgHeight = 640
    val startX = 0
    val startY = 0
    val mOutputColumn = 7 // left, top, right, bottom, score, and 2 class probabilities

    for (i in 0 until mOutputRow) {
        if ((i * PrePostProcessor.mOutputColumn + 4) < 176400) { // 手动添加的越界判断
            if (outputs[i * mOutputColumn + 4] > mThreshold) {
                val x = outputs[i * mOutputColumn]
                val y = outputs[i * mOutputColumn + 1]
                val w = outputs[i * mOutputColumn + 2]
                val h = outputs[i * mOutputColumn + 3]

                // 计算相对Mat的坐标
                val left = startX + (x - w / 2) * imgWidth
                val top = startY + (y - h / 2) * imgHeight
                val right = startX + (x + w / 2) * imgWidth
                val bottom = startY + (y + h / 2) * imgHeight

                var max = outputs[i * mOutputColumn + 5]
                var cls = 0

                innerloop@ for (j in 0 until PrePostProcessor.mOutputColumn - 5) {
                    val indx = i * PrePostProcessor.mOutputColumn + 5 + j
                    if (indx < 176400) { // 手动添加的越界判断
                        if (outputs[i * PrePostProcessor.mOutputColumn + 5 + j] > max) {
                            max = outputs[i * PrePostProcessor.mOutputColumn + 5 + j]
                            cls = j
                        }
                    } else {
                        break@innerloop
                    }
                }

                val rect = Rect(
                    left.toInt(),
                    top.toInt(),
                    right.toInt(),
                    bottom.toInt()
                )

                val result = Result(cls, outputs[i * mOutputColumn + 4], rect)
                results.add(result)
            }
        }
    }

    return nonMaxSuppression(results, mNmsLimit, mThreshold)
}

解决方案

1. 修复IndexOutOfBound异常

硬编码176400是错误根源，必须通过输出数组的实际长度动态计算有效行数，彻底避免越界：

fun outputsToNMSPredictionsMat(outputs: FloatArray): ArrayList<Result> {
    val results = ArrayList<Result>()
    val imgWidth = 640
    val imgHeight = 640
    val mOutputColumn = 7 // left, top, right, bottom, score, 2类概率
    // 动态计算有效行数，适配不同模型输出
    val mOutputRow = outputs.size / mOutputColumn
    val mThreshold = 0.25f // 可根据需求调整置信度阈值
    val mNmsLimit = 50 // NMS最大保留检测框数量

    for (i in 0 until mOutputRow) {
        val scoreIndex = i * mOutputColumn + 4
        // 直接用数组长度判断，无需硬编码
        if (scoreIndex >= outputs.size) break
        
        val score = outputs[scoreIndex]
        if (score > mThreshold) {
            val x = outputs[i * mOutputColumn]
            val y = outputs[i * mOutputColumn + 1]
            val w = outputs[i * mOutputColumn + 2]
            val h = outputs[i * mOutputColumn + 3]

            // YOLOv5输出为归一化中心坐标+宽高，转换为640x640绝对坐标
            val left = (x - w / 2) * imgWidth
            val top = (y - h / 2) * imgHeight
            val right = (x + w / 2) * imgWidth
            val bottom = (y + h / 2) * imgHeight

            // 遍历2类概率，找到最高类别
            var maxProb = outputs[i * mOutputColumn + 5]
            var clsIndex = 0
            for (j in 0 until 2) {
                val probIndex = i * mOutputColumn + 5 + j
                if (probIndex >= outputs.size) break
                val prob = outputs[probIndex]
                if (prob > maxProb) {
                    maxProb = prob
                    clsIndex = j
                }
            }

            // 正确构造OpenCV Rect：参数为x, y, width, height
            val rect = Rect(
                left.toInt(),
                top.toInt(),
                (right - left).toInt(),
                (bottom - top).toInt()
            )

            results.add(Result(clsIndex, score, rect))
        }
    }

    return nonMaxSuppression(results, mNmsLimit, mThreshold)
}

2. 修复BoundingBox映射错误

模型输出坐标基于640x640输入图像，需映射回原始图像尺寸，同时修正Rect构造逻辑：
修改MainActivity中的绘制循环部分：

// 计算原始图像与模型输入的缩放比例
val scaleX = imgRgb.cols() / 640.0
val scaleY = imgRgb.rows() / 640.0

for (result in mResults) {
    // 将640x640的检测框坐标映射回原始图像
    val left = (result.rect.x * scaleX).toInt()
    val top = (result.rect.y * scaleY).toInt()
    val width = (result.rect.width * scaleX).toInt()
    val height = (result.rect.height * scaleY).toInt()
    
    // 正确构造OpenCV Rect
    val rec = Rect(left, top, width, height)
    
    Imgproc.rectangle(imgRgb, rec, color, thickness)
    
    // 文本位置同步缩放，替换错误的字体常量
    val textPoint = Point(left.toDouble(), top.toDouble())
    Imgproc.putText(
        imgRgb,
        String.format("%s %.2f", PrePostProcessor.mClasses[result.classIndex], result.score),
        textPoint,
        Imgproc.FONT_HERSHEY_SIMPLEX,
        0.8, // 适配原始图像的字体大小
        Scalar(255.0, 255.0, 255.0),
        2
    )
}

3. 额外验证点

确认matToFloat32TensorFun的预处理逻辑与YOLOv5训练时完全一致：包括是否将图像归一化到[0,1]、是否为RGB通道顺序、是否做了正确的尺寸缩放。
检查模型输出维度：2类YOLOv5的输出应为(1, 25200, 7)，确保mOutputColumn=7的设置正确。

内容的提问来源于stack exchange，提问作者Arslan

使用PyTorch Mobile(Kotlin)在Mat对象上计算BoundingBox的问题

问题 #

MainActivity核心代码 #

检测结果处理函数 #

解决方案 #

1. 修复IndexOutOfBound异常 #

2. 修复BoundingBox映射错误 #

3. 额外验证点 #