如何修改OpenCV代码实现无标记3D物体稳定投影，解决模型乱飞问题

环境配置 #

Windows 10, python 3.7, opencv-contrib-python==4.5.3.56

问题描述 #

实现将3D物体投影到无标记卡片上的功能时，运行main.py打开摄像头后，3D物体出现无规则漂移，效果如下：

预期效果：仅当摄像头捕获到指定卡片时，才在卡片对应位置稳定投影fox.obj模型。

项目文件结构 #

同一文件夹下共4个文件：

• main.py：主运行代码
• objloader_simple.py：OBJ模型加载工具
• fox.obj：3D狐狸模型文件
• model.jpg：待识别的目标卡片图片

原有代码 #

main.py #

import argparse
import cv2
import numpy as np
import math
import os
from objloader_simple import *

MIN_MATCHES = 10
DEFAULT_COLOR = (0, 0, 0)


def main():
    homography = None 
    camera_parameters = np.array([[800, 0, 320], [0, 800, 240], [0, 0, 1]])
    orb = cv2.ORB_create()
    bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
    dir_name = os.getcwd()
    model = cv2.imread(os.path.join(dir_name, 'model.jpg'), 0)
    kp_model, des_model = orb.detectAndCompute(model, None)
    obj = OBJ(os.path.join(dir_name, 'fox.obj'), swapyz=True)  
    cap = cv2.VideoCapture(0)

    while True:
        ret, frame = cap.read()
        if not ret:
            print("Unable to capture video")
            return 
        kp_frame, des_frame = orb.detectAndCompute(frame, None)
        matches = bf.match(des_model, des_frame)
        matches = sorted(matches, key=lambda x: x.distance)

        if len(matches) > MIN_MATCHES:
            src_pts = np.float32([kp_model[m.queryIdx].pt for m in matches]).reshape(-1, 1, 2)
            dst_pts = np.float32([kp_frame[m.trainIdx].pt for m in matches]).reshape(-1, 1, 2)
            homography, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
            if args.rectangle:
                h, w = model.shape
                pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2)
                dst = cv2.perspectiveTransform(pts, homography)
                frame = cv2.polylines(frame, [np.int32(dst)], True, 255, 3, cv2.LINE_AA)  
            if homography is not None:
                try:
                    projection = projection_matrix(camera_parameters, homography)
                    frame = render(frame, obj, projection, model, False)
                except:
                    pass
            if args.matches:
                frame = cv2.drawMatches(model, kp_model, frame, kp_frame, matches[:10], 0, flags=2)
            cv2.imshow('frame', frame)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break

        else:
            print("Not enough matches found - %d/%d" % (len(matches), MIN_MATCHES))

    cap.release()
    cv2.destroyAllWindows()
    return 0

def render(img, obj, projection, model, color=False):
    vertices = obj.vertices
    scale_matrix = np.eye(3) * 3
    h, w = model.shape

    for face in obj.faces:
        face_vertices = face[0]
        points = np.array([vertices[vertex - 1] for vertex in face_vertices])
        points = np.dot(points, scale_matrix)
        points = np.array([[p[0] + w / 2, p[1] + h / 2, p[2]] for p in points])
        dst = cv2.perspectiveTransform(points.reshape(-1, 1, 3), projection)
        imgpts = np.int32(dst)
        if color is False:
            cv2.fillConvexPoly(img, imgpts, DEFAULT_COLOR)
        else:
            color = hex_to_rgb(face[-1])
            color = color[::-1]  # reverse
            cv2.fillConvexPoly(img, imgpts, color)

    return img

def projection_matrix(camera_parameters, homography):
    homography = homography * (-1)
    rot_and_transl = np.dot(np.linalg.inv(camera_parameters), homography)
    col_1 = rot_and_transl[:, 0]
    col_2 = rot_and_transl[:, 1]
    col_3 = rot_and_transl[:, 2]
    l = math.sqrt(np.linalg.norm(col_1, 2) * np.linalg.norm(col_2, 2))
    rot_1 = col_1 / l
    rot_2 = col_2 / l
    translation = col_3 / l
    c = rot_1 + rot_2
    p = np.cross(rot_1, rot_2)
    d = np.cross(c, p)
    rot_1 = np.dot(c / np.linalg.norm(c, 2) + d / np.linalg.norm(d, 2), 1 / math.sqrt(2))
    rot_2 = np.dot(c / np.linalg.norm(c, 2) - d / np.linalg.norm(d, 2), 1 / math.sqrt(2))
    rot_3 = np.cross(rot_1, rot_2)
    projection = np.stack((rot_1, rot_2, rot_3, translation)).T
    return np.dot(camera_parameters, projection)

def hex_to_rgb(hex_color):
    hex_color = hex_color.lstrip('#')
    h_len = len(hex_color)
    return tuple(int(hex_color[i:i + h_len // 3], 16) for i in range(0, h_len, h_len // 3))

parser = argparse.ArgumentParser(description='Augmented reality application')

parser.add_argument('-r','--rectangle', help = 'draw rectangle delimiting target surface on frame', action = 'store_true')
parser.add_argument('-mk','--model_keypoints', help = 'draw model keypoints', action = 'store_true')
parser.add_argument('-fk','--frame_keypoints', help = 'draw frame keypoints', action = 'store_true')
parser.add_argument('-ma','--matches', help = 'draw matches between keypoints', action = 'store_true')

args = parser.parse_args()

if __name__ == '__main__':
    main()

objloader_simple.py #

class OBJ:
    def __init__(self, filename, swapyz=False):
        """Loads a Wavefront OBJ file. """
        self.vertices = []
        self.normals = []
        self.texcoords = []
        self.faces = []
        material = None
        for line in open(filename, "r"):
            if line.startswith('#'): continue
            values = line.split()
            if not values: continue
            if values[0] == 'v':
                v = list(map(float, values[1:4]))
                if swapyz:
                    v = v[0], v[2], v[1]
                self.vertices.append(v)
            elif values[0] == 'vn':
                v = list(map(float, values[1:4]))
                if swapyz:
                    v = v[0], v[2], v[1]
                self.normals.append(v)
            elif values[0] == 'vt':
                self.texcoords.append(map(float, values[1:3]))
            #elif values[0] in ('usemtl', 'usemat'):
                #material = values[1]
            #elif values[0] == 'mtllib':
                #self.mtl = MTL(values[1])
            elif values[0] == 'f':
                face = []
                texcoords = []
                norms = []
                for v in values[1:]:
                    w = v.split('/')
                    face.append(int(w[0]))
                    if len(w) >= 2 and len(w[1]) > 0:
                        texcoords.append(int(w[1]))
                    else:
                        texcoords.append(0)
                    if len(w) >= 3 and len(w[2]) > 0:
                        norms.append(int(w[2]))
                    else:
                        norms.append(0)
                #self.faces.append((face, norms, texcoords, material))
                self.faces.append((face, norms, texcoords))

修复方案 #

对main.py做以下修改即可解决漂移问题：

1. 提高匹配数量阈值，减少误匹配触发渲染的概率：

# 将原MIN_MATCHES = 10修改为
MIN_MATCHES = 30

2. 新增匹配结果过滤逻辑，剔除距离过大的错误匹配；新增单应性矩阵校验，过滤质量差的变换矩阵；新增单应性平滑逻辑，避免帧间跳变：

def main():
    homography = None 
    # 新增：存储上一帧的有效单应性矩阵用于平滑
    prev_homography = None
    camera_parameters = np.array([[800, 0, 320], [0, 800, 240], [0, 0, 1]])
    # ... 原有代码保持不变 ...
    while True:
        ret, frame = cap.read()
        if not ret:
            print("Unable to capture video")
            return 
        kp_frame, des_frame = orb.detectAndCompute(frame, None)
        matches = bf.match(des_model, des_frame)
        matches = sorted(matches, key=lambda x: x.distance)

        # -------------------- 修改开始 --------------------
        # 过滤低质量匹配
        good_matches = []
        if matches:
            min_dist = matches[0].distance
            # 仅保留距离小于2倍最小距离的匹配，同时设置阈值避免min_dist过小
            good_matches = [m for m in matches if m.distance < max(2 * min_dist, 30)]
        
        if len(good_matches) > MIN_MATCHES:
            src_pts = np.float32([kp_model[m.queryIdx].pt for m in good_matches]).reshape(-1, 1, 2)
            dst_pts = np.float32([kp_frame[m.trainIdx].pt for m in good_matches]).reshape(-1, 1, 2)
            homography, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
            
            # 校验单应性矩阵质量：内点占比低于30%则认为无效
            if homography is not None:
                inlier_ratio = np.sum(mask) / len(mask)
                if inlier_ratio < 0.3:
                    homography = None
            
            # 单应性矩阵帧间平滑，减少跳变
            if homography is not None and prev_homography is not None:
                homography = 0.7 * homography + 0.3 * prev_homography
            prev_homography = homography
        # -------------------- 修改结束 --------------------
        
        # 后续原有代码同步替换matches为good_matches
        if args.rectangle and homography is not None:
            h, w = model.shape
            pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2)
            dst = cv2.perspectiveTransform(pts, homography)
            frame = cv2.polylines(frame, [np.int32(dst)], True, 255, 3, cv2.LINE_AA)  
        if homography is not None:
            try:
                projection = projection_matrix(camera_parameters, homography)
                frame = render(frame, obj, projection, model, False)
            except:
                pass
        if args.matches:
            frame = cv2.drawMatches(model, kp_model, frame, kp_frame, good_matches[:10], 0, flags=2)
        cv2.imshow('frame', frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
        else:
            print("Not enough matches found - %d/%d" % (len(good_matches), MIN_MATCHES))

3. 可选优化：如果依然存在偏移，可使用Opencv摄像头校准方法校准你的设备摄像头，替换代码中写死的camera_parameters内参矩阵，投影精度会进一步提升。

内容的提问来源于stack exchange，提问作者lawrencema Ma