如何使用Python+OpenCV为PAN/Aadhar身份证生成无重叠bounding box实现OCR提取

我正在使用Python搭配OpenCV库编写代码，目的是从PAN卡或Aadhar卡中OCR提取父亲姓名、持卡人姓名、出生日期、PAN编号等数据。但如果卡面内容稍有杂乱，基于预处理后的阈值图像生成的bounding box会出现重叠问题：

• 预处理后的阈值图像
  
• 绘制完边界框的最终图像
  

from __future__ import print_function
from matplotlib import pyplot as plt
from PIL import Image
import re
import numpy as np
from imutils import paths
import os
import pytesseract
import cv2
import argparse
import imutils
import glob
import xlsxwriter
from google.colab.patches import cv2_imshow

'''
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--images", required=True, help="path to images directory")
args = vars(ap.parse_args())
'''
'''
workbook = xlsxwriter.Workbook(r'/Users/vd/Downloads/PAN Number extraction/example.xlsx') 
worksheet = workbook.add_worksheet("My sheet") 
'''

rectKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (13, 5))#返回由1填充的矩形结构矩阵
print(rectKernel)
sqKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (21, 21))
print(sqKernel)
row=0
col=0
i=0

#for imagePath in paths.list_images(args["images"]):
imagePath="/content/Tanvi-Papa-Pan-Card.jpg"
i+=1
print(i)
image = cv2.imread(imagePath)
image = imutils.resize(image, height=600)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)#转灰度图
gray = cv2.GaussianBlur(gray, (3, 3), 0)#3x3高斯模糊降噪
cv2_imshow(gray)
blackhat = cv2.morphologyEx(gray, cv2.MORPH_BLACKHAT, sqKernel)#增强亮背景下的暗文本区域
cv2_imshow(blackhat)
gradX = cv2.Sobel(blackhat, ddepth=cv2.CV_32F, dx=1, dy=0, ksize=-1)#X方向梯度计算
cv2_imshow(gradX)
gradX = np.absolute(gradX)#取绝对值
(minVal, maxVal) = (np.min(gradX), np.max(gradX))#取梯度矩阵的最值
print(maxVal,minVal)
cv2_imshow(gradX)
gradX = (255 * ((gradX - minVal) / (maxVal - minVal))).astype("uint8")#归一化到0-255
cv2_imshow(gradX)
print(gradX)
gradX = cv2.morphologyEx(gradX, cv2.MORPH_CLOSE, rectKernel)#闭运算补全文本区域间隙
cv2_imshow(gradX)


thresh = cv2.threshold(gradX, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]#大津法二值化
cv2_imshow(thresh)
thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, sqKernel)#再次闭运算
cv2_imshow(thresh)
thresh = cv2.erode(thresh, None, iterations=4)#腐蚀操作
cv2_imshow(thresh)



p = int(image.shape[1] * 0.05)
#print(p)
thresh[:, 0:p] = 0
thresh[:, image.shape[1] - p:] = 0
cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
#cv2_imshow(cnts)
#print(cnts)
cnts = imutils.grab_contours(cnts)
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)

for c in cnts:
    (x, y, w, h) = cv2.boundingRect(c)
    ar = w / float(h)
    crWidth = w / float(gray.shape[1])
    pX = int((x + w) * 0.03)
    pY = int((y + h) * 0.03)
    (x, y) = (x - pX, y - pY)
    (w, h) = (w + (pX * 2), h + (pY * 2))

    roi = image[y:y + h, x:x + w].copy()
    cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
    #filename="{}.jpg".format(os.getpid())
    #cv2.imwrite(filename,roi)
    text=pytesseract.image_to_string(roi)
    #os.remove(filename)
    
     
    #p=re.compile('[A-Z][A-Z][A-Z][A-Z][A-Z]\d\d\d\d[A-Z]')

    #q=''.join(p.findall(text))
    print(text)
'''
    if q is not None:
        worksheet.write(row, col, i) 
        worksheet.write(row, col+1, q)
'''
row+=1  
cv2_imshow(image)
cv2_imshow(roi)
#workbook.close()

上述代码在部分场景下会生成重叠的边界框，导致重叠区域的文本无法被正常提取输出。下面给出可行的解决方案：

方案1：优化图像预处理流程 #

从根源减少合并的连通域，避免生成重叠框：

• 调整腐蚀迭代次数：当前4次腐蚀迭代次数过高，会抹除相邻文本行的间隙，导致多行合并为一个连通域，可将迭代次数降到2~3
• 缩小方形结构核尺寸：当前sqKernel为(21,21)，对PAN卡这类行高较小的证件来说过大，可缩小到(15,15)，避免闭运算合并不同行的文本
• 增加开运算步骤：二值化后增加一次开运算，清除小噪点，避免噪点被识别为独立连通域：

# 加在二值化腐蚀操作之前
thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, np.ones((3,3), np.uint8), iterations=1)

方案2：加入非极大值抑制（NMS）过滤重叠框 #

就算预处理存在误差，也可以在生成所有边界框后，通过NMS算法过滤重叠度过高的框：
首先实现NMS函数：

def non_max_suppression(boxes, overlapThresh=0.3):
    if len(boxes) == 0:
        return []
    boxes = np.array(boxes)
    # 提取每个框的坐标
    x1 = boxes[:,0]
    y1 = boxes[:,1]
    x2 = boxes[:,0] + boxes[:,2]
    y2 = boxes[:,1] + boxes[:,3]
    # 计算每个框的面积
    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
    # 按y坐标排序，优先保留靠下的框
    idxs = np.argsort(y2)
    pick = []
    while len(idxs) > 0:
        last = len(idxs) - 1
        i = idxs[last]
        pick.append(i)
        # 计算当前框与剩余框的重叠率
        xx1 = np.maximum(x1[i], x1[idxs[:last]])
        yy1 = np.maximum(y1[i], y1[idxs[:last]])
        xx2 = np.minimum(x2[i], x2[idxs[:last]])
        yy2 = np.minimum(y2[i], y2[idxs[:last]])
        w = np.maximum(0, xx2 - xx1 + 1)
        h = np.maximum(0, yy2 - yy1 + 1)
        overlap = (w * h) / areas[idxs[:last]]
        # 删除重叠率超过阈值的框
        idxs = np.delete(idxs, np.concatenate(([last], np.where(overlap > overlapThresh)[0])))
    return boxes[pick].astype("int")

修改原代码中遍历轮廓的逻辑，先收集所有框再过滤：

boxes = []
for c in cnts:
    (x, y, w, h) = cv2.boundingRect(c)
    ar = w / float(h)
    # 加入长宽比过滤，PAN卡文本行长宽比一般在2~25之间，不符合的直接跳过
    if ar < 2 or ar > 25:
        continue
    pX = int((x + w) * 0.03)
    pY = int((y + h) * 0.03)
    (x, y) = (x - pX, y - pY)
    (w, h) = (w + (pX * 2), h + (pY * 2))
    boxes.append([x, y, w, h])
# 执行NMS过滤重叠框，重叠阈值可根据实际效果调整
filtered_boxes = non_max_suppression(boxes, overlapThresh=0.25)
# 遍历过滤后的框提取文本
for (x,y,w,h) in filtered_boxes:
    roi = image[y:y + h, x:x + w].copy()
    cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
    text=pytesseract.image_to_string(roi, lang='eng')
    print(text)

方案3：直接使用Tesseract自带的文本检测功能 #

对于排版规整的证件类图像，直接用Tesseract自带的文本检测逻辑，不需要自己实现OpenCV形态学处理，准确率更高也不会出现框重叠问题：

# 直接对灰度图做文本检测
d = pytesseract.image_to_data(gray, output_type=pytesseract.Output.DICT)
n_boxes = len(d['text'])
for i in range(n_boxes):
    # 只保留置信度大于60的检测结果
    if int(d['conf'][i]) > 60:
        (x, y, w, h) = (d['left'][i], d['top'][i], d['width'][i], d['height'][i])
        cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
        text = d['text'][i]
        print(text)

如果需要整行提取，可以基于检测结果的top和height字段，把垂直位置接近的框合并为整行再识别。

内容的提问来源于stack exchange，提问作者G-K