MediaPipe人脸渲染程序纹理输出伪影排查与解决

开发简易人脸面具渲染功能时，输出纹理存在多处细小伪影，目前既无法定位伪影产生的根本原因，也没有找到彻底消除伪影的方法，伪影位置已在测试示例图中标注。
当前实现基于开源项目mediapipe_faceswap做小幅修改，依赖项目内triangulation_media_pipe.py文件定义的三角剖分规则，核心逻辑如下：

• 调用MediaPipe FaceMesh接口提取人脸关键点
• 将归一化格式的关键点坐标转换为OpenCV可用的像素坐标
• 逐三角形计算仿射变换矩阵
• 完成源人脸纹理到目标人脸区域的仿射变换与拼接

import cv2
import mediapipe as mp
import triangulation_media_pipe as tmp
import numpy as np

mp_drawing = mp.solutions.drawing_utils
mp_face_mesh = mp.solutions.face_mesh
face = "face_textures/yash.jpg"


def load_base_img(face_mesh, image_file_name, ):
    image = cv2.imread(image_file_name)
    results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
    return {"img": image, "landmarks": results}


def transform_landmarks_from_tf_to_ocv(keypoints, face_width, face_height):
    landmark_list = []
    if (keypoints.multi_face_landmarks != None):
        for face_landmarks in keypoints.multi_face_landmarks:
            for l in face_landmarks.landmark:
                pt = mp_drawing._normalized_to_pixel_coordinates(l.x, l.y,
                                                                 face_width, face_height)

                landmark_list.append(pt)
    return landmark_list


def main():

    # 摄像头输入逻辑
    face_mesh = mp_face_mesh.FaceMesh()
    base_face_handler, landmark_base_ocv, base_input_image = process_base_face_mesh(face_mesh, face)
    cap = cv2.VideoCapture(2)

    while cap.isOpened():
        _, webcam_img = cap.read()

        image_rows, image_cols, _ = webcam_img.shape
        results = face_mesh.process(webcam_img)
        landmark_target_ocv = transform_landmarks_from_tf_to_ocv(results, image_cols, image_rows)

        # 绘制人脸网格标注
        image = webcam_img.copy()
        img2_new_face = np.zeros_like(image)

        if results.multi_face_landmarks:
            if True:
                for i in range(0, int(len(tmp.TRIANGULATION) / 3)):
                    triangle_index = [tmp.TRIANGULATION[i * 3],
                                      tmp.TRIANGULATION[i * 3 + 1],
                                      tmp.TRIANGULATION[i * 3 + 2]]
                    tbas1 = landmark_base_ocv[triangle_index[0]]
                    tbas2 = landmark_base_ocv[triangle_index[1]]
                    tbas3 = landmark_base_ocv[triangle_index[2]]
                    triangle1 = np.array([tbas1, tbas2, tbas3])

                    rect1 = cv2.boundingRect(triangle1)
                    (x1, y1, w1, h1) = rect1
                    cropped_triangle = base_input_image[y1: y1 + h1, x1: x1 + w1]
                    cropped_tr1_mask = np.zeros((h1, w1), np.uint8)

                    points = np.array([[tbas1[0] - x1, tbas1[1] - y1],
                                       [tbas2[0] - x1, tbas2[1] - y1],
                                       [tbas3[0] - x1, tbas3[1] - y1]])

                    cv2.fillConvexPoly(cropped_tr1_mask, points, 255)
                    ttar1 = landmark_target_ocv[triangle_index[0]]
                    ttar2 = landmark_target_ocv[triangle_index[1]]
                    ttar3 = landmark_target_ocv[triangle_index[2]]

                    triangle2 = np.array([ttar1, ttar2, ttar3])

                    rect2 = cv2.boundingRect(triangle2)
                    (x2, y2, w2, h2) = rect2

                    cropped_tr2_mask = np.zeros((h2, w2), np.uint8)

                    points2 = np.array([[ttar1[0] - x2, ttar1[1] - y2],
                                        [ttar2[0] - x2, ttar2[1] - y2],
                                        [ttar3[0] - x2, ttar3[1] - y2]])

                    cv2.fillConvexPoly(cropped_tr2_mask, points2, 255)
                    # 三角形仿射变换
                    points = np.float32(points)
                    points2 = np.float32(points2)
                    M = cv2.getAffineTransform(points, points2)
                    warped_triangle = cv2.warpAffine(cropped_triangle, M, (w2, h2))
                    warped_triangle = cv2.bitwise_and(warped_triangle, warped_triangle, mask=cropped_tr2_mask)

                    # 重建目标人脸区域
                    img2_new_face_rect_area = img2_new_face[y2: y2 + h2, x2: x2 + w2]
                    img2_new_face_rect_area_gray = cv2.cvtColor(img2_new_face_rect_area, cv2.COLOR_BGR2GRAY)
                    _, mask_triangles_designed = cv2.threshold(img2_new_face_rect_area_gray, 0, 255,
                                                               cv2.THRESH_BINARY_INV)

                    warped_triangle = cv2.bitwise_and(warped_triangle, warped_triangle,
                                                      mask=mask_triangles_designed)


                    img2_new_face_rect_area = cv2.add(img2_new_face_rect_area, warped_triangle)
                    img2_new_face[y2: y2 + h2, x2: x2 + w2] = img2_new_face_rect_area

                    cv2.imshow('mask', img2_new_face)

        key = cv2.waitKey(5)

    face_mesh.close()
    cap.release()


def process_base_face_mesh(face_mesh,
                           image_file):
    base_face_handler = load_base_img(face_mesh, image_file)
    base_input_image = base_face_handler["img"].copy()
    image_rows, image_cols, _ = base_face_handler["img"].shape
    landmark_base_ocv = transform_landmarks_from_tf_to_ocv(base_face_handler["landmarks"], image_cols, image_rows)
    return base_face_handler, landmark_base_ocv, base_input_image


if __name__ == "__main__":
    main()

之前尝试过用同图像的模糊版本生成掩码覆盖伪影区域，但该方案会导致像素值偏差，达不到输出像素值尽可能贴近真实图像的要求。
需要在保证像素精度的前提下，找到可彻底消除该类渲染伪影的可行方案。

内容的提问来源于stack exchange，提问作者vwertuzy