1 Difficulties in Face Change
(1) There are obvious differences in the geometric shapes of human faces, and there are some abnormal values.
(2) Illumination makes the skin tone of the face shift.
(3) the influence of head posture;
(4) Skin texture difference.
2 Face Change Scheme
%matplotlib inline from utils.plot_imgs import * #from utils.delaunay_triangulation import * from utils.dlib_face_api import * import cv2 import glob import os
2.1 Face Alignment
The target image f(x, y)f^{ast}(x, y)f(x, y)f(y) face area is covered with the source image g(x, y)g(x, y)g(x, y)face area.
# Read sample images filename_src = "./img/ted_cruz.jpg" filename_dst = "./img/donald_trump.jpg" img_src = cv2.imread(filename_src) img_dst = cv2.imread(filename_dst) img_dst_warped = np.copy(img_dst) idx_fig = 1 plot_imgs(idx_fig, [img_src, img_dst])
2.1.1 Facial Key Point Location
(1) Facial Key Point Location (68, dlib)
dlib.get_frontal_face_detector()
dlib.shape_predictor()
(2) Deloni triangulation
Key points of facial outline (jaw[1 - 17], right_brow[18 - 22], left_brow[23 - 27])
# Read Key Points def read_points(str_file_path): lst_points = [] with open(str_file_path) as fr: for str_line in fr.readlines(): lst_points.append( [int(item) for item in str_line.strip().split(" ")]) return lst_points
# Facial key points points_src = read_points(filename_src + ".txt") points_dst = read_points(filename_dst + ".txt")
2.1.2 convex hull
convex hull: boundary without depression
cv2.convexHull()
# convex hull hull_pt_src = [] hull_pt_dst = [] hull_pt_indices = cv2.convexHull(np.array(points_dst), returnPoints = False) hull_pt_indices = hull_pt_indices.flatten() for idx_pt in hull_pt_indices: hull_pt_src.append(points_src[idx_pt]) hull_pt_dst.append(points_dst[idx_pt])
print(hull_pt_indices) print(hull_pt_src) print(hull_pt_dst)
[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 17 18 19 24 25] [[464, 329], [463, 371], [459, 411], [451, 454], [433, 493], [401, 526], [361, 552], [317, 572], [272, 576], [232, 566], [203, 540], [178, 510], [159, 476], [150, 439], [147, 401], [146, 365], [148, 328], [160, 301], [170, 280], [193, 269], [365, 268], [396, 279]] [[494, 321], [487, 377], [479, 433], [464, 486], [434, 531], [391, 568], [335, 591], [278, 611], [226, 616], [180, 607], [144, 580], [114, 546], [96, 502], [88, 454], [83, 405], [80, 357], [80, 311], [109, 259], [126, 238], [154, 235], [359, 240], [397, 247]]
2.1.3 Deloni Triangulation
cv2.Subdiv2D()
# Whether the checkpoint pt is in rectangular rect def rect_contains(rect, point): if point[0] < rect[0]: return False elif point[1] < rect[1]: return False elif point[0] > rect[2]: return False elif point[1] > rect[3]: return False return True #---------------------------------------------------------------------- def cal_delaunay_tri(rect, points): """Computational Dillonian Triangulation""" subdiv = cv2.Subdiv2D(rect); # Point by point insertion for pt in points: # subdiv.insert input type: tuple subdiv.insert(tuple(pt)) lst_tri = subdiv.getTriangleList(); # Vertex Index of Deloni Triangular Mesh lst_delaunay_tri_pt_indices = [] for tri in lst_tri: lst_tri_pts = [(tri[0], tri[1])] lst_tri_pts.append((tri[2], tri[3])) lst_tri_pts.append((tri[4], tri[5])) # Query Vertex Index of Triangular Mesh lst_pt_indices = [] for tri_pt in lst_tri_pts: for idx_pt in range(len(points)): if (abs(tri_pt[0] - points[idx_pt][0]) < 1) and \ (abs(tri_pt[1] - points[idx_pt][1]) < 1): lst_pt_indices.append(idx_pt) lst_delaunay_tri_pt_indices.append(lst_pt_indices) return lst_delaunay_tri_pt_indices
# delaunay triangulation size_img_dst = img_dst.shape rect = (0, 0, size_img_dst[1], size_img_dst[0]) # The Deloni triangular index corresponds to the convex hull vertex index, which is different from the facial key point index. lst_delaunay_tri_pt_indices = cal_delaunay_tri(rect, hull_pt_dst)
print(lst_delaunay_tri_pt_indices) print(len(lst_delaunay_tri_pt_indices))
[[4, 6, 20], [6, 4, 5], [1, 20, 21], [20, 1, 2], [2, 4, 20], [4, 2, 3], [0, 1, 21], [7, 13, 6], [13, 7, 12], [8, 10, 7], [10, 8, 9], [7, 10, 12], [12, 10, 11], [6, 13, 20], [20, 13, 14], [20, 14, 19], [19, 14, 15], [16, 19, 15], [19, 16, 17], [19, 17, 18]] 20
2.1.4 Triangular Affine Deformation
The triangle of g(x,y)g(x,y)g(x,y) facial region is affined to the target image f (x,y) f ^ {ast} (x,y) f (x,y) facial region.
#---------------------------------------------------------------------- def warp_affine(img_src, tri_src, tri_dst, size): """affine""" # Affine Matrix mat_warp = cv2.getAffineTransform(np.float32(tri_src), np.float32(tri_dst)) # affine transformation img_dst = cv2.warpAffine(img_src, mat_warp, (size[0], size[1]), None, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101) return img_dst #---------------------------------------------------------------------- def warp_tri(img_src, img_dst, tri_src, tri_dst, alpha=1) : """Affine triangulation, source image to target image""" # Triangular area frame rect_src = cv2.boundingRect(np.array(tri_src)) rect_dst = cv2.boundingRect(np.array(tri_dst)) # The migration of triangle vertex relative to triangle region frame tri_src_to_rect = [(item[0] - rect_src[0], item[1] - rect_src[1]) for item in tri_src] tri_dst_to_rect = [(item[0] - rect_dst[0], item[1] - rect_dst[1]) for item in tri_dst] # mask mask = np.zeros((rect_dst[3], rect_dst[2], 3), dtype = np.float32) cv2.fillConvexPoly(mask, np.array(tri_dst_to_rect), (1, 1, 1), 16, 0) # Intercepting the source image in the triangular region box img_src_rect = img_src[rect_src[1] : rect_src[1] + rect_src[3], rect_src[0] : rect_src[0] + rect_src[2]] size = (rect_dst[2], rect_dst[3]) # Triangular region frame affine img_src_rect_warpped = warp_affine(img_src_rect, tri_src_to_rect, tri_dst_to_rect, size) # Mask * Transparency mask *= alpha # Target image = target image * (1 - mask) + source image * mask img_dst[rect_dst[1] : rect_dst[1] + rect_dst[3], rect_dst[0] : rect_dst[0] + rect_dst[2]] = \ img_dst[rect_dst[1] : rect_dst[1] + rect_dst[3], rect_dst[0] : rect_dst[0] + rect_dst[2]] * (1 - mask) + \ img_src_rect_warpped * mask
# Deloni triangulation affine for tri_pt_indices in lst_delaunay_tri_pt_indices: # Triangular vertex coordinates of source image and target image tri_src = [hull_pt_src[tri_pt_indices[idx]] for idx in range(3)] tri_dst = [hull_pt_dst[tri_pt_indices[idx]] for idx in range(3)] warp_tri(img_src, img_dst_warped, tri_src, tri_dst, 1) idx_fig += 1 plot_imgs(idx_fig, [img_dst_warped])
2.2 Seamless Fusion
Eliminate skin color differences
-
Seamless fusion: cv2. seamless Clone ()
-
Mask: cv2.fillConvexPoly()
mask = np.zeros(img_dst.shape, dtype=img_dst.dtype) cv2.fillConvexPoly(mask, np.array(hull_pt_dst), (255, 255, 255)) rect = cv2.boundingRect(np.float32([hull_pt_dst])) center = (rect[0] + rect[2] // 2, rect[1] + rect[3] // 2) img_dst_src_grad = cv2.seamlessClone(img_dst_warped, img_dst, mask, center, cv2.NORMAL_CLONE) img_dst_mix_grad = cv2.seamlessClone(img_dst_warped, img_dst, mask, center, cv2.MIXED_CLONE) img_dst_mono_grad = cv2.seamlessClone(img_dst_warped, img_dst, mask, center, cv2.MONOCHROME_TRANSFER) idx_fig += 1 plot_imgs(idx_fig, [img_src, img_dst, img_dst_warped, img_dst_src_grad, img_dst_mix_grad, img_dst_mono_grad])
