Ruiluo medical

reference resources:

Using paddlehub to realize video face changing - Propeller AI Studio - artificial intelligence learning and training community

Change your face, Cang * Kong becomes Zhao * Ying, single welfare (understand all), get on the bus quickly. Just kidding, it's purely for technical learning and communication.

1. Prepare a face picture and a video, and use opencv and moviepy to divide the video into pictures;
2. Use PaddleHub's face_ landmark_ The localization model obtains 68 personal face feature points of face picture im1 and video picture im2;
3. Obtain the face mask IM1 of the two pictures according to the feature points obtained in the previous step_ Mask and im2_mask；
4. Use 3 of the 68 feature points to affine transform the face image IM1, align its face with the face of the replacement image, and obtain the image affine_im1；
5. Mask IM1 of face image_ Mask performs the same affine transformation to get affine_im1_mask；
6. Mask im2_mask and mask affine_ im1_ Union is obtained by merging the masked part of the mask_ mask；
7. Use the seamlesclone function in opencv to affine transform the affine_im1 and camera image im2 are Poisson fused, and the mask is union_mask to obtain the seamless image after fusion_ im.

To be optimized

1. Target image shape training: the transformation freedom of a single image is too low, and the material is added to synthesize various shapes;

2. Spatial position and layer calibration, facial occlusion, proportion, etc. need to be optimized.

Original video

Target face image

Installation module

pip install moviepy==0.2.1.9.7
hub install face_landmark_localization

Import module

import cv2
import numpy as np
import paddlehub as hub
from moviepy.editor import *
import shutil, os

Get target image size

def get_image_size(image):
    """
    Get picture size (height),Width)
    :param image: image
    :return: (height,Width)
    """
    image_size = (image.shape[0], image.shape[1])
    return image_size

Get face markers, 68 feature points

def get_face_landmarks(image):
    """
    Get face markers, 68 feature points
    :param image: image
    :param face_detector: dlib.get_frontal_face_detector
    :param shape_predictor: dlib.shape_predictor
    :return: np.array([[],[]]), 68 Feature points
    """
    dets = face_landmark.keypoint_detection([image])
    # num_faces = len(dets[0]['data'][0])
    if len(dets) == 0:
        print("Sorry, there were no faces found.")
        return None
    # shape = shape_predictor(image, dets[0])
    face_landmarks = np.array([[p[0], p[1]] for p in dets[0]['data'][0]])
    return face_landmarks

Get face mask

def get_face_mask(image_size, face_landmarks):
    """
    Get face mask
    :param image_size: Picture size
    :param face_landmarks: 68 Feature points
    :return: image_mask, Mask picture
    """
    mask = np.zeros(image_size, dtype=np.int32)
    points = np.concatenate([face_landmarks[0:16], face_landmarks[26:17:-1]])
    points = np.array(points, dtype=np.int32)

    cv2.fillPoly(img=mask, pts=[points], color=255)

    # mask = np.zeros(image_size, dtype=np.uint8)
    # points = cv2.convexHull(face_landmarks)  # convex hull
    # cv2.fillConvexPoly(mask, points, color=255)
    return mask.astype(np.uint8)

Get the image after affine transformation of image 1

def get_affine_image(image1, image2, face_landmarks1, face_landmarks2):
    """
    Get the image after affine transformation of image 1
    :param image1: Picture 1, Image to affine transform
    :param image2: Picture 2, As long as it is used to obtain the image size, an affine transformation image with the same size is generated
    :param face_landmarks1: Face feature points in picture 1
    :param face_landmarks2: Face feature points in picture 2
    :return: Affine transformed image
    """
    three_points_index = [18, 8, 25]
    M = cv2.getAffineTransform(face_landmarks1[three_points_index].astype(np.float32),
                               face_landmarks2[three_points_index].astype(np.float32))
    dsize = (image2.shape[1], image2.shape[0])
    affine_image = cv2.warpAffine(image1, M, dsize)
    return affine_image.astype(np.uint8)

Obtain the center point coordinates of the mask

def get_mask_center_point(image_mask):
    """
    Obtain the center point coordinates of the mask
    :param image_mask: Mask picture
    :return: Mask Center
    """
    image_mask_index = np.argwhere(image_mask > 0)
    miny, minx = np.min(image_mask_index, axis=0)
    maxy, maxx = np.max(image_mask_index, axis=0)
    center_point = ((maxx + minx) // 2, (maxy + miny) // 2)
    return center_point

Gets the union of two mask masking parts

def get_mask_union(mask1, mask2):
    """
    Gets the union of two mask masking parts
    :param mask1: mask_image, Mask 1
    :param mask2: mask_image, Mask 2
    :return: Union of masking parts of two masks
    """
    mask = np.min([mask1, mask2], axis=0)  # Masked partial union
    mask = ((cv2.blur(mask, (5, 5)) == 255) * 255).astype(np.uint8)  # Reduce mask size
    mask = cv2.blur(mask, (3, 3)).astype(np.uint8)  # Blur mask
    return mask

Skin tone adjustment

def skin_color_adjustment(im1, im2, mask=None):
    """
    Skin tone adjustment
    :param im1: Picture 1
    :param im2: Picture 2
    :param mask: Face mask. If it exists, the face partial mean is used to calculate the skin color transformation coefficient; Otherwise, Gaussian blur is used to calculate the skin color transformation coefficient
    :return: Picture 1 adjusted according to the color of picture 2
    """
    if mask is None:
        im1_ksize = 55
        im2_ksize = 55
        im1_factor = cv2.GaussianBlur(im1, (im1_ksize, im1_ksize), 0).astype(np.float)
        im2_factor = cv2.GaussianBlur(im2, (im2_ksize, im2_ksize), 0).astype(np.float)
    else:
        im1_face_image = cv2.bitwise_and(im1, im1, mask=mask)
        im2_face_image = cv2.bitwise_and(im2, im2, mask=mask)
        im1_factor = np.mean(im1_face_image, axis=(0, 1))
        im2_factor = np.mean(im2_face_image, axis=(0, 1))

    im1 = np.clip((im1.astype(np.float) * im2_factor / np.clip(im1_factor, 1e-6, None)), 0, 255).astype(np.uint8)
    return im1

Face replacement

def change_face(im_name1, im_name2, new_path):
    """
    :param im1: Face picture or file name to replace with
    :param im_name2: Original face image file name
    :param new_path: Picture directory after replacement
    """
    if isinstance(im_name1, str):
        im1 = cv2.imread(im_name1)  # face_image
    else:
        im1 = im_name1
    im1 = cv2.resize(im1, (600, im1.shape[0] * 600 // im1.shape[1]))
    landmarks1 = get_face_landmarks(im1)  # 68_face_landmarks

    im1_size = get_image_size(im1)  # Face size
    im1_mask = get_face_mask(im1_size, landmarks1)  # Face mask

    im2 = cv2.imread(im_name2)
    landmarks2 = get_face_landmarks(im2)  # 68_face_landmarks
    if landmarks2 is not None:
        im2_size = get_image_size(im2)  # Camera picture size
        im2_mask = get_face_mask(im2_size, landmarks2)  # Camera image face mask

        affine_im1 = get_affine_image(im1, im2, landmarks1, landmarks2)  # im1 (face image) affine transformed image
        affine_im1_mask = get_affine_image(im1_mask, im2, landmarks1, landmarks2)  # im1 (face map) face mask of affine transformed picture

        union_mask = get_mask_union(im2_mask, affine_im1_mask)  # Mask merging

        affine_im1 = skin_color_adjustment(affine_im1, im2, mask=union_mask)  # Skin tone adjustment
        point = get_mask_center_point(affine_im1_mask)  # im1 (face map) is the center point of the face mask of the affine transformed picture
        seamless_im = cv2.seamlessClone(affine_im1, im2, mask=union_mask, p=point, flags=cv2.NORMAL_CLONE)  # Poisson fusion

        new_im = os.path.join(new_path, os.path.split(im_name2)[-1])
        cv2.imwrite(new_im, seamless_im)
    else:
        shutil.copy(im_name2, new_path)

Decompose video

def cut_video_to_image(video_path, img_path):
    """
    Decompose video into pictures
    input parameter : Split video address,Save picture address
    Output parameters: Two global variables are declared to save the size of the split picture
    Function: divide the video into pictures by frame
    """
    cap = cv2.VideoCapture(video_path)

    index = 0
    global size_y, size_x, fps

    # Find OpenCV version
    (major_ver, minor_ver, subminor_ver) = (cv2.__version__).split('.')

    if int(major_ver) < 3:
        fps = cap.get(cv2.cv.CV_CAP_PROP_FPS)
    else:
        fps = cap.get(cv2.CAP_PROP_FPS)

    while (True):
        ret, frame = cap.read()
        if ret:
            cv2.imwrite(img_path + '/%d.jpg' % index, frame)
            index += 1
        else:
            break
        size_x = frame.shape[0]
        size_y = frame.shape[1]
    cap.release()

    print('Video cut finish, all %d frame' % index)
    print("imge size:x is {1},y is {0}".format(size_x, size_y))

Batch processing

def get_faces_changed(im_name1, old_path, new_path):
    """
    Cycle to read the picture and change your face
    :param im_name1: Face image file name to replace with
    :param old_path: Original picture directory
    :param new_path: Output picture directory after face change
    """
    im1 = cv2.imread(im_name1)
    im_names = os.listdir(old_path)
    for im_name2 in os.listdir(old_path):
        if im_name2.startswith("."):
            continue

        change_face(im1, old_path + "/" + im_name2, new_path)

Video synthesis

def combine_image_to_video(image_path, video_name):
    """
    Video synthesis

    """
    fourcc = cv2.VideoWriter_fourcc(*'MP4V')
    #print(video_name)
    out = cv2.VideoWriter(video_name, fourcc, fps, (size_y, size_x))
    files = os.listdir(image_path)
    print("Altogether{}Frame picture to be synthesized".format(len(files)))
    for i in range(len(files)):
        pic_name = str(i)+".jpg"
        #print(pic_name)
        img = cv2.imread(image_path + "/" + pic_name)
        out.write(img)    # Save frame
    out.release()

Extract the audio from the original video and fuse it with the modified video

def combine_audio_video(orig_video, new_video_wot_audio, final_video):
    """
    Extract the audio from the original video and fuse it with the modified video
    """
    audioclip = AudioFileClip(orig_video)
    clip_finall = VideoFileClip(new_video_wot_audio)
    videoclip = clip_finall.set_audio(audioclip)
    videoclip.write_videofile(final_video)

debugging

def main(orig_video, face_img, out_video, video_to_img_path="./data/frame", face_changed_img_path="./data/frame_changed"):
    ## Cut video into pictures
    print("Cut video into pictures", end="\n\n")
    cut_video_to_image(orig_video, video_to_img_path)
    ## Face change
    print("Start changing faces", end="\n\n")
    get_faces_changed(face_img, video_to_img_path, face_changed_img_path)
    ## Synthesize pictures into video
    print("Synthesize pictures into video", end="\n\n")
    combine_image_to_video(face_changed_img_path, "temp.mp4")
    ## Add audio
    print("Add audio", end="\n\n")
    combine_audio_video(orig_video, "temp.mp4", out_video)
    print("Done!")


##
for i in os.listdir("./data/frame"):
    if i.startswith("."): continue
    os.remove("./data/frame/" + i)

for i in os.listdir("./data/frame_changed"):
    if i.startswith("."): continue
    os.remove("./data/frame_changed/" + i)

## Face key point model



face_landmark = hub.Module(name="face_landmark_localization")
# Picture face replacement
#im1 = cv2.imread("luffy.jpg")
#change_face(im1, "bchelor.jpg", "test")

## Video face replacement
main("./data/video/shejian_zsl.mp4", "./data/zly.jpg", "./data/video/shejian_zly.mp4")
##main("video/popping_Hozin.mp4", "luffy.jpg", "video/popping_luffy.mp4")