face_Switching_dlib_opencv.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 原文\n",
    "源码来源：https://matthewearl.github.io/2015/07/28/switching-eds-with-python/\n",
    "\n",
    "shape_predictor_68_face_landmarks模型下载链接:     \n",
    "https://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2/download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "import dlib\n",
    "import numpy\n",
    "import sys\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "PREDICTOR_PATH = \"/.../shape_predictor_68_face_landmarks.dat\"  # 68个关键点landmarks的模型文件\n",
    "SCALE_FACTOR = 1 # 图像的放缩比\n",
    "FEATHER_AMOUNT = 15  # 羽化边界范围，越大，羽化能力越大，一定要奇数，不能偶数\n",
    "\n",
    "#　68个点\n",
    "FACE_POINTS = list(range(17, 68))  # 脸\n",
    "MOUTH_POINTS = list(range(48, 61))  # 嘴巴\n",
    "RIGHT_BROW_POINTS = list(range(17, 22))  # 右眉毛\n",
    "LEFT_BROW_POINTS = list(range(22, 27))  # 左眉毛\n",
    "RIGHT_EYE_POINTS = list(range(36, 42))  # 右眼睛\n",
    "LEFT_EYE_POINTS = list(range(42, 48))  # 左眼睛\n",
    "NOSE_POINTS = list(range(27, 35))  # 鼻子\n",
    "JAW_POINTS = list(range(0, 17))  # 下巴\n",
    "\n",
    "# 选取用于叠加在第一张脸上的第二张脸的面部特征\n",
    "# 特征点包括左右眼、眉毛、鼻子和嘴巴\n",
    "# 是否数量变多之后，会有什么干扰吗?\n",
    "ALIGN_POINTS = (LEFT_BROW_POINTS + RIGHT_EYE_POINTS + LEFT_EYE_POINTS +\n",
    "                               RIGHT_BROW_POINTS + NOSE_POINTS + MOUTH_POINTS)\n",
    "\n",
    "# Points from the second image to overlay on the first. The convex hull of each\n",
    "# element will be overlaid.\n",
    "OVERLAY_POINTS = [\n",
    "    LEFT_EYE_POINTS + RIGHT_EYE_POINTS + LEFT_BROW_POINTS + RIGHT_BROW_POINTS,\n",
    "    NOSE_POINTS + MOUTH_POINTS,\n",
    "]  \n",
    "# 眼睛 ，眉毛             2 * 22\n",
    "# 鼻子，嘴巴   分开来\n",
    "\n",
    "# 定义用于颜色校正的模糊量，作为瞳孔距离的系数\n",
    "COLOUR_CORRECT_BLUR_FRAC = 0.6\n",
    "\n",
    "# 实例化脸部检测器\n",
    "detector = dlib.get_frontal_face_detector()\n",
    "# 加载训练模型\n",
    "# 并实例化特征提取器\n",
    "predictor = dlib.shape_predictor(PREDICTOR_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 定义了两个类处理意外\n",
    "class TooManyFaces(Exception): \n",
    "    pass\n",
    "\n",
    "class NoFaces(Exception):\n",
    "    pass\n",
    "\n",
    "\n",
    "def get_landmarks(im):\n",
    "    '''\n",
    "    通过predictor 拿到68 landmarks\n",
    "    '''\n",
    "    rects = detector(im, 1)\n",
    "    \n",
    "    if len(rects) > 1:\n",
    "        raise TooManyFaces\n",
    "    if len(rects) == 0:\n",
    "        raise NoFaces\n",
    "\n",
    "    return numpy.matrix([[p.x, p.y] for p in predictor(im, rects[0]).parts()])   # 68*2的矩阵\n",
    "\n",
    "def annotate_landmarks(im, landmarks):\n",
    "    '''\n",
    "    人脸关键点，画图函数\n",
    "    '''\n",
    "    im = im.copy()\n",
    "    for idx, point in enumerate(landmarks):\n",
    "        pos = (point[0, 0], point[0, 1])\n",
    "        cv2.putText(im, str(idx), pos,\n",
    "                    fontFace=cv2.FONT_HERSHEY_SCRIPT_SIMPLEX,\n",
    "                    fontScale=0.4,\n",
    "                    color=(0, 0, 255))\n",
    "        cv2.circle(im, pos, 3, color=(0, 255, 255))\n",
    "    return im\n",
    "\n",
    "def draw_convex_hull(im, points, color):\n",
    "    '''\n",
    "    # 绘制凸多边形 计算凸包\n",
    "    '''\n",
    "    points = cv2.convexHull(points)\n",
    "    cv2.fillConvexPoly(im, points, color=color)\n",
    "\n",
    "def get_face_mask(im, landmarks):\n",
    "    '''获取面部特征部分（眉毛、眼睛、鼻子以及嘴巴）的图像掩码。\n",
    "    图像掩码作用于原图之后，原图中对应掩码部分为白色的部分才能显示出来，黑色的部分则不予显示，因此通过图像掩码我们就能实现对图像“裁剪”。\n",
    "    效果参考：https://dn-anything-about-doc.qbox.me/document-uid242676labid2260timestamp1477921310170.png/wm\n",
    "    get_face_mask()的定义是为一张图像和一个标记矩阵生成一个遮罩，它画出了两个白色的凸多边形：一个是眼睛周围的区域，\n",
    "    一个是鼻子和嘴部周围的区域。之后它由11个(FEATHER_AMOUNT)像素向遮罩的边缘外部羽化扩展，可以帮助隐藏任何不连续的区域。\n",
    "    '''\n",
    "    im = numpy.zeros(im.shape[:2], dtype=numpy.float64)\n",
    "\n",
    "    for group in OVERLAY_POINTS:\n",
    "        draw_convex_hull(im,\n",
    "                         landmarks[group],\n",
    "                         color=1)\n",
    "\n",
    "    im = numpy.array([im, im, im]).transpose((1, 2, 0))\n",
    "\n",
    "    im = (cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0) > 0) * 1.0\n",
    "    im = cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0)\n",
    "\n",
    "    return im\n",
    "    \n",
    "def transformation_from_points(points1, points2):\n",
    "    \"\"\"\n",
    "    Return an affine transformation [s * R | T] such that:\n",
    "        sum ||s*R*p1,i + T - p2,i||^2\n",
    "    is minimized.\n",
    "    \"\"\"\n",
    "    # Solve the procrustes problem by subtracting centroids, scaling by the\n",
    "    # standard deviation, and then using the SVD to calculate the rotation. See\n",
    "    # the following for more details:\n",
    "    #   https://en.wikipedia.org/wiki/Orthogonal_Procrustes_problem\n",
    "\n",
    "    points1 = points1.astype(numpy.float64)\n",
    "    points2 = points2.astype(numpy.float64)\n",
    "\n",
    "    c1 = numpy.mean(points1, axis=0)\n",
    "    c2 = numpy.mean(points2, axis=0)\n",
    "    points1 -= c1\n",
    "    points2 -= c2\n",
    "\n",
    "    s1 = numpy.std(points1)\n",
    "    s2 = numpy.std(points2)\n",
    "    points1 /= s1\n",
    "    points2 /= s2\n",
    "\n",
    "    U, S, Vt = numpy.linalg.svd(points1.T * points2)\n",
    "\n",
    "    # The R we seek is in fact the transpose of the one given by U * Vt. This\n",
    "    # is because the above formulation assumes the matrix goes on the right\n",
    "    # (with row vectors) where as our solution requires the matrix to be on the\n",
    "    # left (with column vectors).\n",
    "    R = (U * Vt).T\n",
    "\n",
    "    return numpy.vstack([numpy.hstack(((s2 / s1) * R,\n",
    "                                       c2.T - (s2 / s1) * R * c1.T)),\n",
    "                         numpy.matrix([0., 0., 1.])])\n",
    "\n",
    "def read_im_and_landmarks(fname):\n",
    "    im = cv2.imread(fname, cv2.IMREAD_COLOR)\n",
    "    im = cv2.resize(im, (im.shape[1] * SCALE_FACTOR,\n",
    "                         im.shape[0] * SCALE_FACTOR))\n",
    "    s = get_landmarks(im)\n",
    "\n",
    "    return im, s\n",
    "\n",
    "def warp_im(im, M, dshape):\n",
    "    '''\n",
    "    由 get_face_mask 获得的图像掩码还不能直接使用，因为一般来讲用户提供的两张图像的分辨率大小很可能不一样，而且即便分辨率一样，\n",
    "    图像中的人脸由于拍摄角度和距离等原因也会呈现出不同的大小以及角度，所以如果不能只是简单地把第二个人的面部特征抠下来直接放在第一个人脸上，\n",
    "    我们还需要根据两者计算所得的面部特征区域进行匹配变换，使得二者的面部特征尽可能重合。\n",
    "    \n",
    "    仿射函数，warpAffine，能对图像进行几何变换\n",
    "        三个主要参数，第一个输入图像，第二个变换矩阵 np.float32 类型，第三个变换之后图像的宽高\n",
    "    \n",
    "    对齐主要函数\n",
    "    '''\n",
    "    output_im = numpy.zeros(dshape, dtype=im.dtype)\n",
    "    cv2.warpAffine(im,\n",
    "                   M[:2],\n",
    "                   (dshape[1], dshape[0]),\n",
    "                   dst=output_im,\n",
    "                   borderMode=cv2.BORDER_TRANSPARENT,\n",
    "                   flags=cv2.WARP_INVERSE_MAP)\n",
    "    return output_im\n",
    "\n",
    "def correct_colours(im1, im2, landmarks1):\n",
    "    '''\n",
    "    修改皮肤颜色，使两张图片在拼接时候显得更加自然。\n",
    "    '''\n",
    "    blur_amount = COLOUR_CORRECT_BLUR_FRAC * numpy.linalg.norm(\n",
    "                              numpy.mean(landmarks1[LEFT_EYE_POINTS], axis=0) -\n",
    "                              numpy.mean(landmarks1[RIGHT_EYE_POINTS], axis=0))\n",
    "    blur_amount = int(blur_amount)\n",
    "    if blur_amount % 2 == 0:\n",
    "        blur_amount += 1\n",
    "    im1_blur = cv2.GaussianBlur(im1, (blur_amount, blur_amount), 0)\n",
    "    im2_blur = cv2.GaussianBlur(im2, (blur_amount, blur_amount), 0)\n",
    "\n",
    "    # Avoid divide-by-zero errors.\n",
    "    im2_blur += (128 * (im2_blur <= 1.0)).astype(im2_blur.dtype)\n",
    "\n",
    "    return (im2.astype(numpy.float64) * im1_blur.astype(numpy.float64) /\n",
    "                                                im2_blur.astype(numpy.float64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 换脸函数\n",
    "def Switch_face(Base_path,cover_path):\n",
    "    im1, landmarks1 = read_im_and_landmarks(Base_path)  # 底图\n",
    "    im2, landmarks2 = read_im_and_landmarks(cover_path)  # 贴上来的图\n",
    "    \n",
    "    if len(landmarks1) == 0 & len(landmarks2) == 0 :\n",
    "        raise ImproperNumber(\"Faces detected is no face!\")\n",
    "    if len(landmarks1) > 1 & len(landmarks2) > 1 :\n",
    "        raise ImproperNumber(\"Faces detected is more than 1!\")\n",
    "    \n",
    "    M = transformation_from_points(landmarks1[ALIGN_POINTS],\n",
    "                                   landmarks2[ALIGN_POINTS])\n",
    "    mask = get_face_mask(im2, landmarks2)\n",
    "    warped_mask = warp_im(mask, M, im1.shape)\n",
    "    combined_mask = numpy.max([get_face_mask(im1, landmarks1), warped_mask],\n",
    "                              axis=0)\n",
    "    warped_im2 = warp_im(im2, M, im1.shape)\n",
    "    warped_corrected_im2 = correct_colours(im1, warped_im2, landmarks1)\n",
    "\n",
    "    output_im = im1 * (1.0 - combined_mask) + warped_corrected_im2 * combined_mask\n",
    "    return output_im\n",
    "\n",
    "# 人脸对齐函数\n",
    "def face_Align(Base_path,cover_path):\n",
    "    im1, landmarks1 = read_im_and_landmarks(Base_path)  # 底图\n",
    "    im2, landmarks2 = read_im_and_landmarks(cover_path)  # 贴上来的图\n",
    "    \n",
    "    if len(landmarks1) == 0 & len(landmarks2) == 0 :\n",
    "        raise ImproperNumber(\"Faces detected is no face!\")\n",
    "    if len(landmarks1) > 1 & len(landmarks2) > 1 :\n",
    "        raise ImproperNumber(\"Faces detected is more than 1!\")\n",
    "    \n",
    "    M = transformation_from_points(landmarks1[ALIGN_POINTS],\n",
    "                                   landmarks2[ALIGN_POINTS])\n",
    "    warped_im2 = warp_im(im2, M, im1.shape)\n",
    "    return warped_im2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "换脸\n",
    "Base_path:是底图\n",
    "cover_path：从该图抽取内容覆盖到底图上\n",
    "'''\n",
    "FEATHER_AMOUNT = 23\n",
    "\n",
    "Base_path = '03.jpg'\n",
    "cover_path = '02.jpg'\n",
    "output_im = Switch_face(Base_path,cover_path)\n",
    "\n",
    "cv2.imwrite('/../output.jpg', output_im)  # 换脸数据保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "人脸对齐-dlib\n",
    "Base_path:模板图\n",
    "cover_path：需对齐的图\n",
    "'''\n",
    "FEATHER_AMOUNT = 19  # 匹配的时候，特征数量，现在是以11个点为基准点  11  15  17 \n",
    "\n",
    "Base_path = '01.jpg'\n",
    "cover_path = '02.jpg'\n",
    "warped_mask = face_Align(Base_path,cover_path)\n",
    "\n",
    "%matplotlib inline\n",
    "plt.subplot(111)\n",
    "plt.imshow(warped_mask)     #　数据展示"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 关键点定位 画图函数\n",
    "    # 画圈 + 数字\n",
    "def annotate_landmarks(im, landmarks):\n",
    "    im = im.copy()\n",
    "    for idx, point in enumerate(landmarks):\n",
    "        pos = (point[0, 0], point[0, 1])\n",
    "        cv2.putText(im, str(idx), pos,\n",
    "                    fontFace=cv2.FONT_HERSHEY_SCRIPT_SIMPLEX,\n",
    "                    fontScale=0.4,\n",
    "                    color=(0, 0, 255))\n",
    "        cv2.circle(im, pos, 3, color=(0, 255, 255))\n",
    "    return im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "im1, landmarks1 = read_im_and_landmarks('02.jpg')  # 底图\n",
    "im1 = annotate_landmarks(im1, landmarks1)\n",
    "\n",
    "%matplotlib inline\n",
    "plt.subplot(111)\n",
    "plt.imshow(im1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}