DHASH AND HAMMING DISTANCE_Lesson.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from scipy.misc import imread, imresize, imshow\n",
    "import cv2\n",
    "import os\n",
    "import time\n",
    "from hashlib import md5\n",
    "import scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\Moondra\\\\Desktop\\\\DATA\\\\IMAGE_2'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IMAGE_DIR = 'C:\\\\Users\\\\Moondra\\\\Desktop\\\\DATA\\\\IMAGE_2\\\\'\n",
    "\n",
    "os.chdir(IMAGE_DIR)\n",
    "os.getcwd()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43\n"
     ]
    }
   ],
   "source": [
    "image_files = os.listdir()\n",
    "print(len(image_files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'%C6%B0n-%C6%B0ebhg - Copy.jpg'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_files[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(420, 640, 3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imread(image_files[0]).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Helper Functions\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def filter_images(images):\n",
    "    image_list = []\n",
    "    for image in images:\n",
    "        try:\n",
    "            assert imread(image).shape[2] == 3\n",
    "            image_list.append(image)\n",
    "        except  AssertionError as e:\n",
    "            print(e)\n",
    "    return image_list\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#First turn the image into a gray scale image\n",
    "\n",
    "\n",
    "def img_gray(image):\n",
    "    image = imread(image)\n",
    "    return np.average(image, weights=[0.299, 0.587, 0.114], axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#resize image and flatten\n",
    "def resize(image, height=30, width=30):\n",
    "    row_res = cv2.resize(image,(height, width), interpolation = cv2.INTER_AREA).flatten()\n",
    "    col_res = cv2.resize(image,(height, width), interpolation = cv2.INTER_AREA).flatten('F')\n",
    "    return row_res, col_res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#gradient direction based on intensity \n",
    "\n",
    "def intensity_diff(row_res, col_res):\n",
    "    difference_row = np.diff(row_res)\n",
    "    difference_col = np.diff(col_res)\n",
    "    difference_row = difference_row > 0\n",
    "    difference_col = difference_col > 0\n",
    "    return np.vstack((difference_row, difference_col)).flatten()\n",
    "    #return difference_row\n",
    "    #return np.vstack((difference_row, difference_col)) #str method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def file_hash(array):\n",
    "    return md5(array).hexdigest()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def difference_score(image, height = 30, width = 30):\n",
    "    gray = img_gray(image)\n",
    "    row_res, col_res = resize(gray, height, width)\n",
    "    difference = intensity_diff(row_res, col_res)\n",
    "    \n",
    "    return difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def difference_score_dict_hash(image_list):\n",
    "    ds_dict = {}\n",
    "    duplicates = []\n",
    "    hash_ds = []\n",
    "    for image in image_list:\n",
    "        ds = difference_score(image)\n",
    "        hash_ds.append(ds)\n",
    "        filehash = md5(ds).hexdigest()\n",
    "        if filehash not in ds_dict:\n",
    "            ds_dict[filehash] = image\n",
    "        else:\n",
    "            duplicates.append((image, ds_dict[filehash]) )\n",
    "    \n",
    "    return  duplicates, ds_dict, hash_ds\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "image_files = filter_images(image_files)\n",
    "duplicates, ds_dict, hash_ds =difference_score_dict_hash(image_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(duplicates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for file_names in duplicates[:30]:\n",
    "    try:\n",
    "    \n",
    "        plt.subplot(121),plt.imshow(imread(file_names[0]))\n",
    "        plt.title('Duplicate'), plt.xticks([]), plt.yticks([])\n",
    "\n",
    "        plt.subplot(122),plt.imshow(imread(file_names[1]))\n",
    "        plt.title('Original'), plt.xticks([]), plt.yticks([])\n",
    "        plt.show()\n",
    "    \n",
    "    except OSError as e:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Now Let's try Hamming distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def hamming_distance(image, image2):\n",
    "    score =scipy.spatial.distance.hamming(image, image2)\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Hamming\n",
    "\n",
    "def difference_score_dict(image_list):\n",
    "    ds_dict = {}\n",
    "    duplicates = []\n",
    "    for image in image_list:\n",
    "        ds = difference_score(image)\n",
    "        \n",
    "        if image not in ds_dict:\n",
    "            ds_dict[image] = ds\n",
    "        else:\n",
    "            duplicates.append((image, ds_dict[image]) )\n",
    "    \n",
    "    return  duplicates, ds_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "image_files = filter_images(image_files)\n",
    "duplicates, ds_dict =difference_score_dict(image_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(duplicates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ds_dict.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "for k1,k2 in itertools.combinations(ds_dict, 2):\n",
    "    if hamming_distance(ds_dict[k1], ds_dict[k2])< .10:\n",
    "        duplicates.append((k1,k2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(duplicates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for file_names in duplicates[:31]:\n",
    "    try:\n",
    "    \n",
    "        plt.subplot(121),plt.imshow(imread(file_names[0]))\n",
    "        plt.title('Duplicate'), plt.xticks([]), plt.yticks([])\n",
    "\n",
    "        plt.subplot(122),plt.imshow(imread(file_names[1]))\n",
    "        plt.title('Original'), plt.xticks([]), plt.yticks([])\n",
    "        plt.show()\n",
    "    \n",
    "    except OSError as e:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "os.remove()  #If we wanted to remove them`"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}