Creating_a_Virtual_Pen_and_Eraser.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# <center> <font style=\"color:rgb(100,109,254)\">   Creating a Virtual Pen & Eraser </font> </center>\n"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "import cv2\r\n",
    "import numpy as np\r\n",
    "import time"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## <font style=\"color:rgb(134,19,348)\">Step 1: Find Color range of target Pen and save it</font>\n"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "# A required callback method that goes into the trackbar function.\r\n",
    "def nothing(x):\r\n",
    "    pass\r\n",
    "\r\n",
    "# Intializing the webcam feed.\r\n",
    "cap = cv2.VideoCapture(0)\r\n",
    "cap.set(3,1280)\r\n",
    "cap.set(4,720)\r\n",
    "\r\n",
    "# Create a window named trackbars.\r\n",
    "cv2.namedWindow(\"Trackbars\")\r\n",
    "\r\n",
    "# Now create 6 tracbars that will control the lower and upper range of H,S & V channels.\r\n",
    "# The Arguments are like this: Name of trackbar, window name, range, callback function.\r\n",
    "# For Hue the range is 0-179 and for S,V its 0-255.\r\n",
    "cv2.createTrackbar(\"L - H\", \"Trackbars\", 0, 179, nothing)\r\n",
    "cv2.createTrackbar(\"L - S\", \"Trackbars\", 0, 255, nothing)\r\n",
    "cv2.createTrackbar(\"L - V\", \"Trackbars\", 0, 255, nothing)\r\n",
    "cv2.createTrackbar(\"U - H\", \"Trackbars\", 179, 179, nothing)\r\n",
    "cv2.createTrackbar(\"U - S\", \"Trackbars\", 255, 255, nothing)\r\n",
    "cv2.createTrackbar(\"U - V\", \"Trackbars\", 255, 255, nothing)\r\n",
    " \r\n",
    " \r\n",
    "while True:\r\n",
    "    \r\n",
    "    # Start reading the webcam feed frame by frame.\r\n",
    "    ret, frame = cap.read()\r\n",
    "    if not ret:\r\n",
    "        break\r\n",
    "    # Flip the frame horizontally (Not required)\r\n",
    "    frame = cv2.flip( frame, 1 ) \r\n",
    "    \r\n",
    "    # Convert the BGR image to HSV image.\r\n",
    "    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)\r\n",
    "    \r\n",
    "    # Get the new values of the trackbar in real time as the user changes them\r\n",
    "    l_h = cv2.getTrackbarPos(\"L - H\", \"Trackbars\")\r\n",
    "    l_s = cv2.getTrackbarPos(\"L - S\", \"Trackbars\")\r\n",
    "    l_v = cv2.getTrackbarPos(\"L - V\", \"Trackbars\")\r\n",
    "    u_h = cv2.getTrackbarPos(\"U - H\", \"Trackbars\")\r\n",
    "    u_s = cv2.getTrackbarPos(\"U - S\", \"Trackbars\")\r\n",
    "    u_v = cv2.getTrackbarPos(\"U - V\", \"Trackbars\")\r\n",
    " \r\n",
    "    # Set the lower and upper HSV range according to the value selected by the trackbar\r\n",
    "    lower_range = np.array([l_h, l_s, l_v])\r\n",
    "    upper_range = np.array([u_h, u_s, u_v])\r\n",
    "    \r\n",
    "    # Filter the image and get the binary mask, where white represents your target color\r\n",
    "    mask = cv2.inRange(hsv, lower_range, upper_range)\r\n",
    " \r\n",
    "    # You can also visualize the real part of the target color (Optional)\r\n",
    "    res = cv2.bitwise_and(frame, frame, mask=mask)\r\n",
    "    \r\n",
    "    # Converting the binary mask to 3 channel image, this is just so we can stack it with the others\r\n",
    "    mask_3 = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)\r\n",
    "    \r\n",
    "    # stack the mask, orginal frame and the filtered result\r\n",
    "    stacked = np.hstack((mask_3,frame,res))\r\n",
    "    \r\n",
    "    # Show this stacked frame at 40% of the size.\r\n",
    "    cv2.imshow('Trackbars',cv2.resize(stacked,None,fx=0.4,fy=0.4))\r\n",
    "    \r\n",
    "    # If the user presses ESC then exit the program\r\n",
    "    key = cv2.waitKey(1)\r\n",
    "    if key == 27:\r\n",
    "        break\r\n",
    "    \r\n",
    "    # If the user presses `s` then print this array.\r\n",
    "    if key == ord('s'):\r\n",
    "        \r\n",
    "        thearray = [[l_h,l_s,l_v],[u_h, u_s, u_v]]\r\n",
    "        print(thearray)\r\n",
    "        \r\n",
    "        # Also save this array as penval.npy\r\n",
    "        np.save('penval',thearray)\r\n",
    "        break\r\n",
    "    \r\n",
    "# Release the camera & destroy the windows.    \r\n",
    "cap.release()\r\n",
    "cv2.destroyAllWindows()"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "error",
     "evalue": "OpenCV(4.5.3) C:\\Users\\runneradmin\\AppData\\Local\\Temp\\pip-req-build-q3d_8t8e\\opencv\\modules\\highgui\\src\\window_w32.cpp:2577: error: (-27:Null pointer) NULL window: 'Trackbars' in function 'cvGetTrackbarPos'\n",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31merror\u001b[0m                                     Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-107af601f1ba>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     35\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m     \u001b[1;31m# Get the new values of the trackbar in real time as the user changes them\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 37\u001b[1;33m     \u001b[0ml_h\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetTrackbarPos\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"L - H\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"Trackbars\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     38\u001b[0m     \u001b[0ml_s\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetTrackbarPos\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"L - S\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"Trackbars\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[0ml_v\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetTrackbarPos\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"L - V\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"Trackbars\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31merror\u001b[0m: OpenCV(4.5.3) C:\\Users\\runneradmin\\AppData\\Local\\Temp\\pip-req-build-q3d_8t8e\\opencv\\modules\\highgui\\src\\window_w32.cpp:2577: error: (-27:Null pointer) NULL window: 'Trackbars' in function 'cvGetTrackbarPos'\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## <font style=\"color:rgb(134,19,348)\">Step 2: Maximizing the Detection Mask and Getting rid of the noise   </font>"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "# This variable determines if we want to load color range from memory or use the ones defined here. \r\n",
    "load_from_disk = True\r\n",
    "\r\n",
    "# If true then load color range from memory\r\n",
    "if load_from_disk:\r\n",
    "    penval = np.load('penval.npy')\r\n",
    "\r\n",
    "cap = cv2.VideoCapture(0)\r\n",
    "cap.set(3,1280)\r\n",
    "cap.set(4,720)\r\n",
    "\r\n",
    "# Creating A 5x5 kernel for morphological operations\r\n",
    "kernel = np.ones((5,5),np.uint8)\r\n",
    "\r\n",
    "while(1):\r\n",
    "    \r\n",
    "    ret, frame = cap.read()\r\n",
    "    if not ret:\r\n",
    "        break\r\n",
    "        \r\n",
    "    frame = cv2.flip( frame, 1 )\r\n",
    "\r\n",
    "    # Convert BGR to HSV\r\n",
    "    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)\r\n",
    "    \r\n",
    "    # If you're reading from memory then load the upper and lower ranges from there\r\n",
    "    if load_from_disk:\r\n",
    "            lower_range = penval[0]\r\n",
    "            upper_range = penval[1]\r\n",
    "            \r\n",
    "    # Otherwise define your own custom values for upper and lower range.\r\n",
    "    else:             \r\n",
    "       lower_range  = np.array([26,80,147])\r\n",
    "       upper_range = np.array([81,255,255])\r\n",
    "    \r\n",
    "    mask = cv2.inRange(hsv, lower_range, upper_range)\r\n",
    "    \r\n",
    "    # Perform the morphological operations to get rid of the noise.\r\n",
    "    # Erosion Eats away the white part while dilation expands it.\r\n",
    "    mask = cv2.erode(mask,kernel,iterations = 1)\r\n",
    "    mask = cv2.dilate(mask,kernel,iterations = 2)\r\n",
    "\r\n",
    "    res = cv2.bitwise_and(frame,frame, mask= mask)\r\n",
    "\r\n",
    "    mask_3 = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)\r\n",
    "    \r\n",
    "    # stack all frames and show it\r\n",
    "    stacked = np.hstack((mask_3,frame,res))\r\n",
    "    cv2.imshow('Trackbars',cv2.resize(stacked,None,fx=0.4,fy=0.4))\r\n",
    "    \r\n",
    "    k = cv2.waitKey(5) & 0xFF\r\n",
    "    if k == 27:\r\n",
    "        break\r\n",
    "\r\n",
    "cv2.destroyAllWindows()\r\n",
    "cap.release()"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-b85b6a009d43>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     49\u001b[0m     \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Trackbars'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstacked\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.4\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m     \u001b[0mk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwaitKey\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m&\u001b[0m \u001b[1;36m0xFF\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     52\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mk\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m27\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m         \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## <font style=\"color:rgb(134,19,348)\">Step 3: Tracking the Target Pen  </font>"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "# This variable determines if we want to load color range from memory or use the ones defined in notebook. \r\n",
    "load_from_disk = True\r\n",
    "\r\n",
    "# If true then load color range from memory\r\n",
    "if load_from_disk:\r\n",
    "    penval = np.load('penval.npy')\r\n",
    "\r\n",
    "cap = cv2.VideoCapture(0)\r\n",
    "cap.set(3,1280)\r\n",
    "cap.set(4,720)\r\n",
    "\r\n",
    "# kernel for morphological operations\r\n",
    "kernel = np.ones((5,5),np.uint8)\r\n",
    "\r\n",
    "# set the window to autosize so we can view this full screen.\r\n",
    "cv2.namedWindow('image', cv2.WINDOW_NORMAL)\r\n",
    "\r\n",
    "# This threshold is used to filter noise, the contour area must be bigger than this to qualify as an actual contour.\r\n",
    "noiseth = 500\r\n",
    "\r\n",
    "while(1):\r\n",
    "    \r\n",
    "    _, frame = cap.read()\r\n",
    "    frame = cv2.flip( frame, 1 )\r\n",
    "\r\n",
    "    # Convert BGR to HSV\r\n",
    "    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)\r\n",
    "    \r\n",
    "    # If you're reading from memory then load the upper and lower ranges from there\r\n",
    "    if load_from_disk:\r\n",
    "            lower_range = penval[0]\r\n",
    "            upper_range = penval[1]\r\n",
    "            \r\n",
    "    # Otherwise define your own custom values for upper and lower range.\r\n",
    "    else:             \r\n",
    "       lower_range  = np.array([26,80,147])\r\n",
    "       upper_range = np.array([81,255,255])\r\n",
    "    \r\n",
    "    mask = cv2.inRange(hsv, lower_range, upper_range)\r\n",
    "    \r\n",
    "    # Perform the morphological operations to get rid of the noise\r\n",
    "    mask = cv2.erode(mask,kernel,iterations = 1)\r\n",
    "    mask = cv2.dilate(mask,kernel,iterations = 2)\r\n",
    "    \r\n",
    "    # Find Contours in the frame.\r\n",
    "    contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL,\r\n",
    "                                           cv2.CHAIN_APPROX_SIMPLE)\r\n",
    "    \r\n",
    "    # Make sure there is a contour present and also make sure its size is bigger than noise threshold.\r\n",
    "    if contours and cv2.contourArea(max(contours, \r\n",
    "                                        key = cv2.contourArea)) > noiseth:\r\n",
    "        \r\n",
    "        # Grab the biggest contour with respect to area\r\n",
    "        c = max(contours, key = cv2.contourArea)\r\n",
    "        \r\n",
    "        # Get bounding box coordinates around that contour\r\n",
    "        x,y,w,h = cv2.boundingRect(c)\r\n",
    "        \r\n",
    "        # Draw that bounding box\r\n",
    "        cv2.rectangle(frame,(x,y),(x+w,y+h),(0,25,255),2)        \r\n",
    "\r\n",
    "    cv2.imshow('image',frame)\r\n",
    "    \r\n",
    "    k = cv2.waitKey(5) & 0xFF\r\n",
    "    if k == 27:\r\n",
    "        break\r\n",
    "\r\n",
    "cv2.destroyAllWindows()\r\n",
    "cap.release()"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-0897a28ab50f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m     \u001b[1;31m# Convert BGR to HSV\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 27\u001b[1;33m     \u001b[0mhsv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcvtColor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCOLOR_BGR2HSV\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     28\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     29\u001b[0m     \u001b[1;31m# If you're reading from memory then load the upper and lower ranges from there\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## <font style=\"color:rgb(134,19,348)\">Step 4: Drawing with the Pen   </font>"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "load_from_disk = True\r\n",
    "if load_from_disk:\r\n",
    "    penval = np.load('penval.npy')\r\n",
    "\r\n",
    "cap = cv2.VideoCapture(0)\r\n",
    "cap.set(3,1280)\r\n",
    "cap.set(4,720)\r\n",
    "\r\n",
    "kernel = np.ones((5,5),np.uint8)\r\n",
    "\r\n",
    "# Initializing the canvas on which we will draw upon\r\n",
    "canvas = None\r\n",
    "\r\n",
    "# Initilize x1,y1 points\r\n",
    "x1,y1=0,0\r\n",
    "\r\n",
    "# Threshold for noise\r\n",
    "noiseth = 800\r\n",
    "\r\n",
    "while(1):\r\n",
    "    _, frame = cap.read()\r\n",
    "    frame = cv2.flip( frame, 1 )\r\n",
    "    \r\n",
    "    # Initilize the canvas as a black image of same size as the frame.\r\n",
    "    if canvas is None:\r\n",
    "        canvas = np.zeros_like(frame)\r\n",
    "\r\n",
    "    # Convert BGR to HSV\r\n",
    "    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)\r\n",
    "    \r\n",
    "    # If you're reading from memory then load the upper and lower ranges from there\r\n",
    "    if load_from_disk:\r\n",
    "            lower_range = penval[0]\r\n",
    "            upper_range = penval[1]\r\n",
    "            \r\n",
    "    # Otherwise define your own custom values for upper and lower range.\r\n",
    "    else:             \r\n",
    "       lower_range  = np.array([26,80,147])\r\n",
    "       upper_range = np.array([81,255,255])\r\n",
    "    \r\n",
    "    mask = cv2.inRange(hsv, lower_range, upper_range)\r\n",
    "    \r\n",
    "    # Perform morphological operations to get rid of the noise\r\n",
    "    mask = cv2.erode(mask,kernel,iterations = 1)\r\n",
    "    mask = cv2.dilate(mask,kernel,iterations = 2)\r\n",
    "    \r\n",
    "    # Find Contours\r\n",
    "    contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\r\n",
    "    \r\n",
    "    # Make sure there is a contour present and also its size is bigger than the noise threshold.\r\n",
    "    if contours and cv2.contourArea(max(contours, key = cv2.contourArea)) > noiseth:\r\n",
    "                \r\n",
    "        c = max(contours, key = cv2.contourArea)    \r\n",
    "        x2,y2,w,h = cv2.boundingRect(c)\r\n",
    "        \r\n",
    "        # If there were no previous points then save the detected x2,y2 coordinates as x1,y1. \r\n",
    "        # This is true when we writing for the first time or when writing again when the pen had disapeared from view.\r\n",
    "        if x1 == 0 and y1 == 0:\r\n",
    "            x1,y1= x2,y2\r\n",
    "            \r\n",
    "        else:\r\n",
    "            # Draw the line on the canvas\r\n",
    "            canvas = cv2.line(canvas, (x1,y1),(x2,y2), [255,0,0], 4)\r\n",
    "        \r\n",
    "        # After the line is drawn the new points become the previous points.\r\n",
    "        x1,y1= x2,y2\r\n",
    "\r\n",
    "    else:\r\n",
    "        # If there were no contours detected then make x1,y1 = 0\r\n",
    "        x1,y1 =0,0\r\n",
    "    \r\n",
    "    # Merge the canvas and the frame.\r\n",
    "    frame = cv2.add(frame,canvas)\r\n",
    "    \r\n",
    "    # Optionally stack both frames and show it.\r\n",
    "    stacked = np.hstack((canvas,frame))\r\n",
    "    cv2.imshow('Trackbars',cv2.resize(stacked,None,fx=0.6,fy=0.6))\r\n",
    "\r\n",
    "    k = cv2.waitKey(1) & 0xFF\r\n",
    "    if k == 27:\r\n",
    "        break\r\n",
    "        \r\n",
    "    # When c is pressed clear the canvas\r\n",
    "    if k == ord('c'):\r\n",
    "        canvas = None\r\n",
    "\r\n",
    "cv2.destroyAllWindows()\r\n",
    "cap.release()"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-744f7dca7c6d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     77\u001b[0m     \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Trackbars'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstacked\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.6\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.6\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     78\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 79\u001b[1;33m     \u001b[0mk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwaitKey\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m&\u001b[0m \u001b[1;36m0xFF\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     80\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mk\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m27\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     81\u001b[0m         \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## <font style=\"color:rgb(134,19,348)\">Step 5: Adding An Image Wiper  </font>\n"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "load_from_disk = True\r\n",
    "if load_from_disk:\r\n",
    "    penval = np.load('penval.npy')\r\n",
    "\r\n",
    "cap = cv2.VideoCapture(0)\r\n",
    "cap.set(3,1280)\r\n",
    "cap.set(4,720)\r\n",
    "\r\n",
    "kernel = np.ones((5,5),np.uint8)\r\n",
    "\r\n",
    "# Making window size adjustable\r\n",
    "cv2.namedWindow('image', cv2.WINDOW_NORMAL)\r\n",
    "\r\n",
    "# This is the canvas on which we will draw upon\r\n",
    "canvas=None\r\n",
    "\r\n",
    "# Initilize x1,y1 points\r\n",
    "x1,y1=0,0\r\n",
    "\r\n",
    "# Threshold for noise\r\n",
    "noiseth = 800\r\n",
    "\r\n",
    "# Threshold for wiper, the size of the contour must be bigger than for us to clear the canvas\r\n",
    "wiper_thresh = 40000\r\n",
    "\r\n",
    "# A varaible which tells when to clear canvas, if its True then we clear the canvas\r\n",
    "clear = False\r\n",
    "\r\n",
    "while(1):\r\n",
    "    _ , frame = cap.read()\r\n",
    "    frame = cv2.flip( frame, 1 )\r\n",
    "    \r\n",
    "    # Initilize the canvas as a black image\r\n",
    "    if canvas is None:\r\n",
    "        canvas = np.zeros_like(frame)\r\n",
    "\r\n",
    "    # Convert BGR to HSV\r\n",
    "    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)\r\n",
    "    \r\n",
    "    # If you're reading from memory then load the upper and lower ranges from there\r\n",
    "    if load_from_disk:\r\n",
    "            lower_range = penval[0]\r\n",
    "            upper_range = penval[1]\r\n",
    "            \r\n",
    "    # Otherwise define your own custom values for upper and lower range.\r\n",
    "    else:             \r\n",
    "       lower_range  = np.array([26,80,147])\r\n",
    "       upper_range = np.array([81,255,255])\r\n",
    "    \r\n",
    "    mask = cv2.inRange(hsv, lower_range, upper_range)\r\n",
    "    \r\n",
    "    # perform the morphological operations to get rid of the noise\r\n",
    "    mask = cv2.erode(mask,kernel,iterations = 1)\r\n",
    "    mask = cv2.dilate(mask,kernel,iterations = 2)\r\n",
    "    \r\n",
    "    # Find Contours.\r\n",
    "    contours, hierarchy = cv2.findContours(mask,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\r\n",
    "    \r\n",
    "    # Make sure there is a contour present and also its size is bigger than the noise threshold.\r\n",
    "    if contours and cv2.contourArea(max(contours, key = cv2.contourArea)) > noiseth:\r\n",
    "                \r\n",
    "        c = max(contours, key = cv2.contourArea)    \r\n",
    "        x2,y2,w,h = cv2.boundingRect(c)\r\n",
    "        \r\n",
    "        # Get the area of the contour\r\n",
    "        area = cv2.contourArea(c)\r\n",
    "        \r\n",
    "        # If there were no previous points then save the detected x2,y2 coordinates as x1,y1. \r\n",
    "        if x1 == 0 and y1 == 0:\r\n",
    "            x1,y1= x2,y2\r\n",
    "            \r\n",
    "        else:\r\n",
    "            # Draw the line on the canvas\r\n",
    "            canvas = cv2.line(canvas, (x1,y1),(x2,y2), [255,0,0], 5)\r\n",
    "        \r\n",
    "        # After the line is drawn the new points become the previous points.\r\n",
    "        x1,y1= x2,y2\r\n",
    "        \r\n",
    "        # Now if the area is greater than the wiper threshold then set the clear variable to True and warn User.\r\n",
    "        if area > wiper_thresh:\r\n",
    "           cv2.putText(canvas,'Clearing Canvas',(100,200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0,0,255), 5, cv2.LINE_AA)\r\n",
    "           clear = True \r\n",
    "\r\n",
    "    else:\r\n",
    "        # If there were no contours detected then make x1,y1 = 0\r\n",
    "        x1,y1 =0,0\r\n",
    "    \r\n",
    "   \r\n",
    "    # Now this piece of code is just for smooth drawing. (Optional)\r\n",
    "    _ ,mask = cv2.threshold(cv2.cvtColor(canvas, cv2.COLOR_BGR2GRAY), 20, 255, cv2.THRESH_BINARY)\r\n",
    "    foreground = cv2.bitwise_and(canvas, canvas, mask = mask)\r\n",
    "    background = cv2.bitwise_and(frame, frame, mask = cv2.bitwise_not(mask))\r\n",
    "    frame = cv2.add(foreground,background)\r\n",
    "\r\n",
    "    cv2.imshow('image',frame)\r\n",
    "    \r\n",
    "    k = cv2.waitKey(5) & 0xFF\r\n",
    "    if k == 27:\r\n",
    "        break\r\n",
    "    \r\n",
    "    # Clear the canvas after 1 second if the clear variable is true\r\n",
    "    if clear == True:\r\n",
    "        \r\n",
    "        time.sleep(1)\r\n",
    "        canvas = None\r\n",
    "        \r\n",
    "        # And then set clear to false\r\n",
    "        clear = False\r\n",
    "        \r\n",
    "cv2.destroyAllWindows()\r\n",
    "cap.release()"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-d4a4b1c671fb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     89\u001b[0m     \u001b[1;31m# Now this piece of code is just for smooth drawing. (Optional)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     90\u001b[0m     \u001b[0m_\u001b[0m \u001b[1;33m,\u001b[0m\u001b[0mmask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mthreshold\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcvtColor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcanvas\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCOLOR_BGR2GRAY\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m20\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m255\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTHRESH_BINARY\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 91\u001b[1;33m     \u001b[0mforeground\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbitwise_and\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcanvas\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcanvas\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     92\u001b[0m     \u001b[0mbackground\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbitwise_and\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mframe\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbitwise_not\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     93\u001b[0m     \u001b[0mframe\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mforeground\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbackground\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## <font style=\"color:rgb(134,19,348)\">Step 6: Adding the Eraser Functionality</font>\n"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "load_from_disk = True\r\n",
    "if load_from_disk:\r\n",
    "    penval = np.load('penval.npy')\r\n",
    "\r\n",
    "cap = cv2.VideoCapture(0)\r\n",
    "cap.set(3,1280)\r\n",
    "cap.set(4,720)\r\n",
    "\r\n",
    "# Load these 2 images and resize them to the same size.\r\n",
    "pen_img = cv2.resize(cv2.imread('pen.png',1), (50, 50))\r\n",
    "eraser_img = cv2.resize(cv2.imread('eraser.jpg',1), (50, 50))\r\n",
    "\r\n",
    "kernel = np.ones((5,5),np.uint8)\r\n",
    "\r\n",
    "# Making window size adjustable\r\n",
    "cv2.namedWindow('image', cv2.WINDOW_NORMAL)\r\n",
    "\r\n",
    "# This is the canvas on which we will draw upon\r\n",
    "canvas = None\r\n",
    "\r\n",
    "# Create a background subtractor Object\r\n",
    "backgroundobject = cv2.createBackgroundSubtractorMOG2( detectShadows = False )\r\n",
    "\r\n",
    "# This threshold determines the amount of disruption in background.\r\n",
    "background_threshold = 600\r\n",
    "\r\n",
    "# A variable which tells you if you're using a pen or an eraser.\r\n",
    "switch = 'Pen'\r\n",
    "\r\n",
    "# With this variable we will monitor the time between previous switch.\r\n",
    "last_switch = time.time()\r\n",
    "\r\n",
    "# Initilize x1,y1 points\r\n",
    "x1,y1=0,0\r\n",
    "\r\n",
    "# Threshold for noise\r\n",
    "noiseth = 800\r\n",
    "\r\n",
    "# Threshold for wiper, the size of the contour must be bigger than this for us to clear the canvas\r\n",
    "wiper_thresh = 40000\r\n",
    "\r\n",
    "# A varaible which tells when to clear canvas\r\n",
    "clear = False\r\n",
    "\r\n",
    "while(1):\r\n",
    "    _, frame = cap.read()\r\n",
    "    frame = cv2.flip(frame, 1 )\r\n",
    "    \r\n",
    "    # Initilize the canvas as a black image\r\n",
    "    if canvas is None:\r\n",
    "        canvas = np.zeros_like(frame)\r\n",
    "        \r\n",
    "    # Take the top left of the frame and apply the background subtractor there    \r\n",
    "    top_left = frame[0: 50, 0: 50]\r\n",
    "    fgmask = backgroundobject.apply(top_left)\r\n",
    "    \r\n",
    "    # Note the number of pixels that are white, this is the level of disruption.\r\n",
    "    switch_thresh = np.sum(fgmask==255)\r\n",
    "    \r\n",
    "    # If the disruption is greater than background threshold and there has been some time after the previous switch then you \r\n",
    "    # can change the object type.\r\n",
    "    if switch_thresh > background_threshold  and (time.time() - last_switch) > 1:\r\n",
    "        \r\n",
    "        # Save the time of the switch. \r\n",
    "        last_switch = time.time()\r\n",
    "        \r\n",
    "        if switch == 'Pen':\r\n",
    "            switch = 'Eraser'\r\n",
    "        else:\r\n",
    "            switch = 'Pen'\r\n",
    "\r\n",
    "    # Convert BGR to HSV\r\n",
    "    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)\r\n",
    "    \r\n",
    "    # If you're reading from memory then load the upper and lower ranges from there\r\n",
    "    if load_from_disk:\r\n",
    "            lower_range = penval[0]\r\n",
    "            upper_range = penval[1]\r\n",
    "            \r\n",
    "    # Otherwise define your own custom values for upper and lower range.\r\n",
    "    else:             \r\n",
    "       lower_range  = np.array([26,80,147])\r\n",
    "       upper_range = np.array([81,255,255])\r\n",
    "    \r\n",
    "    mask = cv2.inRange(hsv, lower_range, upper_range)\r\n",
    "    \r\n",
    "    # Perform morphological operations to get rid of the noise\r\n",
    "    mask = cv2.erode(mask,kernel,iterations = 1)\r\n",
    "    mask = cv2.dilate(mask,kernel,iterations = 2)\r\n",
    "    \r\n",
    "    # Find Contours\r\n",
    "    contours, hierarchy = cv2.findContours(mask,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\r\n",
    "    \r\n",
    "    # Make sure there is a contour present and also its size is bigger than noise threshold.\r\n",
    "    if contours and cv2.contourArea(max(contours, key = cv2.contourArea)) > noiseth:\r\n",
    "                \r\n",
    "        c = max(contours, key = cv2.contourArea)    \r\n",
    "        x2,y2,w,h = cv2.boundingRect(c)\r\n",
    "        \r\n",
    "        # Get the area of the contour\r\n",
    "        area = cv2.contourArea(c)\r\n",
    "        \r\n",
    "        # If there were no previous points then save the detected x2,y2 coordinates as x1,y1. \r\n",
    "        if x1 == 0 and y1 == 0:\r\n",
    "            x1,y1= x2,y2\r\n",
    "            \r\n",
    "        else:\r\n",
    "            \r\n",
    "            if switch == 'Pen':\r\n",
    "                # Draw the line on the canvas\r\n",
    "                canvas = cv2.line(canvas, (x1,y1), (x2,y2), [255,0,0], 5)\r\n",
    "                \r\n",
    "            else:\r\n",
    "                cv2.circle(canvas, (x2, y2), 20, (0,0,0), -1)\r\n",
    "            \r\n",
    "            \r\n",
    "        \r\n",
    "        # After the line is drawn the new points become the previous points.\r\n",
    "        x1,y1= x2,y2\r\n",
    "        \r\n",
    "        # Now if the area is greater than the wiper threshold then set the clear variable to True\r\n",
    "        if area > wiper_thresh:\r\n",
    "           cv2.putText(canvas,'Clearing Canvas',(0,200), cv2.FONT_HERSHEY_SIMPLEX, 2, (0,0,255), 1, cv2.LINE_AA)\r\n",
    "           clear = True \r\n",
    "\r\n",
    "    else:\r\n",
    "        # If there were no contours detected then make x1,y1 = 0\r\n",
    "        x1,y1 =0,0\r\n",
    "    \r\n",
    "   \r\n",
    "    # Now this piece of code is just for smooth drawing. (Optional)\r\n",
    "    _,mask = cv2.threshold(cv2.cvtColor(canvas, cv2.COLOR_BGR2GRAY), 20, 255, cv2.THRESH_BINARY)\r\n",
    "    foreground = cv2.bitwise_and(canvas, canvas, mask = mask)\r\n",
    "    background = cv2.bitwise_and(frame, frame, mask = cv2.bitwise_not(mask))\r\n",
    "    frame = cv2.add(foreground,background)\r\n",
    "\r\n",
    "    \r\n",
    "    # Switch the images depending upon what we're using, pen or eraser.\r\n",
    "    if switch != 'Pen':\r\n",
    "        cv2.circle(frame, (x1, y1), 20, (255,255,255), -1)\r\n",
    "        frame[0: 50, 0: 50] = eraser_img\r\n",
    "    else:\r\n",
    "        frame[0: 50, 0: 50] = pen_img\r\n",
    "\r\n",
    "    \r\n",
    "    cv2.imshow('image',frame)\r\n",
    "\r\n",
    "    k = cv2.waitKey(5) & 0xFF\r\n",
    "    if k == 27:\r\n",
    "        break\r\n",
    "    \r\n",
    "    # Clear the canvas after 1 second, if the clear variable is true\r\n",
    "    if clear == True:\r\n",
    "        \r\n",
    "        time.sleep(1)\r\n",
    "        canvas = None\r\n",
    "        \r\n",
    "        # And then set clear to false\r\n",
    "        clear = False\r\n",
    "        \r\n",
    "cv2.destroyAllWindows()\r\n",
    "cap.release()"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-dbe6a4f07520>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     44\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     45\u001b[0m \u001b[1;32mwhile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 46\u001b[1;33m     \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mframe\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcap\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     47\u001b[0m     \u001b[0mframe\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     48\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit ('base': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "interpreter": {
   "hash": "b3ba2566441a7c06988d0923437866b63cedc61552a5af99d1f4fb67d367b25f"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}