module_3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Release the Kraken!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The next library we're going to look at is called Kraken, which was developed by Université \n",
    "# PSL in Paris. It's actually based on a slightly older code base, OCRopus. You can see how the\n",
    "# flexible open-source licenses allow new ideas to grow by building upon older ideas. And, in\n",
    "# this case, I fully support the idea that the Kraken - a mythical massive sea creature - is the\n",
    "# natural progression of an octopus!\n",
    "#\n",
    "# What we are going to use Kraken for is to detect lines of text as bounding boxes in a given\n",
    "# image. The biggest limitation of tesseract is the lack of a layout engine inside of it. Tesseract\n",
    "# expects to be using fairly clean text, and gets confused if we don't crop out other artifacts.\n",
    "# It's not bad, but Kraken can help us out be segmenting pages. Lets take a look."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'kraken'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-c423ca8076f0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# First, we'll take a look at the kraken module itself\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mkraken\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mhelp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkraken\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'kraken'"
     ]
    }
   ],
   "source": [
    "# First, we'll take a look at the kraken module itself\n",
    "import kraken\n",
    "help(kraken)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There isn't much of a discussion here, but there are a number of sub-modules that look\n",
    "# interesting. I spend a bit of time on their website, and I think the pageseg module, which\n",
    "# handles all of the page segmentation, is the one we want to use. Lets look at it\n",
    "from kraken import pageseg\n",
    "help(pageseg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# So it looks like there are a few different functions we can call, and the segment\n",
    "# function looks particularly appropriate. I love how expressive this library is on the\n",
    "# documentation front -- I can see immediately that we are working with PIL.Image files,\n",
    "# and the author has even indicated that we need to pass in either a binarized (e.g. '1')\n",
    "# or grayscale (e.g. 'L') image. We can also see that the return value is a dictionary\n",
    "# object with two keys, \"text_direction\" which will return to us a string of the\n",
    "# direction of the text, and \"boxes\" which appears to be a list of tuples, where each\n",
    "# tuple is a box in the original image.\n",
    "#\n",
    "# Lets try this on the image of text. I have a simple bit of text in a file called\n",
    "# two_col.png which is from a newspaper on campus here\n",
    "from PIL import Image\n",
    "im=Image.open(\"readonly/two_col.png\")\n",
    "# Lets display the image inline\n",
    "display(im)\n",
    "# Lets now convert it to black and white and segment it up into lines with kraken\n",
    "bounding_boxes=pageseg.segment(im.convert('1'))['boxes']\n",
    "# And lets print those lines to the screen\n",
    "print(bounding_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok, pretty simple two column text and then a list of lists which are the bounding boxes of \n",
    "# lines of that text. Lets write a little routine to try and see the effects a bit more\n",
    "# clearly. I'm going to clean up my act a bit and write real documentation too, it's a good\n",
    "# practice\n",
    "def show_boxes(img):\n",
    "    '''Modifies the passed image to show a series of bounding boxes on an image as run by kraken\n",
    "    \n",
    "    :param img: A PIL.Image object\n",
    "    :return img: The modified PIL.Image object\n",
    "    '''\n",
    "    # Lets bring in our ImageDraw object\n",
    "    from PIL import ImageDraw\n",
    "    # And grab a drawing object to annotate that image\n",
    "    drawing_object=ImageDraw.Draw(img)\n",
    "    # We can create a set of boxes using pageseg.segment\n",
    "    bounding_boxes=pageseg.segment(img.convert('1'))['boxes']\n",
    "    # Now lets go through the list of bounding boxes\n",
    "    for box in bounding_boxes:\n",
    "        # An just draw a nice rectangle\n",
    "        drawing_object.rectangle(box, fill = None, outline ='red')\n",
    "    # And to make it easy, lets return the image object\n",
    "    return img\n",
    "\n",
    "# To test this, lets use display\n",
    "display(show_boxes(Image.open(\"readonly/two_col.png\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not bad at all! It's interesting to see that kraken isn't completely sure what to do with this\n",
    "# two column format. In some cases, kraken has identified a line in just a single column, while\n",
    "# in other cases kraken has spanned the line marker all the way across the page. Does this matter?\n",
    "# Well, it really depends on our goal. In this case, I want to see if we can improve a bit on this.\n",
    "#\n",
    "# So we're going to go a bit off script here. While this week of lectures is about libraries, the\n",
    "# goal of this last course is to give you confidence that you can apply your knowledge to actual\n",
    "# programming tasks, even if the library you are using doesn't quite do what you want. \n",
    "#\n",
    "# I'd like to pause the video for the moment and collect your thoughts. Looking at the image above,\n",
    "# with the two column example and red boxes, how do you think we might modify this image to improve\n",
    "# kraken's ability to text lines?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Thanks for sharing your thoughts, I'm looking forward to seeing the breadth of ideas that everyone\n",
    "# in the course comes up with. Here's my partial solution -- while looking through the kraken docs on \n",
    "# the pageseg() function I saw that there are a few parameters we can supply in order to improve \n",
    "# segmentation. One of these is the black_colseps parameter. If set to True, kraken will assume that \n",
    "# columns will be separated by black lines. This isn't our case here, but, I think we have all of the\n",
    "# tools to go through and actually change the source image to have a black separator between columns.\n",
    "#\n",
    "# The first step is that I want to update the show_boxes() function. I'm just going to do a quick\n",
    "# copy and paste from the above but add in the black_colseps=True parameter\n",
    "def show_boxes(img):\n",
    "    '''Modifies the passed image to show a series of bounding boxes on an image as run by kraken\n",
    "    \n",
    "    :param img: A PIL.Image object\n",
    "    :return img: The modified PIL.Image object\n",
    "    '''\n",
    "    # Lets bring in our ImageDraw object\n",
    "    from PIL import ImageDraw\n",
    "    # And grab a drawing object to annotate that image\n",
    "    drawing_object=ImageDraw.Draw(img)\n",
    "    # We can create a set of boxes using pageseg.segment\n",
    "    bounding_boxes=pageseg.segment(img.convert('1'), black_colseps=True)['boxes']\n",
    "    # Now lets go through the list of bounding boxes\n",
    "    for box in bounding_boxes:\n",
    "        # An just draw a nice rectangle\n",
    "        drawing_object.rectangle(box, fill = None, outline ='red')\n",
    "    # And to make it easy, lets return the image object\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The next step is to think of the algorithm we want to apply to detect a white column separator.\n",
    "# In experimenting a bit I decided that I only wanted to add the separator if the space of was\n",
    "# at least 25 pixels wide, which is roughly the width of a character, and six lines high. The\n",
    "# width is easy, lets just make a variable\n",
    "char_width=25\n",
    "# The height is harder, since it depends on the height of the text. I'm going to write a routine\n",
    "# to calculate the average height of a line\n",
    "def calculate_line_height(img):\n",
    "    '''Calculates the average height of a line from a given image\n",
    "    :param img: A PIL.Image object\n",
    "    :return: The average line height in pixels\n",
    "    '''\n",
    "    # Lets get a list of bounding boxes for this image\n",
    "    bounding_boxes=pageseg.segment(img.convert('1'))['boxes']\n",
    "    # Each box is a tuple of (top, left, bottom, right) so the height is just top - bottom\n",
    "    # So lets just calculate this over the set of all boxes\n",
    "    height_accumulator=0\n",
    "    for box in bounding_boxes:\n",
    "        height_accumulator=height_accumulator+box[3]-box[1]\n",
    "        # this is a bit tricky, remember that we start counting at the upper left corner in PIL!\n",
    "    # now lets just return the average height\n",
    "    # lets change it to the nearest full pixel by making it an integer\n",
    "    return int(height_accumulator/len(bounding_boxes))\n",
    "\n",
    "# And lets test this with the image with have been using\n",
    "line_height=calculate_line_height(Image.open(\"readonly/two_col.png\"))\n",
    "print(line_height)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok, so the average height of a line is 31.\n",
    "# Now, we want to scan through the image - looking at each pixel in turn - to determine if there\n",
    "# is a block of whitespace. How bit of a block should we look for? That's a bit more of an art\n",
    "# than a science. Looking at our sample image, I'm going to say an appropriate block should be\n",
    "# one char_width wide, and six line_heights tall. But, I honestly just made this up by eyeballing\n",
    "# the image, so I would encourage you to play with values as you explore.\n",
    "# Lets create a new box called gap box that represents this area\n",
    "gap_box=(0,0,char_width,line_height*6)\n",
    "gap_box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# It seems we will want to have a function which, given a pixel in an image, can check to see\n",
    "# if that pixel has whitespace to the right and below it. Essentially, we want to test to see\n",
    "# if the pixel is the upper left corner of something that looks like the gap_box. If so, then\n",
    "# we should insert a line to \"break up\" this box before sending to kraken\n",
    "#\n",
    "# Lets call this new function gap_check\n",
    "def gap_check(img, location):\n",
    "    '''Checks the img in a given (x,y) location to see if it fits the description\n",
    "    of a gap_box\n",
    "    :param img: A PIL.Image file\n",
    "    :param location: A tuple (x,y) which is a pixel location in that image\n",
    "    :return: True if that fits the definition of a gap_box, otherwise False\n",
    "    '''\n",
    "    # Recall that we can get a pixel using the img.getpixel() function. It returns this value\n",
    "    # as a tuple of integers, one for each color channel. Our tools all work with binarized\n",
    "    # images (black and white), so we should just get one value. If the value is 0 it's a black\n",
    "    # pixel, if it's white then the value should be 255\n",
    "    #\n",
    "    # We're going to assume that the image is in the correct mode already, e.g. it has been\n",
    "    # binarized. The algorithm to check our bounding box is fairly easy: we have a single location \n",
    "    # which is our start and then we want to check all the pixels to the right of that location \n",
    "    # up to gap_box[2]\n",
    "    for x in range(location[0], location[0]+gap_box[2]):\n",
    "        # the height is similar, so lets iterate a y variable to gap_box[3]\n",
    "        for y in range(location[1], location[1]+gap_box[3]):\n",
    "            # we want to check if the pixel is white, but only if we are still within the image\n",
    "            if x < img.width and y < img.height:\n",
    "                # if the pixel is white we don't do anything, if it's black, we just want to\n",
    "                # finish and return False\n",
    "                if img.getpixel((x,y)) != 255:\n",
    "                    return False\n",
    "    # If we have managed to walk all through the gap_box without finding any non-white pixels\n",
    "    # then we can return true -- this is a gap!\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Alright, we have a function to check for a gap, called gap_check. What should we do once\n",
    "# we find a gap? For this, lets just draw a line in the middle of it. Lets create a new function\n",
    "def draw_sep(img,location):\n",
    "    '''Draws a line in img in the middle of the gap discovered at location. Note that\n",
    "    this doesn't draw the line in location, but draws it at the middle of a gap_box\n",
    "    starting at location.\n",
    "    :param img: A PIL.Image file\n",
    "    :param location: A tuple(x,y) which is a pixel location in the image\n",
    "    '''\n",
    "    # First lets bring in all of our drawing code\n",
    "    from PIL import ImageDraw\n",
    "    drawing_object=ImageDraw.Draw(img)\n",
    "    # next, lets decide what the middle means in terms of coordinates in the image\n",
    "    x1=location[0]+int(gap_box[2]/2)\n",
    "    # and our x2 is just the same thing, since this is a one pixel vertical line\n",
    "    x2=x1\n",
    "    # our starting y coordinate is just the y coordinate which was passed in, the top of the box\n",
    "    y1=location[1]\n",
    "    # but we want our final y coordinate to be the bottom of the box\n",
    "    y2=y1+gap_box[3]\n",
    "    drawing_object.rectangle((x1,y1,x2,y2), fill = 'black', outline ='black')\n",
    "    # and we don't have anything we need to return from this, because we modified the image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, lets try it all out. This is pretty easy, we can just iterate through each pixel \n",
    "# in the image, check if there is a gap, then insert a line if there is.\n",
    "def process_image(img):\n",
    "    '''Takes in an image of text and adds black vertical bars to break up columns\n",
    "    :param img: A PIL.Image file\n",
    "    :return: A modified PIL.Image file\n",
    "    '''\n",
    "    # we'll start with a familiar iteration process\n",
    "    for x in range(img.width):\n",
    "        for y in range(img.height):\n",
    "            # check if there is a gap at this point\n",
    "            if (gap_check(img, (x,y))):\n",
    "                # then update image to one which has a separator drawn on it\n",
    "                draw_sep(img, (x,y))\n",
    "    # and for good measure we'll return the image we modified\n",
    "    return img\n",
    "\n",
    "# Lets read in our test image and convert it through binarization\n",
    "i=Image.open(\"readonly/two_col.png\").convert(\"L\")\n",
    "i=process_image(i)\n",
    "display(i)\n",
    "\n",
    "#Note: This will take some time to run! Be patient!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not bad at all! The effect at the bottom of the image is a bit unexpected to me, but it makes\n",
    "# sense. You can imagine that there are several ways we might try and control this. Lets see how \n",
    "# this new image works when run through the kraken layout engine\n",
    "display(show_boxes(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Looks like that is pretty accurate, and fixes the problem we faced. Feel free to experiment\n",
    "# with different settings for the gap heights and width and share in the forums. You'll notice though \n",
    "# method we created is really quite slow, which is a bit of a problem if we wanted to use\n",
    "# this on larger text. But I wanted to show you how you can mix your own logic and work with\n",
    "# libraries you're using. Just because Kraken didn't work perfectly, doesn't mean we can't\n",
    "# build something more specific to our use case on top of it.\n",
    "#\n",
    "# I want to end this lecture with a pause and to ask you to reflect on the code we've written\n",
    "# here. We started this course with some pretty simple use of libraries, but now we're\n",
    "# digging in deeper and solving problems ourselves with the help of these libraries. Before we\n",
    "# go on to our last library, how well prepared do you think you are to take your python\n",
    "# skills out into the wild?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Comparing Image Data Structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# OpenCV supports reading of images in most file formats, such as JPEG, PNG, and TIFF. Most image and \n",
    "# video analysis requires converting images into grayscale first. This simplifies the image and reduces \n",
    "# noise allowing for improved analysis. Let's write some code that reads an image of as person, Floyd \n",
    "# Mayweather and converts it into greyscale.\n",
    "\n",
    "# First we will import the open cv package cv2 \n",
    "import cv2 as cv\n",
    "# We'll load the floyd.jpg image \n",
    "img = cv.imread('readonly/floyd.jpg')\n",
    "# And we'll convert it to grayscale using the cvtColor image\n",
    "gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)\n",
    "\n",
    "# Now, before we get to the result, lets talk about docs. Just like tesseract, opencv is an external\n",
    "# package written in C++, and the docs for python are really poor. This is unfortunatly quite common\n",
    "# when python is being used as a wrapper. Thankfully, the web docs for opencv are actually pretty good,\n",
    "# so hit the website docs.opencv.org when you want to learn more about a particular function. In this\n",
    "# case cvtColor converts from one color space to another, and we are convering our image to grayscale.\n",
    "# Of course, we already know at least two different ways of doing this, using binarization and PIL\n",
    "# color spaces conversions\n",
    "\n",
    "# Lets instpec this object that has been returned.\n",
    "import inspect\n",
    "inspect.getmro(type(gray))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We see that it is of type ndarray, which is a fundamental list type coming from the numerical\n",
    "# python project. That's a bit surprising - up until this point we have been used to working with\n",
    "# PIL.Image objects. OpenCV, however, wants to represent an image as a two dimensional sequence \n",
    "# of bytes, and the ndarray, which stands for n dimensional array, is the ideal way to do this.\n",
    "# Lets look at the array contents.\n",
    "gray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The array is shown here as a list of lists, where the inner lists are filled with integers.\n",
    "# The dtype=uint8 definition indicates that each of the items in an array is an 8 bit unsigned\n",
    "# integer, which is very common for black and white images. So this is a pixel by pixel definition\n",
    "# of the image.\n",
    "#\n",
    "# The display package, however, doesn't know what to do with this image. So lets convert it\n",
    "# into a PIL object to render it in the browser.\n",
    "from PIL import Image\n",
    "\n",
    "# PIL can take an array of data with a given color format and convert this into a PIL object.\n",
    "# This is perfect for our situation, as the PIL color mode, \"L\" is just an array of luminance\n",
    "# values in unsigned integers\n",
    "image = Image.fromarray(gray, \"L\")\n",
    "display(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets talk a bit more about images for a moment. Numpy arrays are multidimensional. For \n",
    "# instance, we can define an array in a single dimension:\n",
    "import numpy as np\n",
    "single_dim = np.array([25, 50 , 25, 10, 10])\n",
    "\n",
    "# In an image, this is analagous to a single row of 5 pixels each in grayscale. But actually,\n",
    "# all imaging libraries tend to expect at least two dimensions, a width and a height, and to\n",
    "# show a matrix. So if we put the single_dim inside of another array, this would be a two\n",
    "# dimensional array with element in the height direction, and five in the width direction\n",
    "double_dim = np.array([single_dim])\n",
    "\n",
    "double_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This should look pretty familiar, it's a lot like a list of lists! Lets see what this new\n",
    "# two dimensional array looks like if we display it\n",
    "display(Image.fromarray(double_dim, \"L\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pretty unexciting - it's just a little line. Five pixels in a row to be exact, of different\n",
    "# levels of black. The numpy library has a nice attribute called shape that allows us to see how\n",
    "# many dimensions big an array is. The shape attribute returns a tuple that shows the height of\n",
    "# the image, by the width of the image\n",
    "double_dim.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets take a look at the shape of our initial image which we loaded into the img variable\n",
    "img.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This image has three dimensions! That's because it has a width, a height, and what's called\n",
    "# a color depth. In this case, the color is represented as an array of three values. Lets take a \n",
    "# look at the color of the first pixel\n",
    "first_pixel=img[0][0]\n",
    "first_pixel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here we see that the color value is provided in full RGB using an unsigned integer. This\n",
    "# means that each color can have one of 256 values, and the total number of unique colors\n",
    "# that can be represented by this data is 256 * 256 *256 which is roughly 16 million colors.\n",
    "# We call this 24 bit color, which is 8+8+8.\n",
    "#\n",
    "# If you find yourself shopping for a television, you might notice that some expensive models\n",
    "# are advertised as having 10 bit or even 12 bit panels. These are televisions where each of\n",
    "# the red, green, and blue color channels are represented by 10 or 12 bits instead of 8. For\n",
    "# ten bit panels this means that there are 1 billion colors capable, and 12 bit panels are\n",
    "# capable of over 68 billion colors!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We're not going to talk much more about color in this course, but it's a fun subject. Instead,\n",
    "# lets go back to this array representation of images, because we can do some interesting things\n",
    "# with this.\n",
    "#\n",
    "# One of the most common things to do with an ndarray is to reshape it -- to change the number\n",
    "# of rows and columns that are represented so that we can do different kinds of operations.\n",
    "# Here is our original two dimensional image\n",
    "print(\"Original image\")\n",
    "print(gray)\n",
    "# If we wanted to represent that as a one dimensional image, we just call reshape\n",
    "print(\"New image\")\n",
    "# And reshape takes the image as the first parameter, and a new shape as the second\n",
    "image1d=np.reshape(gray,(1,gray.shape[0]*gray.shape[1]))\n",
    "print(image1d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# So, why are we talking about these nested arrays of bytes, we were supposed to be talking\n",
    "# about OpenCV as a library. Well, I wanted to show you that often libraries working on the\n",
    "# same kind of principles, in this case images stored as arrays of bytes, are not representing\n",
    "# data in the same way in their APIs. But, by exploring a bit you can learn how the internal\n",
    "# representation of data is stored, and build routines to convert between formats.\n",
    "#\n",
    "# For instance, remember in the last lecture when we wanted to look for gaps in an image so\n",
    "# that we could draw lines to feed into kraken? Well, we use PIL to do this, using getpixel()\n",
    "# to look at individual pixels and see what the luminosity was, then ImageDraw.rectangle to\n",
    "# actually fill in a black bar separator. This was a nice high level API, and let us write\n",
    "# routines to do the work we wanted without having to understand too much about how the images\n",
    "# were being stored. But it was computationally very slow.\n",
    "#\n",
    "# Instead, we could write the code to do this using matrix features within numpy. Lets take\n",
    "# a look.\n",
    "import cv2 as cv\n",
    "# We'll load the 2 column image\n",
    "img = cv.imread('readonly/two_col.png')\n",
    "# And we'll convert it to grayscale using the cvtColor image\n",
    "gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, remember how slicing on a list works, if you have a list of number such as \n",
    "# a=[0,1,2,3,4,5] then a[2:4] will return the sublist of numbers at position 2 through 4 \n",
    "# inclusive - don't forget that lists start indexing at 0!\n",
    "# If we have a two dimensional array, we can slice out a smaller piece of that using the\n",
    "# format a[2:4,1:3]. You can think of this as first slicing along the rows dimension, then\n",
    "# in the columns dimension. So in this example, that would be a matrix of rows 2, and 3,\n",
    "# and columns 1, and 2. Here's a look at our image.\n",
    "gray[2:4,1:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# So we see that it is all white. We can use this as a \"window\" and move it around our\n",
    "# our big image.\n",
    "#\n",
    "# Finally, the ndarray library has lots of matrix functions which are generally very fast\n",
    "# to run. One that we want to consider in this case is count_nonzero(), which just returns\n",
    "# the number of entries in the matrix which are not zero.\n",
    "np.count_nonzero(gray[2:4,1:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok, the last benefit of going to this low level approach to images is that we can change\n",
    "# pixels very fast as well. Previously we were drawing rectangles and setting a fill and line\n",
    "# width. This is nice if you want to do something like change the color of the fill from the\n",
    "# line, or draw complex shapes. But we really just want a line here. That's really easy to\n",
    "# do - we just want to change a number of luminosity values from 255 to 0.\n",
    "#\n",
    "# As an example, lets create a big white matrix\n",
    "white_matrix=np.full((12,12),255,dtype=np.uint8)\n",
    "display(Image.fromarray(white_matrix,\"L\"))\n",
    "white_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# looks pretty boring, it's just a giant white square we can't see. But if we want, we can\n",
    "# easily color a column to be black\n",
    "white_matrix[:,6]=np.full((1,12),0,dtype=np.uint8)\n",
    "display(Image.fromarray(white_matrix,\"L\"))\n",
    "white_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# And that's exactly what we wanted to do. So, why do it this way, when it seems so much\n",
    "# more low level? Really, the answer is speed. This paradigm of using matricies to store\n",
    "# and manipulate bytes of data for images is much closer to how low level API and hardware\n",
    "# developers think about storing files and bytes in memory.\n",
    "#\n",
    "# How much faster is it? Well, that's up to you to discover; there's an optional assignment\n",
    "# for this week to convert our old code over into this new format, to compare both the\n",
    "# readability and speed of the two different approaches."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OpenCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok, we're just about at the project for this course. If you reflect on the specialization \n",
    "# as a whole you'll realize that you started with probably little or no understanding of python,\n",
    "# progressed through the basic control structures and libraries included with the language\n",
    "# with the help of a digital textbook, moved on to more high level representations of data\n",
    "# and functions with objects, and now started to explore third party libraries that exist for\n",
    "# python which allow you to manipulate and display images. This is quite an achievement!\n",
    "#\n",
    "# You have also no doubt found that as you have progressed the demands on you to engage in self-\n",
    "# discovery have also increased. Where the first assignments were maybe straight forward, the\n",
    "# ones in this week require you to struggle a bit more with planning and debugging code as\n",
    "# you develop.\n",
    "#\n",
    "# But, you've persisted, and I'd like to share with you just one more set of features before\n",
    "# we head over to a project. The OpenCV library contains mechanisms to do face detection on\n",
    "# images. The technique used is based on Haar cascades, which is a machine learning approach.\n",
    "# Now, we're not going to go into the machine learning bits, we have another specialization on\n",
    "# Applied Data Science with Python which you can take after this if you're interested in that topic.\n",
    "# But here we'll treat OpenCV like a black box.\n",
    "#\n",
    "# OpenCV comes with trained models for detecting faces, eyes, and smiles which we'll be using.\n",
    "# You can train models for detecting other things - like hot dogs or flutes - and if you're\n",
    "# interested in that I'd recommend you check out the Open CV docs on how to train a cascade\n",
    "# classifier: https://docs.opencv.org/3.4/dc/d88/tutorial_traincascade.html\n",
    "# However, in this lecture we just want to use the current classifiers and see if we can detect\n",
    "# portions of an image which are interesting.\n",
    "#\n",
    "# First step is to load opencv and the XML-based classifiers\n",
    "import cv2 as cv\n",
    "face_cascade = cv.CascadeClassifier('readonly/haarcascade_frontalface_default.xml')\n",
    "eye_cascade = cv.CascadeClassifier('readonly/haarcascade_eye.xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok, with the classifiers loaded, we now want to try and detect a face. Lets pull in the\n",
    "# picture we played with last time\n",
    "img = cv.imread('readonly/floyd.jpg')\n",
    "# And we'll convert it to grayscale using the cvtColor image\n",
    "gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)\n",
    "# The next step is to use the face_cascade classifier. I'll let you go explore the docs if you\n",
    "# would like to, but the norm is to use the detectMultiScale() function. This function returns\n",
    "# a list of objects as rectangles. The first parameter is an ndarray of the image.\n",
    "faces = face_cascade.detectMultiScale(gray)\n",
    "# And lets just print those faces out to the screen\n",
    "faces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "faces.tolist()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The resulting rectangles are in the format of (x,y,w,h) where x and y denote the upper\n",
    "# left hand point for the image and the width and height represent the bounding box. We know\n",
    "# how to handle this in PIL\n",
    "from PIL import Image\n",
    "\n",
    "# Lets create a PIL image object\n",
    "pil_img=Image.fromarray(gray,mode=\"L\")\n",
    "\n",
    "# Now lets bring in our drawing object\n",
    "from PIL import ImageDraw\n",
    "# And lets create our drawing context\n",
    "drawing=ImageDraw.Draw(pil_img)\n",
    "\n",
    "# Now lets pull the rectangle out of the faces object\n",
    "rec=faces.tolist()[0]\n",
    "\n",
    "# Now we just draw a rectangle around the bounds\n",
    "drawing.rectangle(rec, outline=\"white\")\n",
    "\n",
    "# And display\n",
    "display(pil_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# So, not quite what we were looking for. What do you think went wrong?\n",
    "# Well, a quick double check of the docs and it is apparent that OpenCV is return the coordinates\n",
    "# as (x,y,w,h), while PIL.ImageDraw is looking for (x1,y1,x2,y2). Looks like an easy fix\n",
    "# Wipe our old image\n",
    "pil_img=Image.fromarray(gray,mode=\"L\")\n",
    "# Setup our drawing context\n",
    "drawing=ImageDraw.Draw(pil_img)\n",
    "# And draw the new box\n",
    "drawing.rectangle((rec[0],rec[1],rec[0]+rec[2],rec[1]+rec[3]), outline=\"white\")\n",
    "# And display\n",
    "display(pil_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We see the face detection works pretty good on this image! Note that it's apparent that this is\n",
    "# not head detection, but that the haarcascades file we used is looking for eyes and a mouth.\n",
    "# Lets try this on something a bit more complex, lets read in our MSI recruitment image\n",
    "img = cv.imread('readonly/msi_recruitment.gif')\n",
    "# And lets take a look at that image\n",
    "display(Image.fromarray(img))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Whoa, what's that error about? It looks like there is an error on a line deep within the PIL\n",
    "# Image.py file, and it is trying to call an internal private member called __array_interface__\n",
    "# on the img object, but this object is None\n",
    "#\n",
    "# It turns out that the root of this error is that OpenCV can't work with Gif images. This is\n",
    "# kind of a pain and unfortunate. But we know how to fix that right? One was is that we could\n",
    "# just open this in PIL and then save it as a png, then open that in open cv.\n",
    "#\n",
    "# Lets use PIL to open our image\n",
    "pil_img=Image.open('readonly/msi_recruitment.gif')\n",
    "# now lets convert it to greyscale for opencv, and get the bytestream\n",
    "open_cv_version=pil_img.convert(\"L\")\n",
    "# now lets just write that to a file\n",
    "open_cv_version.save(\"msi_recruitment.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok, now that the conversion of format is done, lets try reading this back into opencv\n",
    "cv_img=cv.imread('msi_recruitment.png')\n",
    "# We don't need to color convert this, because we saved it as grayscale\n",
    "# lets try and detect faces in that image\n",
    "faces = face_cascade.detectMultiScale(cv_img)\n",
    "\n",
    "# Now, we still have our PIL color version in a gif\n",
    "pil_img=Image.open('readonly/msi_recruitment.gif')\n",
    "# Set our drawing context\n",
    "drawing=ImageDraw.Draw(pil_img)\n",
    "\n",
    "# For each item in faces, lets surround it with a red box\n",
    "for x,y,w,h in faces:\n",
    "    # That might be new syntax for you! Recall that faces is a list of rectangles in (x,y,w,h)\n",
    "    # format, that is, a list of lists. Instead of having to do an iteration and then manually\n",
    "    # pull out each item, we can use tuple unpacking to pull out individual items in the sublist\n",
    "    # directly to variables. A really nice python feature\n",
    "    #\n",
    "    # Now we just need to draw our box\n",
    "    drawing.rectangle((x,y,x+w,y+h), outline=\"white\")\n",
    "display(pil_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# What happened here!? We see that we have detected faces, and that we have drawn boxes\n",
    "# around those faces on the image, but that the colors have gone all weird! This, it turns\n",
    "# out, has to do with color limitations for gif images. In short, a gif image has a very\n",
    "# limited number of colors. This is called a color pallette after the pallette artists\n",
    "# use to mix paints. For gifs the pallette can only be 256 colors -- but they can be *any*\n",
    "# 256 colors. When a new color is introduced, is has to take the space of an old color.\n",
    "# In this case, PIL adds white to the pallette but doesn't know which color to replace and\n",
    "# thus messes up the image.\n",
    "#\n",
    "# Who knew there was so much to learn about image formats? We can see what mode the image\n",
    "# is in with the .mode attribute\n",
    "pil_img.mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can see a list of modes in the PILLOW documentation, and they correspond with the\n",
    "# color spaces we have been using. For the moment though, lets change back to RGB, which\n",
    "# represents color as a three byte tuple instead of in a pallette.\n",
    "# Lets read in the image\n",
    "pil_img=Image.open('readonly/msi_recruitment.gif')\n",
    "# Lets convert it to RGB mode\n",
    "pil_img = pil_img.convert(\"RGB\")\n",
    "# And lets print out the mode\n",
    "pil_img.mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok, now lets go back to drawing rectangles. Lets get our drawing object\n",
    "drawing=ImageDraw.Draw(pil_img)\n",
    "# And iterate through the faces sequence, tuple unpacking as we go\n",
    "for x,y,w,h in faces:\n",
    "    # And remember this is width and height so we have to add those appropriately.\n",
    "    drawing.rectangle((x,y,x+w,y+h), outline=\"white\")\n",
    "display(pil_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Awesome! We managed to detect a bunch of faces in that image. Looks like we have missed \n",
    "# four faces. In the machine learning world we would call these false negatives - something\n",
    "# which the machine thought was not a face (so a negative), but that it was incorrect on.\n",
    "# Consequently, we would call the actual faces that were detected as true positives -\n",
    "# something that the machine thought was a face and it was correct on. This leaves us with\n",
    "# false positives - something the machine thought was a face but it wasn't. We see there are\n",
    "# two of these in the image, picking up shadow patterns or textures in shirts and matching\n",
    "# them with the haarcascades. Finally, we have true negatives, or the set of all possible\n",
    "# rectangles the machine learning classifier could consider where it correctly indicated that\n",
    "# the result was not a face. In this case there are many many true negatives."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There are a few ways we could try and improve this, and really, it requires a lot of \n",
    "# experimentation to find good values for a given image. First, lets create a function\n",
    "# which will plot rectanges for us over the image\n",
    "def show_rects(faces):\n",
    "    #Lets read in our gif and convert it\n",
    "    pil_img=Image.open('readonly/msi_recruitment.gif').convert(\"RGB\")\n",
    "    # Set our drawing context\n",
    "    drawing=ImageDraw.Draw(pil_img)\n",
    "    # And plot all of the rectangles in faces\n",
    "    for x,y,w,h in faces:\n",
    "        drawing.rectangle((x,y,x+w,y+h), outline=\"white\")\n",
    "    #Finally lets display this\n",
    "    display(pil_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok, first up, we could try and binarize this image. It turns out that opencv has a built in\n",
    "# binarization function called threshold(). You simply pass in the image, the midpoint, and\n",
    "# the maximum value, as well as a flag which indicates whether the threshold should be\n",
    "# binary or something else. Lets try this.\n",
    "cv_img_bin=cv.threshold(img,120,255,cv.THRESH_BINARY)[1] # returns a list, we want the second value\n",
    "# Now do the actual face detection\n",
    "faces = face_cascade.detectMultiScale(cv_img_bin)\n",
    "# Now lets see the results\n",
    "show_rects(faces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# That's kind of interesting. Not better, but we do see that there is one false positive\n",
    "# towards the bottom, where the classifier detected the sunglasses as eyes and the dark shadow\n",
    "# line below as a mouth.\n",
    "#\n",
    "# If you're following in the notebook with this video, why don't you pause things and try a\n",
    "# few different parameters for the thresholding value?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The detectMultiScale() function from OpenCV also has a couple of parameters. The first of\n",
    "# these is the scale factor. The scale factor changes the size of rectangles which are\n",
    "# considered against the model, that is, the haarcascades XML file. You can think of it as if\n",
    "# it were changing the size of the rectangles which are on the screen.\n",
    "#\n",
    "# Lets experiment with the scale factor. Usually it's a small value, lets try 1.05\n",
    "faces = face_cascade.detectMultiScale(cv_img,1.05)\n",
    "# Show those results\n",
    "show_rects(faces)\n",
    "# Now lets also try 1.15\n",
    "faces = face_cascade.detectMultiScale(cv_img,1.15)\n",
    "# Show those results\n",
    "show_rects(faces)\n",
    "# Finally lets also try 1.25\n",
    "faces = face_cascade.detectMultiScale(cv_img,1.25)\n",
    "# Show those results\n",
    "show_rects(faces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can see that as we change the scale factor we change the number of true and \n",
    "# false positives and negatives. With the scale set to 1.05, we have 7 true positives,\n",
    "# which are correctly identified faces, and 3 false negatives, which are faces which\n",
    "# are there but not detected, and 3 false positives, where are non-faces which\n",
    "# opencv thinks are faces. When we change this to 1.15 we lose the false positives but\n",
    "# also lose one of the true positives, the person to the right wearing a hat. And\n",
    "# when we change this to 1.25 we lost more true positives as well.\n",
    "#\n",
    "# This is actually a really interesting phenomena in machine learning and artificial\n",
    "# intelligence. There is a trade off between not only how accurate a model is, but how\n",
    "# the inaccuracy actually happens. Which of these three models do you think is best?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Well, the answer to that question is really, \"it depends\". It depends why you are trying\n",
    "# to detect faces, and what you are going to do with them. If you think these issues\n",
    "# are interesting, you might want to check out the Applied Data Science with Python\n",
    "# specialization Michigan offers on Coursera.\n",
    "#\n",
    "# Ok, beyond an opportunity to advertise, did you notice anything else that happened when\n",
    "# we changed the scale factor? It's subtle, but the speed at which the processing ran\n",
    "# took longer at smaller scale factors. This is because more subimages are being considered\n",
    "# for these scales. This could also affect which method we might use.\n",
    "#\n",
    "# Jupyter has nice support for timing commands. You might have seen this before, a line\n",
    "# that starts with a percentage sign in jupyter is called a \"magic function\". This isn't\n",
    "# normal python - it's actually a shorthand way of writing a function which Jupyter\n",
    "# has predefined. It looks a lot like the decorators we talked about in a previous\n",
    "# lecture, but the magic functions were around long before decorators were part of the\n",
    "# python language. One of the built-in magic functions in juptyer is called timeit, and this\n",
    "# repeats a piece of python ten times (by default) and tells you the average speed it\n",
    "# took to complete.\n",
    "#\n",
    "# Lets time the speed of detectmultiscale when using a scale of 1.05\n",
    "%timeit face_cascade.detectMultiScale(cv_img,1.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok, now lets compare that to the speed at scale = 1.15\n",
    "%timeit face_cascade.detectMultiScale(cv_img,1.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can see that this is a dramatic difference, roughly two and a half times slower\n",
    "# when using the smaller scale!\n",
    "#\n",
    "# This wraps up our discussion of detecting faces in opencv. You'll see that, like OCR, this\n",
    "# is not a foolproof process. But we can build on the work others have done in machine learning\n",
    "# and leverage powerful libraries to bring us closer to building a turn key python-based\n",
    "# solution. Remember that the detection mechanism isn't specific to faces, that's just the\n",
    "# haarcascades training data we used. On the web you'll be able to find other training data\n",
    "# to detect other objects, including eyes, animals, and so forth."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## More Jupyter Widgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'ipywebrtc'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-5a35d324c168>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;31m# First, lets import from this library two different classes which we'll use in a\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;31m# demo, one for the camera and one for images.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mipywebrtc\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mCameraStream\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mImageRecorder\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[1;31m# Then lets take a look at the camera stream object\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[0mhelp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCameraStream\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'ipywebrtc'"
     ]
    }
   ],
   "source": [
    "# One of the nice things about using the Jupyter notebook systems is that there is a\n",
    "# rich set of contributed plugins that seek to extend this system. In this lecture I\n",
    "# want to introduce you to one such plugin, call ipy web rtc. Webrtc is a fairly new\n",
    "# protocol for real time communication on the web. Yup, I'm talking about chatting.\n",
    "# The widget brings this to the Jupyter notebook system. Lets take a look.\n",
    "#\n",
    "# First, lets import from this library two different classes which we'll use in a\n",
    "# demo, one for the camera and one for images.\n",
    "from ipywebrtc import CameraStream, ImageRecorder\n",
    "# Then lets take a look at the camera stream object\n",
    "help(CameraStream)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We see from the docs that it's east to get a camera facing the user, and we can have\n",
    "# the audio on or off. We don't need audio for this demo, so lets create a new camera\n",
    "# instance\n",
    "camera = CameraStream.facing_user(audio=False)\n",
    "# The next object we want to look at is the ImageRecorder\n",
    "help(ImageRecorder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The image recorder lets us actually grab images from the camera stream. There are features\n",
    "# for downloading and using the image as well. We see that the default format is a png file.\n",
    "# Lets hook up the ImageRecorder to our stream\n",
    "image_recorder = ImageRecorder(stream=camera)\n",
    "# Now, the docs are a little unclear how to use this within Jupyter, but if we call the\n",
    "# download() function it will actually store the results of the camera which is hooked up\n",
    "# in image_recorder.image. Lets try it out\n",
    "# First, lets tell the recorder to start capturing data\n",
    "image_recorder.recording=True\n",
    "# Now lets download the image\n",
    "image_recorder.download()\n",
    "# Then lets inspect the type of the image\n",
    "type(image_recorder.image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok, the object that it stores is an ipywidgets.widgets.widget_media.Image. How do we do\n",
    "# something useful with this? Well, an inspection of the object shows that there is a handy\n",
    "# value field which actually holds the bytes behind the image. And we know how to display\n",
    "# those.\n",
    "# Lets import PIL Image\n",
    "import PIL.Image\n",
    "# And lets import io\n",
    "import io\n",
    "# And now lets create a PIL image from the bytes\n",
    "img = PIL.Image.open(io.BytesIO(image_recorder.image.value))\n",
    "# And render it to the screen\n",
    "display(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Great, you see a picture! Hopefully you are following along in one of the notebooks\n",
    "# and have been able to try this out for yourself!\n",
    "#\n",
    "# What can you do with this? This is a great way to get started with a bit of computer vision.\n",
    "# You already know how to identify a face in the webcam picture, or try and capture text\n",
    "# from within the picture. With OpenCV there are any number of other things you can do, simply\n",
    "# with a webcam, the Jupyter notebooks, and python!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}